diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index 38cbcb0e9e5..e421618325a 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -4377,6 +4377,121 @@ jobs: docker logout docker.io || true rm -rf "${DOCKER_CONFIG}" + tunnel-lifecycle-vitest: + needs: generate-matrix + if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',tunnel-lifecycle-vitest,') || contains(format(',{0},', inputs.scenarios), ',tunnel-lifecycle,') }} + runs-on: ubuntu-latest + timeout-minutes: 75 + env: + FREE_STANDING_VITEST_JOB: "1" + FREE_STANDING_SCENARIO_ID: "tunnel-lifecycle" + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/tunnel-lifecycle + NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js + NEMOCLAW_RUN_E2E_SCENARIOS: "1" + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_SANDBOX_NAME: "e2e-tunnel-lifecycle" + OPENSHELL_GATEWAY: "nemoclaw" + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Configure isolated Docker auth directory + run: echo "DOCKER_CONFIG=${RUNNER_TEMP}/docker-config-tunnel-lifecycle" >> "$GITHUB_ENV" + + - name: Authenticate to Docker Hub + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + shell: bash + run: | + set -euo pipefail + if [[ -z "${DOCKERHUB_USERNAME}" || -z "${DOCKERHUB_TOKEN}" ]]; then + echo "::notice::Docker Hub credentials not configured; continuing with anonymous pulls." + exit 0 + fi + mkdir -p "${DOCKER_CONFIG}" + chmod 700 "${DOCKER_CONFIG}" + login_succeeded=0 + for attempt in 1 2 3; do + if echo "${DOCKERHUB_TOKEN}" | timeout 30s docker login docker.io --username "${DOCKERHUB_USERNAME}" --password-stdin; then + login_succeeded=1 + break + fi + if [[ "$attempt" -lt 3 ]]; then + echo "::warning::Docker Hub login attempt ${attempt} failed; retrying." + sleep 5 + fi + done + if [[ "$login_succeeded" -ne 1 ]]; then + echo "::warning::Docker Hub login failed after 3 attempts; continuing with anonymous pulls." + fi + + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: 22 + cache: npm + + - name: Install root dependencies + run: npm ci --ignore-scripts + + - name: Build CLI + run: npm run build:cli + + - name: Install and verify cloudflared prerequisite + run: | + set -euo pipefail + if command -v cloudflared >/dev/null 2>&1; then + cloudflared --version + exit 0 + fi + source test/e2e/lib/cloudflared-version-resolver.sh + sudo mkdir -p --mode=0755 /usr/share/keyrings + curl -fsSL https://pkg.cloudflare.com/cloudflare-main.gpg | sudo tee /usr/share/keyrings/cloudflare-main.gpg >/dev/null + echo "deb [signed-by=/usr/share/keyrings/cloudflare-main.gpg] https://pkg.cloudflare.com/cloudflared $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/cloudflared.list >/dev/null + sudo apt-get update -qq + available_versions="$(apt-cache madison cloudflared | awk '{print $3}')" + cf_min_version="${CLOUDFLARED_MIN_VERSION:-$CLOUDFLARED_DEFAULT_MIN_VERSION}" + if [ -n "${CLOUDFLARED_VERSION:-}" ]; then + cf_version="$(cloudflared_resolve_package_version "$available_versions" "$cf_min_version" "$CLOUDFLARED_VERSION")" + else + cf_version="$(cloudflared_resolve_package_version "$available_versions" "$cf_min_version")" + fi + sudo apt-get install -y "cloudflared=${cf_version}" + cloudflared --version + + - name: Run tunnel lifecycle live test + # Migrated from test/e2e/test-tunnel-lifecycle.sh. This preserves the + # real Docker/OpenShell onboard, host cloudflared quick-tunnel, + # local-dashboard readiness, public tunnel probe, and stop/status + # cleanup boundaries under Vitest. + env: + NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} + run: | + set -euo pipefail + npx vitest run --project e2e-scenarios-live \ + test/e2e-scenario/live/tunnel-lifecycle.test.ts \ + --silent=false --reporter=default + + - name: Upload tunnel lifecycle artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-vitest-scenarios-tunnel-lifecycle + path: e2e-artifacts/vitest/tunnel-lifecycle/ + include-hidden-files: false + if-no-files-found: ignore + retention-days: 14 + + - name: Clean up Docker auth + if: always() + run: | + set -euo pipefail + docker logout docker.io || true + rm -rf "${DOCKER_CONFIG}" + # ── PR result comment (mirrors nightly-e2e.yaml's report-to-pr) ─────────── # Posts a results table on the open PR for the dispatching branch (or the # PR identified by `inputs.pr_number`). `if: always()` so the comment lands @@ -4444,6 +4559,7 @@ jobs: gateway-health-honest-vitest, device-auth-health-vitest, channels-add-remove-vitest, + tunnel-lifecycle-vitest, telegram-injection-vitest, channels-stop-start-vitest, ] diff --git a/test/cli/sandbox-status-json.test.ts b/test/cli/sandbox-status-json.test.ts index c3f3ffc5603..daf8de10fba 100644 --- a/test/cli/sandbox-status-json.test.ts +++ b/test/cli/sandbox-status-json.test.ts @@ -7,9 +7,9 @@ import net from "node:net"; import os from "node:os"; import path from "node:path"; -import { runWithEnv, writeSandboxRegistry } from "./helpers"; +import { runWithEnv, testTimeoutOptions, writeSandboxRegistry } from "./helpers"; -describe("CLI sandbox status JSON output", () => { +describe("CLI sandbox status JSON output", testTimeoutOptions(20_000), () => { it("sandbox status --json emits structured per-sandbox report", () => { const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-sandbox-status-json-")); const localBin = path.join(home, "bin"); diff --git a/test/cloudflared-version-resolver.test.ts b/test/cloudflared-version-resolver.test.ts index 12a820a4190..750bc345cd0 100644 --- a/test/cloudflared-version-resolver.test.ts +++ b/test/cloudflared-version-resolver.test.ts @@ -23,6 +23,7 @@ if [[ "\${1:-}" != "--compare-versions" ]]; then fi rank() { case "\${1:-}" in + 2020.1.1) printf '20200101' ;; 2026.4.30) printf '20260430' ;; 2026.5.1~rc1) printf '20260500' ;; 2026.5.1) printf '20260501' ;; @@ -89,13 +90,21 @@ describe("cloudflared APT package resolver", () => { expect(result.stderr).toContain("meets minimum 2026.5.1"); }); - it("preserves exact CLOUDFLARED_VERSION overrides for emergency repro", () => { + it("preserves syntactically valid exact CLOUDFLARED_VERSION overrides for emergency repro", () => { const result = runResolver("2026.5.1\n2026.5.10", "2026.5.1", "2020.1.1"); expect(result.status).toBe(0); expect(result.stdout.trim()).toBe("2020.1.1"); }); + it("rejects invalid CLOUDFLARED_VERSION overrides before apt install", () => { + const result = runResolver("2026.5.1\n2026.5.10", "2026.5.1", "bad/min"); + + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("invalid CLOUDFLARED_VERSION"); + expect(result.stdout.trim()).toBe(""); + }); + it("rejects invalid minimum versions before comparing package versions", () => { const result = runResolver("2026.5.1", "bad/min"); diff --git a/test/e2e-scenario/live/tunnel-lifecycle-helpers.ts b/test/e2e-scenario/live/tunnel-lifecycle-helpers.ts new file mode 100644 index 00000000000..587be74ffe1 --- /dev/null +++ b/test/e2e-scenario/live/tunnel-lifecycle-helpers.ts @@ -0,0 +1,479 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Live Vitest replacement for test/e2e/test-tunnel-lifecycle.sh. + * + * Preserves the legacy real boundaries: Docker/OpenShell onboarding, the + * installed/source NemoClaw CLI, host `cloudflared`, the local dashboard origin, + * public trycloudflare reachability, cloudflared log diagnosis, and tunnel stop + * cleanup/status removal. + */ + +import fs from "node:fs"; +import path from "node:path"; + +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { resultText } from "../fixtures/clients/index.ts"; +import { validateSandboxName } from "../fixtures/clients/sandbox.ts"; +import type { E2EScenarioFixtures } from "../fixtures/e2e-test.ts"; +import { expect } from "../fixtures/e2e-test.ts"; +import { requireHostedInferenceConfig } from "../fixtures/hosted-inference.ts"; +import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; + +const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); +const TEST_SANDBOX_PREFIX = "e2e-tunnel-lifecycle"; +const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? TEST_SANDBOX_PREFIX; +const LOCAL_DASHBOARD_PORT = process.env.NEMOCLAW_DASHBOARD_PORT ?? "18789"; +const TEST_TIMEOUT_MS = Number(process.env.NEMOCLAW_E2E_TIMEOUT_SECONDS ?? 3_600) * 1_000; +const ONBOARD_TIMEOUT_MS = 30 * 60_000; +const COMMAND_TIMEOUT_MS = 60_000; +const TUNNEL_URL_PATTERN = /https:\/\/[a-z0-9-]+\.trycloudflare\.com\b[\w./?%&=-]*/i; +const DASHBOARD_MARKER_PATTERN = /OpenClaw Control<\/title>|<openclaw-app/i; + +validateSandboxName(SANDBOX_NAME); + +type CurlProbe = { + httpCode: string; + body: string; + result: ShellProbeResult; +}; + +function assertTestOwnedSandboxName(): void { + if (!SANDBOX_NAME.startsWith(TEST_SANDBOX_PREFIX)) { + throw new Error( + `tunnel-lifecycle live test is destructive and only accepts sandbox names with prefix ${TEST_SANDBOX_PREFIX}; got ${SANDBOX_NAME}`, + ); + } +} + +function sleep(ms: number): Promise<void> { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function commandEnv(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { + return { + ...buildAvailabilityProbeEnv(), + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, + NEMOCLAW_POLICY_TIER: "open", + NEMOCLAW_AGENT: "openclaw", + NEMOCLAW_PROVIDER: "cloud", + OPENSHELL_GATEWAY: "nemoclaw", + ...(process.env.NEMOCLAW_DASHBOARD_PORT + ? { NEMOCLAW_DASHBOARD_PORT: process.env.NEMOCLAW_DASHBOARD_PORT } + : {}), + ...extra, + }; +} + +function isCloudflareTransientText(text: string): boolean { + return /failed to unmarshal quick Tunnel|quick tunnels? (are )?(temporarily )?disabled|failed to (dial|register)|tunnel server.*error|i\/o timeout|EOF.*tunnel|couldn.?t start tunnel|tunnel creation failed|bad gateway|\b50[234]\b/i.test( + text, + ); +} + +function isCloudflareTransientHttpCode(code: string): boolean { + return ["000", "502", "503", "504"].includes(code); +} + +export function getCloudflaredLogPath( + logRoot = "/tmp", + sandboxName = SANDBOX_NAME, +): string | undefined { + // Source boundary: NemoClaw owns the per-sandbox cloudflared service log at + // /tmp/nemoclaw-services-${sandboxName}/cloudflared.log. If that exact file + // is missing, this live contract classifies the invalid state as + // `nemoclaw_no_spawn` instead of falling back to the newest /tmp log, because + // unrelated parallel/stale sandboxes can otherwise corrupt fault attribution. + // Remove this filesystem fallback point entirely once NemoClaw exposes + // machine-readable tunnel diagnostics from `nemoclaw status --json`. + const sandboxLog = path.join(logRoot, `nemoclaw-services-${sandboxName}`, "cloudflared.log"); + return fs.existsSync(sandboxLog) ? sandboxLog : undefined; +} + +function readCloudflaredLog(): string { + const logPath = getCloudflaredLogPath(); + if (!logPath) return ""; + return fs.readFileSync(logPath, "utf8"); +} + +function cloudflaredLogTail(lines = 80): string { + const logPath = getCloudflaredLogPath(); + if (!logPath) return "(no cloudflared.log found under /tmp/nemoclaw-services-*/)"; + const text = fs.readFileSync(logPath, "utf8"); + return [ + `--- cloudflared.log (${logPath}, last ${lines} lines) ---`, + ...text.split(/\r?\n/).slice(-lines), + ].join("\n"); +} + +export function classifyCloudflaredLog( + logRoot = "/tmp", + sandboxName = SANDBOX_NAME, +): "nemoclaw_no_spawn" | "nemoclaw_capture_bug" | "nemoclaw_local" | "cloudflare" | "unknown" { + const logPath = getCloudflaredLogPath(logRoot, sandboxName); + if (!logPath) return "nemoclaw_no_spawn"; + const log = fs.readFileSync(logPath, "utf8"); + if (TUNNEL_URL_PATTERN.test(log)) return "nemoclaw_capture_bug"; + if ( + /unable to reach the origin|connection refused.*127\.0\.0\.1|connection refused.*localhost|dial tcp.*127\.0\.0\.1.*refused/i.test( + log, + ) + ) { + return "nemoclaw_local"; + } + if (isCloudflareTransientText(log)) return "cloudflare"; + return "unknown"; +} + +function extractTunnelUrl(text: string): string | undefined { + return text.match(TUNNEL_URL_PATTERN)?.[0]; +} + +export function publicTunnelProbeCurlArgs(tunnelUrl: string): string[] { + // Source boundary: the public tunnel URL already came from `nemoclaw status` + // and matched `*.trycloudflare.com`. Do not ask curl to follow redirects; + // a 3xx response is a tunnel/output contract failure unless NemoClaw grows a + // documented same-host redirect requirement. If that happens, replace this + // with explicit redirect target inspection before issuing a second request. + return ["-sS", "--max-time", "30", "-w", "\n__HTTP_CODE:%{http_code}\n", tunnelUrl]; +} + +function parseCurlProbe(result: ShellProbeResult): CurlProbe { + const text = result.stdout; + const match = text.match(/\n__HTTP_CODE:(\d{3})\s*$/); + const httpCode = match?.[1] ?? "000"; + const body = match ? text.slice(0, match.index) : text; + return { httpCode, body, result }; +} + +async function bestEffort(run: () => Promise<unknown>): Promise<void> { + try { + await run(); + } catch { + // Inline recovery remains best-effort so the primary E2E failure stays visible. + } +} + +function isBenignTunnelStopFailure(text: string): boolean { + return /no active tunnel|no tunnel.*running|tunnel.*not.*running|already stopped|cloudflared.*not.*running|no cloudflared/i.test( + text, + ); +} + +export const TUNNEL_LIFECYCLE_TEST_TIMEOUT_MS = TEST_TIMEOUT_MS; + +type TunnelLifecycleFixtures = Pick< + E2EScenarioFixtures, + "artifacts" | "cleanup" | "host" | "secrets" +> & { + skip: (note?: string) => never; +}; + +type TunnelLifecycleCleanupHost = Pick<E2EScenarioFixtures["host"], "cleanupSandbox" | "nemoclaw">; + +type TunnelLifecycleCleanupRegistry = Pick<E2EScenarioFixtures["cleanup"], "add">; + +export function registerTunnelLifecycleCleanup( + cleanup: TunnelLifecycleCleanupRegistry, + host: TunnelLifecycleCleanupHost, +): void { + // CleanupRegistry runs callbacks in reverse registration order. Register the + // sandbox destroy first so host `cloudflared` is stopped before the sandbox is + // torn down on early failures. Source boundary: `nemoclaw tunnel stop` owns + // quick-tunnel process cleanup; `cleanupSandbox` owns the Docker/OpenShell + // sandbox and only suppresses already-missing sandboxes. Keep both callbacks + // strict so unexpected cleanup failures surface in cleanup.json. Removal + // condition: replace this ordering guard once NemoClaw exposes one atomic + // machine-readable lifecycle cleanup that stops tunnels before destroying the + // sandbox. + cleanup.add(`destroy sandbox ${SANDBOX_NAME}`, async () => { + if (process.env.NEMOCLAW_E2E_KEEP_SANDBOX === "1") return; + await host.cleanupSandbox(SANDBOX_NAME, { + artifactName: "cleanup-nemoclaw-destroy-tunnel-lifecycle", + timeoutMs: 15 * 60_000, + }); + }); + cleanup.add("stop cloudflared quick tunnel", async () => { + const stop = await host.nemoclaw(["tunnel", "stop"], { + artifactName: "cleanup-tunnel-stop", + env: commandEnv(), + timeoutMs: COMMAND_TIMEOUT_MS, + }); + if (stop.exitCode === 0) return; + const text = resultText(stop); + if (isBenignTunnelStopFailure(text)) return; + throw new Error( + `[NemoClaw fault] cleanup tunnel stop failed with exit ${stop.exitCode ?? "unknown"}: ${text}`, + ); + }); +} + +export async function runTunnelLifecycleContract({ + artifacts, + cleanup, + host, + secrets, + skip, +}: TunnelLifecycleFixtures): Promise<void> { + assertTestOwnedSandboxName(); + const hosted = requireHostedInferenceConfig(secrets); + const apiKey = hosted.apiKey; + + await artifacts.writeJson("contract.json", { + legacySource: "test/e2e/test-tunnel-lifecycle.sh", + sandboxName: SANDBOX_NAME, + localDashboardPort: LOCAL_DASHBOARD_PORT, + preservedBoundaries: [ + "real Docker/OpenShell OpenClaw sandbox onboarding", + "host cloudflared binary and quick-tunnel registration", + "nemoclaw tunnel start/status/stop CLI commands", + "local dashboard origin readiness before tunnel attribution", + "public trycloudflare HTTP probe with dashboard marker assertion", + "cloudflared.log classification for NemoClaw-vs-Cloudflare failures", + ], + inferenceCredential: hosted.contractLabel, + }); + + registerTunnelLifecycleCleanup(cleanup, host); + + const docker = await host.command("docker", ["info"], { + artifactName: "prereq-docker-info-tunnel-lifecycle", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + if (docker.exitCode !== 0) { + if (process.env.GITHUB_ACTIONS === "true") { + throw new Error(`Docker is required for tunnel lifecycle E2E: ${resultText(docker)}`); + } + skip("Docker is required for tunnel lifecycle E2E"); + } + + const cloudflared = await host.command("cloudflared", ["--version"], { + artifactName: "prereq-cloudflared-version", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + if (cloudflared.exitCode !== 0) { + if (process.env.GITHUB_ACTIONS === "true") { + throw new Error( + `cloudflared is required for tunnel lifecycle E2E: ${resultText(cloudflared)}`, + ); + } + skip("cloudflared is required for tunnel lifecycle E2E"); + } + + expect(fs.existsSync(path.join(REPO_ROOT, "install.sh"))).toBe(true); + await host.bestEffortCleanupSandbox(SANDBOX_NAME, { + artifactName: "pre-cleanup-nemoclaw-destroy-tunnel-lifecycle", + timeoutMs: 15 * 60_000, + }); + + const install = await host.command( + "bash", + ["install.sh", "--non-interactive", "--yes-i-accept-third-party-software"], + { + artifactName: "install-sh-tunnel-lifecycle", + cwd: REPO_ROOT, + env: commandEnv({ + ...hosted.env, + NVIDIA_INFERENCE_API_KEY: apiKey, + NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1", + }), + redactionValues: [apiKey], + timeoutMs: ONBOARD_TIMEOUT_MS, + }, + ); + expect(install.exitCode, resultText(install)).toBe(0); + + await host.expectListed(SANDBOX_NAME, { artifactName: "post-install-nemoclaw-list" }); + + let localReady = false; + for (let attempt = 1; attempt <= 30; attempt += 1) { + const local = await host.command( + "curl", + [ + "-sS", + "-o", + "/dev/null", + "-w", + "%{http_code}", + "--max-time", + "5", + `http://localhost:${LOCAL_DASHBOARD_PORT}/`, + ], + { + artifactName: `local-dashboard-ready-${attempt}`, + env: buildAvailabilityProbeEnv(), + timeoutMs: 10_000, + }, + ); + const code = local.stdout.trim() || "000"; + if (code !== "000") { + localReady = true; + break; + } + await sleep(1_000); + } + expect( + localReady, + `[NemoClaw fault] Local OpenClaw dashboard not reachable on localhost:${LOCAL_DASHBOARD_PORT} after 30s; tunnel cannot proxy a dead origin.`, + ).toBe(true); + + const start = await host.nemoclaw(["tunnel", "start"], { + artifactName: "tunnel-start", + env: commandEnv(), + timeoutMs: 90_000, + }); + if (start.exitCode !== 0) { + await artifacts.writeText("cloudflared-log-after-start-failure.txt", cloudflaredLogTail()); + if (isCloudflareTransientText(resultText(start)) || classifyCloudflaredLog() === "cloudflare") { + await bestEffort(() => + host.nemoclaw(["tunnel", "stop"], { + artifactName: "tunnel-stop-after-cloudflare-start-failure", + env: commandEnv(), + timeoutMs: COMMAND_TIMEOUT_MS, + }), + ); + skip( + `[Cloudflare fault] nemoclaw tunnel start exited ${start.exitCode ?? "unknown"} because quick-tunnel registration returned a transient external error.`, + ); + } + throw new Error( + `[NemoClaw fault] nemoclaw tunnel start failed with exit ${start.exitCode ?? "unknown"}: ${resultText(start)}`, + ); + } + + let tunnelUrl: string | undefined; + let lastStatusText = ""; + for (let attempt = 1; attempt <= 15; attempt += 1) { + const status = await host.nemoclaw(["status"], { + artifactName: `status-with-tunnel-url-${attempt}`, + env: commandEnv(), + timeoutMs: COMMAND_TIMEOUT_MS, + }); + lastStatusText = resultText(status); + tunnelUrl = extractTunnelUrl(lastStatusText); + if (tunnelUrl) break; + await sleep(1_000); + } + + if (!tunnelUrl) { + await artifacts.writeText("cloudflared-log-without-status-url.txt", cloudflaredLogTail()); + const cfClass = classifyCloudflaredLog(); + await bestEffort(() => + host.nemoclaw(["tunnel", "stop"], { + artifactName: "tunnel-stop-after-missing-url", + env: commandEnv(), + timeoutMs: COMMAND_TIMEOUT_MS, + }), + ); + if (cfClass === "cloudflare") { + skip("[Cloudflare fault] cloudflared failed to register a quick tunnel URL."); + } + let reason: string; + switch (cfClass) { + case "nemoclaw_no_spawn": + reason = "cloudflared.log missing — NemoClaw failed to spawn the cloudflared process"; + break; + case "nemoclaw_capture_bug": + reason = "cloudflared.log has a trycloudflare URL but nemoclaw status did not surface it"; + break; + case "nemoclaw_local": + reason = `cloudflared.log reports it cannot reach localhost:${LOCAL_DASHBOARD_PORT}`; + break; + default: + reason = `tunnel URL did not surface and cloudflared.log did not match a known pattern; status was:\n${lastStatusText}`; + } + throw new Error(`[NemoClaw fault] ${reason}`); + } + + let lastPublicProbe: CurlProbe | undefined; + let backoffMs = 2_000; + for (let attempt = 1; attempt <= 15; attempt += 1) { + const probe = parseCurlProbe( + await host.command("curl", publicTunnelProbeCurlArgs(tunnelUrl), { + artifactName: `public-tunnel-probe-${attempt}`, + env: buildAvailabilityProbeEnv(), + timeoutMs: 35_000, + }), + ); + lastPublicProbe = probe; + if (probe.httpCode === "200") break; + + const local = await host.command( + "curl", + [ + "-sS", + "-o", + "/dev/null", + "-w", + "%{http_code}", + "--max-time", + "5", + `http://localhost:${LOCAL_DASHBOARD_PORT}/`, + ], + { + artifactName: `local-dashboard-recheck-${attempt}`, + env: buildAvailabilityProbeEnv(), + timeoutMs: 10_000, + }, + ); + const localCode = local.stdout.trim() || "000"; + if (localCode === "000") { + throw new Error( + `[NemoClaw fault] Tunnel returned ${probe.httpCode} and local dashboard regressed during retry loop; likely sandbox/dashboard crash, not Cloudflare.`, + ); + } + await sleep(backoffMs); + backoffMs = Math.min(backoffMs * 2, 30_000); + } + + expect(lastPublicProbe, "public tunnel probe should have run").toBeTruthy(); + if (lastPublicProbe!.httpCode !== "200") { + if ( + isCloudflareTransientHttpCode(lastPublicProbe!.httpCode) || + isCloudflareTransientText(lastPublicProbe!.body) || + isCloudflareTransientText(readCloudflaredLog()) + ) { + skip( + `[Cloudflare fault] Tunnel URL never became reachable while local stayed healthy; last HTTP status ${lastPublicProbe!.httpCode}.`, + ); + } + throw new Error( + `[NemoClaw fault] Tunnel returned unexpected HTTP ${lastPublicProbe!.httpCode} while local stayed healthy; body prefix: ${lastPublicProbe!.body.slice(0, 200)}`, + ); + } + expect(lastPublicProbe!.body, "public tunnel must serve OpenClaw dashboard markers").toMatch( + DASHBOARD_MARKER_PATTERN, + ); + + const stop = await host.nemoclaw(["tunnel", "stop"], { + artifactName: "tunnel-stop", + env: commandEnv(), + timeoutMs: COMMAND_TIMEOUT_MS, + }); + expect(stop.exitCode, resultText(stop)).toBe(0); + + let postStopUrl: string | undefined; + let statusReadable = false; + for (let attempt = 1; attempt <= 10; attempt += 1) { + const status = await host.nemoclaw(["status"], { + artifactName: `status-after-tunnel-stop-${attempt}`, + env: commandEnv(), + timeoutMs: COMMAND_TIMEOUT_MS, + }); + if (status.exitCode !== 0) { + await sleep(1_000); + continue; + } + statusReadable = true; + postStopUrl = extractTunnelUrl(resultText(status)); + if (!postStopUrl) break; + await sleep(1_000); + } + expect(statusReadable, "nemoclaw status should be readable after tunnel stop").toBe(true); + expect(postStopUrl, "tunnel URL must be absent after nemoclaw tunnel stop").toBeUndefined(); +} diff --git a/test/e2e-scenario/live/tunnel-lifecycle.test.ts b/test/e2e-scenario/live/tunnel-lifecycle.test.ts new file mode 100644 index 00000000000..fd257436fa9 --- /dev/null +++ b/test/e2e-scenario/live/tunnel-lifecycle.test.ts @@ -0,0 +1,24 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Live Vitest replacement for test/e2e/test-tunnel-lifecycle.sh. + * + * Preserves the legacy real boundaries: Docker/OpenShell onboarding, the + * installed/source NemoClaw CLI, host `cloudflared`, the local dashboard origin, + * public trycloudflare reachability, cloudflared log diagnosis, and tunnel stop + * cleanup/status removal. + */ + +import { test } from "../fixtures/e2e-test.ts"; +import { shouldRunLiveE2EScenarios } from "../fixtures/live-project-gate.ts"; +import { + runTunnelLifecycleContract, + TUNNEL_LIFECYCLE_TEST_TIMEOUT_MS, +} from "./tunnel-lifecycle-helpers.ts"; + +test.skipIf(!shouldRunLiveE2EScenarios())( + "tunnel-lifecycle: cloudflared quick tunnel starts, serves OpenClaw, and stops cleanly", + { timeout: TUNNEL_LIFECYCLE_TEST_TIMEOUT_MS }, + runTunnelLifecycleContract, +); diff --git a/test/e2e-scenario/support-tests/tunnel-lifecycle-helpers.test.ts b/test/e2e-scenario/support-tests/tunnel-lifecycle-helpers.test.ts new file mode 100644 index 00000000000..e683b614019 --- /dev/null +++ b/test/e2e-scenario/support-tests/tunnel-lifecycle-helpers.test.ts @@ -0,0 +1,191 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { CleanupRegistry } from "../fixtures/cleanup.ts"; +import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; +import { + classifyCloudflaredLog, + getCloudflaredLogPath, + publicTunnelProbeCurlArgs, + registerTunnelLifecycleCleanup, +} from "../live/tunnel-lifecycle-helpers.ts"; + +function shellResult(overrides: Partial<ShellProbeResult> = {}): ShellProbeResult { + return { + command: ["nemoclaw"], + exitCode: 0, + signal: null, + timedOut: false, + stdout: "", + stderr: "", + artifacts: { + stdout: "stdout.txt", + stderr: "stderr.txt", + result: "result.json", + }, + ...overrides, + }; +} + +describe("tunnel lifecycle cleanup registration", () => { + it("stops the tunnel before destroying the sandbox during registered cleanup", async () => { + const calls: string[] = []; + const cleanup = new CleanupRegistry(); + registerTunnelLifecycleCleanup(cleanup, { + cleanupSandbox: async () => { + calls.push("destroy"); + }, + nemoclaw: async () => { + calls.push("stop"); + return shellResult(); + }, + }); + + const result = await cleanup.runAll(); + + expect(result.failures).toEqual([]); + expect(calls).toEqual(["stop", "destroy"]); + }); + + it("surfaces unexpected tunnel-stop cleanup failures", async () => { + const cleanup = new CleanupRegistry(); + registerTunnelLifecycleCleanup(cleanup, { + cleanupSandbox: async () => {}, + nemoclaw: async () => + shellResult({ + exitCode: 1, + stderr: "permission denied while stopping cloudflared", + }), + }); + + const result = await cleanup.runAll(); + + expect(result.failures).toEqual([ + { + name: "stop cloudflared quick tunnel", + message: + "[NemoClaw fault] cleanup tunnel stop failed with exit 1: permission denied while stopping cloudflared", + }, + ]); + }); + + it("surfaces unexpected sandbox-destroy cleanup failures", async () => { + const cleanup = new CleanupRegistry(); + registerTunnelLifecycleCleanup(cleanup, { + cleanupSandbox: async () => { + throw new Error("docker daemon denied sandbox destroy"); + }, + nemoclaw: async () => shellResult(), + }); + + const result = await cleanup.runAll(); + + expect(result.failures).toEqual([ + { + name: "destroy sandbox e2e-tunnel-lifecycle", + message: "docker daemon denied sandbox destroy", + }, + ]); + }); + + it("suppresses already-stopped tunnel cleanup states", async () => { + const cleanup = new CleanupRegistry(); + registerTunnelLifecycleCleanup(cleanup, { + cleanupSandbox: async () => {}, + nemoclaw: async () => shellResult({ exitCode: 1, stderr: "no active tunnel" }), + }); + + const result = await cleanup.runAll(); + + expect(result.failures).toEqual([]); + }); +}); + +describe("tunnel lifecycle cloudflared log attribution", () => { + it("does not follow redirects from the public trycloudflare probe", () => { + expect(publicTunnelProbeCurlArgs("https://current.trycloudflare.com/")).toEqual([ + "-sS", + "--max-time", + "30", + "-w", + "\n__HTTP_CODE:%{http_code}\n", + "https://current.trycloudflare.com/", + ]); + }); + + it("does not attribute an unrelated newer cloudflared log to the current sandbox", () => { + const logRoot = fs.mkdtempSync(path.join(os.tmpdir(), "tunnel-lifecycle-logs-")); + const unrelatedDir = path.join(logRoot, "nemoclaw-services-other-sandbox"); + fs.mkdirSync(unrelatedDir, { recursive: true }); + fs.writeFileSync( + path.join(unrelatedDir, "cloudflared.log"), + "https://unrelated.trycloudflare.com captured by another run\n", + ); + + try { + expect(getCloudflaredLogPath(logRoot, "e2e-tunnel-lifecycle-current")).toBeUndefined(); + expect(classifyCloudflaredLog(logRoot, "e2e-tunnel-lifecycle-current")).toBe( + "nemoclaw_no_spawn", + ); + } finally { + fs.rmSync(logRoot, { recursive: true, force: true }); + } + }); + + it("classifies only the sandbox-specific cloudflared log", () => { + const logRoot = fs.mkdtempSync(path.join(os.tmpdir(), "tunnel-lifecycle-logs-")); + const sandboxDir = path.join(logRoot, "nemoclaw-services-e2e-tunnel-lifecycle-current"); + fs.mkdirSync(sandboxDir, { recursive: true }); + const sandboxLog = path.join(sandboxDir, "cloudflared.log"); + fs.writeFileSync(sandboxLog, "https://current.trycloudflare.com\n"); + + try { + expect(getCloudflaredLogPath(logRoot, "e2e-tunnel-lifecycle-current")).toBe(sandboxLog); + expect(classifyCloudflaredLog(logRoot, "e2e-tunnel-lifecycle-current")).toBe( + "nemoclaw_capture_bug", + ); + } finally { + fs.rmSync(logRoot, { recursive: true, force: true }); + } + }); + + it("classifies localhost/origin-refused logs as a NemoClaw local-origin fault", () => { + const logRoot = fs.mkdtempSync(path.join(os.tmpdir(), "tunnel-lifecycle-logs-")); + const sandboxDir = path.join(logRoot, "nemoclaw-services-e2e-tunnel-lifecycle-current"); + fs.mkdirSync(sandboxDir, { recursive: true }); + fs.writeFileSync( + path.join(sandboxDir, "cloudflared.log"), + 'ERR Request failed error="Unable to reach the origin service. dial tcp 127.0.0.1:18789: connect: connection refused"\n', + ); + + try { + expect(classifyCloudflaredLog(logRoot, "e2e-tunnel-lifecycle-current")).toBe( + "nemoclaw_local", + ); + } finally { + fs.rmSync(logRoot, { recursive: true, force: true }); + } + }); + + it("classifies representative quick-tunnel registration failures as Cloudflare faults", () => { + const logRoot = fs.mkdtempSync(path.join(os.tmpdir(), "tunnel-lifecycle-logs-")); + const sandboxDir = path.join(logRoot, "nemoclaw-services-e2e-tunnel-lifecycle-current"); + fs.mkdirSync(sandboxDir, { recursive: true }); + fs.writeFileSync( + path.join(sandboxDir, "cloudflared.log"), + "ERR failed to unmarshal quick Tunnel response: tunnel server returned 503 bad gateway\n", + ); + + try { + expect(classifyCloudflaredLog(logRoot, "e2e-tunnel-lifecycle-current")).toBe("cloudflare"); + } finally { + fs.rmSync(logRoot, { recursive: true, force: true }); + } + }); +}); diff --git a/test/e2e-scenario/support-tests/tunnel-lifecycle-workflow-boundary.test.ts b/test/e2e-scenario/support-tests/tunnel-lifecycle-workflow-boundary.test.ts new file mode 100644 index 00000000000..35cc3d88e29 --- /dev/null +++ b/test/e2e-scenario/support-tests/tunnel-lifecycle-workflow-boundary.test.ts @@ -0,0 +1,163 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; +import YAML from "yaml"; + +import { + evaluateE2eVitestWorkflowDispatchSelectors, + validateE2eVitestScenariosWorkflowBoundary, +} from "../../../tools/e2e-scenarios/workflow-boundary.mts"; + +function readWorkflow(): Record<string, unknown> { + return YAML.parse( + fs.readFileSync( + path.join(process.cwd(), ".github/workflows/e2e-vitest-scenarios.yaml"), + "utf-8", + ), + ) as Record<string, unknown>; +} + +describe("tunnel lifecycle workflow boundary", () => { + it("maps the tunnel lifecycle selector to its free-standing Vitest job", () => { + expect( + evaluateE2eVitestWorkflowDispatchSelectors({ scenarios: "tunnel-lifecycle" }), + ).toMatchObject({ + valid: true, + liveScenariosRuns: false, + selectedFreeStandingJobs: ["tunnel-lifecycle-vitest"], + registryScenarios: [], + }); + expect( + evaluateE2eVitestWorkflowDispatchSelectors({ jobs: "tunnel-lifecycle-vitest" }), + ).toMatchObject({ + valid: true, + liveScenariosRuns: false, + selectedFreeStandingJobs: ["tunnel-lifecycle-vitest"], + registryScenarios: [], + }); + }); + + it("requires the tunnel lifecycle job to use the repo NemoClaw CLI boundary", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-vitest-workflow-")); + const workflowPath = path.join(tmp, "workflow.yaml"); + const workflow = readWorkflow() as { + jobs: Record<string, { env?: Record<string, unknown> }>; + }; + const job = workflow.jobs["tunnel-lifecycle-vitest"]; + expect(job).toBeDefined(); + job.env = { ...job.env }; + delete job.env.NEMOCLAW_CLI_BIN; + fs.writeFileSync(workflowPath, YAML.stringify(workflow)); + + try { + expect(validateE2eVitestScenariosWorkflowBoundary(workflowPath)).toContain( + "tunnel-lifecycle-vitest job must point NEMOCLAW_CLI_BIN at the repo CLI", + ); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + + it("rejects tunnel lifecycle trusted-boundary drift", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-vitest-workflow-")); + const workflowPath = path.join(tmp, "workflow.yaml"); + const workflow = readWorkflow() as { + jobs: Record< + string, + { env?: Record<string, unknown>; steps: Array<Record<string, unknown>> } + >; + }; + const job = workflow.jobs["tunnel-lifecycle-vitest"]; + expect(job).toBeDefined(); + job.env = { + ...job.env, + DOCKER_CONFIG: "${{ github.workspace }}/e2e-artifacts/vitest/tunnel-lifecycle/docker-config", + }; + + const checkout = job.steps.find((step) => + String(step.uses ?? "").startsWith("actions/checkout@"), + ); + expect(checkout).toBeDefined(); + checkout!.with = { + ...(checkout!.with as Record<string, unknown>), + "persist-credentials": true, + }; + + const configureDockerAuth = job.steps.find( + (step) => step.name === "Configure isolated Docker auth directory", + ); + expect(configureDockerAuth).toBeDefined(); + configureDockerAuth!.run = + 'echo "DOCKER_CONFIG=${{ github.workspace }}/docker-config-tunnel-lifecycle" >> "$GITHUB_ENV"'; + + const install = job.steps.find((step) => step.name === "Install root dependencies"); + expect(install).toBeDefined(); + install!.env = { + NVIDIA_INFERENCE_API_KEY: "${{ secrets.NVIDIA_INFERENCE_API_KEY }}", + NVIDIA_API_KEY: "${{ secrets.NVIDIA_API_KEY }}", + }; + install!.run = "npm install"; + + const cloudflared = job.steps.find( + (step) => step.name === "Install and verify cloudflared prerequisite", + ); + expect(cloudflared).toBeDefined(); + cloudflared!.env = { + NVIDIA_INFERENCE_API_KEY: "${{ secrets.NVIDIA_INFERENCE_API_KEY }}", + NVIDIA_API_KEY: "${{ secrets.NVIDIA_API_KEY }}", + }; + cloudflared!.run = "cloudflared --version"; + + const runTunnel = job.steps.find((step) => step.name === "Run tunnel lifecycle live test"); + expect(runTunnel).toBeDefined(); + runTunnel!.run = `${String(runTunnel!.run ?? "")}\nsudo apt-get install -y cloudflared`; + + const upload = job.steps.find((step) => step.name === "Upload tunnel lifecycle artifacts"); + expect(upload).toBeDefined(); + upload!.with = { + ...(upload!.with as Record<string, unknown>), + path: "e2e-artifacts/vitest/", + "include-hidden-files": true, + }; + + const cleanup = job.steps.find((step) => step.name === "Clean up Docker auth"); + expect(cleanup).toBeDefined(); + cleanup!.if = "success()"; + cleanup!.run = 'set -euo pipefail\necho "missing Docker auth cleanup"\n'; + fs.writeFileSync(workflowPath, YAML.stringify(workflow)); + + try { + expect(validateE2eVitestScenariosWorkflowBoundary(workflowPath)).toEqual( + expect.arrayContaining([ + "tunnel-lifecycle-vitest job must not set DOCKER_CONFIG at job level", + 'step \'Configure isolated Docker auth directory\' run script must include echo "DOCKER_CONFIG=${RUNNER_TEMP}/docker-config-tunnel-lifecycle" >> "$GITHUB_ENV"', + "step 'Configure isolated Docker auth directory' run script must not include ${{ github.workspace }}", + "tunnel-lifecycle-vitest checkout step must set persist-credentials=false", + "tunnel-lifecycle-vitest step 'Install root dependencies' env must not include NVIDIA_INFERENCE_API_KEY", + "tunnel-lifecycle-vitest step 'Install root dependencies' env must not include NVIDIA_API_KEY", + "step 'Install root dependencies' run script must include npm ci --ignore-scripts", + "tunnel-lifecycle-vitest step 'Install and verify cloudflared prerequisite' env must not include NVIDIA_INFERENCE_API_KEY", + "tunnel-lifecycle-vitest step 'Install and verify cloudflared prerequisite' env must not include NVIDIA_API_KEY", + "tunnel-lifecycle-vitest cloudflared prerequisite step env must not include NVIDIA_INFERENCE_API_KEY", + "tunnel-lifecycle-vitest cloudflared prerequisite step env must not include NVIDIA_API_KEY", + "step 'Install and verify cloudflared prerequisite' run script must include test/e2e/lib/cloudflared-version-resolver.sh", + "step 'Install and verify cloudflared prerequisite' run script must include sudo apt-get install -y", + "step 'Install and verify cloudflared prerequisite' run script must include cloudflared=${cf_version}", + "tunnel-lifecycle-vitest Vitest step must not run cloudflared APT installation with NVIDIA_INFERENCE_API_KEY in scope", + "artifact upload path must include e2e-artifacts/vitest/tunnel-lifecycle/", + "tunnel-lifecycle-vitest artifact upload must set include-hidden-files: false", + "tunnel-lifecycle-vitest Docker auth cleanup must always run", + "step 'Clean up Docker auth' run script must include docker logout docker.io", + "step 'Clean up Docker auth' run script must include rm -rf \"${DOCKER_CONFIG}\"", + ]), + ); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); +}); diff --git a/test/e2e/lib/cloudflared-version-resolver.sh b/test/e2e/lib/cloudflared-version-resolver.sh index a56a93d252f..64e88aca36b 100755 --- a/test/e2e/lib/cloudflared-version-resolver.sh +++ b/test/e2e/lib/cloudflared-version-resolver.sh @@ -28,7 +28,12 @@ cloudflared_resolve_package_version() { # Emergency repro knob: install the exact requested version and let APT report # unavailable overrides, rather than silently substituting another package. + # Still validate Debian-version syntax before the sudo apt install boundary. if [[ -n "$override_version" ]]; then + if ! cloudflared_is_debian_version "$override_version"; then + printf 'ERROR: invalid CLOUDFLARED_VERSION %q\n' "$override_version" >&2 + return 1 + fi printf '%s\n' "$override_version" return 0 fi diff --git a/tools/e2e-scenarios/workflow-boundary.mts b/tools/e2e-scenarios/workflow-boundary.mts index 0b9abb2dd11..17f7381a0cf 100644 --- a/tools/e2e-scenarios/workflow-boundary.mts +++ b/tools/e2e-scenarios/workflow-boundary.mts @@ -3138,6 +3138,192 @@ function validateModelRouterProviderRoutedInferenceVitestJob( requireRunContains(errors, cleanup, 'rm -rf "${DOCKER_CONFIG}"'); } +function runContainsCloudflaredAptInstall(run: string): boolean { + return /apt-get\s+install[\s\S]*cloudflared|apt\s+install[\s\S]*cloudflared|pkg\.cloudflare\.com\/cloudflared/.test( + run, + ); +} + +function validateTunnelLifecycleVitestJob(errors: string[], jobs: WorkflowRecord): void { + const jobName = "tunnel-lifecycle-vitest"; + const scenarioName = "tunnel-lifecycle"; + const job = asRecord(jobs[jobName]); + if (Object.keys(job).length === 0) { + errors.push("workflow missing tunnel-lifecycle-vitest job"); + return; + } + + if (job["runs-on"] !== "ubuntu-latest") { + errors.push("tunnel-lifecycle-vitest job must run on ubuntu-latest"); + } + if (job["timeout-minutes"] !== 75) { + errors.push("tunnel-lifecycle-vitest job must keep the 75 minute timeout"); + } + validateFreeStandingJobSelector(errors, jobs, jobName, scenarioName); + + const jobEnv = asRecord(job.env); + if ("DOCKER_CONFIG" in jobEnv) { + errors.push("tunnel-lifecycle-vitest job must not set DOCKER_CONFIG at job level"); + } + if (jobEnv.NEMOCLAW_CLI_BIN !== "${{ github.workspace }}/bin/nemoclaw.js") { + errors.push("tunnel-lifecycle-vitest job must point NEMOCLAW_CLI_BIN at the repo CLI"); + } + if (jobEnv.FREE_STANDING_VITEST_JOB !== "1") { + errors.push("tunnel-lifecycle-vitest job must set FREE_STANDING_VITEST_JOB=1"); + } + if (jobEnv.FREE_STANDING_SCENARIO_ID !== scenarioName) { + errors.push(`tunnel-lifecycle-vitest job must set FREE_STANDING_SCENARIO_ID=${scenarioName}`); + } + if (jobEnv.NEMOCLAW_RUN_E2E_SCENARIOS !== "1") { + errors.push("tunnel-lifecycle-vitest job must set NEMOCLAW_RUN_E2E_SCENARIOS=1"); + } + requireEnvDoesNotExposeSecret( + errors, + "tunnel-lifecycle-vitest job", + jobEnv, + "NVIDIA_INFERENCE_API_KEY", + ); + + const steps = asSteps(job.steps); + requireNoDispatchInputInterpolation(errors, steps); + for (const step of steps) { + const stepName = `tunnel-lifecycle-vitest step '${step.name ?? step.uses ?? "<unnamed>"}'`; + const stepEnv = asRecord(step.env); + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "GITHUB_TOKEN"); + if (step.name !== "Run tunnel lifecycle live test") { + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "NVIDIA_INFERENCE_API_KEY"); + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "NVIDIA_API_KEY"); + } + if (step.name !== "Authenticate to Docker Hub") { + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "DOCKERHUB_USERNAME"); + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "DOCKERHUB_TOKEN"); + requireNoDockerHubAuthInRun(errors, stepName, stringValue(step.run)); + } + } + + const checkout = steps.find((step) => stringValue(step.uses).startsWith("actions/checkout@")); + if (!checkout) { + errors.push("tunnel-lifecycle-vitest job missing checkout step"); + } + requireFullShaAction(errors, checkout, "tunnel-lifecycle-vitest checkout"); + if (asRecord(checkout?.with)["persist-credentials"] !== false) { + errors.push("tunnel-lifecycle-vitest checkout step must set persist-credentials=false"); + } + + const configureDockerAuth = requireJobStep( + errors, + jobName, + steps, + "Configure isolated Docker auth directory", + ); + requireRunContains( + errors, + configureDockerAuth, + 'echo "DOCKER_CONFIG=${RUNNER_TEMP}/docker-config-tunnel-lifecycle" >> "$GITHUB_ENV"', + ); + requireRunDoesNotContain(errors, configureDockerAuth, "${{ runner.temp }}"); + requireRunDoesNotContain(errors, configureDockerAuth, "${{ github.workspace }}"); + + const dockerLogin = requireJobStep(errors, jobName, steps, "Authenticate to Docker Hub"); + const dockerLoginEnv = asRecord(dockerLogin?.env); + if (dockerLoginEnv.DOCKERHUB_USERNAME !== "${{ secrets.DOCKERHUB_USERNAME }}") { + errors.push( + "tunnel-lifecycle-vitest Docker Hub auth must receive DOCKERHUB_USERNAME from secrets", + ); + } + if (dockerLoginEnv.DOCKERHUB_TOKEN !== "${{ secrets.DOCKERHUB_TOKEN }}") { + errors.push( + "tunnel-lifecycle-vitest Docker Hub auth must receive DOCKERHUB_TOKEN from secrets", + ); + } + requireRunContains(errors, dockerLogin, 'mkdir -p "${DOCKER_CONFIG}"'); + requireRunContains(errors, dockerLogin, 'chmod 700 "${DOCKER_CONFIG}"'); + requireRunContains(errors, dockerLogin, "docker login docker.io"); + requireRunContains(errors, dockerLogin, "--password-stdin"); + requireRunContains(errors, dockerLogin, "continuing with anonymous pulls"); + + const setupNode = namedStep(steps, "Set up Node"); + if (!setupNode) { + errors.push("tunnel-lifecycle-vitest job missing step: Set up Node"); + } + requireFullShaAction(errors, setupNode, "tunnel-lifecycle-vitest setup-node"); + + const installRootDependencies = requireJobStep( + errors, + jobName, + steps, + "Install root dependencies", + ); + requireRunContains(errors, installRootDependencies, "npm ci --ignore-scripts"); + + const buildCli = requireJobStep(errors, jobName, steps, "Build CLI"); + requireRunContains(errors, buildCli, "npm run build:cli"); + + const cloudflaredPrereq = requireJobStep( + errors, + jobName, + steps, + "Install and verify cloudflared prerequisite", + ); + const cloudflaredPrereqEnv = asRecord(cloudflaredPrereq?.env); + requireEnvDoesNotExposeSecret( + errors, + "tunnel-lifecycle-vitest cloudflared prerequisite step", + cloudflaredPrereqEnv, + "NVIDIA_INFERENCE_API_KEY", + ); + requireEnvDoesNotExposeSecret( + errors, + "tunnel-lifecycle-vitest cloudflared prerequisite step", + cloudflaredPrereqEnv, + "NVIDIA_API_KEY", + ); + requireRunContains(errors, cloudflaredPrereq, "cloudflared --version"); + requireRunContains(errors, cloudflaredPrereq, "test/e2e/lib/cloudflared-version-resolver.sh"); + requireRunContains(errors, cloudflaredPrereq, "sudo apt-get install -y"); + requireRunContains(errors, cloudflaredPrereq, "cloudflared=${cf_version}"); + + const runVitest = requireJobStep(errors, jobName, steps, "Run tunnel lifecycle live test"); + const runVitestEnv = asRecord(runVitest?.env); + if (runVitestEnv.NVIDIA_INFERENCE_API_KEY !== "${{ secrets.NVIDIA_INFERENCE_API_KEY }}") { + errors.push( + "tunnel-lifecycle-vitest Vitest step must receive NVIDIA_INFERENCE_API_KEY from secrets", + ); + } + if (runContainsCloudflaredAptInstall(stringValue(runVitest?.run))) { + errors.push( + "tunnel-lifecycle-vitest Vitest step must not run cloudflared APT installation with NVIDIA_INFERENCE_API_KEY in scope", + ); + } + requireRunContains(errors, runVitest, "npx vitest run --project e2e-scenarios-live"); + requireRunContains(errors, runVitest, "test/e2e-scenario/live/tunnel-lifecycle.test.ts"); + + const upload = requireJobStep(errors, jobName, steps, "Upload tunnel lifecycle artifacts"); + requireFullShaAction(errors, upload, "tunnel-lifecycle-vitest upload-artifact"); + const uploadWith = asRecord(upload?.with); + if (uploadWith.name !== "e2e-vitest-scenarios-tunnel-lifecycle") { + errors.push("tunnel-lifecycle-vitest artifact upload name must be stable"); + } + const uploadPath = stringValue(uploadWith.path); + requireUploadPathContains(errors, uploadPath, "e2e-artifacts/vitest/tunnel-lifecycle/"); + if (uploadWith["include-hidden-files"] !== false) { + errors.push("tunnel-lifecycle-vitest artifact upload must set include-hidden-files: false"); + } + if (uploadWith["if-no-files-found"] !== "ignore") { + errors.push("tunnel-lifecycle-vitest artifact upload must ignore missing fixture artifacts"); + } + if (uploadWith["retention-days"] !== 14) { + errors.push("tunnel-lifecycle-vitest artifact upload retention-days must be 14"); + } + + const cleanup = requireJobStep(errors, jobName, steps, "Clean up Docker auth"); + if (cleanup?.if !== "always()") { + errors.push("tunnel-lifecycle-vitest Docker auth cleanup must always run"); + } + requireRunContains(errors, cleanup, "docker logout docker.io"); + requireRunContains(errors, cleanup, 'rm -rf "${DOCKER_CONFIG}"'); +} + function validateIssue2478CrashLoopRecoveryVitestJob(errors: string[], jobs: WorkflowRecord): void { const jobName = "issue-2478-crash-loop-recovery-vitest"; const scenarioName = "issue-2478-crash-loop-recovery"; @@ -4328,6 +4514,8 @@ export function validateE2eVitestScenariosWorkflowBoundary( validateIssue2478CrashLoopRecoveryVitestJob(errors, jobs); + validateTunnelLifecycleVitestJob(errors, jobs); + validateFreeStandingJobSelector( errors, jobs,