diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml
index 38cbcb0e9e5..e421618325a 100644
--- a/.github/workflows/e2e-vitest-scenarios.yaml
+++ b/.github/workflows/e2e-vitest-scenarios.yaml
@@ -4377,6 +4377,121 @@ jobs:
docker logout docker.io || true
rm -rf "${DOCKER_CONFIG}"
+ tunnel-lifecycle-vitest:
+ needs: generate-matrix
+ if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',tunnel-lifecycle-vitest,') || contains(format(',{0},', inputs.scenarios), ',tunnel-lifecycle,') }}
+ runs-on: ubuntu-latest
+ timeout-minutes: 75
+ env:
+ FREE_STANDING_VITEST_JOB: "1"
+ FREE_STANDING_SCENARIO_ID: "tunnel-lifecycle"
+ E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/tunnel-lifecycle
+ NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
+ NEMOCLAW_RUN_E2E_SCENARIOS: "1"
+ NEMOCLAW_NON_INTERACTIVE: "1"
+ NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
+ NEMOCLAW_SANDBOX_NAME: "e2e-tunnel-lifecycle"
+ OPENSHELL_GATEWAY: "nemoclaw"
+ steps:
+ - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+ with:
+ persist-credentials: false
+
+ - name: Configure isolated Docker auth directory
+ run: echo "DOCKER_CONFIG=${RUNNER_TEMP}/docker-config-tunnel-lifecycle" >> "$GITHUB_ENV"
+
+ - name: Authenticate to Docker Hub
+ env:
+ DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+ DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+ shell: bash
+ run: |
+ set -euo pipefail
+ if [[ -z "${DOCKERHUB_USERNAME}" || -z "${DOCKERHUB_TOKEN}" ]]; then
+ echo "::notice::Docker Hub credentials not configured; continuing with anonymous pulls."
+ exit 0
+ fi
+ mkdir -p "${DOCKER_CONFIG}"
+ chmod 700 "${DOCKER_CONFIG}"
+ login_succeeded=0
+ for attempt in 1 2 3; do
+ if echo "${DOCKERHUB_TOKEN}" | timeout 30s docker login docker.io --username "${DOCKERHUB_USERNAME}" --password-stdin; then
+ login_succeeded=1
+ break
+ fi
+ if [[ "$attempt" -lt 3 ]]; then
+ echo "::warning::Docker Hub login attempt ${attempt} failed; retrying."
+ sleep 5
+ fi
+ done
+ if [[ "$login_succeeded" -ne 1 ]]; then
+ echo "::warning::Docker Hub login failed after 3 attempts; continuing with anonymous pulls."
+ fi
+
+ - name: Set up Node
+ uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0
+ with:
+ node-version: 22
+ cache: npm
+
+ - name: Install root dependencies
+ run: npm ci --ignore-scripts
+
+ - name: Build CLI
+ run: npm run build:cli
+
+ - name: Install and verify cloudflared prerequisite
+ run: |
+ set -euo pipefail
+ if command -v cloudflared >/dev/null 2>&1; then
+ cloudflared --version
+ exit 0
+ fi
+ source test/e2e/lib/cloudflared-version-resolver.sh
+ sudo mkdir -p --mode=0755 /usr/share/keyrings
+ curl -fsSL https://pkg.cloudflare.com/cloudflare-main.gpg | sudo tee /usr/share/keyrings/cloudflare-main.gpg >/dev/null
+ echo "deb [signed-by=/usr/share/keyrings/cloudflare-main.gpg] https://pkg.cloudflare.com/cloudflared $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/cloudflared.list >/dev/null
+ sudo apt-get update -qq
+ available_versions="$(apt-cache madison cloudflared | awk '{print $3}')"
+ cf_min_version="${CLOUDFLARED_MIN_VERSION:-$CLOUDFLARED_DEFAULT_MIN_VERSION}"
+ if [ -n "${CLOUDFLARED_VERSION:-}" ]; then
+ cf_version="$(cloudflared_resolve_package_version "$available_versions" "$cf_min_version" "$CLOUDFLARED_VERSION")"
+ else
+ cf_version="$(cloudflared_resolve_package_version "$available_versions" "$cf_min_version")"
+ fi
+ sudo apt-get install -y "cloudflared=${cf_version}"
+ cloudflared --version
+
+ - name: Run tunnel lifecycle live test
+ # Migrated from test/e2e/test-tunnel-lifecycle.sh. This preserves the
+ # real Docker/OpenShell onboard, host cloudflared quick-tunnel,
+ # local-dashboard readiness, public tunnel probe, and stop/status
+ # cleanup boundaries under Vitest.
+ env:
+ NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }}
+ run: |
+ set -euo pipefail
+ npx vitest run --project e2e-scenarios-live \
+ test/e2e-scenario/live/tunnel-lifecycle.test.ts \
+ --silent=false --reporter=default
+
+ - name: Upload tunnel lifecycle artifacts
+ if: always()
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: e2e-vitest-scenarios-tunnel-lifecycle
+ path: e2e-artifacts/vitest/tunnel-lifecycle/
+ include-hidden-files: false
+ if-no-files-found: ignore
+ retention-days: 14
+
+ - name: Clean up Docker auth
+ if: always()
+ run: |
+ set -euo pipefail
+ docker logout docker.io || true
+ rm -rf "${DOCKER_CONFIG}"
+
# ── PR result comment (mirrors nightly-e2e.yaml's report-to-pr) ───────────
# Posts a results table on the open PR for the dispatching branch (or the
# PR identified by `inputs.pr_number`). `if: always()` so the comment lands
@@ -4444,6 +4559,7 @@ jobs:
gateway-health-honest-vitest,
device-auth-health-vitest,
channels-add-remove-vitest,
+ tunnel-lifecycle-vitest,
telegram-injection-vitest,
channels-stop-start-vitest,
]
diff --git a/test/cli/sandbox-status-json.test.ts b/test/cli/sandbox-status-json.test.ts
index c3f3ffc5603..daf8de10fba 100644
--- a/test/cli/sandbox-status-json.test.ts
+++ b/test/cli/sandbox-status-json.test.ts
@@ -7,9 +7,9 @@ import net from "node:net";
import os from "node:os";
import path from "node:path";
-import { runWithEnv, writeSandboxRegistry } from "./helpers";
+import { runWithEnv, testTimeoutOptions, writeSandboxRegistry } from "./helpers";
-describe("CLI sandbox status JSON output", () => {
+describe("CLI sandbox status JSON output", testTimeoutOptions(20_000), () => {
it("sandbox status --json emits structured per-sandbox report", () => {
const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-sandbox-status-json-"));
const localBin = path.join(home, "bin");
diff --git a/test/cloudflared-version-resolver.test.ts b/test/cloudflared-version-resolver.test.ts
index 12a820a4190..750bc345cd0 100644
--- a/test/cloudflared-version-resolver.test.ts
+++ b/test/cloudflared-version-resolver.test.ts
@@ -23,6 +23,7 @@ if [[ "\${1:-}" != "--compare-versions" ]]; then
fi
rank() {
case "\${1:-}" in
+ 2020.1.1) printf '20200101' ;;
2026.4.30) printf '20260430' ;;
2026.5.1~rc1) printf '20260500' ;;
2026.5.1) printf '20260501' ;;
@@ -89,13 +90,21 @@ describe("cloudflared APT package resolver", () => {
expect(result.stderr).toContain("meets minimum 2026.5.1");
});
- it("preserves exact CLOUDFLARED_VERSION overrides for emergency repro", () => {
+ it("preserves syntactically valid exact CLOUDFLARED_VERSION overrides for emergency repro", () => {
const result = runResolver("2026.5.1\n2026.5.10", "2026.5.1", "2020.1.1");
expect(result.status).toBe(0);
expect(result.stdout.trim()).toBe("2020.1.1");
});
+ it("rejects invalid CLOUDFLARED_VERSION overrides before apt install", () => {
+ const result = runResolver("2026.5.1\n2026.5.10", "2026.5.1", "bad/min");
+
+ expect(result.status).not.toBe(0);
+ expect(result.stderr).toContain("invalid CLOUDFLARED_VERSION");
+ expect(result.stdout.trim()).toBe("");
+ });
+
it("rejects invalid minimum versions before comparing package versions", () => {
const result = runResolver("2026.5.1", "bad/min");
diff --git a/test/e2e-scenario/live/tunnel-lifecycle-helpers.ts b/test/e2e-scenario/live/tunnel-lifecycle-helpers.ts
new file mode 100644
index 00000000000..587be74ffe1
--- /dev/null
+++ b/test/e2e-scenario/live/tunnel-lifecycle-helpers.ts
@@ -0,0 +1,479 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * Live Vitest replacement for test/e2e/test-tunnel-lifecycle.sh.
+ *
+ * Preserves the legacy real boundaries: Docker/OpenShell onboarding, the
+ * installed/source NemoClaw CLI, host `cloudflared`, the local dashboard origin,
+ * public trycloudflare reachability, cloudflared log diagnosis, and tunnel stop
+ * cleanup/status removal.
+ */
+
+import fs from "node:fs";
+import path from "node:path";
+
+import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts";
+import { resultText } from "../fixtures/clients/index.ts";
+import { validateSandboxName } from "../fixtures/clients/sandbox.ts";
+import type { E2EScenarioFixtures } from "../fixtures/e2e-test.ts";
+import { expect } from "../fixtures/e2e-test.ts";
+import { requireHostedInferenceConfig } from "../fixtures/hosted-inference.ts";
+import type { ShellProbeResult } from "../fixtures/shell-probe.ts";
+
+const REPO_ROOT = path.resolve(import.meta.dirname, "../../..");
+const TEST_SANDBOX_PREFIX = "e2e-tunnel-lifecycle";
+const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? TEST_SANDBOX_PREFIX;
+const LOCAL_DASHBOARD_PORT = process.env.NEMOCLAW_DASHBOARD_PORT ?? "18789";
+const TEST_TIMEOUT_MS = Number(process.env.NEMOCLAW_E2E_TIMEOUT_SECONDS ?? 3_600) * 1_000;
+const ONBOARD_TIMEOUT_MS = 30 * 60_000;
+const COMMAND_TIMEOUT_MS = 60_000;
+const TUNNEL_URL_PATTERN = /https:\/\/[a-z0-9-]+\.trycloudflare\.com\b[\w./?%&=-]*/i;
+const DASHBOARD_MARKER_PATTERN = /
OpenClaw Control<\/title>| {
+ return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function commandEnv(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv {
+ return {
+ ...buildAvailabilityProbeEnv(),
+ NEMOCLAW_NON_INTERACTIVE: "1",
+ NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1",
+ NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME,
+ NEMOCLAW_POLICY_TIER: "open",
+ NEMOCLAW_AGENT: "openclaw",
+ NEMOCLAW_PROVIDER: "cloud",
+ OPENSHELL_GATEWAY: "nemoclaw",
+ ...(process.env.NEMOCLAW_DASHBOARD_PORT
+ ? { NEMOCLAW_DASHBOARD_PORT: process.env.NEMOCLAW_DASHBOARD_PORT }
+ : {}),
+ ...extra,
+ };
+}
+
+function isCloudflareTransientText(text: string): boolean {
+ return /failed to unmarshal quick Tunnel|quick tunnels? (are )?(temporarily )?disabled|failed to (dial|register)|tunnel server.*error|i\/o timeout|EOF.*tunnel|couldn.?t start tunnel|tunnel creation failed|bad gateway|\b50[234]\b/i.test(
+ text,
+ );
+}
+
+function isCloudflareTransientHttpCode(code: string): boolean {
+ return ["000", "502", "503", "504"].includes(code);
+}
+
+export function getCloudflaredLogPath(
+ logRoot = "/tmp",
+ sandboxName = SANDBOX_NAME,
+): string | undefined {
+ // Source boundary: NemoClaw owns the per-sandbox cloudflared service log at
+ // /tmp/nemoclaw-services-${sandboxName}/cloudflared.log. If that exact file
+ // is missing, this live contract classifies the invalid state as
+ // `nemoclaw_no_spawn` instead of falling back to the newest /tmp log, because
+ // unrelated parallel/stale sandboxes can otherwise corrupt fault attribution.
+ // Remove this filesystem fallback point entirely once NemoClaw exposes
+ // machine-readable tunnel diagnostics from `nemoclaw status --json`.
+ const sandboxLog = path.join(logRoot, `nemoclaw-services-${sandboxName}`, "cloudflared.log");
+ return fs.existsSync(sandboxLog) ? sandboxLog : undefined;
+}
+
+function readCloudflaredLog(): string {
+ const logPath = getCloudflaredLogPath();
+ if (!logPath) return "";
+ return fs.readFileSync(logPath, "utf8");
+}
+
+function cloudflaredLogTail(lines = 80): string {
+ const logPath = getCloudflaredLogPath();
+ if (!logPath) return "(no cloudflared.log found under /tmp/nemoclaw-services-*/)";
+ const text = fs.readFileSync(logPath, "utf8");
+ return [
+ `--- cloudflared.log (${logPath}, last ${lines} lines) ---`,
+ ...text.split(/\r?\n/).slice(-lines),
+ ].join("\n");
+}
+
+export function classifyCloudflaredLog(
+ logRoot = "/tmp",
+ sandboxName = SANDBOX_NAME,
+): "nemoclaw_no_spawn" | "nemoclaw_capture_bug" | "nemoclaw_local" | "cloudflare" | "unknown" {
+ const logPath = getCloudflaredLogPath(logRoot, sandboxName);
+ if (!logPath) return "nemoclaw_no_spawn";
+ const log = fs.readFileSync(logPath, "utf8");
+ if (TUNNEL_URL_PATTERN.test(log)) return "nemoclaw_capture_bug";
+ if (
+ /unable to reach the origin|connection refused.*127\.0\.0\.1|connection refused.*localhost|dial tcp.*127\.0\.0\.1.*refused/i.test(
+ log,
+ )
+ ) {
+ return "nemoclaw_local";
+ }
+ if (isCloudflareTransientText(log)) return "cloudflare";
+ return "unknown";
+}
+
+function extractTunnelUrl(text: string): string | undefined {
+ return text.match(TUNNEL_URL_PATTERN)?.[0];
+}
+
+export function publicTunnelProbeCurlArgs(tunnelUrl: string): string[] {
+ // Source boundary: the public tunnel URL already came from `nemoclaw status`
+ // and matched `*.trycloudflare.com`. Do not ask curl to follow redirects;
+ // a 3xx response is a tunnel/output contract failure unless NemoClaw grows a
+ // documented same-host redirect requirement. If that happens, replace this
+ // with explicit redirect target inspection before issuing a second request.
+ return ["-sS", "--max-time", "30", "-w", "\n__HTTP_CODE:%{http_code}\n", tunnelUrl];
+}
+
+function parseCurlProbe(result: ShellProbeResult): CurlProbe {
+ const text = result.stdout;
+ const match = text.match(/\n__HTTP_CODE:(\d{3})\s*$/);
+ const httpCode = match?.[1] ?? "000";
+ const body = match ? text.slice(0, match.index) : text;
+ return { httpCode, body, result };
+}
+
+async function bestEffort(run: () => Promise): Promise {
+ try {
+ await run();
+ } catch {
+ // Inline recovery remains best-effort so the primary E2E failure stays visible.
+ }
+}
+
+function isBenignTunnelStopFailure(text: string): boolean {
+ return /no active tunnel|no tunnel.*running|tunnel.*not.*running|already stopped|cloudflared.*not.*running|no cloudflared/i.test(
+ text,
+ );
+}
+
+export const TUNNEL_LIFECYCLE_TEST_TIMEOUT_MS = TEST_TIMEOUT_MS;
+
+type TunnelLifecycleFixtures = Pick<
+ E2EScenarioFixtures,
+ "artifacts" | "cleanup" | "host" | "secrets"
+> & {
+ skip: (note?: string) => never;
+};
+
+type TunnelLifecycleCleanupHost = Pick;
+
+type TunnelLifecycleCleanupRegistry = Pick;
+
+export function registerTunnelLifecycleCleanup(
+ cleanup: TunnelLifecycleCleanupRegistry,
+ host: TunnelLifecycleCleanupHost,
+): void {
+ // CleanupRegistry runs callbacks in reverse registration order. Register the
+ // sandbox destroy first so host `cloudflared` is stopped before the sandbox is
+ // torn down on early failures. Source boundary: `nemoclaw tunnel stop` owns
+ // quick-tunnel process cleanup; `cleanupSandbox` owns the Docker/OpenShell
+ // sandbox and only suppresses already-missing sandboxes. Keep both callbacks
+ // strict so unexpected cleanup failures surface in cleanup.json. Removal
+ // condition: replace this ordering guard once NemoClaw exposes one atomic
+ // machine-readable lifecycle cleanup that stops tunnels before destroying the
+ // sandbox.
+ cleanup.add(`destroy sandbox ${SANDBOX_NAME}`, async () => {
+ if (process.env.NEMOCLAW_E2E_KEEP_SANDBOX === "1") return;
+ await host.cleanupSandbox(SANDBOX_NAME, {
+ artifactName: "cleanup-nemoclaw-destroy-tunnel-lifecycle",
+ timeoutMs: 15 * 60_000,
+ });
+ });
+ cleanup.add("stop cloudflared quick tunnel", async () => {
+ const stop = await host.nemoclaw(["tunnel", "stop"], {
+ artifactName: "cleanup-tunnel-stop",
+ env: commandEnv(),
+ timeoutMs: COMMAND_TIMEOUT_MS,
+ });
+ if (stop.exitCode === 0) return;
+ const text = resultText(stop);
+ if (isBenignTunnelStopFailure(text)) return;
+ throw new Error(
+ `[NemoClaw fault] cleanup tunnel stop failed with exit ${stop.exitCode ?? "unknown"}: ${text}`,
+ );
+ });
+}
+
+export async function runTunnelLifecycleContract({
+ artifacts,
+ cleanup,
+ host,
+ secrets,
+ skip,
+}: TunnelLifecycleFixtures): Promise {
+ assertTestOwnedSandboxName();
+ const hosted = requireHostedInferenceConfig(secrets);
+ const apiKey = hosted.apiKey;
+
+ await artifacts.writeJson("contract.json", {
+ legacySource: "test/e2e/test-tunnel-lifecycle.sh",
+ sandboxName: SANDBOX_NAME,
+ localDashboardPort: LOCAL_DASHBOARD_PORT,
+ preservedBoundaries: [
+ "real Docker/OpenShell OpenClaw sandbox onboarding",
+ "host cloudflared binary and quick-tunnel registration",
+ "nemoclaw tunnel start/status/stop CLI commands",
+ "local dashboard origin readiness before tunnel attribution",
+ "public trycloudflare HTTP probe with dashboard marker assertion",
+ "cloudflared.log classification for NemoClaw-vs-Cloudflare failures",
+ ],
+ inferenceCredential: hosted.contractLabel,
+ });
+
+ registerTunnelLifecycleCleanup(cleanup, host);
+
+ const docker = await host.command("docker", ["info"], {
+ artifactName: "prereq-docker-info-tunnel-lifecycle",
+ env: buildAvailabilityProbeEnv(),
+ timeoutMs: 30_000,
+ });
+ if (docker.exitCode !== 0) {
+ if (process.env.GITHUB_ACTIONS === "true") {
+ throw new Error(`Docker is required for tunnel lifecycle E2E: ${resultText(docker)}`);
+ }
+ skip("Docker is required for tunnel lifecycle E2E");
+ }
+
+ const cloudflared = await host.command("cloudflared", ["--version"], {
+ artifactName: "prereq-cloudflared-version",
+ env: buildAvailabilityProbeEnv(),
+ timeoutMs: 30_000,
+ });
+ if (cloudflared.exitCode !== 0) {
+ if (process.env.GITHUB_ACTIONS === "true") {
+ throw new Error(
+ `cloudflared is required for tunnel lifecycle E2E: ${resultText(cloudflared)}`,
+ );
+ }
+ skip("cloudflared is required for tunnel lifecycle E2E");
+ }
+
+ expect(fs.existsSync(path.join(REPO_ROOT, "install.sh"))).toBe(true);
+ await host.bestEffortCleanupSandbox(SANDBOX_NAME, {
+ artifactName: "pre-cleanup-nemoclaw-destroy-tunnel-lifecycle",
+ timeoutMs: 15 * 60_000,
+ });
+
+ const install = await host.command(
+ "bash",
+ ["install.sh", "--non-interactive", "--yes-i-accept-third-party-software"],
+ {
+ artifactName: "install-sh-tunnel-lifecycle",
+ cwd: REPO_ROOT,
+ env: commandEnv({
+ ...hosted.env,
+ NVIDIA_INFERENCE_API_KEY: apiKey,
+ NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1",
+ }),
+ redactionValues: [apiKey],
+ timeoutMs: ONBOARD_TIMEOUT_MS,
+ },
+ );
+ expect(install.exitCode, resultText(install)).toBe(0);
+
+ await host.expectListed(SANDBOX_NAME, { artifactName: "post-install-nemoclaw-list" });
+
+ let localReady = false;
+ for (let attempt = 1; attempt <= 30; attempt += 1) {
+ const local = await host.command(
+ "curl",
+ [
+ "-sS",
+ "-o",
+ "/dev/null",
+ "-w",
+ "%{http_code}",
+ "--max-time",
+ "5",
+ `http://localhost:${LOCAL_DASHBOARD_PORT}/`,
+ ],
+ {
+ artifactName: `local-dashboard-ready-${attempt}`,
+ env: buildAvailabilityProbeEnv(),
+ timeoutMs: 10_000,
+ },
+ );
+ const code = local.stdout.trim() || "000";
+ if (code !== "000") {
+ localReady = true;
+ break;
+ }
+ await sleep(1_000);
+ }
+ expect(
+ localReady,
+ `[NemoClaw fault] Local OpenClaw dashboard not reachable on localhost:${LOCAL_DASHBOARD_PORT} after 30s; tunnel cannot proxy a dead origin.`,
+ ).toBe(true);
+
+ const start = await host.nemoclaw(["tunnel", "start"], {
+ artifactName: "tunnel-start",
+ env: commandEnv(),
+ timeoutMs: 90_000,
+ });
+ if (start.exitCode !== 0) {
+ await artifacts.writeText("cloudflared-log-after-start-failure.txt", cloudflaredLogTail());
+ if (isCloudflareTransientText(resultText(start)) || classifyCloudflaredLog() === "cloudflare") {
+ await bestEffort(() =>
+ host.nemoclaw(["tunnel", "stop"], {
+ artifactName: "tunnel-stop-after-cloudflare-start-failure",
+ env: commandEnv(),
+ timeoutMs: COMMAND_TIMEOUT_MS,
+ }),
+ );
+ skip(
+ `[Cloudflare fault] nemoclaw tunnel start exited ${start.exitCode ?? "unknown"} because quick-tunnel registration returned a transient external error.`,
+ );
+ }
+ throw new Error(
+ `[NemoClaw fault] nemoclaw tunnel start failed with exit ${start.exitCode ?? "unknown"}: ${resultText(start)}`,
+ );
+ }
+
+ let tunnelUrl: string | undefined;
+ let lastStatusText = "";
+ for (let attempt = 1; attempt <= 15; attempt += 1) {
+ const status = await host.nemoclaw(["status"], {
+ artifactName: `status-with-tunnel-url-${attempt}`,
+ env: commandEnv(),
+ timeoutMs: COMMAND_TIMEOUT_MS,
+ });
+ lastStatusText = resultText(status);
+ tunnelUrl = extractTunnelUrl(lastStatusText);
+ if (tunnelUrl) break;
+ await sleep(1_000);
+ }
+
+ if (!tunnelUrl) {
+ await artifacts.writeText("cloudflared-log-without-status-url.txt", cloudflaredLogTail());
+ const cfClass = classifyCloudflaredLog();
+ await bestEffort(() =>
+ host.nemoclaw(["tunnel", "stop"], {
+ artifactName: "tunnel-stop-after-missing-url",
+ env: commandEnv(),
+ timeoutMs: COMMAND_TIMEOUT_MS,
+ }),
+ );
+ if (cfClass === "cloudflare") {
+ skip("[Cloudflare fault] cloudflared failed to register a quick tunnel URL.");
+ }
+ let reason: string;
+ switch (cfClass) {
+ case "nemoclaw_no_spawn":
+ reason = "cloudflared.log missing — NemoClaw failed to spawn the cloudflared process";
+ break;
+ case "nemoclaw_capture_bug":
+ reason = "cloudflared.log has a trycloudflare URL but nemoclaw status did not surface it";
+ break;
+ case "nemoclaw_local":
+ reason = `cloudflared.log reports it cannot reach localhost:${LOCAL_DASHBOARD_PORT}`;
+ break;
+ default:
+ reason = `tunnel URL did not surface and cloudflared.log did not match a known pattern; status was:\n${lastStatusText}`;
+ }
+ throw new Error(`[NemoClaw fault] ${reason}`);
+ }
+
+ let lastPublicProbe: CurlProbe | undefined;
+ let backoffMs = 2_000;
+ for (let attempt = 1; attempt <= 15; attempt += 1) {
+ const probe = parseCurlProbe(
+ await host.command("curl", publicTunnelProbeCurlArgs(tunnelUrl), {
+ artifactName: `public-tunnel-probe-${attempt}`,
+ env: buildAvailabilityProbeEnv(),
+ timeoutMs: 35_000,
+ }),
+ );
+ lastPublicProbe = probe;
+ if (probe.httpCode === "200") break;
+
+ const local = await host.command(
+ "curl",
+ [
+ "-sS",
+ "-o",
+ "/dev/null",
+ "-w",
+ "%{http_code}",
+ "--max-time",
+ "5",
+ `http://localhost:${LOCAL_DASHBOARD_PORT}/`,
+ ],
+ {
+ artifactName: `local-dashboard-recheck-${attempt}`,
+ env: buildAvailabilityProbeEnv(),
+ timeoutMs: 10_000,
+ },
+ );
+ const localCode = local.stdout.trim() || "000";
+ if (localCode === "000") {
+ throw new Error(
+ `[NemoClaw fault] Tunnel returned ${probe.httpCode} and local dashboard regressed during retry loop; likely sandbox/dashboard crash, not Cloudflare.`,
+ );
+ }
+ await sleep(backoffMs);
+ backoffMs = Math.min(backoffMs * 2, 30_000);
+ }
+
+ expect(lastPublicProbe, "public tunnel probe should have run").toBeTruthy();
+ if (lastPublicProbe!.httpCode !== "200") {
+ if (
+ isCloudflareTransientHttpCode(lastPublicProbe!.httpCode) ||
+ isCloudflareTransientText(lastPublicProbe!.body) ||
+ isCloudflareTransientText(readCloudflaredLog())
+ ) {
+ skip(
+ `[Cloudflare fault] Tunnel URL never became reachable while local stayed healthy; last HTTP status ${lastPublicProbe!.httpCode}.`,
+ );
+ }
+ throw new Error(
+ `[NemoClaw fault] Tunnel returned unexpected HTTP ${lastPublicProbe!.httpCode} while local stayed healthy; body prefix: ${lastPublicProbe!.body.slice(0, 200)}`,
+ );
+ }
+ expect(lastPublicProbe!.body, "public tunnel must serve OpenClaw dashboard markers").toMatch(
+ DASHBOARD_MARKER_PATTERN,
+ );
+
+ const stop = await host.nemoclaw(["tunnel", "stop"], {
+ artifactName: "tunnel-stop",
+ env: commandEnv(),
+ timeoutMs: COMMAND_TIMEOUT_MS,
+ });
+ expect(stop.exitCode, resultText(stop)).toBe(0);
+
+ let postStopUrl: string | undefined;
+ let statusReadable = false;
+ for (let attempt = 1; attempt <= 10; attempt += 1) {
+ const status = await host.nemoclaw(["status"], {
+ artifactName: `status-after-tunnel-stop-${attempt}`,
+ env: commandEnv(),
+ timeoutMs: COMMAND_TIMEOUT_MS,
+ });
+ if (status.exitCode !== 0) {
+ await sleep(1_000);
+ continue;
+ }
+ statusReadable = true;
+ postStopUrl = extractTunnelUrl(resultText(status));
+ if (!postStopUrl) break;
+ await sleep(1_000);
+ }
+ expect(statusReadable, "nemoclaw status should be readable after tunnel stop").toBe(true);
+ expect(postStopUrl, "tunnel URL must be absent after nemoclaw tunnel stop").toBeUndefined();
+}
diff --git a/test/e2e-scenario/live/tunnel-lifecycle.test.ts b/test/e2e-scenario/live/tunnel-lifecycle.test.ts
new file mode 100644
index 00000000000..fd257436fa9
--- /dev/null
+++ b/test/e2e-scenario/live/tunnel-lifecycle.test.ts
@@ -0,0 +1,24 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * Live Vitest replacement for test/e2e/test-tunnel-lifecycle.sh.
+ *
+ * Preserves the legacy real boundaries: Docker/OpenShell onboarding, the
+ * installed/source NemoClaw CLI, host `cloudflared`, the local dashboard origin,
+ * public trycloudflare reachability, cloudflared log diagnosis, and tunnel stop
+ * cleanup/status removal.
+ */
+
+import { test } from "../fixtures/e2e-test.ts";
+import { shouldRunLiveE2EScenarios } from "../fixtures/live-project-gate.ts";
+import {
+ runTunnelLifecycleContract,
+ TUNNEL_LIFECYCLE_TEST_TIMEOUT_MS,
+} from "./tunnel-lifecycle-helpers.ts";
+
+test.skipIf(!shouldRunLiveE2EScenarios())(
+ "tunnel-lifecycle: cloudflared quick tunnel starts, serves OpenClaw, and stops cleanly",
+ { timeout: TUNNEL_LIFECYCLE_TEST_TIMEOUT_MS },
+ runTunnelLifecycleContract,
+);
diff --git a/test/e2e-scenario/support-tests/tunnel-lifecycle-helpers.test.ts b/test/e2e-scenario/support-tests/tunnel-lifecycle-helpers.test.ts
new file mode 100644
index 00000000000..e683b614019
--- /dev/null
+++ b/test/e2e-scenario/support-tests/tunnel-lifecycle-helpers.test.ts
@@ -0,0 +1,191 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+import { describe, expect, it } from "vitest";
+
+import { CleanupRegistry } from "../fixtures/cleanup.ts";
+import type { ShellProbeResult } from "../fixtures/shell-probe.ts";
+import {
+ classifyCloudflaredLog,
+ getCloudflaredLogPath,
+ publicTunnelProbeCurlArgs,
+ registerTunnelLifecycleCleanup,
+} from "../live/tunnel-lifecycle-helpers.ts";
+
+function shellResult(overrides: Partial = {}): ShellProbeResult {
+ return {
+ command: ["nemoclaw"],
+ exitCode: 0,
+ signal: null,
+ timedOut: false,
+ stdout: "",
+ stderr: "",
+ artifacts: {
+ stdout: "stdout.txt",
+ stderr: "stderr.txt",
+ result: "result.json",
+ },
+ ...overrides,
+ };
+}
+
+describe("tunnel lifecycle cleanup registration", () => {
+ it("stops the tunnel before destroying the sandbox during registered cleanup", async () => {
+ const calls: string[] = [];
+ const cleanup = new CleanupRegistry();
+ registerTunnelLifecycleCleanup(cleanup, {
+ cleanupSandbox: async () => {
+ calls.push("destroy");
+ },
+ nemoclaw: async () => {
+ calls.push("stop");
+ return shellResult();
+ },
+ });
+
+ const result = await cleanup.runAll();
+
+ expect(result.failures).toEqual([]);
+ expect(calls).toEqual(["stop", "destroy"]);
+ });
+
+ it("surfaces unexpected tunnel-stop cleanup failures", async () => {
+ const cleanup = new CleanupRegistry();
+ registerTunnelLifecycleCleanup(cleanup, {
+ cleanupSandbox: async () => {},
+ nemoclaw: async () =>
+ shellResult({
+ exitCode: 1,
+ stderr: "permission denied while stopping cloudflared",
+ }),
+ });
+
+ const result = await cleanup.runAll();
+
+ expect(result.failures).toEqual([
+ {
+ name: "stop cloudflared quick tunnel",
+ message:
+ "[NemoClaw fault] cleanup tunnel stop failed with exit 1: permission denied while stopping cloudflared",
+ },
+ ]);
+ });
+
+ it("surfaces unexpected sandbox-destroy cleanup failures", async () => {
+ const cleanup = new CleanupRegistry();
+ registerTunnelLifecycleCleanup(cleanup, {
+ cleanupSandbox: async () => {
+ throw new Error("docker daemon denied sandbox destroy");
+ },
+ nemoclaw: async () => shellResult(),
+ });
+
+ const result = await cleanup.runAll();
+
+ expect(result.failures).toEqual([
+ {
+ name: "destroy sandbox e2e-tunnel-lifecycle",
+ message: "docker daemon denied sandbox destroy",
+ },
+ ]);
+ });
+
+ it("suppresses already-stopped tunnel cleanup states", async () => {
+ const cleanup = new CleanupRegistry();
+ registerTunnelLifecycleCleanup(cleanup, {
+ cleanupSandbox: async () => {},
+ nemoclaw: async () => shellResult({ exitCode: 1, stderr: "no active tunnel" }),
+ });
+
+ const result = await cleanup.runAll();
+
+ expect(result.failures).toEqual([]);
+ });
+});
+
+describe("tunnel lifecycle cloudflared log attribution", () => {
+ it("does not follow redirects from the public trycloudflare probe", () => {
+ expect(publicTunnelProbeCurlArgs("https://current.trycloudflare.com/")).toEqual([
+ "-sS",
+ "--max-time",
+ "30",
+ "-w",
+ "\n__HTTP_CODE:%{http_code}\n",
+ "https://current.trycloudflare.com/",
+ ]);
+ });
+
+ it("does not attribute an unrelated newer cloudflared log to the current sandbox", () => {
+ const logRoot = fs.mkdtempSync(path.join(os.tmpdir(), "tunnel-lifecycle-logs-"));
+ const unrelatedDir = path.join(logRoot, "nemoclaw-services-other-sandbox");
+ fs.mkdirSync(unrelatedDir, { recursive: true });
+ fs.writeFileSync(
+ path.join(unrelatedDir, "cloudflared.log"),
+ "https://unrelated.trycloudflare.com captured by another run\n",
+ );
+
+ try {
+ expect(getCloudflaredLogPath(logRoot, "e2e-tunnel-lifecycle-current")).toBeUndefined();
+ expect(classifyCloudflaredLog(logRoot, "e2e-tunnel-lifecycle-current")).toBe(
+ "nemoclaw_no_spawn",
+ );
+ } finally {
+ fs.rmSync(logRoot, { recursive: true, force: true });
+ }
+ });
+
+ it("classifies only the sandbox-specific cloudflared log", () => {
+ const logRoot = fs.mkdtempSync(path.join(os.tmpdir(), "tunnel-lifecycle-logs-"));
+ const sandboxDir = path.join(logRoot, "nemoclaw-services-e2e-tunnel-lifecycle-current");
+ fs.mkdirSync(sandboxDir, { recursive: true });
+ const sandboxLog = path.join(sandboxDir, "cloudflared.log");
+ fs.writeFileSync(sandboxLog, "https://current.trycloudflare.com\n");
+
+ try {
+ expect(getCloudflaredLogPath(logRoot, "e2e-tunnel-lifecycle-current")).toBe(sandboxLog);
+ expect(classifyCloudflaredLog(logRoot, "e2e-tunnel-lifecycle-current")).toBe(
+ "nemoclaw_capture_bug",
+ );
+ } finally {
+ fs.rmSync(logRoot, { recursive: true, force: true });
+ }
+ });
+
+ it("classifies localhost/origin-refused logs as a NemoClaw local-origin fault", () => {
+ const logRoot = fs.mkdtempSync(path.join(os.tmpdir(), "tunnel-lifecycle-logs-"));
+ const sandboxDir = path.join(logRoot, "nemoclaw-services-e2e-tunnel-lifecycle-current");
+ fs.mkdirSync(sandboxDir, { recursive: true });
+ fs.writeFileSync(
+ path.join(sandboxDir, "cloudflared.log"),
+ 'ERR Request failed error="Unable to reach the origin service. dial tcp 127.0.0.1:18789: connect: connection refused"\n',
+ );
+
+ try {
+ expect(classifyCloudflaredLog(logRoot, "e2e-tunnel-lifecycle-current")).toBe(
+ "nemoclaw_local",
+ );
+ } finally {
+ fs.rmSync(logRoot, { recursive: true, force: true });
+ }
+ });
+
+ it("classifies representative quick-tunnel registration failures as Cloudflare faults", () => {
+ const logRoot = fs.mkdtempSync(path.join(os.tmpdir(), "tunnel-lifecycle-logs-"));
+ const sandboxDir = path.join(logRoot, "nemoclaw-services-e2e-tunnel-lifecycle-current");
+ fs.mkdirSync(sandboxDir, { recursive: true });
+ fs.writeFileSync(
+ path.join(sandboxDir, "cloudflared.log"),
+ "ERR failed to unmarshal quick Tunnel response: tunnel server returned 503 bad gateway\n",
+ );
+
+ try {
+ expect(classifyCloudflaredLog(logRoot, "e2e-tunnel-lifecycle-current")).toBe("cloudflare");
+ } finally {
+ fs.rmSync(logRoot, { recursive: true, force: true });
+ }
+ });
+});
diff --git a/test/e2e-scenario/support-tests/tunnel-lifecycle-workflow-boundary.test.ts b/test/e2e-scenario/support-tests/tunnel-lifecycle-workflow-boundary.test.ts
new file mode 100644
index 00000000000..35cc3d88e29
--- /dev/null
+++ b/test/e2e-scenario/support-tests/tunnel-lifecycle-workflow-boundary.test.ts
@@ -0,0 +1,163 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+import { describe, expect, it } from "vitest";
+import YAML from "yaml";
+
+import {
+ evaluateE2eVitestWorkflowDispatchSelectors,
+ validateE2eVitestScenariosWorkflowBoundary,
+} from "../../../tools/e2e-scenarios/workflow-boundary.mts";
+
+function readWorkflow(): Record {
+ return YAML.parse(
+ fs.readFileSync(
+ path.join(process.cwd(), ".github/workflows/e2e-vitest-scenarios.yaml"),
+ "utf-8",
+ ),
+ ) as Record;
+}
+
+describe("tunnel lifecycle workflow boundary", () => {
+ it("maps the tunnel lifecycle selector to its free-standing Vitest job", () => {
+ expect(
+ evaluateE2eVitestWorkflowDispatchSelectors({ scenarios: "tunnel-lifecycle" }),
+ ).toMatchObject({
+ valid: true,
+ liveScenariosRuns: false,
+ selectedFreeStandingJobs: ["tunnel-lifecycle-vitest"],
+ registryScenarios: [],
+ });
+ expect(
+ evaluateE2eVitestWorkflowDispatchSelectors({ jobs: "tunnel-lifecycle-vitest" }),
+ ).toMatchObject({
+ valid: true,
+ liveScenariosRuns: false,
+ selectedFreeStandingJobs: ["tunnel-lifecycle-vitest"],
+ registryScenarios: [],
+ });
+ });
+
+ it("requires the tunnel lifecycle job to use the repo NemoClaw CLI boundary", () => {
+ const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-vitest-workflow-"));
+ const workflowPath = path.join(tmp, "workflow.yaml");
+ const workflow = readWorkflow() as {
+ jobs: Record }>;
+ };
+ const job = workflow.jobs["tunnel-lifecycle-vitest"];
+ expect(job).toBeDefined();
+ job.env = { ...job.env };
+ delete job.env.NEMOCLAW_CLI_BIN;
+ fs.writeFileSync(workflowPath, YAML.stringify(workflow));
+
+ try {
+ expect(validateE2eVitestScenariosWorkflowBoundary(workflowPath)).toContain(
+ "tunnel-lifecycle-vitest job must point NEMOCLAW_CLI_BIN at the repo CLI",
+ );
+ } finally {
+ fs.rmSync(tmp, { recursive: true, force: true });
+ }
+ });
+
+ it("rejects tunnel lifecycle trusted-boundary drift", () => {
+ const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-vitest-workflow-"));
+ const workflowPath = path.join(tmp, "workflow.yaml");
+ const workflow = readWorkflow() as {
+ jobs: Record<
+ string,
+ { env?: Record; steps: Array> }
+ >;
+ };
+ const job = workflow.jobs["tunnel-lifecycle-vitest"];
+ expect(job).toBeDefined();
+ job.env = {
+ ...job.env,
+ DOCKER_CONFIG: "${{ github.workspace }}/e2e-artifacts/vitest/tunnel-lifecycle/docker-config",
+ };
+
+ const checkout = job.steps.find((step) =>
+ String(step.uses ?? "").startsWith("actions/checkout@"),
+ );
+ expect(checkout).toBeDefined();
+ checkout!.with = {
+ ...(checkout!.with as Record),
+ "persist-credentials": true,
+ };
+
+ const configureDockerAuth = job.steps.find(
+ (step) => step.name === "Configure isolated Docker auth directory",
+ );
+ expect(configureDockerAuth).toBeDefined();
+ configureDockerAuth!.run =
+ 'echo "DOCKER_CONFIG=${{ github.workspace }}/docker-config-tunnel-lifecycle" >> "$GITHUB_ENV"';
+
+ const install = job.steps.find((step) => step.name === "Install root dependencies");
+ expect(install).toBeDefined();
+ install!.env = {
+ NVIDIA_INFERENCE_API_KEY: "${{ secrets.NVIDIA_INFERENCE_API_KEY }}",
+ NVIDIA_API_KEY: "${{ secrets.NVIDIA_API_KEY }}",
+ };
+ install!.run = "npm install";
+
+ const cloudflared = job.steps.find(
+ (step) => step.name === "Install and verify cloudflared prerequisite",
+ );
+ expect(cloudflared).toBeDefined();
+ cloudflared!.env = {
+ NVIDIA_INFERENCE_API_KEY: "${{ secrets.NVIDIA_INFERENCE_API_KEY }}",
+ NVIDIA_API_KEY: "${{ secrets.NVIDIA_API_KEY }}",
+ };
+ cloudflared!.run = "cloudflared --version";
+
+ const runTunnel = job.steps.find((step) => step.name === "Run tunnel lifecycle live test");
+ expect(runTunnel).toBeDefined();
+ runTunnel!.run = `${String(runTunnel!.run ?? "")}\nsudo apt-get install -y cloudflared`;
+
+ const upload = job.steps.find((step) => step.name === "Upload tunnel lifecycle artifacts");
+ expect(upload).toBeDefined();
+ upload!.with = {
+ ...(upload!.with as Record),
+ path: "e2e-artifacts/vitest/",
+ "include-hidden-files": true,
+ };
+
+ const cleanup = job.steps.find((step) => step.name === "Clean up Docker auth");
+ expect(cleanup).toBeDefined();
+ cleanup!.if = "success()";
+ cleanup!.run = 'set -euo pipefail\necho "missing Docker auth cleanup"\n';
+ fs.writeFileSync(workflowPath, YAML.stringify(workflow));
+
+ try {
+ expect(validateE2eVitestScenariosWorkflowBoundary(workflowPath)).toEqual(
+ expect.arrayContaining([
+ "tunnel-lifecycle-vitest job must not set DOCKER_CONFIG at job level",
+ 'step \'Configure isolated Docker auth directory\' run script must include echo "DOCKER_CONFIG=${RUNNER_TEMP}/docker-config-tunnel-lifecycle" >> "$GITHUB_ENV"',
+ "step 'Configure isolated Docker auth directory' run script must not include ${{ github.workspace }}",
+ "tunnel-lifecycle-vitest checkout step must set persist-credentials=false",
+ "tunnel-lifecycle-vitest step 'Install root dependencies' env must not include NVIDIA_INFERENCE_API_KEY",
+ "tunnel-lifecycle-vitest step 'Install root dependencies' env must not include NVIDIA_API_KEY",
+ "step 'Install root dependencies' run script must include npm ci --ignore-scripts",
+ "tunnel-lifecycle-vitest step 'Install and verify cloudflared prerequisite' env must not include NVIDIA_INFERENCE_API_KEY",
+ "tunnel-lifecycle-vitest step 'Install and verify cloudflared prerequisite' env must not include NVIDIA_API_KEY",
+ "tunnel-lifecycle-vitest cloudflared prerequisite step env must not include NVIDIA_INFERENCE_API_KEY",
+ "tunnel-lifecycle-vitest cloudflared prerequisite step env must not include NVIDIA_API_KEY",
+ "step 'Install and verify cloudflared prerequisite' run script must include test/e2e/lib/cloudflared-version-resolver.sh",
+ "step 'Install and verify cloudflared prerequisite' run script must include sudo apt-get install -y",
+ "step 'Install and verify cloudflared prerequisite' run script must include cloudflared=${cf_version}",
+ "tunnel-lifecycle-vitest Vitest step must not run cloudflared APT installation with NVIDIA_INFERENCE_API_KEY in scope",
+ "artifact upload path must include e2e-artifacts/vitest/tunnel-lifecycle/",
+ "tunnel-lifecycle-vitest artifact upload must set include-hidden-files: false",
+ "tunnel-lifecycle-vitest Docker auth cleanup must always run",
+ "step 'Clean up Docker auth' run script must include docker logout docker.io",
+ "step 'Clean up Docker auth' run script must include rm -rf \"${DOCKER_CONFIG}\"",
+ ]),
+ );
+ } finally {
+ fs.rmSync(tmp, { recursive: true, force: true });
+ }
+ });
+});
diff --git a/test/e2e/lib/cloudflared-version-resolver.sh b/test/e2e/lib/cloudflared-version-resolver.sh
index a56a93d252f..64e88aca36b 100755
--- a/test/e2e/lib/cloudflared-version-resolver.sh
+++ b/test/e2e/lib/cloudflared-version-resolver.sh
@@ -28,7 +28,12 @@ cloudflared_resolve_package_version() {
# Emergency repro knob: install the exact requested version and let APT report
# unavailable overrides, rather than silently substituting another package.
+ # Still validate Debian-version syntax before the sudo apt install boundary.
if [[ -n "$override_version" ]]; then
+ if ! cloudflared_is_debian_version "$override_version"; then
+ printf 'ERROR: invalid CLOUDFLARED_VERSION %q\n' "$override_version" >&2
+ return 1
+ fi
printf '%s\n' "$override_version"
return 0
fi
diff --git a/tools/e2e-scenarios/workflow-boundary.mts b/tools/e2e-scenarios/workflow-boundary.mts
index 0b9abb2dd11..17f7381a0cf 100644
--- a/tools/e2e-scenarios/workflow-boundary.mts
+++ b/tools/e2e-scenarios/workflow-boundary.mts
@@ -3138,6 +3138,192 @@ function validateModelRouterProviderRoutedInferenceVitestJob(
requireRunContains(errors, cleanup, 'rm -rf "${DOCKER_CONFIG}"');
}
+function runContainsCloudflaredAptInstall(run: string): boolean {
+ return /apt-get\s+install[\s\S]*cloudflared|apt\s+install[\s\S]*cloudflared|pkg\.cloudflare\.com\/cloudflared/.test(
+ run,
+ );
+}
+
+function validateTunnelLifecycleVitestJob(errors: string[], jobs: WorkflowRecord): void {
+ const jobName = "tunnel-lifecycle-vitest";
+ const scenarioName = "tunnel-lifecycle";
+ const job = asRecord(jobs[jobName]);
+ if (Object.keys(job).length === 0) {
+ errors.push("workflow missing tunnel-lifecycle-vitest job");
+ return;
+ }
+
+ if (job["runs-on"] !== "ubuntu-latest") {
+ errors.push("tunnel-lifecycle-vitest job must run on ubuntu-latest");
+ }
+ if (job["timeout-minutes"] !== 75) {
+ errors.push("tunnel-lifecycle-vitest job must keep the 75 minute timeout");
+ }
+ validateFreeStandingJobSelector(errors, jobs, jobName, scenarioName);
+
+ const jobEnv = asRecord(job.env);
+ if ("DOCKER_CONFIG" in jobEnv) {
+ errors.push("tunnel-lifecycle-vitest job must not set DOCKER_CONFIG at job level");
+ }
+ if (jobEnv.NEMOCLAW_CLI_BIN !== "${{ github.workspace }}/bin/nemoclaw.js") {
+ errors.push("tunnel-lifecycle-vitest job must point NEMOCLAW_CLI_BIN at the repo CLI");
+ }
+ if (jobEnv.FREE_STANDING_VITEST_JOB !== "1") {
+ errors.push("tunnel-lifecycle-vitest job must set FREE_STANDING_VITEST_JOB=1");
+ }
+ if (jobEnv.FREE_STANDING_SCENARIO_ID !== scenarioName) {
+ errors.push(`tunnel-lifecycle-vitest job must set FREE_STANDING_SCENARIO_ID=${scenarioName}`);
+ }
+ if (jobEnv.NEMOCLAW_RUN_E2E_SCENARIOS !== "1") {
+ errors.push("tunnel-lifecycle-vitest job must set NEMOCLAW_RUN_E2E_SCENARIOS=1");
+ }
+ requireEnvDoesNotExposeSecret(
+ errors,
+ "tunnel-lifecycle-vitest job",
+ jobEnv,
+ "NVIDIA_INFERENCE_API_KEY",
+ );
+
+ const steps = asSteps(job.steps);
+ requireNoDispatchInputInterpolation(errors, steps);
+ for (const step of steps) {
+ const stepName = `tunnel-lifecycle-vitest step '${step.name ?? step.uses ?? ""}'`;
+ const stepEnv = asRecord(step.env);
+ requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "GITHUB_TOKEN");
+ if (step.name !== "Run tunnel lifecycle live test") {
+ requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "NVIDIA_INFERENCE_API_KEY");
+ requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "NVIDIA_API_KEY");
+ }
+ if (step.name !== "Authenticate to Docker Hub") {
+ requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "DOCKERHUB_USERNAME");
+ requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "DOCKERHUB_TOKEN");
+ requireNoDockerHubAuthInRun(errors, stepName, stringValue(step.run));
+ }
+ }
+
+ const checkout = steps.find((step) => stringValue(step.uses).startsWith("actions/checkout@"));
+ if (!checkout) {
+ errors.push("tunnel-lifecycle-vitest job missing checkout step");
+ }
+ requireFullShaAction(errors, checkout, "tunnel-lifecycle-vitest checkout");
+ if (asRecord(checkout?.with)["persist-credentials"] !== false) {
+ errors.push("tunnel-lifecycle-vitest checkout step must set persist-credentials=false");
+ }
+
+ const configureDockerAuth = requireJobStep(
+ errors,
+ jobName,
+ steps,
+ "Configure isolated Docker auth directory",
+ );
+ requireRunContains(
+ errors,
+ configureDockerAuth,
+ 'echo "DOCKER_CONFIG=${RUNNER_TEMP}/docker-config-tunnel-lifecycle" >> "$GITHUB_ENV"',
+ );
+ requireRunDoesNotContain(errors, configureDockerAuth, "${{ runner.temp }}");
+ requireRunDoesNotContain(errors, configureDockerAuth, "${{ github.workspace }}");
+
+ const dockerLogin = requireJobStep(errors, jobName, steps, "Authenticate to Docker Hub");
+ const dockerLoginEnv = asRecord(dockerLogin?.env);
+ if (dockerLoginEnv.DOCKERHUB_USERNAME !== "${{ secrets.DOCKERHUB_USERNAME }}") {
+ errors.push(
+ "tunnel-lifecycle-vitest Docker Hub auth must receive DOCKERHUB_USERNAME from secrets",
+ );
+ }
+ if (dockerLoginEnv.DOCKERHUB_TOKEN !== "${{ secrets.DOCKERHUB_TOKEN }}") {
+ errors.push(
+ "tunnel-lifecycle-vitest Docker Hub auth must receive DOCKERHUB_TOKEN from secrets",
+ );
+ }
+ requireRunContains(errors, dockerLogin, 'mkdir -p "${DOCKER_CONFIG}"');
+ requireRunContains(errors, dockerLogin, 'chmod 700 "${DOCKER_CONFIG}"');
+ requireRunContains(errors, dockerLogin, "docker login docker.io");
+ requireRunContains(errors, dockerLogin, "--password-stdin");
+ requireRunContains(errors, dockerLogin, "continuing with anonymous pulls");
+
+ const setupNode = namedStep(steps, "Set up Node");
+ if (!setupNode) {
+ errors.push("tunnel-lifecycle-vitest job missing step: Set up Node");
+ }
+ requireFullShaAction(errors, setupNode, "tunnel-lifecycle-vitest setup-node");
+
+ const installRootDependencies = requireJobStep(
+ errors,
+ jobName,
+ steps,
+ "Install root dependencies",
+ );
+ requireRunContains(errors, installRootDependencies, "npm ci --ignore-scripts");
+
+ const buildCli = requireJobStep(errors, jobName, steps, "Build CLI");
+ requireRunContains(errors, buildCli, "npm run build:cli");
+
+ const cloudflaredPrereq = requireJobStep(
+ errors,
+ jobName,
+ steps,
+ "Install and verify cloudflared prerequisite",
+ );
+ const cloudflaredPrereqEnv = asRecord(cloudflaredPrereq?.env);
+ requireEnvDoesNotExposeSecret(
+ errors,
+ "tunnel-lifecycle-vitest cloudflared prerequisite step",
+ cloudflaredPrereqEnv,
+ "NVIDIA_INFERENCE_API_KEY",
+ );
+ requireEnvDoesNotExposeSecret(
+ errors,
+ "tunnel-lifecycle-vitest cloudflared prerequisite step",
+ cloudflaredPrereqEnv,
+ "NVIDIA_API_KEY",
+ );
+ requireRunContains(errors, cloudflaredPrereq, "cloudflared --version");
+ requireRunContains(errors, cloudflaredPrereq, "test/e2e/lib/cloudflared-version-resolver.sh");
+ requireRunContains(errors, cloudflaredPrereq, "sudo apt-get install -y");
+ requireRunContains(errors, cloudflaredPrereq, "cloudflared=${cf_version}");
+
+ const runVitest = requireJobStep(errors, jobName, steps, "Run tunnel lifecycle live test");
+ const runVitestEnv = asRecord(runVitest?.env);
+ if (runVitestEnv.NVIDIA_INFERENCE_API_KEY !== "${{ secrets.NVIDIA_INFERENCE_API_KEY }}") {
+ errors.push(
+ "tunnel-lifecycle-vitest Vitest step must receive NVIDIA_INFERENCE_API_KEY from secrets",
+ );
+ }
+ if (runContainsCloudflaredAptInstall(stringValue(runVitest?.run))) {
+ errors.push(
+ "tunnel-lifecycle-vitest Vitest step must not run cloudflared APT installation with NVIDIA_INFERENCE_API_KEY in scope",
+ );
+ }
+ requireRunContains(errors, runVitest, "npx vitest run --project e2e-scenarios-live");
+ requireRunContains(errors, runVitest, "test/e2e-scenario/live/tunnel-lifecycle.test.ts");
+
+ const upload = requireJobStep(errors, jobName, steps, "Upload tunnel lifecycle artifacts");
+ requireFullShaAction(errors, upload, "tunnel-lifecycle-vitest upload-artifact");
+ const uploadWith = asRecord(upload?.with);
+ if (uploadWith.name !== "e2e-vitest-scenarios-tunnel-lifecycle") {
+ errors.push("tunnel-lifecycle-vitest artifact upload name must be stable");
+ }
+ const uploadPath = stringValue(uploadWith.path);
+ requireUploadPathContains(errors, uploadPath, "e2e-artifacts/vitest/tunnel-lifecycle/");
+ if (uploadWith["include-hidden-files"] !== false) {
+ errors.push("tunnel-lifecycle-vitest artifact upload must set include-hidden-files: false");
+ }
+ if (uploadWith["if-no-files-found"] !== "ignore") {
+ errors.push("tunnel-lifecycle-vitest artifact upload must ignore missing fixture artifacts");
+ }
+ if (uploadWith["retention-days"] !== 14) {
+ errors.push("tunnel-lifecycle-vitest artifact upload retention-days must be 14");
+ }
+
+ const cleanup = requireJobStep(errors, jobName, steps, "Clean up Docker auth");
+ if (cleanup?.if !== "always()") {
+ errors.push("tunnel-lifecycle-vitest Docker auth cleanup must always run");
+ }
+ requireRunContains(errors, cleanup, "docker logout docker.io");
+ requireRunContains(errors, cleanup, 'rm -rf "${DOCKER_CONFIG}"');
+}
+
function validateIssue2478CrashLoopRecoveryVitestJob(errors: string[], jobs: WorkflowRecord): void {
const jobName = "issue-2478-crash-loop-recovery-vitest";
const scenarioName = "issue-2478-crash-loop-recovery";
@@ -4328,6 +4514,8 @@ export function validateE2eVitestScenariosWorkflowBoundary(
validateIssue2478CrashLoopRecoveryVitestJob(errors, jobs);
+ validateTunnelLifecycleVitestJob(errors, jobs);
+
validateFreeStandingJobSelector(
errors,
jobs,