From 5dde375d1114f0e35812a84b473149d800610a5c Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Tue, 11 Aug 2026 23:06:24 -0400 Subject: [PATCH 1/4] test(e2e): wait for Brev host SSH --- test/brev-launchable-e2e.test.ts | 33 +++++++++++++++++++++++++++++--- tools/e2e/brev-launchable-e2e.sh | 19 ++++++++++++++++++ 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/test/brev-launchable-e2e.test.ts b/test/brev-launchable-e2e.test.ts index 64ae1ee91fa..f081a6d5029 100644 --- a/test/brev-launchable-e2e.test.ts +++ b/test/brev-launchable-e2e.test.ts @@ -36,6 +36,7 @@ function fixture( repoSha?: string; runtimeOverrides?: boolean; schemaVersion?: number; + sshReadyAfter?: number; sourceRepository?: string; sourcePath?: string; } = {}, @@ -46,6 +47,7 @@ function fixture( const workDir = path.join(root, "evidence"); const state = path.join(root, "workspace.json"); const calls = path.join(root, "calls.log"); + const sshAttempts = path.join(root, "ssh-attempts"); fs.mkdirSync(bin); fs.mkdirSync(workDir); @@ -131,6 +133,15 @@ esac path.join(bin, "ssh"), `#!/usr/bin/env bash set -euo pipefail +if [ "\${*: -1}" = true ]; then + attempts=0 + [ ! -f "$FAKE_SSH_ATTEMPTS" ] || attempts="$(cat "$FAKE_SSH_ATTEMPTS")" + attempts=$((attempts + 1)) + printf '%s\n' "$attempts" > "$FAKE_SSH_ATTEMPTS" + printf 'ssh host readiness attempt %s\n' "$attempts" >> "$FAKE_CALLS" + [ "$attempts" -ge "$FAKE_SSH_READY_AFTER" ] + exit +fi script="$(cat)" grep -q 'NEMOCLAW_E2E_SETUP_MODE=preinstalled-launchable' <<<"$script" grep -q 'NEMOCLAW_SOURCE_PATH=/opt/nemoclaw-image/NemoClaw' <<<"$script" @@ -166,6 +177,8 @@ printf 'NEMOCLAW_FULL_E2E_PASSED\\n' FAKE_REPO_SHA: options.repoSha ?? candidateSha, FAKE_RUNTIME_OVERRIDES: options.runtimeOverrides ? "true" : "false", FAKE_SCHEMA_VERSION: String(options.schemaVersion ?? 1), + FAKE_SSH_ATTEMPTS: sshAttempts, + FAKE_SSH_READY_AFTER: String(options.sshReadyAfter ?? 1), FAKE_SOURCE_REPOSITORY: options.sourceRepository ?? "NVIDIA/NemoClaw", FAKE_SOURCE_PATH: options.sourcePath ?? "/opt/nemoclaw-image/NemoClaw", FAKE_STATE: state, @@ -178,7 +191,7 @@ printf 'NEMOCLAW_FULL_E2E_PASSED\\n' RUNNER_TEMP: root, WORK_DIR: workDir, }; - return { calls, env, state, workDir }; + return { calls, env, sshAttempts, state, workDir }; } function run(env: NodeJS.ProcessEnv) { @@ -187,7 +200,7 @@ function run(env: NodeJS.ProcessEnv) { describe("focused staging Brev Launchable lane", () => { it("binds the producer run, verifies the clean booted SHA, runs E2E, and deletes (#6943)", () => { - const { calls, env, state, workDir } = fixture(); + const { calls, env, sshAttempts, state, workDir } = fixture({ sshReadyAfter: 3 }); const result = run(env); expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); const commands = fs.readFileSync(calls, "utf8"); @@ -197,6 +210,8 @@ describe("focused staging Brev Launchable lane", () => { commands.indexOf("create nclaw-e2e-test-1 --launchable env-staging123"), ); expect(commands).toContain("create nclaw-e2e-test-1 --launchable env-staging123"); + expect(commands.match(/ssh host readiness attempt/gu)).toHaveLength(3); + expect(fs.readFileSync(sshAttempts, "utf8").trim()).toBe("3"); expect(commands).toContain("ssh preinstalled full-e2e.test.ts"); expect(commands).not.toContain("nvapi-test-value"); expect(commands).not.toMatch(/rsync|install\.sh|npm (?:ci|install)|git clone/u); @@ -281,7 +296,7 @@ describe("focused staging Brev Launchable lane", () => { expect(fs.readFileSync(boot.calls, "utf8")).not.toContain("full-e2e.test.ts"); expect(fs.existsSync(boot.state)).toBe(false); } - }); + }, 90_000); it("reports E2E failure only after verified workspace cleanup", () => { const { env, state, workDir } = fixture({ e2eFails: true }); @@ -294,6 +309,18 @@ describe("focused staging Brev Launchable lane", () => { }); }); + it("fails after host SSH readiness times out and deletes the workspace", () => { + const { calls, env, state, workDir } = fixture({ sshReadyAfter: Number.MAX_SAFE_INTEGER }); + const result = run({ ...env, BREV_HOST_SSH_TIMEOUT_SECONDS: "1" }); + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("host SSH readiness timed out"); + expect(fs.readFileSync(calls, "utf8")).not.toMatch(/brev exec|full-e2e\.test\.ts/u); + expect(fs.existsSync(state)).toBe(false); + expect(JSON.parse(fs.readFileSync(path.join(workDir, "cleanup.json"), "utf8"))).toMatchObject({ + status: "ABSENT", + }); + }); + it("preserves the booted image when the provision receipt is missing", () => { const { calls, env, state, workDir } = fixture({ missingProvisionReceipt: true }); const result = run(env); diff --git a/tools/e2e/brev-launchable-e2e.sh b/tools/e2e/brev-launchable-e2e.sh index 0f5abdb0a19..afa61eaf52f 100755 --- a/tools/e2e/brev-launchable-e2e.sh +++ b/tools/e2e/brev-launchable-e2e.sh @@ -38,6 +38,24 @@ workspace() { else error("workspace name is ambiguous") end' } +wait_for_host_ssh() { + local timeout_seconds="${BREV_HOST_SSH_TIMEOUT_SECONDS:-600}" + local deadline=$((SECONDS + timeout_seconds)) + log "Waiting for host SSH access" + while [ "$SECONDS" -lt "$deadline" ]; do + timeout 60s brev refresh >/dev/null 2>&1 || true + if timeout 15s ssh -T -o BatchMode=yes -o ConnectTimeout=10 \ + -o ConnectionAttempts=1 -o NumberOfPasswordPrompts=0 \ + -o RequestTTY=no -o LogLevel=ERROR "${INSTANCE_NAME}-host" true \ + >/dev/null 2>&1; then + log "SSH access to ${INSTANCE_NAME}-host succeeded" + return 0 + fi + sleep "${POLL_SECONDS:-15}" + done + die "host SSH readiness timed out" +} + cleanup() { local record deadline absent=0 workspace_id="" record="$(workspace || true)" @@ -176,6 +194,7 @@ jq -e '.status == "RUNNING" and (.shell_status // .shellStatus) == "READY" and <<<"${ready:-null}" >/dev/null || die "workspace readiness timed out" workspace_id="$(jq -r '.id // ""' <<<"$ready")" log "Workspace $INSTANCE_NAME ($workspace_id) is ready" +wait_for_host_ssh # Record the booted image before reading the baked runtime receipt so a stale # Launchable image remains visible when the receipt is absent. From 8018c3e5311d6e3f5eb123dc9d9042c2731e3985 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Tue, 11 Aug 2026 23:34:09 -0400 Subject: [PATCH 2/4] test(e2e): enforce Brev SSH deadline --- test/brev-launchable-e2e.test.ts | 47 ++++++++++++++++++++++++++++++-- tools/e2e/brev-launchable-e2e.sh | 16 +++++++++-- 2 files changed, 58 insertions(+), 5 deletions(-) diff --git a/test/brev-launchable-e2e.test.ts b/test/brev-launchable-e2e.test.ts index f081a6d5029..1cf70443cd6 100644 --- a/test/brev-launchable-e2e.test.ts +++ b/test/brev-launchable-e2e.test.ts @@ -22,6 +22,7 @@ function executable(file: string, source: string): void { function fixture( options: { + blockRefresh?: boolean; bootImage?: string; deleteFails?: boolean; e2eFails?: boolean; @@ -48,10 +49,26 @@ function fixture( const state = path.join(root, "workspace.json"); const calls = path.join(root, "calls.log"); const sshAttempts = path.join(root, "ssh-attempts"); + const timeoutBlock = path.join(root, "timeout-block"); fs.mkdirSync(bin); fs.mkdirSync(workDir); + if (options.blockRefresh) fs.writeFileSync(timeoutBlock, "block\n"); - executable(path.join(bin, "timeout"), '#!/usr/bin/env bash\nshift\nexec "$@"\n'); + executable( + path.join(bin, "timeout"), + `#!/usr/bin/env bash +set -euo pipefail +duration="$1" +shift +printf 'timeout %s %s\n' "$duration" "$*" >> "$FAKE_CALLS" +if [ -f "$FAKE_TIMEOUT_BLOCK" ] && [ "\${1:-} \${2:-}" = "brev refresh" ]; then + rm -f "$FAKE_TIMEOUT_BLOCK" + /bin/sleep "\${duration%s}" + exit 124 +fi +exec "$@" +`, + ); executable( path.join(bin, "sleep"), '#!/usr/bin/env bash\nprintf "sleep %s\\n" "$*" >> "$FAKE_CALLS"\n', @@ -134,11 +151,17 @@ esac `#!/usr/bin/env bash set -euo pipefail if [ "\${*: -1}" = true ]; then + expected=(-T -o BatchMode=yes -o ConnectTimeout=10 -o ConnectionAttempts=1 -o NumberOfPasswordPrompts=0 -o RequestTTY=no -o LogLevel=ERROR "$INSTANCE_NAME-host" true) + [ "$#" -eq "\${#expected[@]}" ] + received=("$@") + for index in "\${!expected[@]}"; do + [ "\${received[$index]}" = "\${expected[$index]}" ] + done attempts=0 [ ! -f "$FAKE_SSH_ATTEMPTS" ] || attempts="$(cat "$FAKE_SSH_ATTEMPTS")" attempts=$((attempts + 1)) printf '%s\n' "$attempts" > "$FAKE_SSH_ATTEMPTS" - printf 'ssh host readiness attempt %s\n' "$attempts" >> "$FAKE_CALLS" + printf 'ssh host readiness attempt %s: %s\n' "$attempts" "$*" >> "$FAKE_CALLS" [ "$attempts" -ge "$FAKE_SSH_READY_AFTER" ] exit fi @@ -182,6 +205,7 @@ printf 'NEMOCLAW_FULL_E2E_PASSED\\n' FAKE_SOURCE_REPOSITORY: options.sourceRepository ?? "NVIDIA/NemoClaw", FAKE_SOURCE_PATH: options.sourcePath ?? "/opt/nemoclaw-image/NemoClaw", FAKE_STATE: state, + FAKE_TIMEOUT_BLOCK: timeoutBlock, GH_TOKEN: "github-test-token", GITHUB_RUN_ATTEMPT: "1", GITHUB_RUN_ID: "789", @@ -211,6 +235,9 @@ describe("focused staging Brev Launchable lane", () => { ); expect(commands).toContain("create nclaw-e2e-test-1 --launchable env-staging123"); expect(commands.match(/ssh host readiness attempt/gu)).toHaveLength(3); + expect(commands).toContain( + "ssh host readiness attempt 1: -T -o BatchMode=yes -o ConnectTimeout=10 -o ConnectionAttempts=1 -o NumberOfPasswordPrompts=0 -o RequestTTY=no -o LogLevel=ERROR nclaw-e2e-test-1-host true", + ); expect(fs.readFileSync(sshAttempts, "utf8").trim()).toBe("3"); expect(commands).toContain("ssh preinstalled full-e2e.test.ts"); expect(commands).not.toContain("nvapi-test-value"); @@ -321,6 +348,22 @@ describe("focused staging Brev Launchable lane", () => { }); }); + it("caps a blocking refresh by the host SSH deadline and deletes the workspace", () => { + const { calls, env, state, workDir } = fixture({ blockRefresh: true }); + const startedAt = performance.now(); + const result = run({ ...env, BREV_HOST_SSH_TIMEOUT_SECONDS: "1" }); + const elapsedMs = performance.now() - startedAt; + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("host SSH readiness timed out"); + expect(elapsedMs).toBeLessThan(10_000); + expect(fs.readFileSync(calls, "utf8")).toContain("timeout 1s brev refresh"); + expect(fs.readFileSync(calls, "utf8")).not.toMatch(/brev exec|full-e2e\.test\.ts/u); + expect(fs.existsSync(state)).toBe(false); + expect(JSON.parse(fs.readFileSync(path.join(workDir, "cleanup.json"), "utf8"))).toMatchObject({ + status: "ABSENT", + }); + }, 90_000); + it("preserves the booted image when the provision receipt is missing", () => { const { calls, env, state, workDir } = fixture({ missingProvisionReceipt: true }); const result = run(env); diff --git a/tools/e2e/brev-launchable-e2e.sh b/tools/e2e/brev-launchable-e2e.sh index afa61eaf52f..96b3f505d36 100755 --- a/tools/e2e/brev-launchable-e2e.sh +++ b/tools/e2e/brev-launchable-e2e.sh @@ -41,17 +41,27 @@ workspace() { wait_for_host_ssh() { local timeout_seconds="${BREV_HOST_SSH_TIMEOUT_SECONDS:-600}" local deadline=$((SECONDS + timeout_seconds)) + local remaining refresh_timeout sleep_seconds ssh_timeout log "Waiting for host SSH access" while [ "$SECONDS" -lt "$deadline" ]; do - timeout 60s brev refresh >/dev/null 2>&1 || true - if timeout 15s ssh -T -o BatchMode=yes -o ConnectTimeout=10 \ + remaining=$((deadline - SECONDS)) + [ "$remaining" -gt 0 ] || break + refresh_timeout=$((remaining < 60 ? remaining : 60)) + timeout "${refresh_timeout}s" brev refresh >/dev/null 2>&1 || true + remaining=$((deadline - SECONDS)) + [ "$remaining" -gt 0 ] || break + ssh_timeout=$((remaining < 15 ? remaining : 15)) + if timeout "${ssh_timeout}s" ssh -T -o BatchMode=yes -o ConnectTimeout=10 \ -o ConnectionAttempts=1 -o NumberOfPasswordPrompts=0 \ -o RequestTTY=no -o LogLevel=ERROR "${INSTANCE_NAME}-host" true \ >/dev/null 2>&1; then log "SSH access to ${INSTANCE_NAME}-host succeeded" return 0 fi - sleep "${POLL_SECONDS:-15}" + remaining=$((deadline - SECONDS)) + [ "$remaining" -gt 0 ] || break + sleep_seconds="${POLL_SECONDS:-15}" + sleep "$((sleep_seconds < remaining ? sleep_seconds : remaining))" done die "host SSH readiness timed out" } From 74852d8aefdda824c2e645eb524f276ab1d8f90a Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Tue, 11 Aug 2026 23:42:14 -0400 Subject: [PATCH 3/4] test(e2e): keep Brev fixture setup linear --- test/brev-launchable-e2e.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/brev-launchable-e2e.test.ts b/test/brev-launchable-e2e.test.ts index 1cf70443cd6..6acbc8a8190 100644 --- a/test/brev-launchable-e2e.test.ts +++ b/test/brev-launchable-e2e.test.ts @@ -52,7 +52,7 @@ function fixture( const timeoutBlock = path.join(root, "timeout-block"); fs.mkdirSync(bin); fs.mkdirSync(workDir); - if (options.blockRefresh) fs.writeFileSync(timeoutBlock, "block\n"); + fs.writeFileSync(timeoutBlock, "block\n"); executable( path.join(bin, "timeout"), @@ -205,7 +205,7 @@ printf 'NEMOCLAW_FULL_E2E_PASSED\\n' FAKE_SOURCE_REPOSITORY: options.sourceRepository ?? "NVIDIA/NemoClaw", FAKE_SOURCE_PATH: options.sourcePath ?? "/opt/nemoclaw-image/NemoClaw", FAKE_STATE: state, - FAKE_TIMEOUT_BLOCK: timeoutBlock, + FAKE_TIMEOUT_BLOCK: options.blockRefresh ? timeoutBlock : path.join(root, "timeout-disabled"), GH_TOKEN: "github-test-token", GITHUB_RUN_ATTEMPT: "1", GITHUB_RUN_ID: "789", From c28af0738fe0d1ef0621b3888028dee6fbe6a2bb Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Tue, 11 Aug 2026 23:57:23 -0400 Subject: [PATCH 4/4] test(e2e): cover blocked Brev SSH probe --- test/brev-launchable-e2e.test.ts | 54 +++++++++++++++++++++++++------- 1 file changed, 43 insertions(+), 11 deletions(-) diff --git a/test/brev-launchable-e2e.test.ts b/test/brev-launchable-e2e.test.ts index 6acbc8a8190..fdfbd96fca9 100644 --- a/test/brev-launchable-e2e.test.ts +++ b/test/brev-launchable-e2e.test.ts @@ -22,7 +22,6 @@ function executable(file: string, source: string): void { function fixture( options: { - blockRefresh?: boolean; bootImage?: string; deleteFails?: boolean; e2eFails?: boolean; @@ -40,6 +39,7 @@ function fixture( sshReadyAfter?: number; sourceRepository?: string; sourcePath?: string; + timeoutBlockCommand?: "brev refresh" | "ssh -T"; } = {}, ) { const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-launchable-e2e-")); @@ -61,7 +61,7 @@ set -euo pipefail duration="$1" shift printf 'timeout %s %s\n' "$duration" "$*" >> "$FAKE_CALLS" -if [ -f "$FAKE_TIMEOUT_BLOCK" ] && [ "\${1:-} \${2:-}" = "brev refresh" ]; then +if [ -f "$FAKE_TIMEOUT_BLOCK" ] && [ "\${1:-} \${2:-}" = "$FAKE_TIMEOUT_BLOCK_COMMAND" ]; then rm -f "$FAKE_TIMEOUT_BLOCK" /bin/sleep "\${duration%s}" exit 124 @@ -151,12 +151,11 @@ esac `#!/usr/bin/env bash set -euo pipefail if [ "\${*: -1}" = true ]; then - expected=(-T -o BatchMode=yes -o ConnectTimeout=10 -o ConnectionAttempts=1 -o NumberOfPasswordPrompts=0 -o RequestTTY=no -o LogLevel=ERROR "$INSTANCE_NAME-host" true) - [ "$#" -eq "\${#expected[@]}" ] - received=("$@") - for index in "\${!expected[@]}"; do - [ "\${received[$index]}" = "\${expected[$index]}" ] + required=(-T "-o BatchMode=yes" "-o ConnectTimeout=10" "-o ConnectionAttempts=1" "-o NumberOfPasswordPrompts=0" "-o RequestTTY=no" "-o LogLevel=ERROR") + for argument in "\${required[@]}"; do + [[ " $* " == *" $argument "* ]] done + [ "\${*: -2:1}" = "$INSTANCE_NAME-host" ] attempts=0 [ ! -f "$FAKE_SSH_ATTEMPTS" ] || attempts="$(cat "$FAKE_SSH_ATTEMPTS")" attempts=$((attempts + 1)) @@ -205,7 +204,10 @@ printf 'NEMOCLAW_FULL_E2E_PASSED\\n' FAKE_SOURCE_REPOSITORY: options.sourceRepository ?? "NVIDIA/NemoClaw", FAKE_SOURCE_PATH: options.sourcePath ?? "/opt/nemoclaw-image/NemoClaw", FAKE_STATE: state, - FAKE_TIMEOUT_BLOCK: options.blockRefresh ? timeoutBlock : path.join(root, "timeout-disabled"), + FAKE_TIMEOUT_BLOCK: options.timeoutBlockCommand + ? timeoutBlock + : path.join(root, "timeout-disabled"), + FAKE_TIMEOUT_BLOCK_COMMAND: options.timeoutBlockCommand ?? "", GH_TOKEN: "github-test-token", GITHUB_RUN_ATTEMPT: "1", GITHUB_RUN_ID: "789", @@ -235,9 +237,23 @@ describe("focused staging Brev Launchable lane", () => { ); expect(commands).toContain("create nclaw-e2e-test-1 --launchable env-staging123"); expect(commands.match(/ssh host readiness attempt/gu)).toHaveLength(3); - expect(commands).toContain( - "ssh host readiness attempt 1: -T -o BatchMode=yes -o ConnectTimeout=10 -o ConnectionAttempts=1 -o NumberOfPasswordPrompts=0 -o RequestTTY=no -o LogLevel=ERROR nclaw-e2e-test-1-host true", + const readinessCall = commands + .split("\n") + .find((line) => line.startsWith("ssh host readiness attempt 1: ")); + expect(readinessCall).toBeDefined(); + const readinessArgs = readinessCall?.split(": ").at(1)?.split(" ") ?? []; + expect(readinessArgs).toEqual( + expect.arrayContaining([ + "-T", + "BatchMode=yes", + "ConnectTimeout=10", + "ConnectionAttempts=1", + "NumberOfPasswordPrompts=0", + "RequestTTY=no", + "LogLevel=ERROR", + ]), ); + expect(readinessArgs.slice(-2)).toEqual(["nclaw-e2e-test-1-host", "true"]); expect(fs.readFileSync(sshAttempts, "utf8").trim()).toBe("3"); expect(commands).toContain("ssh preinstalled full-e2e.test.ts"); expect(commands).not.toContain("nvapi-test-value"); @@ -349,7 +365,7 @@ describe("focused staging Brev Launchable lane", () => { }); it("caps a blocking refresh by the host SSH deadline and deletes the workspace", () => { - const { calls, env, state, workDir } = fixture({ blockRefresh: true }); + const { calls, env, state, workDir } = fixture({ timeoutBlockCommand: "brev refresh" }); const startedAt = performance.now(); const result = run({ ...env, BREV_HOST_SSH_TIMEOUT_SECONDS: "1" }); const elapsedMs = performance.now() - startedAt; @@ -364,6 +380,22 @@ describe("focused staging Brev Launchable lane", () => { }); }, 90_000); + it("caps a blocking SSH probe by the host SSH deadline and deletes the workspace", () => { + const { calls, env, state, workDir } = fixture({ timeoutBlockCommand: "ssh -T" }); + const startedAt = performance.now(); + const result = run({ ...env, BREV_HOST_SSH_TIMEOUT_SECONDS: "2" }); + const elapsedMs = performance.now() - startedAt; + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("host SSH readiness timed out"); + expect(elapsedMs).toBeLessThan(10_000); + expect(fs.readFileSync(calls, "utf8")).toContain("timeout 2s ssh -T"); + expect(fs.readFileSync(calls, "utf8")).not.toMatch(/brev exec|full-e2e\.test\.ts/u); + expect(fs.existsSync(state)).toBe(false); + expect(JSON.parse(fs.readFileSync(path.join(workDir, "cleanup.json"), "utf8"))).toMatchObject({ + status: "ABSENT", + }); + }, 90_000); + it("preserves the booted image when the provision receipt is missing", () => { const { calls, env, state, workDir } = fixture({ missingProvisionReceipt: true }); const result = run(env);