From 7089183498158347153e5ffe5e2d774d9c06446b Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 11 May 2026 17:22:06 -0700 Subject: [PATCH] Revert "ci(nightly): enable brev-e2e job with long-lived BREV_API_TOKEN (#3350)" This reverts commit 0776ea063547cca80908dcacac5d07562de0ad82. --- .github/workflows/e2e-branch-validation.yaml | 160 +++-------- .github/workflows/nightly-e2e.yaml | 56 +--- test/e2e/brev-e2e.test.ts | 270 ++++--------------- vitest.config.ts | 18 +- 4 files changed, 97 insertions(+), 407 deletions(-) diff --git a/.github/workflows/e2e-branch-validation.yaml b/.github/workflows/e2e-branch-validation.yaml index fdef5d92830..bf5b6b74ca5 100644 --- a/.github/workflows/e2e-branch-validation.yaml +++ b/.github/workflows/e2e-branch-validation.yaml @@ -3,26 +3,18 @@ name: e2e-branch-validation -# ─── Branch Validation E2E ─────────────────────────────────────────── +# ─── Branch Validation E2E ─────────────────────────────────────────────────── # -# PURPOSE: Validates a specific git branch by rsync-ing it onto a real Brev -# cloud instance provisioned from the PUBLISHED NemoClaw launchable image, -# then running the selected security / messaging / full suites against it. -# Answers: "Does this branch work on the image customers actually get?" +# PURPOSE: Validates a specific git branch by installing NemoClaw FROM SOURCE +# on an ephemeral Brev cloud instance. Answers the question: "Does this branch +# work if you install from source on a clean machine?" # # HOW IT WORKS: -# 1. Provisions a Brev CPU instance from the published NemoClaw launchable -# (env-3Azt0aYgVNFEuz7opyx3gscmowS) — a pre-baked GCP image built 3x -# daily by brevdev/nemoclaw-image with NemoClaw, OpenShell, and the -# sandbox image already on disk. Boots in ~2 min. -# 2. Rsyncs the checked-out branch code over the image's ~/NemoClaw. -# 3. Rebuilds CLI dist/, runs onboard, and executes the selected suite. -# 4. Tears down the instance (unless keep_alive=true). -# -# FALLBACK: set use_published_launchable=false to provision a bare Ubuntu -# VM and bootstrap via scripts/brev-launchable-ci-cpu.sh. Slower (~5 min) -# and does NOT test the image itself — useful only for validating changes -# to the startup script without touching the published image. +# 1. Provisions a fresh Brev CPU instance (~4 vCPU, 16 GB RAM) +# 2. Rsyncs the checked-out branch code to the VM +# 3. Runs install.sh from source + onboards a sandbox +# 4. Executes the selected test suite against the live sandbox +# 5. Tears down the instance (unless keep_alive=true) # # WHEN TO USE: # - Before merging a PR that touches onboard, sandbox, security, or infra @@ -53,27 +45,14 @@ name: e2e-branch-validation # all — Runs credential-sanitization + telegram-injection (NOT full, # which destroys the sandbox the security tests need). # -# Required secrets: BREV_API_KEY, BREV_ORG_ID, NVIDIA_API_KEY -# BREV_API_KEY — long-lived Brev API key (format: bak-...). Replaces the -# legacy short-lived BREV_API_TOKEN refresh-token secret. -# BREV_ORG_ID — Brev organization id (format: org-...). Required alongside -# the API key for brev login. -# -# Optional inputs: -# launchable_id — override the published launchable. Default (empty → -# env-3Azt0aYgVNFEuz7opyx3gscmowS) matches docs/deployment/ -# brev-web-ui.md. Override to test a staging image. +# Required secrets: BREV_API_TOKEN, NVIDIA_API_KEY # Instance cost: Brev CPU credits (~$0.10/run for 4x16 instance) on: workflow_dispatch: inputs: - branch: - description: "Branch to test (default: the branch this dispatch targets via --ref)" - required: false - default: "" pr_number: - description: "PR number (resolves branch automatically, overrides branch input)" + description: "PR number (resolves branch automatically)" required: false default: "" test_suite: @@ -88,24 +67,19 @@ on: - messaging-providers - all use_launchable: - description: "Deprecated — always true. Kept for backward-compat with existing dispatches." + description: "Use CI launchable (true) or bare brev create + brev-setup.sh (false)" required: false type: boolean default: true - use_published_launchable: - description: "Provision from the published NemoClaw launchable image (recommended) vs. the repo-local startup-script fallback" - required: false - type: boolean - default: true - launchable_id: - description: "Published launchable ID (default: the public NemoClaw launchable env-3Azt0aYgVNFEuz7opyx3gscmowS)" - required: false - default: "" keep_alive: description: "Keep Brev instance alive after tests (for SSH debugging)" required: false type: boolean default: false + brev_token: + description: "Brev refresh token (overrides BREV_API_TOKEN secret if provided)" + required: false + default: "" workflow_call: inputs: branch: @@ -123,14 +97,6 @@ on: required: false type: boolean default: true - use_published_launchable: - required: false - type: boolean - default: true - launchable_id: - required: false - type: string - default: "" setup_script_url: required: false type: string @@ -140,9 +106,7 @@ on: type: boolean default: true secrets: - BREV_API_KEY: - required: true - BREV_ORG_ID: + BREV_API_TOKEN: required: true NVIDIA_API_KEY: required: true @@ -158,14 +122,7 @@ concurrency: jobs: e2e-branch-validation: - # Allow both upstream (NVIDIA/NemoClaw) and the known CI fork (jyaunches/NemoClaw). - # The fork is used to validate the long-lived BREV_API_KEY + nightly wiring - # before the secret is rotated into the upstream repo. Other forks get a no-op - # skip so community PRs that touch this file don't try to run against someone - # else's Brev account. - if: >- - github.repository == 'NVIDIA/NemoClaw' || - github.repository == 'jyaunches/NemoClaw' + # if: github.repository == 'NVIDIA/NemoClaw' # Disabled for fork testing — re-enable before merge runs-on: ubuntu-latest timeout-minutes: 90 steps: @@ -181,15 +138,7 @@ jobs: - name: Checkout target branch uses: actions/checkout@v6 with: - # Precedence: - # 1. PR-resolved branch (from pr_number lookup above) - # 2. Explicit branch input (workflow_dispatch / workflow_call) - # 3. github.ref_name (the ref that was dispatched against) — - # this is what `gh workflow run --ref X` passes. Without this - # fallback, workflow_dispatch without explicit inputs silently - # checks out 'main' even when the dispatch targeted a feature - # branch, making branch changes invisible to the run. - ref: ${{ env.RESOLVED_BRANCH || inputs.branch || github.ref_name }} + ref: ${{ env.RESOLVED_BRANCH || inputs.branch || 'main' }} - name: Create check run (pending) if: inputs.pr_number != '' @@ -215,79 +164,40 @@ jobs: - name: Install Brev CLI env: - # GitHub Actions inputs are plaintext run metadata; Brev credentials - # must only come from repository/org secrets so they remain masked. - BREV_API_KEY: ${{ secrets.BREV_API_KEY }} - BREV_ORG_ID: ${{ secrets.BREV_ORG_ID }} + BREV_API_TOKEN: ${{ inputs.brev_token || secrets.BREV_API_TOKEN }} run: | - # Brev CLI v0.6.324+ — first release to support `brev login --api-key` - # for long-lived CI auth (bak-... keys). Earlier v0.6.322 pin required - # writing ~/.brev/credentials.json by hand because `brev login` had - # been removed in #1470 and only supported interactive OAuth. - curl -fsSL -o /tmp/brev.tar.gz "https://github.com/brevdev/brev-cli/releases/download/v0.6.324/brev-cli_0.6.324_linux_amd64.tar.gz" + # Brev CLI v0.6.322+ — CPU instances use `brev search cpu | brev create` + # Startup scripts use `brev create --startup-script @file` (not brev start --cpu) + curl -fsSL -o /tmp/brev.tar.gz "https://github.com/brevdev/brev-cli/releases/download/v0.6.322/brev-cli_0.6.322_linux_amd64.tar.gz" tar -xzf /tmp/brev.tar.gz -C /usr/local/bin brev chmod +x /usr/local/bin/brev - if [ -z "${BREV_API_KEY:-}" ]; then - echo "::error::BREV_API_KEY is empty — cannot authenticate Brev CLI." + # Brev CLI does not read BREV_API_TOKEN from env — it requires + # ~/.brev/credentials.json. The login call was removed in #1470 + # (374a847d), breaking CI. Write the credentials file so `brev ls` + # works in the test harness's hasAuthenticatedBrev check. + if [ -z "${BREV_API_TOKEN:-}" ]; then + echo "::error::BREV_API_TOKEN is empty — cannot authenticate Brev CLI." exit 1 fi - if [ -z "${BREV_ORG_ID:-}" ]; then - echo "::error::BREV_ORG_ID is empty — required for API-key login." - exit 1 - fi - # Long-lived API-key login. Persists api_key + api_key_org_id into - # ~/.brev/credentials.json; no browser, no refresh-token expiry. - brev login --api-key "$BREV_API_KEY" --org-id "$BREV_ORG_ID" + mkdir -p ~/.brev + umask 077 + printf '{"refresh_token":"%s"}' "$BREV_API_TOKEN" > ~/.brev/credentials.json chmod 600 ~/.brev/credentials.json - # Skip the first-run onboarding wizard that blocks on stdin (removed in #1470) + # Skip the first-run onboarding wizard that blocks on stdin (also removed in #1470) printf '{"step":1,"hasRunBrevShell":true,"hasRunBrevOpen":true}' > ~/.brev/onboarding_step.json - # Verify credentials with brev ls. The Brev backend periodically - # returns 'context deadline exceeded' on the workspaces/list - # endpoint under platform load — retry a few times before - # surfacing as a hard failure. Observed on 2026-05-11 during PR - # #3350 validation: up to ~30 min windows where brev ls failed - # across multiple orgs with working credentials. - for attempt in 1 2 3 4 5; do - if brev ls >/dev/null 2>/tmp/brev-ls-err; then - echo "brev ls succeeded on attempt $attempt" - break - fi - err=$(cat /tmp/brev-ls-err) - if [ "$attempt" -eq 5 ]; then - echo "::warning::brev ls failed 5 times (last error: $err) — proceeding anyway; the test harness will retry with brev refresh if needed." - break - fi - echo "brev ls attempt $attempt failed ($err) — retrying in 15s..." - sleep 15 - done + brev ls >/dev/null - name: Install dependencies run: npm install --ignore-scripts - - name: Build CLI (dist/) - # The Brev E2E harness invokes bin/nemoclaw.js locally for pre-flight - # sandbox-name validation before touching Brev. bin/nemoclaw.js - # requires '../dist/nemoclaw', which is produced by the `prepare` - # npm script — skipped above because --ignore-scripts is set to - # avoid the git-hook installer running in CI. Build explicitly. - run: npm run build:cli - - name: Run ephemeral Brev E2E env: - BREV_API_KEY: ${{ secrets.BREV_API_KEY }} - BREV_ORG_ID: ${{ secrets.BREV_ORG_ID }} + BREV_API_TOKEN: ${{ inputs.brev_token || secrets.BREV_API_TOKEN }} NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} GITHUB_TOKEN: ${{ github.token }} INSTANCE_NAME: e2e-pr-${{ inputs.pr_number || github.run_id }} TEST_SUITE: ${{ inputs.test_suite }} USE_LAUNCHABLE: ${{ inputs.use_launchable && '1' || '0' }} - # Use the published NemoClaw launchable (pre-baked GCP image, ~2 min - # boot) by default. Set to '0' to fall back to the repo-local - # startup-script path (bare VM, ~5 min bootstrap). - USE_PUBLISHED_LAUNCHABLE: ${{ inputs.use_published_launchable && '1' || '0' }} - # Launchable ID override. Empty string → test harness default - # (env-3Azt0aYgVNFEuz7opyx3gscmowS, the public NemoClaw launchable). - BREV_LAUNCHABLE_ID: ${{ inputs.launchable_id || '' }} LAUNCHABLE_SETUP_SCRIPT: ${{ inputs.setup_script_url || '' }} BREV_PROVIDER: gcp KEEP_ALIVE: ${{ inputs.keep_alive }} diff --git a/.github/workflows/nightly-e2e.yaml b/.github/workflows/nightly-e2e.yaml index 3118e298289..e11d4141dd7 100644 --- a/.github/workflows/nightly-e2e.yaml +++ b/.github/workflows/nightly-e2e.yaml @@ -43,13 +43,6 @@ # OpenShell gateway, secure zero-fill on unlink, allowlist filter # on non-credential env keys, and symlink-safe deletion. # launchable-smoke-e2e Community install path (brev-launchable-ci-cpu.sh) on ubuntu-latest. -# brev-e2e Ephemeral Brev cloud instance end-to-end — provisions a real -# Brev VM from the PUBLISHED NemoClaw launchable image -# (env-3Azt0aYgVNFEuz7opyx3gscmowS, rebuilt 3x daily by -# brevdev/nemoclaw-image), rsyncs main over it, runs the -# selected test suites, and tears down. This is the only -# coverage that validates the image customers actually get. -# Requires BREV_API_KEY (long-lived bak-... CI key) + BREV_ORG_ID. # gpu-e2e Local Ollama inference on an NVKS ephemeral GPU runner. # gpu-double-onboard-e2e Ollama proxy token consistency after re-onboard (#2553). # notify-on-failure Auto-creates a GitHub issue when any E2E job fails. @@ -92,7 +85,7 @@ on: onboard-repair-e2e, onboard-resume-e2e, runtime-overrides-e2e, credential-sanitization-e2e, telegram-injection-e2e, overlayfs-autofix-e2e, device-auth-health-e2e, - launchable-smoke-e2e, brev-e2e, gpu-e2e, gpu-double-onboard-e2e, + launchable-smoke-e2e, gpu-e2e, gpu-double-onboard-e2e, brave-search-e2e required: false type: string @@ -1780,50 +1773,6 @@ jobs: path: /tmp/nemoclaw-launchable-test.log if-no-files-found: ignore - # ── Ephemeral Brev E2E (real cloud instance + launchable) ──────── - # Provisions a fresh Brev CPU instance via the CI launchable, installs - # NemoClaw from main, and runs the selected security + messaging suites. - # Calls the reusable e2e-branch-validation.yaml workflow. - # - # Matrix rationale: - # - `all` covers credential-sanitization + telegram-injection - # (the two security regression suites that need a shared sandbox). - # - `messaging-providers` creates its own sandbox and exercises the - # Telegram + Discord L7 proxy chain. - # - `full` is the end-to-end install → onboard → inference → CLI path. - # Listed last because it destroys its sandbox on completion. - # - # Cost: ~$0.10 per Brev instance × 3 suites ≈ $9/month at nightly cadence. - # Secrets: BREV_API_KEY (long-lived bak-... CI key), BREV_ORG_ID (org-...), - # NVIDIA_API_KEY. - brev-e2e: - if: >- - github.repository == 'NVIDIA/NemoClaw' && - (github.event_name != 'workflow_dispatch' || - inputs.jobs == '' || - contains(format(',{0},', inputs.jobs), ',brev-e2e,')) - strategy: - fail-fast: false - matrix: - test_suite: [all, messaging-providers, full] - uses: ./.github/workflows/e2e-branch-validation.yaml - with: - # Respect the selected ref for manual dispatches; scheduled runs execute - # on main, so this remains main there. - branch: ${{ github.ref_name }} - test_suite: ${{ matrix.test_suite }} - use_launchable: true - # Provision from the published NemoClaw launchable image (pre-baked - # GCP image, ~2 min boot), not the startup-script fallback. Empty - # launchable_id → harness default env-3Azt0aYgVNFEuz7opyx3gscmowS. - use_published_launchable: true - launchable_id: "" - keep_alive: false - secrets: - BREV_API_KEY: ${{ secrets.BREV_API_KEY }} - BREV_ORG_ID: ${{ secrets.BREV_ORG_ID }} - NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} - # ── GPU E2E (Ollama local inference) ────────────────────────── # Runs on an NVKS ephemeral GPU runner (RTX Pro 6000, 36 GB VRAM). # Each job gets a fresh VM — no state leakage between runs. @@ -1981,7 +1930,6 @@ jobs: overlayfs-autofix-e2e, device-auth-health-e2e, launchable-smoke-e2e, - brev-e2e, gpu-e2e, gpu-double-onboard-e2e, ] @@ -2074,7 +2022,6 @@ jobs: overlayfs-autofix-e2e, device-auth-health-e2e, launchable-smoke-e2e, - brev-e2e, gpu-e2e, gpu-double-onboard-e2e, ] @@ -2215,7 +2162,6 @@ jobs: overlayfs-autofix-e2e, device-auth-health-e2e, launchable-smoke-e2e, - brev-e2e, gpu-e2e, gpu-double-onboard-e2e, ] diff --git a/test/e2e/brev-e2e.test.ts b/test/e2e/brev-e2e.test.ts index d6f8f97093a..5eca7c38188 100644 --- a/test/e2e/brev-e2e.test.ts +++ b/test/e2e/brev-e2e.test.ts @@ -67,39 +67,15 @@ function requireInstanceName(): string { } // Launchable configuration -// -// Two provisioning modes are supported: -// -// 1. PUBLISHED LAUNCHABLE (default, preferred): -// `brev create INSTANCE --launchable $BREV_LAUNCHABLE_ID` -// Uses the pre-baked GCP machine image published by brevdev/nemoclaw-image -// (rebuilt 3x daily). Customer-identical — NemoClaw, OpenShell, sandbox -// image all pre-imported into containerd. Boots in ~2 min. -// BREV_LAUNCHABLE_ID defaults to the public NemoClaw launchable: -// env-3Azt0aYgVNFEuz7opyx3gscmowS -// (same ID surfaced in docs/deployment/brev-web-ui.md). -// -// 2. STARTUP SCRIPT (fallback, legacy): -// `brev search cpu | brev create INSTANCE --startup-script @file` -// Provisions a bare Ubuntu VM and bootstraps with the repo-local -// scripts/brev-launchable-ci-cpu.sh. Useful for validating launchable -// setup logic without the published image, or as a fallback if the -// published launchable is down. Opt in with USE_PUBLISHED_LAUNCHABLE=0 -// or by setting LAUNCHABLE_SETUP_SCRIPT to a custom path/URL. -// -const PUBLISHED_LAUNCHABLE_ID_DEFAULT = "env-3Azt0aYgVNFEuz7opyx3gscmowS"; -const BREV_LAUNCHABLE_ID = process.env.BREV_LAUNCHABLE_ID || PUBLISHED_LAUNCHABLE_ID_DEFAULT; -// Default to the published launchable. Set USE_PUBLISHED_LAUNCHABLE=0 (or "false") -// to fall back to the startup-script path. -const USE_PUBLISHED_LAUNCHABLE = - process.env.USE_PUBLISHED_LAUNCHABLE !== "0" && - process.env.USE_PUBLISHED_LAUNCHABLE !== "false"; +// CI-Ready CPU setup script: pre-bakes Docker, Node.js, OpenShell CLI, npm deps, Docker images. +// The Brev CLI (v0.6.322+) uses `brev search cpu | brev create --startup-script @file`. +// Default: use the repo-local script (hermetic — always matches the checked-out branch). +// Override via LAUNCHABLE_SETUP_SCRIPT env var to test a remote URL instead. const DEFAULT_SETUP_SCRIPT_PATH = process.env.LAUNCHABLE_SETUP_SCRIPT || path.join(REPO_DIR, "scripts", "brev-launchable-ci-cpu.sh"); -// Sentinel file written by brev-launchable-ci-cpu.sh when startup-script setup -// completes. Not present on the published launchable image — that path uses -// a systemd/CLI-based readiness probe instead. +// Sentinel file written by brev-launchable-ci-cpu.sh when setup is complete. +// More reliable than grepping log files. const LAUNCHABLE_SENTINEL = "/var/run/nemoclaw-launchable-ready"; let remoteDir = ""; @@ -120,42 +96,11 @@ function brev(...args: string[]): string { } function listBrevInstances(): Array<{ name: string; status?: string }> { - // `brev ls --json` output shape varies between CLI versions: - // v0.6.322 returned a bare JSON array. - // v0.6.324 may return an object wrapper (workspaces/instances/data/ - // result) or null on empty state. Be defensive so hasBrevInstance() - // never throws on non-array shapes just because we upgraded the CLI. - let raw: string; try { - raw = brev("ls", "--json"); - } catch (err) { - console.log(`[listBrevInstances] brev ls --json failed: ${err}`); - return []; - } - let parsed: unknown; - try { - parsed = JSON.parse(raw); - } catch (err) { - console.log( - `[listBrevInstances] JSON.parse failed: ${err}; raw(first 500 chars)=${raw.slice(0, 500)}`, - ); + return JSON.parse(brev("ls", "--json")); + } catch { return []; } - if (Array.isArray(parsed)) return parsed as Array<{ name: string; status?: string }>; - if (parsed && typeof parsed === "object") { - const obj = parsed as Record; - for (const key of ["workspaces", "instances", "data", "result", "items"]) { - if (Array.isArray(obj[key])) { - return obj[key] as Array<{ name: string; status?: string }>; - } - } - } - console.log( - `[listBrevInstances] unrecognized shape; typeof=${typeof parsed}; keys=${ - parsed && typeof parsed === "object" ? Object.keys(parsed).join(",") : "n/a" - }; raw(first 500 chars)=${raw.slice(0, 500)}`, - ); - return []; } function hasBrevInstance(instanceName: string): boolean { @@ -244,16 +189,7 @@ function sshEnv( return ssh(`${envPrefix} && ${cmd}`, { timeout, stream }); } -function waitForSsh(maxAttempts = 0, intervalMs = 5_000): void { - // Launchable instances need more time than bare VMs: the pre-baked image - // boots in ~2 min but the Brev platform's SSH-proxy registration can lag - // another minute or two before `brev refresh` picks the instance up. The - // startup-script path is faster to register because the VM is a plain - // GCP instance that comes up in the default Brev flow. Default the - // attempt count per mode unless the caller passes an explicit override. - if (maxAttempts === 0) { - maxAttempts = USE_PUBLISHED_LAUNCHABLE ? 96 : 40; - } +function waitForSsh(maxAttempts = 40, intervalMs = 5_000): void { for (let i = 1; i <= maxAttempts; i++) { try { ssh("echo ok", { timeout: 10_000 }); @@ -278,69 +214,37 @@ function waitForSsh(maxAttempts = 0, intervalMs = 5_000): void { } /** - * Wait for the launchable VM to be ready for tests. - * - * Two probe strategies depending on provisioning mode: - * - * - PUBLISHED_LAUNCHABLE: check that the `nemoclaw` and `openshell` CLIs - * are on PATH and the ~/NemoClaw clone is present. The pre-baked image - * has these at image-build time; we just need the VM to have booted - * and SSH to be reachable. We do NOT wait for openshell-gateway to be - * active — `nemoclaw onboard` in the test's bootstrap will start it. - * - * - STARTUP_SCRIPT: poll for /var/run/nemoclaw-launchable-ready, the - * sentinel file written by brev-launchable-ci-cpu.sh on completion. + * Wait for the launchable setup script to finish by checking a sentinel file. + * Much more reliable than grepping log files. */ function waitForLaunchableReady(maxWaitMs = 1_200_000, pollIntervalMs = 15_000): void { const start = Date.now(); const elapsed = () => `${Math.round((Date.now() - start) / 1000)}s`; let consecutiveSshFailures = 0; - const mode = USE_PUBLISHED_LAUNCHABLE ? "published-launchable" : "startup-script"; - console.log(`[${elapsed()}] Waiting for VM to be ready (mode: ${mode})...`); - - // Readiness probe varies by provisioning mode. Both return "READY" when - // the VM is ready for tests and "PENDING" otherwise. Published-launchable - // also emits a short diagnostic suffix so the CI log shows which prereq - // (nemoclaw / openshell / ~/NemoClaw) is still missing. - const probeCmd = USE_PUBLISHED_LAUNCHABLE - ? `{ command -v nemoclaw >/dev/null 2>&1 \ - && command -v openshell >/dev/null 2>&1 \ - && test -d "$HOME/NemoClaw" \ - && echo READY; } \ - || { \ - have_nc=$(command -v nemoclaw >/dev/null 2>&1 && echo y || echo n); \ - have_os=$(command -v openshell >/dev/null 2>&1 && echo y || echo n); \ - have_dir=$(test -d "$HOME/NemoClaw" && echo y || echo n); \ - echo "PENDING nemoclaw=$have_nc openshell=$have_os repo=$have_dir"; \ - }` - : `test -f ${LAUNCHABLE_SENTINEL} && echo READY || echo PENDING`; - while (Date.now() - start < maxWaitMs) { try { - const result = ssh(probeCmd, { timeout: 15_000 }); + const result = ssh(`test -f ${LAUNCHABLE_SENTINEL} && echo READY || echo PENDING`, { + timeout: 15_000, + }); consecutiveSshFailures = 0; // reset on success if (result.includes("READY")) { - console.log(`[${elapsed()}] VM ready (${mode})`); + console.log(`[${elapsed()}] Launchable setup complete (sentinel file found)`); return; } - const progress = USE_PUBLISHED_LAUNCHABLE - ? result.trim() - : (() => { - try { - return ssh( - "tail -2 /tmp/launch-plugin.log 2>/dev/null || echo '(no log yet)'", - { timeout: 10_000 }, - ).replace(/\n/g, " | "); - } catch { - return "(log fetch failed)"; - } - })(); - console.log(`[${elapsed()}] Still pending... ${progress}`); + // Show progress from the setup log + try { + const tail = ssh("tail -2 /tmp/launch-plugin.log 2>/dev/null || echo '(no log yet)'", { + timeout: 10_000, + }); + console.log(`[${elapsed()}] Setup still running... ${tail.replace(/\n/g, " | ")}`); + } catch { + /* ignore */ + } } catch { consecutiveSshFailures++; console.log( - `[${elapsed()}] Readiness poll: SSH command failed (${consecutiveSshFailures} consecutive), retrying...`, + `[${elapsed()}] Setup poll: SSH command failed (${consecutiveSshFailures} consecutive), retrying...`, ); // Brev VMs sometimes reboot during setup (kernel upgrades, etc.) // Refresh the SSH config every 3 consecutive failures to pick up @@ -359,12 +263,9 @@ function waitForLaunchableReady(maxWaitMs = 1_200_000, pollIntervalMs = 15_000): execSync(`sleep ${pollIntervalMs / 1000}`); } - const readinessHint = USE_PUBLISHED_LAUNCHABLE - ? `nemoclaw+openshell CLIs + ~/NemoClaw (published launchable ${BREV_LAUNCHABLE_ID})` - : `sentinel ${LAUNCHABLE_SENTINEL}`; throw new Error( - `Launchable VM did not become ready within ${maxWaitMs / 60_000} minutes. ` + - `Readiness check: ${readinessHint}`, + `Launchable setup did not complete within ${maxWaitMs / 60_000} minutes. ` + + `Sentinel file ${LAUNCHABLE_SENTINEL} not found.`, ); } @@ -437,11 +338,7 @@ function refreshAndWaitForSsh(elapsed: () => string): void { } /** - * Create a Brev instance. - * - * Prefers the published NemoClaw launchable (pre-baked GCP image, ~2 min boot) - * unless explicitly opted out via USE_PUBLISHED_LAUNCHABLE=0. The startup-script - * path is kept as a fallback for validating setup-script changes themselves. + * Create a Brev instance via `brev search cpu | brev create` with a startup script. * * The Brev API sometimes returns "unexpected EOF" after the instance is actually * created server-side. The CLI then falls back to the next instance type, which @@ -449,12 +346,36 @@ function refreshAndWaitForSsh(elapsed: () => string): void { * check if the instance exists anyway. */ function createBrevInstance(elapsed: () => string): void { + console.log( + `[${elapsed()}] Creating instance via launchable (brev search cpu | brev create + startup-script)...`, + ); + console.log(`[${elapsed()}] setup-script: ${DEFAULT_SETUP_SCRIPT_PATH}`); + console.log( + `[${elapsed()}] cpu: min ${BREV_MIN_VCPU} vCPU, ${BREV_MIN_RAM} GB RAM, ${BREV_MIN_DISK} GB disk, provider: ${BREV_PROVIDER}`, + ); + + // Resolve the setup script to a local file path. + // Default: repo-local scripts/brev-launchable-ci-cpu.sh (hermetic). + // Override: set LAUNCHABLE_SETUP_SCRIPT to a URL and it gets downloaded. + let setupScriptPath: string; + if (DEFAULT_SETUP_SCRIPT_PATH.startsWith("http")) { + setupScriptPath = "/tmp/brev-ci-setup.sh"; + execSync(`curl -fsSL -o ${setupScriptPath} "${DEFAULT_SETUP_SCRIPT_PATH}"`, { + encoding: "utf-8", + timeout: 30_000, + }); + console.log(`[${elapsed()}] Setup script downloaded to ${setupScriptPath}`); + } else { + setupScriptPath = DEFAULT_SETUP_SCRIPT_PATH; + console.log(`[${elapsed()}] Using repo-local setup script`); + } + try { - if (USE_PUBLISHED_LAUNCHABLE) { - createPublishedLaunchableInstance(elapsed); - } else { - createStartupScriptInstance(elapsed); - } + execSync( + `brev search cpu --min-vcpu ${BREV_MIN_VCPU} --min-ram ${BREV_MIN_RAM} --min-disk ${BREV_MIN_DISK} --provider ${BREV_PROVIDER} --sort price | ` + + `brev create ${INSTANCE_NAME} --startup-script @${setupScriptPath} --detached`, + { encoding: "utf-8", timeout: 180_000, stdio: PIPE_INPUT_STDIO }, + ); } catch (createErr) { console.log( `[${elapsed()}] brev create exited with error — checking if instance was created anyway...`, @@ -464,8 +385,7 @@ function createBrevInstance(elapsed: () => string): void { } catch { /* ignore */ } - const lsResult = spawnSync("brev", ["ls"], { encoding: "utf-8", timeout: 30_000 }); - const lsOutput = `${lsResult.stdout || ""}\n${lsResult.stderr || ""}`; + const lsOutput = execSync(`brev ls 2>&1 || true`, { encoding: "utf-8", timeout: 30_000 }); const instanceName = requireInstanceName(); if (!lsOutput.includes(instanceName)) { const createMessage = createErr instanceof Error ? createErr.message : String(createErr); @@ -482,80 +402,6 @@ function createBrevInstance(elapsed: () => string): void { console.log(`[${elapsed()}] brev create returned (instance provisioning in background)`); } -/** - * Create via `brev create --launchable ...`. Uses the published NemoClaw - * launchable image (env-... id) which boots from a pre-baked GCP machine - * image in ~2 min with NemoClaw + OpenShell + sandbox image already on disk. - */ -function createPublishedLaunchableInstance(elapsed: () => string): void { - console.log( - `[${elapsed()}] Creating instance via published launchable (pre-baked image, ~2 min boot)...`, - ); - console.log(`[${elapsed()}] launchable-id: ${BREV_LAUNCHABLE_ID}`); - execFileSync( - "brev", - ["create", requireInstanceName(), "--launchable", BREV_LAUNCHABLE_ID, "--detached"], - { encoding: "utf-8", timeout: 180_000, stdio: CAPTURE_STDIO }, - ); -} - -/** - * Create via `brev search cpu | brev create --startup-script @file` without - * shell interpolation. Legacy path — provisions a bare Ubuntu VM and runs the - * repo-local brev-launchable-ci-cpu.sh to replicate what the published - * launchable pre-bakes. Kept as a fallback for validating setup-script changes. - */ -function createStartupScriptInstance(elapsed: () => string): void { - console.log( - `[${elapsed()}] Creating instance via startup-script (bare VM + bootstrap, fallback mode)...`, - ); - console.log(`[${elapsed()}] setup-script: ${DEFAULT_SETUP_SCRIPT_PATH}`); - console.log( - `[${elapsed()}] cpu: min ${BREV_MIN_VCPU} vCPU, ${BREV_MIN_RAM} GB RAM, ${BREV_MIN_DISK} GB disk, provider: ${BREV_PROVIDER}`, - ); - let setupScriptPath: string; - if (DEFAULT_SETUP_SCRIPT_PATH.startsWith("http")) { - setupScriptPath = "/tmp/brev-ci-setup.sh"; - execFileSync("curl", ["-fsSL", "-o", setupScriptPath, DEFAULT_SETUP_SCRIPT_PATH], { - encoding: "utf-8", - timeout: 30_000, - }); - console.log(`[${elapsed()}] Setup script downloaded to ${setupScriptPath}`); - } else { - setupScriptPath = DEFAULT_SETUP_SCRIPT_PATH; - console.log(`[${elapsed()}] Using repo-local setup script`); - } - - const searchOutput = execFileSync( - "brev", - [ - "search", - "cpu", - "--min-vcpu", - String(BREV_MIN_VCPU), - "--min-ram", - String(BREV_MIN_RAM), - "--min-disk", - String(BREV_MIN_DISK), - "--provider", - BREV_PROVIDER, - "--sort", - "price", - ], - { encoding: "utf-8", timeout: 120_000, stdio: ["pipe", "pipe", "inherit"] }, - ); - execFileSync( - "brev", - ["create", requireInstanceName(), "--startup-script", `@${setupScriptPath}`, "--detached"], - { - encoding: "utf-8", - timeout: 180_000, - stdio: PIPE_INPUT_STDIO, - input: searchOutput, - }, - ); -} - /** * Bootstrap the launchable environment on the remote VM: * rsync branch code, install deps, build plugin, and npm link the CLI. diff --git a/vitest.config.ts b/vitest.config.ts index e276df2f5ca..c832c0c4bb6 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -56,21 +56,9 @@ export default defineConfig({ test: { name: "e2e-branch-validation", include: ["test/e2e/brev-e2e.test.ts"], - // Branch validation E2E: rsyncs the branch over a Brev instance - // provisioned from the published NemoClaw launchable image and - // runs the selected test suites. Only run when explicitly - // targeted: `npx vitest run --project e2e-branch-validation`. - // - // Override the project-root `silent: isCi` setting — diagnostic - // output from createBrevInstance / waitForSsh / waitForLaunchableReady - // is essential for debugging Brev provisioning timing and the - // overall suite runs in a single `describe` block, so there's no - // test chatter to suppress anyway. - silent: false, - // Gate on the new long-lived API key secret. Historically this was - // BREV_API_TOKEN (short-lived refresh token); renamed in the - // nightly-enable PR to match the new `brev login --api-key` flow. - enabled: !!process.env.BREV_API_KEY || !!process.env.BREV_API_TOKEN, + // Branch validation E2E: installs from source on a Brev instance. + // Only run when explicitly targeted: npx vitest run --project e2e-branch-validation + enabled: !!process.env.BREV_API_TOKEN, }, }, ],