diff --git a/.github/workflows/e2e-brev.yaml b/.github/workflows/e2e-brev.yaml index c91f64910d..6399e2afd6 100644 --- a/.github/workflows/e2e-brev.yaml +++ b/.github/workflows/e2e-brev.yaml @@ -46,16 +46,15 @@ on: - credential-sanitization - telegram-injection - all - use_launchable: - description: "Use NemoClaw launchable (true) or bare brev-setup.sh (false)" - required: false - type: boolean - default: true keep_alive: description: "Keep Brev instance alive after tests (for SSH debugging)" required: false type: boolean default: true + brev_token: + description: "Brev refresh token (overrides BREV_API_TOKEN secret if provided)" + required: false + default: "" workflow_call: inputs: branch: @@ -69,10 +68,6 @@ on: required: false type: string default: "full" - use_launchable: - required: false - type: boolean - default: true keep_alive: required: false type: boolean @@ -127,8 +122,8 @@ jobs: - name: Install Brev CLI run: | - # Pin to v0.6.310 — v0.6.322 removed --cpu flag and defaults to GPU instances - curl -fsSL -o /tmp/brev.tar.gz "https://github.com/brevdev/brev-cli/releases/download/v0.6.310/brev-cli_0.6.310_linux_amd64.tar.gz" + # Use latest Brev CLI (v0.6.322+) — CPU instances require `brev search cpu | brev create` + curl -fsSL -o /tmp/brev.tar.gz "https://github.com/brevdev/brev-cli/releases/download/v0.6.322/brev-cli_0.6.322_linux_amd64.tar.gz" tar -xzf /tmp/brev.tar.gz -C /usr/local/bin brev chmod +x /usr/local/bin/brev @@ -137,12 +132,11 @@ jobs: - name: Run ephemeral Brev E2E env: - BREV_API_TOKEN: ${{ secrets.BREV_API_TOKEN }} + BREV_API_TOKEN: ${{ inputs.brev_token || secrets.BREV_API_TOKEN }} NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} GITHUB_TOKEN: ${{ github.token }} INSTANCE_NAME: e2e-pr-${{ inputs.pr_number || github.run_id }} TEST_SUITE: ${{ inputs.test_suite }} - USE_LAUNCHABLE: ${{ inputs.use_launchable && '1' || '0' }} KEEP_ALIVE: ${{ inputs.keep_alive }} run: npx vitest run --project e2e-brev --reporter=verbose diff --git a/test/e2e/brev-e2e.test.js b/test/e2e/brev-e2e.test.js index 98f8560f3b..cd298ddd75 100644 --- a/test/e2e/brev-e2e.test.js +++ b/test/e2e/brev-e2e.test.js @@ -4,7 +4,7 @@ /** * Ephemeral Brev E2E test suite. * - * Creates a fresh Brev instance, bootstraps it, runs E2E tests remotely, + * Creates a fresh Brev CPU instance, bootstraps it, runs E2E tests remotely, * then tears it down. Intended to be run from CI via: * * npx vitest run --project e2e-brev @@ -16,8 +16,9 @@ * INSTANCE_NAME — Brev instance name (e.g. pr-156-test) * * Optional env vars: - * TEST_SUITE — which test to run: full (default), credential-sanitization, all - * BREV_CPU — CPU spec (default: 4x16) + * TEST_SUITE — which test to run: full (default), credential-sanitization, telegram-injection, all + * BREV_MIN_VCPU — Minimum vCPUs for CPU instance (default: 4) + * BREV_MIN_RAM — Minimum RAM in GB for CPU instance (default: 16) */ import { describe, it, expect, beforeAll, afterAll } from "vitest"; @@ -26,21 +27,13 @@ import { mkdirSync, writeFileSync } from "node:fs"; import { homedir } from "node:os"; import path from "node:path"; -const BREV_CPU = process.env.BREV_CPU || "4x16"; +// CPU instance specs: min vCPUs and RAM for the instance search +const BREV_MIN_VCPU = parseInt(process.env.BREV_MIN_VCPU || "4", 10); +const BREV_MIN_RAM = parseInt(process.env.BREV_MIN_RAM || "16", 10); const INSTANCE_NAME = process.env.INSTANCE_NAME; const TEST_SUITE = process.env.TEST_SUITE || "full"; const REPO_DIR = path.resolve(import.meta.dirname, "../.."); -// NemoClaw launchable — uses the OpenShell-Community launch script which -// goes through `nemoclaw onboard` (potentially pre-built images / faster path) -// instead of our manual brev-setup.sh bootstrap. -const LAUNCHABLE_SETUP_SCRIPT = - "https://raw.githubusercontent.com/NVIDIA/OpenShell-Community/refs/heads/feat/brev-nemoclaw-plugin/brev/launch-nemoclaw.sh"; -const NEMOCLAW_REPO_URL = "https://github.com/NVIDIA/NemoClaw.git"; - -// Use launchable by default; set USE_LAUNCHABLE=0 or USE_LAUNCHABLE=false to fall back to brev-setup.sh -const USE_LAUNCHABLE = !["0", "false"].includes(process.env.USE_LAUNCHABLE?.toLowerCase()); - let remoteDir; let instanceCreated = false; @@ -54,47 +47,38 @@ function brev(...args) { }).trim(); } -function ssh(cmd, { timeout = 120_000 } = {}) { - // Use single quotes to prevent local shell expansion of remote commands +function ssh(cmd, { timeout = 120_000, stream = false } = {}) { const escaped = cmd.replace(/'/g, "'\\''"); - return execSync( + /** @type {import("child_process").StdioOptions} */ + const stdio = stream ? ["inherit", "inherit", "inherit"] : ["pipe", "pipe", "pipe"]; + const result = execSync( `ssh -o StrictHostKeyChecking=no -o LogLevel=ERROR "${INSTANCE_NAME}" '${escaped}'`, - { encoding: "utf-8", timeout, stdio: ["pipe", "pipe", "pipe"] }, - ).trim(); + { encoding: "utf-8", timeout, stdio }, + ); + return stream ? "" : result.trim(); } +/** + * Escape a value for safe inclusion in a single-quoted shell string. + * Replaces single quotes with the shell-safe sequence: '\'' + */ function shellEscape(value) { - return value.replace(/'/g, "'\\''"); + return String(value).replace(/'/g, "'\\''"); } -/** Run a command on the remote VM with secrets passed via stdin (not CLI args). */ -function sshWithSecrets(cmd, { timeout = 600_000, stream = false } = {}) { - const secretPreamble = [ +/** Run a command on the remote VM with env vars set for NemoClaw. */ +function sshEnv(cmd, { timeout = 600_000, stream = false } = {}) { + const envPrefix = [ `export NVIDIA_API_KEY='${shellEscape(process.env.NVIDIA_API_KEY)}'`, `export GITHUB_TOKEN='${shellEscape(process.env.GITHUB_TOKEN)}'`, `export NEMOCLAW_NON_INTERACTIVE=1`, `export NEMOCLAW_SANDBOX_NAME=e2e-test`, - ].join("\n"); - - // When stream=true, pipe stdout/stderr to the CI log in real time - // so long-running steps (bootstrap) show progress instead of silence. - /** @type {import("child_process").StdioOptions} */ - const stdio = stream ? ["pipe", "inherit", "inherit"] : ["pipe", "pipe", "pipe"]; + ].join(" && "); - // Pipe secrets via stdin so they don't appear in ps/process listings - const result = execSync( - `ssh -o StrictHostKeyChecking=no -o LogLevel=ERROR "${INSTANCE_NAME}" 'eval "$(cat)" && ${cmd.replace(/'/g, "'\\''")}'`, - { - encoding: "utf-8", - timeout, - input: secretPreamble, - stdio, - }, - ); - return stream ? "" : result.trim(); + return ssh(`${envPrefix} && ${cmd}`, { timeout, stream }); } -function waitForSsh(maxAttempts = 60, intervalMs = 5_000) { +function waitForSsh(maxAttempts = 90, intervalMs = 5_000) { for (let i = 1; i <= maxAttempts; i++) { try { ssh("echo ok", { timeout: 10_000 }); @@ -124,7 +108,7 @@ function runRemoteTest(scriptPath) { ].join(" && "); // Stream test output to CI log AND capture it for assertions - sshWithSecrets(cmd, { timeout: 900_000, stream: true }); + sshEnv(cmd, { timeout: 900_000, stream: true }); // Retrieve the captured output for assertion checking return ssh("cat /tmp/test-output.log", { timeout: 30_000 }); } @@ -147,199 +131,70 @@ describe.runIf(hasRequiredVars)("Brev E2E", () => { ); brev("login", "--token", process.env.BREV_API_TOKEN); - if (USE_LAUNCHABLE) { - // --- Launchable path: brev start with the NemoClaw launch script --- - // This uses the OpenShell-Community launch-nemoclaw.sh which goes through - // nemoclaw's own install/onboard flow — potentially faster than our manual - // brev-setup.sh (different sandbox build strategy, pre-built images, etc.) - console.log(`[${elapsed()}] Creating instance via launchable (brev start + setup-script)...`); - console.log(`[${elapsed()}] setup-script: ${LAUNCHABLE_SETUP_SCRIPT}`); - console.log(`[${elapsed()}] repo: ${NEMOCLAW_REPO_URL}`); - console.log(`[${elapsed()}] cpu: ${BREV_CPU}`); - - // brev start with a git URL may take longer than the default 60s brev() timeout - // (it registers the instance + kicks off provisioning before returning) - execFileSync( - "brev", - [ - "start", - NEMOCLAW_REPO_URL, - "--name", - INSTANCE_NAME, - "--cpu", - BREV_CPU, - "--setup-script", - LAUNCHABLE_SETUP_SCRIPT, - "--detached", - ], - { encoding: "utf-8", timeout: 180_000, stdio: ["pipe", "inherit", "inherit"] }, - ); - instanceCreated = true; - console.log(`[${elapsed()}] brev start returned (instance provisioning in background)`); - - // Wait for SSH - try { - brev("refresh"); - } catch { - /* ignore */ - } - waitForSsh(); - console.log(`[${elapsed()}] SSH is up`); - - // The launchable clones NemoClaw to ~/NemoClaw. We need to find where it landed - // and then rsync our branch code over it. - const remoteHome = ssh("echo $HOME"); - // The launch script clones to $HOME/NemoClaw (PLUGIN_DIR default) - remoteDir = `${remoteHome}/NemoClaw`; - - // Wait for the launch script to finish — it runs as the VM's startup script - // and may still be in progress when SSH becomes available. Poll for completion. - console.log(`[${elapsed()}] Waiting for launchable setup to complete...`); - const setupMaxWait = 2_400_000; // 40 min max - const setupStart = Date.now(); - const setupPollInterval = 15_000; // check every 15s - while (Date.now() - setupStart < setupMaxWait) { - try { - // The launch script writes to /tmp/launch-plugin.log and the last step - // prints "=== Ready ===" when complete - const log = ssh("cat /tmp/launch-plugin.log 2>/dev/null || echo 'NO_LOG'", { - timeout: 15_000, - }); - if (log.includes("=== Ready ===")) { - console.log( - `[${elapsed()}] Launchable setup complete (detected '=== Ready ===' in log)`, - ); - break; - } - // Also check if nemoclaw onboard has run (install marker) - const markerCheck = ssh( - "test -f ~/.cache/nemoclaw-plugin/install-ran && echo DONE || echo PENDING", - { timeout: 10_000 }, - ); - if (markerCheck.includes("DONE")) { - console.log(`[${elapsed()}] Launchable setup complete (install-ran marker found)`); - break; - } - // Print last few lines of log for progress visibility - const tail = ssh("tail -3 /tmp/launch-plugin.log 2>/dev/null || echo '(no log yet)'", { - timeout: 10_000, - }); - console.log(`[${elapsed()}] Setup still running... ${tail.replace(/\n/g, " | ")}`); - } catch { - console.log(`[${elapsed()}] Setup poll: SSH command failed, retrying...`); - } - execSync(`sleep ${setupPollInterval / 1000}`); - } - - // Fail fast if neither readiness marker appeared within the timeout - if (Date.now() - setupStart >= setupMaxWait) { - throw new Error( - `Launchable setup did not complete within ${setupMaxWait / 60_000} minutes. ` + - `Neither '=== Ready ===' in /tmp/launch-plugin.log nor install-ran marker found.`, - ); - } - - // The launch script installs Docker, OpenShell CLI, clones NemoClaw main, - // and sets up code-server — but it does NOT run `nemoclaw onboard` (that's - // deferred to an interactive code-server terminal). So at this point we have: - // ✅ Docker, OpenShell CLI, Node.js, NemoClaw repo (main) - // ❌ No sandbox yet - // - // Now: rsync our PR branch code over the main clone, then run onboard ourselves. - - console.log(`[${elapsed()}] Syncing PR branch code over launchable's clone...`); - execSync( - `rsync -az --delete --exclude node_modules --exclude .git --exclude dist --exclude .venv "${REPO_DIR}/" "${INSTANCE_NAME}:${remoteDir}/"`, - { encoding: "utf-8", timeout: 120_000 }, - ); - console.log(`[${elapsed()}] Code synced`); - - // Install deps for our branch - console.log(`[${elapsed()}] Running npm ci to sync dependencies...`); - sshWithSecrets( - `set -o pipefail && source ~/.nvm/nvm.sh 2>/dev/null || true && cd ${remoteDir} && npm ci --ignore-scripts 2>&1 | tail -5`, - { timeout: 300_000, stream: true }, - ); - console.log(`[${elapsed()}] Dependencies synced`); - - // Run nemoclaw onboard (non-interactive) — this is the path real users take. - // It installs the nemoclaw CLI, builds the sandbox via `nemoclaw onboard`, - // which may use a different (faster) strategy than our manual setup.sh. - // Source nvm first — the launchable installs Node.js via nvm which sets up - // PATH in .bashrc/.nvm/nvm.sh, but non-interactive SSH doesn't source these. - console.log(`[${elapsed()}] Running nemoclaw install + onboard (the user-facing path)...`); - sshWithSecrets( - `source ~/.nvm/nvm.sh 2>/dev/null || true && cd ${remoteDir} && npm link && nemoclaw onboard --non-interactive 2>&1`, - { timeout: 2_400_000, stream: true }, - ); - console.log(`[${elapsed()}] nemoclaw onboard complete`); - - // Verify sandbox is ready - try { - const sandboxStatus = ssh("openshell sandbox list 2>&1 | head -5", { timeout: 15_000 }); - console.log(`[${elapsed()}] Sandbox status: ${sandboxStatus}`); - } catch (e) { - console.log(`[${elapsed()}] Warning: could not check sandbox status: ${e.message}`); - } - } else { - // --- Legacy path: bare brev create + brev-setup.sh --- - console.log(`[${elapsed()}] Creating bare instance via brev create...`); - brev("create", INSTANCE_NAME, "--cpu", BREV_CPU, "--detached"); - instanceCreated = true; - - // Wait for SSH - try { - brev("refresh"); - } catch { - /* ignore */ - } - waitForSsh(); - console.log(`[${elapsed()}] SSH is up`); - - // Sync code - const remoteHome = ssh("echo $HOME"); - remoteDir = `${remoteHome}/nemoclaw`; - ssh(`mkdir -p ${remoteDir}`); - execSync( - `rsync -az --delete --exclude node_modules --exclude .git --exclude dist --exclude .venv "${REPO_DIR}/" "${INSTANCE_NAME}:${remoteDir}/"`, - { encoding: "utf-8", timeout: 120_000 }, - ); - console.log(`[${elapsed()}] Code synced`); - - // Bootstrap VM — stream output to CI log so we can see progress - console.log(`[${elapsed()}] Running brev-setup.sh (manual bootstrap)...`); - sshWithSecrets(`cd ${remoteDir} && SKIP_VLLM=1 bash scripts/brev-setup.sh`, { - timeout: 2_400_000, - stream: true, - }); - console.log(`[${elapsed()}] Bootstrap complete`); - - // Install nemoclaw CLI — brev-setup.sh creates the sandbox but doesn't - // install the host-side CLI that the test scripts need for `nemoclaw status`. - // The `bin` field is in the root package.json (not nemoclaw/), so we need to: - // 1. Build the TypeScript plugin (in nemoclaw/) - // 2. npm link from the repo root (where bin.nemoclaw is defined) - // Use npm_config_prefix so npm link writes to ~/.local/bin (no sudo needed), - // which is already on PATH in runRemoteTest. - console.log(`[${elapsed()}] Installing nemoclaw CLI...`); - ssh( - [ - `export npm_config_prefix=$HOME/.local`, - `export PATH=$HOME/.local/bin:$PATH`, - `cd ${remoteDir}/nemoclaw && npm install && npm run build`, - `cd ${remoteDir} && npm install --ignore-scripts && npm link`, - `which nemoclaw && nemoclaw --version`, - ].join(" && "), - { timeout: 120_000 }, - ); - console.log(`[${elapsed()}] nemoclaw CLI installed`); + // Create bare CPU instance via brev search cpu | brev create + console.log(`[${elapsed()}] Creating CPU instance via brev search cpu | brev create...`); + console.log(`[${elapsed()}] min-vcpu: ${BREV_MIN_VCPU}, min-ram: ${BREV_MIN_RAM}GB`); + execSync( + `brev search cpu --min-vcpu ${BREV_MIN_VCPU} --min-ram ${BREV_MIN_RAM} --sort price | ` + + `brev create ${INSTANCE_NAME} --detached`, + { encoding: "utf-8", timeout: 180_000, stdio: ["pipe", "inherit", "inherit"] }, + ); + instanceCreated = true; + console.log(`[${elapsed()}] brev create returned (instance provisioning in background)`); - // Register the sandbox in nemoclaw's local registry. - // setup.sh creates the sandbox via openshell directly but doesn't write - // ~/.nemoclaw/sandboxes.json, which `nemoclaw status` needs. - console.log(`[${elapsed()}] Registering sandbox in nemoclaw registry...`); - ssh( - `mkdir -p ~/.nemoclaw && cat > ~/.nemoclaw/sandboxes.json << 'REGISTRY' + // Wait for SSH + try { + brev("refresh"); + } catch { + /* ignore */ + } + waitForSsh(); + console.log(`[${elapsed()}] SSH is up`); + + // Sync code + const remoteHome = ssh("echo $HOME"); + remoteDir = `${remoteHome}/nemoclaw`; + ssh(`mkdir -p ${remoteDir}`); + execSync( + `rsync -az --delete --exclude node_modules --exclude .git --exclude dist --exclude .venv "${REPO_DIR}/" "${INSTANCE_NAME}:${remoteDir}/"`, + { encoding: "utf-8", timeout: 120_000 }, + ); + console.log(`[${elapsed()}] Code synced`); + + // Bootstrap VM — stream output to CI log so we can see progress + console.log(`[${elapsed()}] Running brev-setup.sh (bootstrap)...`); + sshEnv(`cd ${remoteDir} && SKIP_VLLM=1 bash scripts/brev-setup.sh`, { + timeout: 2_400_000, + stream: true, + }); + console.log(`[${elapsed()}] Bootstrap complete`); + + // Install nemoclaw CLI — brev-setup.sh creates the sandbox but doesn't + // install the host-side CLI that the test scripts need for `nemoclaw status`. + // The `bin` field is in the root package.json (not nemoclaw/), so we need to: + // 1. Build the TypeScript plugin (in nemoclaw/) + // 2. npm link from the repo root (where bin.nemoclaw is defined) + // Use npm_config_prefix so npm link writes to ~/.local/bin (no sudo needed), + // which is already on PATH in runRemoteTest. + console.log(`[${elapsed()}] Installing nemoclaw CLI...`); + ssh( + [ + `export npm_config_prefix=$HOME/.local`, + `export PATH=$HOME/.local/bin:$PATH`, + `cd ${remoteDir}/nemoclaw && npm install && npm run build`, + `cd ${remoteDir} && npm install --ignore-scripts && npm link`, + `which nemoclaw && nemoclaw --version`, + ].join(" && "), + { timeout: 120_000 }, + ); + console.log(`[${elapsed()}] nemoclaw CLI installed`); + + // Register the sandbox in nemoclaw's local registry. + // setup.sh creates the sandbox via openshell directly but doesn't write + // ~/.nemoclaw/sandboxes.json, which `nemoclaw status` needs. + console.log(`[${elapsed()}] Registering sandbox in nemoclaw registry...`); + ssh( + `mkdir -p ~/.nemoclaw && cat > ~/.nemoclaw/sandboxes.json << 'REGISTRY' { "sandboxes": { "e2e-test": { @@ -355,13 +210,12 @@ describe.runIf(hasRequiredVars)("Brev E2E", () => { "defaultSandbox": "e2e-test" } REGISTRY`, - { timeout: 10_000 }, - ); - console.log(`[${elapsed()}] Sandbox registered`); - } + { timeout: 10_000 }, + ); + console.log(`[${elapsed()}] Sandbox registered`); console.log(`[${elapsed()}] beforeAll complete — total bootstrap time: ${elapsed()}`); - }, 2_700_000); // 45 min — covers both paths + }, 2_700_000); // 45 min afterAll(() => { if (!instanceCreated) return;