Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 12 additions & 3 deletions docs/deployment/deploy-to-remote-gpu.md
Original file line number Diff line number Diff line change
Expand Up @@ -138,12 +138,21 @@ Changing the proxy after onboarding requires re-running `nemoclaw onboard`.

## GPU Configuration

The deploy script uses the `NEMOCLAW_GPU` environment variable to select the GPU type.
The deploy script uses the `NEMOCLAW_GPU` environment variable to select the Brev instance type and GPU name.
The default value is `a2-highgpu-1g:nvidia-tesla-a100:1`.
Set this variable before running `nemoclaw deploy` to use a different GPU configuration:
If you want to keep using the legacy combined setting, export `NEMOCLAW_GPU` before running `nemoclaw deploy`:

```console
$ export NEMOCLAW_GPU="a2-highgpu-1g:nvidia-tesla-a100:2"
$ export NEMOCLAW_GPU="a2-highgpu-1g:nvidia-tesla-a100:1"
$ nemoclaw deploy <instance-name>
```

For direct overrides, set `NEMOCLAW_BREV_TYPE` and `NEMOCLAW_BREV_GPU_NAME` instead.
These take precedence over `NEMOCLAW_GPU` when both are set:

```console
$ export NEMOCLAW_BREV_TYPE="a2-highgpu-1g"
$ export NEMOCLAW_BREV_GPU_NAME="A100"
$ nemoclaw deploy <instance-name>
```

Expand Down
40 changes: 36 additions & 4 deletions src/lib/deploy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,37 @@ export function buildDeployEnvLines(opts: {
return envLines;
}

function normalizeBrevGpuName(value: string | undefined): string {
const trimmed = String(value || "").trim();
if (!trimmed) return "";
return trimmed.replace(/^nvidia-(?:tesla-)?/i, "").toUpperCase();
}

function firstNonEmptyValue(...values: Array<string | undefined>): string {
for (const value of values) {
const trimmed = String(value ?? "").trim();
if (trimmed) return trimmed;
}
return "";
}

function resolveBrevCreateConfig(env: NodeJS.ProcessEnv): {
type: string;
gpuName: string;
provider: string;
} {
const legacyGpu = String(env.NEMOCLAW_GPU || "").trim();
const legacyParts = legacyGpu.includes(":") ? legacyGpu.split(":") : [legacyGpu];
const type = firstNonEmptyValue(env.NEMOCLAW_BREV_TYPE, legacyParts[0], "a2-highgpu-1g");
const gpuName = firstNonEmptyValue(
normalizeBrevGpuName(env.NEMOCLAW_BREV_GPU_NAME),
normalizeBrevGpuName(legacyParts[1]),
"A100",
);
const provider = firstNonEmptyValue(env.NEMOCLAW_BREV_PROVIDER, "gcp").toLowerCase();
return { type, gpuName, provider };
}

function outputHasExactLine(output: string | undefined, expected: string): boolean {
return String(output || "")
.split(/\r?\n/)
Expand Down Expand Up @@ -240,8 +271,7 @@ export async function executeDeploy(opts: DeployExecutionOptions): Promise<void>

const name = validateName(instanceName, "instance name");
const qname = shellQuote(name);
const gpu = env.NEMOCLAW_GPU || "a2-highgpu-1g:nvidia-tesla-a100:1";
const brevProvider = String(env.NEMOCLAW_BREV_PROVIDER || "gcp").trim().toLowerCase();
const { type, gpuName, provider: brevProvider } = resolveBrevCreateConfig(env);
const skipConnect = ["1", "true"].includes(
String(env.NEMOCLAW_DEPLOY_NO_CONNECT || "").toLowerCase(),
);
Expand Down Expand Up @@ -296,8 +326,10 @@ export async function executeDeploy(opts: DeployExecutionOptions): Promise<void>
}

if (!exists) {
log(` Creating Brev instance '${name}' (${gpu}, provider=${brevProvider})...`);
run(`brev create ${qname} --type ${shellQuote(gpu)} --provider ${shellQuote(brevProvider)}`);
log(` Creating Brev instance '${name}' (${type}, ${gpuName}, provider=${brevProvider})...`);
run(
`brev create ${qname} --type ${shellQuote(type)} --gpu-name ${shellQuote(gpuName)} --provider ${shellQuote(brevProvider)}`,
);
} else {
log(` Brev instance '${name}' already exists.`);
}
Expand Down
225 changes: 225 additions & 0 deletions test/deploy-brev.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,225 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { describe, expect, it } from "vitest";
import { execSync } from "node:child_process";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";

const CLI = path.join(import.meta.dirname, "..", "bin", "nemoclaw.js");

function runWithEnv(args: string, env: Record<string, string> = {}, timeout = 25_000) {
try {
const out = execSync(`node "${CLI}" ${args}`, {
encoding: "utf-8",
timeout,
env: {
...process.env,
HOME: fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-brev-home-")),
NEMOCLAW_HEALTH_POLL_COUNT: "1",
NEMOCLAW_HEALTH_POLL_INTERVAL: "0",
NEMOCLAW_DEPLOY_NO_CONNECT: "1",
...env,
},
});
return { code: 0, out };
} catch (err: any) {
return { code: err.status, out: (err.stdout || "") + (err.stderr || "") };
}
}

function writeStub(binDir: string, name: string, lines: string[]) {
fs.writeFileSync(path.join(binDir, name), lines.join("\n"), { mode: 0o755 });
}

function readBrevCalls(markerFile: string) {
return fs.readFileSync(markerFile, "utf8").trim().split("\n").filter(Boolean);
}

function setupDeployStubs({ brevLsOutput = "" }: { brevLsOutput?: string } = {}) {
const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-deploy-brev-"));
const localBin = path.join(home, "bin");
const markerFile = path.join(home, "brev-args");

fs.mkdirSync(localBin, { recursive: true });

writeStub(localBin, "brev", [
"#!/usr/bin/env bash",
`marker_file=${JSON.stringify(markerFile)}`,
'printf \'%s\\n\' "$*" >> "$marker_file"',
'if [ "$1" = "ls" ] && [ "$2" = "--json" ]; then',
' created_name=$(awk \'/^create / { print $2; exit }\' "$marker_file")',
` existing_name=${JSON.stringify(brevLsOutput.split(/\r?\n/).find(Boolean) || "")}`,
' instance_name="${created_name:-$existing_name}"',
' if [ -n "$instance_name" ]; then',
' printf \'[{"name":"%s","status":"RUNNING","build_status":"COMPLETED","shell_status":"READY"}]\' "$instance_name"',
" else",
" printf '[]'",
" fi",
" exit 0",
"fi",
'if [ "$1" = "ls" ]; then',
` printf '%b' ${JSON.stringify(brevLsOutput)}`,
" exit 0",
"fi",
"exit 0",
]);
writeStub(localBin, "ssh", [
"#!/usr/bin/env bash",
'if [ "$1" = "-G" ]; then',
" printf 'hostname 127.0.0.1\\n'",
" exit 0",
"fi",
'if [ "${@: -2:1}" = "echo" ] && [ "${@: -1}" = "$HOME" ]; then',
' printf "%s\\n" "$HOME"',
" exit 0",
"fi",
"exit 0",
]);
writeStub(localBin, "ssh-keyscan", [
"#!/usr/bin/env bash",
'printf "%s\\n" "127.0.0.1 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAITestKeyForDeployHarness"',
]);
writeStub(localBin, "rsync", ["#!/usr/bin/env bash", "exit 0"]);
writeStub(localBin, "scp", ["#!/usr/bin/env bash", "exit 0"]);

return {
home,
localBin,
markerFile,
};
}

describe("deploy brev compatibility", () => {
it("uses --type and --gpu-name for legacy combined NEMOCLAW_GPU", () => {
const { home, localBin, markerFile } = setupDeployStubs();

const result = runWithEnv("deploy pr-1377-legacy", {
HOME: home,
PATH: `${localBin}:${process.env.PATH || ""}`,
NVIDIA_API_KEY: "nvapi-test",
NEMOCLAW_GPU: "a2-highgpu-1g:nvidia-tesla-a100:1",
});

expect(result.code).toBe(0);
const calls = readBrevCalls(markerFile);
expect(calls).toContain("ls");
expect(calls).toContain(
"create pr-1377-legacy --type a2-highgpu-1g --gpu-name A100 --provider gcp",
);
expect(calls.join("\n")).not.toContain("--gpu ");
});

it("prefers explicit Brev override env vars over the legacy combined value", () => {
const { home, localBin, markerFile } = setupDeployStubs();

const result = runWithEnv("deploy pr-1377-overrides", {
HOME: home,
PATH: `${localBin}:${process.env.PATH || ""}`,
NVIDIA_API_KEY: "nvapi-test",
NEMOCLAW_GPU: "a2-highgpu-1g:nvidia-tesla-a100:1",
NEMOCLAW_BREV_TYPE: "a3-highgpu-1g",
NEMOCLAW_BREV_GPU_NAME: "h100",
});

expect(result.code).toBe(0);
const calls = readBrevCalls(markerFile);
expect(calls).toContain(
"create pr-1377-overrides --type a3-highgpu-1g --gpu-name H100 --provider gcp",
);
expect(calls.join("\n")).not.toContain("--gpu ");
});

it("falls back to default Brev type and GPU name when no env vars are set", () => {
const { home, localBin, markerFile } = setupDeployStubs();

const result = runWithEnv("deploy pr-1377-defaults", {
HOME: home,
PATH: `${localBin}:${process.env.PATH || ""}`,
NVIDIA_API_KEY: "nvapi-test",
});

expect(result.code).toBe(0);
const calls = readBrevCalls(markerFile);
expect(calls).toContain(
"create pr-1377-defaults --type a2-highgpu-1g --gpu-name A100 --provider gcp",
);
expect(calls.join("\n")).not.toContain("--gpu ");
});

it("treats whitespace-only overrides as unset and normalizes generic nvidia GPU names", () => {
const { home, localBin, markerFile } = setupDeployStubs();

const result = runWithEnv("deploy pr-1377-l40s", {
HOME: home,
PATH: `${localBin}:${process.env.PATH || ""}`,
NVIDIA_API_KEY: "nvapi-test",
NEMOCLAW_GPU: "a3-highgpu-1g:nvidia-l40s:1",
NEMOCLAW_BREV_TYPE: " ",
NEMOCLAW_BREV_GPU_NAME: " ",
});

expect(result.code).toBe(0);
const calls = readBrevCalls(markerFile);
expect(calls).toContain(
"create pr-1377-l40s --type a3-highgpu-1g --gpu-name L40S --provider gcp",
);
expect(calls.join("\n")).not.toContain("--gpu ");
});

it("skips brev create when the instance already exists", () => {
const { home, localBin, markerFile } = setupDeployStubs({
brevLsOutput: "pr-1377-existing\n",
});

const result = runWithEnv("deploy pr-1377-existing", {
HOME: home,
PATH: `${localBin}:${process.env.PATH || ""}`,
NVIDIA_API_KEY: "nvapi-test",
});

expect(result.code).toBe(0);
const calls = readBrevCalls(markerFile);
expect(calls).toContain("ls");
expect(calls).toContain("refresh");
expect(calls.some((call) => call.startsWith("create "))).toBe(false);
});

it("falls back to the legacy GPU name when an explicit override normalizes to empty", () => {
const { home, localBin, markerFile } = setupDeployStubs();

const result = runWithEnv("deploy pr-1377-empty-gpu-name", {
HOME: home,
PATH: `${localBin}:${process.env.PATH || ""}`,
NVIDIA_API_KEY: "nvapi-test",
NEMOCLAW_GPU: "a2-highgpu-1g:nvidia-l40s:1",
NEMOCLAW_BREV_TYPE: "a3-highgpu-1g",
NEMOCLAW_BREV_GPU_NAME: "nvidia-",
});

expect(result.code).toBe(0);
const calls = readBrevCalls(markerFile);
expect(calls).toContain(
"create pr-1377-empty-gpu-name --type a3-highgpu-1g --gpu-name L40S --provider gcp",
);
expect(calls.join("\n")).not.toContain("--gpu ");
});

it("passes through an explicit Brev provider override", () => {
const { home, localBin, markerFile } = setupDeployStubs();

const result = runWithEnv("deploy pr-1377-provider", {
HOME: home,
PATH: `${localBin}:${process.env.PATH || ""}`,
NVIDIA_API_KEY: "nvapi-test",
NEMOCLAW_BREV_PROVIDER: "aws",
});

expect(result.code).toBe(0);
const calls = readBrevCalls(markerFile);
expect(calls).toContain(
"create pr-1377-provider --type a2-highgpu-1g --gpu-name A100 --provider aws",
);
});
});
5 changes: 3 additions & 2 deletions test/runner.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -440,7 +440,7 @@ describe("regression guards", () => {
const result = spawnSync(
"node",
[
path.join(import.meta.dirname, "..", "bin", "nemoclaw.js"),
path.join(import.meta.dirname, "..", "src", "nemoclaw.ts"),
`test; touch ${canary}`,
"connect",
],
Expand Down Expand Up @@ -668,7 +668,8 @@ describe("regression guards", () => {
expect(src).not.toContain("--exclude src");
expect(src).toContain('"${rootDir}/"');
expect(src).toContain("--exclude dist");
expect(src).toContain('const brevProvider = String(env.NEMOCLAW_BREV_PROVIDER || "gcp")');
expect(src).toContain("resolveBrevCreateConfig(env)");
expect(src).toContain("--gpu-name ${shellQuote(gpuName)}");
expect(src).toContain("--provider ${shellQuote(brevProvider)}");
});

Expand Down