Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion .github/workflows/e2e-vitest-scenarios.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1719,7 +1719,6 @@ jobs:
NEMOCLAW_NON_INTERACTIVE: "1"
NEMOCLAW_RECREATE_SANDBOX: "1"
NEMOCLAW_SANDBOX_NAME: e2e-hermes
NEMOCLAW_MODEL: minimaxai/minimax-m2.7
NEMOCLAW_ONBOARD_VALIDATION_TIMEOUT_SECONDS: "60"
steps:
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
Expand Down
7 changes: 5 additions & 2 deletions test/e2e-scenario/live/cron-preflight-inference-local.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,17 @@ import { resultText } from "../fixtures/clients/index.ts";
import { type SandboxClient, validateSandboxName } from "../fixtures/clients/sandbox.ts";
import { expect, test } from "../fixtures/e2e-test.ts";
import { shouldRunLiveE2EScenarios } from "../fixtures/live-project-gate.ts";
import { requireHostedInferenceConfig } from "../fixtures/hosted-inference.ts";
import {
DEFAULT_HOSTED_INFERENCE_MODEL,
requireHostedInferenceConfig,
} from "../fixtures/hosted-inference.ts";
import type { ShellProbeResult } from "../fixtures/shell-probe.ts";
import { isTransientProviderValidationFailure } from "./network-policy-transient-provider.ts";

const REPO_ROOT = path.resolve(import.meta.dirname, "../../..");
const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-cron-preflight";
validateSandboxName(SANDBOX_NAME);
const MODEL = process.env.NEMOCLAW_CRON_PREFLIGHT_MODEL ?? "nvidia/nemotron-3-super-120b-a12b";
const MODEL = process.env.NEMOCLAW_CRON_PREFLIGHT_MODEL ?? DEFAULT_HOSTED_INFERENCE_MODEL;
const INSTALL_ATTEMPTS = process.env.CI === "true" || process.env.GITHUB_ACTIONS === "true" ? 3 : 1;
const LIVE_TIMEOUT_MS = 35 * 60_000;
const PROBE_SOURCE = String.raw`
Expand Down
11 changes: 8 additions & 3 deletions test/e2e-scenario/live/diagnostics.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,17 @@ import path from "node:path";
import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts";
import { validateSandboxName } from "../fixtures/clients/sandbox.ts";
import { expect, test } from "../fixtures/e2e-test.ts";
import {
DEFAULT_HOSTED_INFERENCE_MODEL,
requireHostedInferenceConfig,
} from "../fixtures/hosted-inference.ts";
import { shouldRunLiveE2EScenarios } from "../fixtures/live-project-gate.ts";
import type { ShellProbeResult } from "../fixtures/shell-probe.ts";

const REPO_ROOT = path.resolve(import.meta.dirname, "../../..");
const CLI_ENTRYPOINT = path.join(REPO_ROOT, "bin", "nemoclaw.js");
const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? `e2e-diag-${process.pid}`;
const DIAGNOSTICS_MODEL = process.env.NEMOCLAW_MODEL ?? "openai/gpt-oss-120b";
const DIAGNOSTICS_MODEL = process.env.NEMOCLAW_MODEL ?? DEFAULT_HOSTED_INFERENCE_MODEL;
const DEBUG_QUICK_TIMEOUT_MS = 30_000;
const INSTALL_TIMEOUT_MS = 35 * 60_000;
const TEST_TIMEOUT_MS = 55 * 60_000;
Expand Down Expand Up @@ -138,7 +142,8 @@ runDiagnosticsTest(
"run `npm run build:cli` before live repo CLI scenarios",
).toBe(true);

const apiKey = secrets.required("NVIDIA_INFERENCE_API_KEY");
const hosted = requireHostedInferenceConfig(secrets, process.env, { model: DIAGNOSTICS_MODEL });
const apiKey = hosted.apiKey;
await artifacts.writeJson("scenario.json", {
id: "diagnostics",
runner: "vitest",
Expand Down Expand Up @@ -195,7 +200,7 @@ runDiagnosticsTest(
fs.rmSync(home, { recursive: true, force: true });
});

const env = testEnv(home, { NVIDIA_INFERENCE_API_KEY: apiKey });
const env = testEnv(home, hosted.env);
await bestEffort(() =>
host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], {
artifactName: "pre-cleanup-nemoclaw-destroy-diagnostics",
Expand Down
15 changes: 9 additions & 6 deletions test/e2e-scenario/live/hermes-e2e.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,10 @@ import { trustedProviderEndpoint } from "../fixtures/clients/provider.ts";
import { trustedSandboxShellScript, validateSandboxName } from "../fixtures/clients/sandbox.ts";
import { expect, test } from "../fixtures/e2e-test.ts";
import { shouldRunLiveE2EScenarios } from "../fixtures/live-project-gate.ts";
import { requireHostedInferenceConfig } from "../fixtures/hosted-inference.ts";
import {
DEFAULT_HOSTED_INFERENCE_MODEL,
requireHostedInferenceConfig,
} from "../fixtures/hosted-inference.ts";
import type { ShellProbeResult } from "../fixtures/shell-probe.ts";

// Migrated from test/e2e/test-hermes-e2e.sh.
Expand All @@ -33,7 +36,7 @@ const HERMES_DASHBOARD_INTERNAL_PORT =
const SESSION_FILE = path.join(os.homedir(), ".nemoclaw", "onboard-session.json");
const REGISTRY_FILE = path.join(os.homedir(), ".nemoclaw", "sandboxes.json");
const LIVE_TIMEOUT_MS = 70 * 60_000;
const CHAT_MODEL = process.env.NEMOCLAW_MODEL ?? "nvidia/nemotron-3-super-120b-a12b";
const CHAT_MODEL = process.env.NEMOCLAW_MODEL ?? DEFAULT_HOSTED_INFERENCE_MODEL;
const ONBOARD_VALIDATION_TIMEOUT_SECONDS =
process.env.NEMOCLAW_ONBOARD_VALIDATION_TIMEOUT_SECONDS ?? "60";

Expand Down Expand Up @@ -65,18 +68,18 @@ function hermesDashboardE2eEnabled(): boolean {
);
}

function commandEnv(apiKey?: string): NodeJS.ProcessEnv {
function commandEnv(hostedEnv: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv {
const env: NodeJS.ProcessEnv = {
...buildAvailabilityProbeEnv(),
...hostedEnv,
NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1",
NEMOCLAW_AGENT: "hermes",
NEMOCLAW_NON_INTERACTIVE: "1",
NEMOCLAW_RECREATE_SANDBOX: "1",
NEMOCLAW_MODEL: CHAT_MODEL,
NEMOCLAW_MODEL: hostedEnv.NEMOCLAW_MODEL ?? CHAT_MODEL,
NEMOCLAW_ONBOARD_VALIDATION_TIMEOUT_SECONDS: ONBOARD_VALIDATION_TIMEOUT_SECONDS,
NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME,
};
if (apiKey) env.NVIDIA_INFERENCE_API_KEY = apiKey;
if (process.env.NEMOCLAW_E2E_HERMES_DASHBOARD) {
env.NEMOCLAW_E2E_HERMES_DASHBOARD = process.env.NEMOCLAW_E2E_HERMES_DASHBOARD;
}
Expand Down Expand Up @@ -232,7 +235,7 @@ test.skipIf(!shouldRunLiveE2EScenarios())(
dashboardEnabled: hermesDashboardE2eEnabled(),
});

const env = commandEnv(apiKey);
const env = commandEnv(hosted.env);
const redactionValues = [apiKey];

const cleanupHermes = async (label: string) => {
Expand Down
13 changes: 6 additions & 7 deletions test/e2e-scenario/live/upgrade-stale-sandbox-helpers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,18 +43,17 @@ function assertSafeSandboxName(): void {
}
}

export function commandEnv(apiKey?: string): NodeJS.ProcessEnv {
const env: NodeJS.ProcessEnv = {
export function commandEnv(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv {
return {
...buildAvailabilityProbeEnv(),
...extra,
NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1",
NEMOCLAW_NON_INTERACTIVE: "1",
NEMOCLAW_REBUILD_VERBOSE: "1",
NEMOCLAW_RECREATE_SANDBOX: "1",
NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME,
OPENSHELL_GATEWAY: process.env.OPENSHELL_GATEWAY ?? "nemoclaw",
};
apiKey && Object.assign(env, { NVIDIA_INFERENCE_API_KEY: apiKey });
return env;
}

export async function bestEffort(run: () => Promise<unknown>): Promise<void> {
Expand Down Expand Up @@ -201,7 +200,7 @@ export async function cleanupOldImage(host: HostCliClient): Promise<void> {

export async function installCurrentNemoclaw(
host: HostCliClient,
apiKey: string,
hosted: { apiKey: string; env: NodeJS.ProcessEnv },
): Promise<ShellProbeResult> {
let install: ShellProbeResult | undefined;
for (let attempt = 1; attempt <= INSTALL_ATTEMPTS; attempt += 1) {
Expand All @@ -211,8 +210,8 @@ export async function installCurrentNemoclaw(
? "phase-1-install-current-nemoclaw"
: `phase-1-install-current-nemoclaw-attempt-${attempt}`,
cwd: REPO_ROOT,
env: commandEnv(apiKey),
redactionValues: [apiKey],
env: commandEnv(hosted.env),
redactionValues: [hosted.apiKey],
timeoutMs: 20 * 60_000,
});
const retry =
Expand Down
17 changes: 9 additions & 8 deletions test/e2e-scenario/live/upgrade-stale-sandbox.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts";
import { resultText } from "../fixtures/clients/index.ts";
import { expect, test } from "../fixtures/e2e-test.ts";
import { requireHostedInferenceConfig } from "../fixtures/hosted-inference.ts";
import { shouldRunLiveE2EScenarios } from "../fixtures/live-project-gate.ts";
import {
assertDeleteInstalledSandboxAllowed,
Expand All @@ -37,7 +38,7 @@ test.skipIf(!shouldRunLiveE2EScenarios())(
"upgrade-sandboxes detects and rebuilds stale OpenClaw sandboxes (#1904)",
{ timeout: LIVE_TIMEOUT_MS },
async ({ artifacts, cleanup, host, sandbox, secrets, skip }) => {
const apiKey = secrets.required("NVIDIA_INFERENCE_API_KEY");
const hosted = requireHostedInferenceConfig(secrets);

await artifacts.writeJson("scenario.json", {
id: "upgrade-stale-sandbox",
Expand Down Expand Up @@ -67,7 +68,7 @@ test.skipIf(!shouldRunLiveE2EScenarios())(
cleanup.add("remove stale OpenClaw test image", () => cleanupOldImage(host));
await cleanupStaleSandbox(host, sandbox);

const install = await installCurrentNemoclaw(host, apiKey);
const install = await installCurrentNemoclaw(host, hosted);
expect(install.exitCode, resultText(install)).toBe(0);

const deleteInstalledSandbox = await sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], {
Expand Down Expand Up @@ -119,8 +120,8 @@ test.skipIf(!shouldRunLiveE2EScenarios())(

const staleCheck = await host.nemoclaw(["upgrade-sandboxes", "--check"], {
artifactName: "phase-5-upgrade-sandboxes-check-stale",
env: commandEnv(apiKey),
redactionValues: [apiKey],
env: commandEnv(hosted.env),
redactionValues: [hosted.apiKey],
timeoutMs: 120_000,
});
expect(staleCheck.exitCode, resultText(staleCheck)).toBe(0);
Expand All @@ -129,8 +130,8 @@ test.skipIf(!shouldRunLiveE2EScenarios())(

const rebuild = await host.nemoclaw([SANDBOX_NAME, "rebuild", "--yes"], {
artifactName: "phase-6-rebuild-stale-sandbox",
env: commandEnv(apiKey),
redactionValues: [apiKey],
env: commandEnv(hosted.env),
redactionValues: [hosted.apiKey],
timeoutMs: 25 * 60_000,
});
expect(rebuild.exitCode, resultText(rebuild)).toBe(0);
Expand All @@ -148,8 +149,8 @@ test.skipIf(!shouldRunLiveE2EScenarios())(

const cleanCheck = await host.nemoclaw(["upgrade-sandboxes", "--check"], {
artifactName: "phase-7-upgrade-sandboxes-check-clean",
env: commandEnv(apiKey),
redactionValues: [apiKey],
env: commandEnv(hosted.env),
redactionValues: [hosted.apiKey],
timeoutMs: 120_000,
});
expect(cleanCheck.exitCode, resultText(cleanCheck)).toBe(0);
Expand Down
Loading
Loading