diff --git a/test/e2e/fixtures/cleanup-resources.ts b/test/e2e/fixtures/cleanup-resources.ts new file mode 100644 index 00000000000..7b939ff71cb --- /dev/null +++ b/test/e2e/fixtures/cleanup-resources.ts @@ -0,0 +1,70 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; + +import { assertExitZero, type CommandExitResult, resultText } from "./clients/command.ts"; +import type { HostCliClient } from "./clients/host.ts"; +import type { ShellProbeRunOptions } from "./shell-probe.ts"; + +type CleanupRun = () => Promise | void; + +export function assertCleanupSucceededOrAbsent( + result: CommandExitResult, + absent: boolean | RegExp, + label: string, +): void { + const alreadyAbsent = typeof absent === "boolean" ? absent : absent.test(resultText(result)); + if (result.exitCode === 0 || alreadyAbsent) return; + assertExitZero(result, label); +} + +export async function cleanupAcquiredResource( + acquired: boolean, + cleanup: CleanupRun, +): Promise { + if (acquired) await cleanup(); +} + +export async function cleanupExistingPath(pathname: string, cleanup: CleanupRun): Promise { + if (fs.existsSync(pathname)) await cleanup(); +} + +export async function cleanupUnlessVerified( + cleanupVerified: boolean, + cleanup: CleanupRun, +): Promise { + if (!cleanupVerified) await cleanup(); +} + +export async function cleanupWhenCommandAvailable( + host: Pick, + command: string, + probeOptions: ShellProbeRunOptions, + cleanup: CleanupRun, +): Promise { + if (!(await host.isCommandAvailable(command, probeOptions))) return; + await cleanup(); +} + +export async function cleanupWhenOpenShellAvailable( + host: Pick, + probeOptions: ShellProbeRunOptions, + cleanup: CleanupRun, +): Promise { + await cleanupWhenCommandAvailable(host, host.openshellCommandPath, probeOptions, cleanup); +} + +export function registerSandboxCleanupUnlessKept(keepSandbox: boolean, register: () => void): void { + if (keepSandbox) return; + register(); +} + +export function terminateProcessIfRunning(pid: number, signal: NodeJS.Signals = "SIGTERM"): void { + try { + process.kill(pid, signal); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ESRCH") return; + throw error; + } +} diff --git a/test/e2e/fixtures/cleanup.ts b/test/e2e/fixtures/cleanup.ts index 5c8b48d01cc..2125bc2ed9f 100644 --- a/test/e2e/fixtures/cleanup.ts +++ b/test/e2e/fixtures/cleanup.ts @@ -42,15 +42,27 @@ export class CleanupRegistry { this.entries.push({ name, run }); } - trackSandbox(host: CleanupHost, name: string, options: ShellProbeRunOptions = {}): void { + trackSandbox( + host: Pick, + name: string, + options: ShellProbeRunOptions = {}, + ): void { this.add(`destroy sandbox ${name}`, () => host.cleanupSandbox(name, options)); } - trackGateway(host: CleanupHost, name: string, options: ShellProbeRunOptions = {}): void { + trackGateway( + host: Pick, + name: string, + options: ShellProbeRunOptions = {}, + ): void { this.add(`remove gateway ${name}`, () => host.cleanupGatewayRegistration(name, options)); } - trackForward(host: CleanupHost, port: number, options: ShellProbeRunOptions = {}): void { + trackForward( + host: Pick, + port: number, + options: ShellProbeRunOptions = {}, + ): void { this.add(`stop forward ${port}`, () => host.cleanupForward(port, options)); } diff --git a/test/e2e/fixtures/clients/host.ts b/test/e2e/fixtures/clients/host.ts index 5ab206fcfbb..2f199d67511 100644 --- a/test/e2e/fixtures/clients/host.ts +++ b/test/e2e/fixtures/clients/host.ts @@ -15,6 +15,7 @@ import { export interface HostClientOptions { cliPath?: string; cwd?: string; + openshellPath?: string; } const GATEWAY_ALREADY_ABSENT = @@ -28,17 +29,23 @@ export class HostCliClient { private readonly runner: CommandRunner; private readonly cliPath: string; private readonly cwd?: string; + private readonly openshellPath: string; constructor(runner: CommandRunner, options: HostClientOptions = {}) { this.runner = runner; this.cliPath = options.cliPath ?? process.env.NEMOCLAW_CLI_BIN ?? "nemoclaw"; this.cwd = options.cwd; + this.openshellPath = options.openshellPath ?? process.env.OPENSHELL_BIN ?? "openshell"; } get commandPath(): string { return this.cliPath; } + get openshellCommandPath(): string { + return this.openshellPath; + } + command( command: string, args: string[] = [], @@ -58,6 +65,23 @@ export class HostCliClient { ); } + async isCommandAvailable(command: string, options: ShellProbeRunOptions = {}): Promise { + const result = await this.command( + "bash", + ["-lc", 'command -v "$1" >/dev/null 2>&1', "command-availability-probe", command], + { + artifactName: `command-available-${artifactLabel(command)}`, + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + ...options, + }, + ); + if (result.exitCode === 0) return true; + if (result.exitCode === 1) return false; + assertExitZero(result, `probe command availability for ${command}`); + return false; + } + nemoclaw(args: string[] = [], options: ShellProbeRunOptions = {}): Promise { return this.command(this.cliPath, args, { artifactName: `nemoclaw-${artifactLabel(args.join("-") || "default")}`, @@ -134,7 +158,7 @@ export class HostCliClient { options: ShellProbeRunOptions = {}, ): Promise { const artifactName = options.artifactName ?? `cleanup-gateway-${artifactLabel(gatewayName)}`; - const remove = await this.command("openshell", ["gateway", "remove", gatewayName], { + const remove = await this.command(this.openshellPath, ["gateway", "remove", gatewayName], { ...options, artifactName: `${artifactName}-remove`, }); @@ -145,16 +169,20 @@ export class HostCliClient { // Remove this fallback once the supported OpenShell floor no longer // includes builds whose local-registration verb was `gateway destroy`. - const destroy = await this.command("openshell", ["gateway", "destroy", "-g", gatewayName], { - ...options, - artifactName: `${artifactName}-legacy-destroy`, - }); + const destroy = await this.command( + this.openshellPath, + ["gateway", "destroy", "-g", gatewayName], + { + ...options, + artifactName: `${artifactName}-legacy-destroy`, + }, + ); if (destroy.exitCode === 0 || GATEWAY_ALREADY_ABSENT.test(resultText(destroy))) return; assertExitZero(destroy, `cleanup gateway registration ${gatewayName}`); } async cleanupForward(port: number, options: ShellProbeRunOptions = {}): Promise { - const result = await this.command("openshell", ["forward", "stop", String(port)], { + const result = await this.command(this.openshellPath, ["forward", "stop", String(port)], { ...options, artifactName: options.artifactName ?? `cleanup-forward-${port}`, }); diff --git a/test/e2e/fixtures/clients/sandbox.ts b/test/e2e/fixtures/clients/sandbox.ts index 0100eca7637..5666d2db40f 100644 --- a/test/e2e/fixtures/clients/sandbox.ts +++ b/test/e2e/fixtures/clients/sandbox.ts @@ -10,8 +10,12 @@ import { assertExitZero, type CommandRunner, outputContainsSandbox, + resultText, } from "./command.ts"; +const SANDBOX_ALREADY_ABSENT = + /\bNotFound\b|\bNot Found\b|sandbox[^\n]*(?:not found|not present|does not exist)|no such sandbox/i; + /** * Default env for openshell-targeted spawns. ShellProbe filters env via * the framework allowlist (HOME, PATH, …) which excludes OPENSHELL_GATEWAY, @@ -99,6 +103,16 @@ export class SandboxClient { }); } + async cleanupSandbox(name: string, options: ShellProbeRunOptions = {}): Promise { + validateSandboxName(name); + const result = await this.openshell(["sandbox", "delete", name], { + artifactName: `cleanup-openshell-sandbox-${name}`, + ...options, + }); + if (result.exitCode === 0 || SANDBOX_ALREADY_ABSENT.test(resultText(result))) return; + assertExitZero(result, `cleanup OpenShell sandbox ${name}`); + } + exec( name: string, command: string[], diff --git a/test/e2e/live/agent-turn-latency-helpers.ts b/test/e2e/live/agent-turn-latency-helpers.ts index 4b984cad67c..dc3fa918867 100644 --- a/test/e2e/live/agent-turn-latency-helpers.ts +++ b/test/e2e/live/agent-turn-latency-helpers.ts @@ -4,6 +4,7 @@ import path from "node:path"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { resultText } from "../fixtures/clients/command.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; import { type SandboxClient, @@ -73,7 +74,10 @@ export function env( return out; } -export async function bestEffort(label: string, run: () => Promise): Promise { +export async function bestEffortPreclean( + label: string, + run: () => Promise, +): Promise { try { await run(); } catch (error) { @@ -283,14 +287,10 @@ export async function cleanupTurnSandboxes( [OPENCLAW_SANDBOX, "openclaw"], [HERMES_SANDBOX, "hermes"], ] as const) { - await bestEffort(`destroy ${agent} sandbox`, () => - host.command("node", [CLI, name, "destroy", "--yes"], { - artifactName: `cleanup-${agent}-destroy`, - env: env(name, agent), - timeoutMs: 120_000, - }), + await bestEffortPreclean(`destroy ${agent} sandbox`, () => + cleanupTurnSandbox(host, name, agent), ); - await bestEffort(`delete ${agent} sandbox`, () => + await bestEffortPreclean(`delete ${agent} sandbox`, () => sandbox.openshell(["sandbox", "delete", name], { artifactName: `cleanup-${agent}-delete`, env: env(name, agent), @@ -298,14 +298,14 @@ export async function cleanupTurnSandboxes( }), ); } - await bestEffort("stop Hermes API forward", () => + await bestEffortPreclean("stop Hermes API forward", () => sandbox.openshell(["forward", "stop", "8642"], { artifactName: "cleanup-forward-stop-hermes-api", env: buildAvailabilityProbeEnv(), timeoutMs: 30_000, }), ); - await bestEffort("destroy OpenShell gateway", () => + await bestEffortPreclean("destroy OpenShell gateway", () => sandbox.openshell(["gateway", "destroy", "-g", "nemoclaw"], { artifactName: "cleanup-gateway-destroy-turn-latency", env: buildAvailabilityProbeEnv(), @@ -314,6 +314,26 @@ export async function cleanupTurnSandboxes( ); } +export async function cleanupTurnSandbox( + host: HostCliClient, + name: string, + agent: "openclaw" | "hermes", +): Promise { + const result = await host.command("node", [CLI, name, "destroy", "--yes"], { + artifactName: `cleanup-${agent}-destroy`, + env: env(name, agent), + timeoutMs: 120_000, + }); + const output = resultText(result); + expect( + result.exitCode === 0 || + /Sandbox '.+' does not exist|Run 'nemoclaw onboard' to create one|sandbox .* not found|no such sandbox/iu.test( + output, + ), + `cleanup ${agent} sandbox ${name}: ${output}`, + ).toBe(true); +} + export async function route( sandbox: SandboxClient, sandboxName: string, diff --git a/test/e2e/live/agent-turn-latency.test.ts b/test/e2e/live/agent-turn-latency.test.ts index 1acbac694fb..539a7bfd21d 100644 --- a/test/e2e/live/agent-turn-latency.test.ts +++ b/test/e2e/live/agent-turn-latency.test.ts @@ -14,6 +14,7 @@ import { assertOpenClawConfig, CLI, chatContent, + cleanupTurnSandbox, cleanupTurnSandboxes, EXPECTED_ROUTE_PROVIDER, env, @@ -45,7 +46,36 @@ test("OpenClaw and Hermes complete real hosted inference turns within the latenc hermesSandbox: HERMES_SANDBOX, }); - cleanup.add("destroy turn latency sandboxes", () => cleanupTurnSandboxes(host, sandbox)); + cleanup.trackGateway(host, "nemoclaw", { + artifactName: "cleanup-gateway-destroy-turn-latency", + env: buildAvailabilityProbeEnv(), + timeoutMs: 60_000, + }); + cleanup.trackForward(host, 8642, { + artifactName: "cleanup-forward-stop-hermes-api", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + cleanup.trackDisposable("delete Hermes OpenShell sandbox", () => + sandbox.cleanupSandbox(HERMES_SANDBOX, { + artifactName: "cleanup-hermes-delete", + env: env(HERMES_SANDBOX, "hermes"), + timeoutMs: 60_000, + }), + ); + cleanup.trackDisposable("destroy Hermes sandbox", () => + cleanupTurnSandbox(host, HERMES_SANDBOX, "hermes"), + ); + cleanup.trackDisposable("delete OpenClaw OpenShell sandbox", () => + sandbox.cleanupSandbox(OPENCLAW_SANDBOX, { + artifactName: "cleanup-openclaw-delete", + env: env(OPENCLAW_SANDBOX, "openclaw"), + timeoutMs: 60_000, + }), + ); + cleanup.trackDisposable("destroy OpenClaw sandbox", () => + cleanupTurnSandbox(host, OPENCLAW_SANDBOX, "openclaw"), + ); const docker = await host.command("docker", ["info"], { artifactName: "docker-info", diff --git a/test/e2e/live/bedrock-runtime-compatible-anthropic.test.ts b/test/e2e/live/bedrock-runtime-compatible-anthropic.test.ts index cb04a6440bb..48da2c017aa 100644 --- a/test/e2e/live/bedrock-runtime-compatible-anthropic.test.ts +++ b/test/e2e/live/bedrock-runtime-compatible-anthropic.test.ts @@ -11,6 +11,11 @@ import os from "node:os"; import path from "node:path"; import type { ArtifactSink } from "../fixtures/artifacts.ts"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { + cleanupAcquiredResource, + cleanupExistingPath, + terminateProcessIfRunning, +} from "../fixtures/cleanup-resources.ts"; import { assertExitZero as expectExitZero, resultText, @@ -496,7 +501,7 @@ async function startFakeBedrockRuntimeMock(options: { }; } -async function bestEffort(run: () => Promise | unknown): Promise { +async function bestEffortPreclean(run: () => Promise | unknown): Promise { try { await run(); } catch { @@ -549,12 +554,12 @@ async function cleanupOpenShellGateway(host: HostCliClient, home: string): Promi } async function cleanupSandboxState(host: HostCliClient, home: string): Promise { - await bestEffort(() => cleanupNemoClawSandbox(host, home)); - await bestEffort(() => cleanupOpenShellSandbox(host, home)); - await bestEffort(() => cleanupOpenShellGateway(host, home)); + await bestEffortPreclean(() => cleanupNemoClawSandbox(host, home)); + await bestEffortPreclean(() => cleanupOpenShellSandbox(host, home)); + await bestEffortPreclean(() => cleanupOpenShellGateway(host, home)); } -function stopBedrockAdapterBestEffort(home: string): void { +function stopBedrockAdapter(home: string): void { const stateFile = path.join(home, ".nemoclaw", "bedrock-runtime-adapter.json"); const pidFile = path.join(home, ".nemoclaw", "bedrock-runtime-adapter.pid"); const tokenFile = path.join(home, ".nemoclaw", "bedrock-runtime-adapter-token"); @@ -566,11 +571,7 @@ function stopBedrockAdapterBestEffort(home: string): void { if (fs.existsSync(pidFile)) { const pid = Number(fs.readFileSync(pidFile, "utf8").trim()); if (Number.isInteger(pid) && pid > 0 && isBedrockAdapterProcess(pid)) { - try { - process.kill(pid, "SIGTERM"); - } catch { - // Already stopped. - } + terminateProcessIfRunning(pid); } } } finally { @@ -602,21 +603,23 @@ async function restoreHostsFile( backupDir: string, home: string, ): Promise { - await bestEffort(() => - host.command("sudo", ["cp", backupPath, "/etc/hosts"], { + expectExitZero( + await host.command("sudo", ["cp", backupPath, "/etc/hosts"], { artifactName: "restore-etc-hosts-bedrock-runtime", env: testEnv(home), timeoutMs: 30_000, }), + "restore /etc/hosts after Bedrock Runtime mapping", ); - await bestEffort(() => - host.command("sudo", ["rm", "-rf", backupDir], { + expectExitZero( + await host.command("sudo", ["rm", "-rf", backupDir], { artifactName: "remove-etc-hosts-backup-bedrock-runtime", env: testEnv(home), timeoutMs: 30_000, }), + "remove Bedrock Runtime hosts backup", ); - await bestEffort(() => fs.rmSync(backupDir, { recursive: true, force: true })); + fs.rmSync(backupDir, { recursive: true, force: true }); } async function mapBedrockHostToLoopback( @@ -715,8 +718,8 @@ async function installBedrockTlsRedirect(host: HostCliClient, home: string): Pro } async function removeBedrockTlsRedirect(host: HostCliClient, home: string): Promise { - await bestEffort(() => - host.command( + expectExitZero( + await host.command( "sudo", [ "-n", @@ -731,6 +734,7 @@ async function removeBedrockTlsRedirect(host: HostCliClient, home: string): Prom timeoutMs: 30_000, }, ), + "remove Bedrock Runtime canonical TLS port redirect", ); } @@ -1310,24 +1314,39 @@ test("bedrock runtime compatible Anthropic endpoint routes through managed infer const hostsBackup = path.join(hostsBackupDir, "hosts"); let mock: MockBedrockRuntime | undefined; let onboarding: RawRunResult | undefined; + let tlsRedirectInstalled = false; - cleanup.add(`remove Bedrock Runtime test home ${home}`, () => + cleanup.trackDisposable(`remove Bedrock Runtime test home ${home}`, () => fs.rmSync(home, { recursive: true, force: true }), ); - cleanup.add(`destroy Bedrock Runtime sandbox ${SANDBOX_NAME}`, () => - cleanupSandboxState(host, home), + cleanup.trackGateway(host, "nemoclaw", { + artifactName: "cleanup-openshell-gateway-destroy-bedrock-runtime", + env: testEnv(home), + timeoutMs: 120_000, + }); + cleanup.trackDisposable(`delete Bedrock Runtime OpenShell sandbox ${SANDBOX_NAME}`, () => + sandbox.cleanupSandbox(SANDBOX_NAME, { + artifactName: "cleanup-openshell-sandbox-delete-bedrock-runtime", + env: testEnv(home), + timeoutMs: 60_000, + }), + ); + cleanup.trackDisposable(`destroy Bedrock Runtime sandbox ${SANDBOX_NAME}`, () => + cleanupNemoClawSandbox(host, home), ); - cleanup.add("restore /etc/hosts after Bedrock Runtime mapping", () => - restoreHostsFile(host, hostsBackup, hostsBackupDir, home), + cleanup.trackDisposable("restore /etc/hosts after Bedrock Runtime mapping", () => + cleanupExistingPath(hostsBackup, () => + restoreHostsFile(host, hostsBackup, hostsBackupDir, home), + ), ); - cleanup.add("remove Bedrock Runtime canonical TLS port redirect", () => - removeBedrockTlsRedirect(host, home), + cleanup.trackDisposable("remove Bedrock Runtime canonical TLS port redirect", () => + cleanupAcquiredResource(tlsRedirectInstalled, () => removeBedrockTlsRedirect(host, home)), ); - cleanup.add("stop Bedrock Runtime adapter", () => stopBedrockAdapterBestEffort(home)); - cleanup.add("stop fake Bedrock Runtime endpoint", async () => { + cleanup.trackDisposable("stop Bedrock Runtime adapter", () => stopBedrockAdapter(home)); + cleanup.trackDisposable("stop fake Bedrock Runtime endpoint", async () => { if (mock) await mock.close(); }); - cleanup.add("write fake Bedrock Runtime log", async () => { + cleanup.trackDisposable("write fake Bedrock Runtime log", async () => { if (mock) { await artifacts.writeText( "fake-bedrock-runtime.log", @@ -1379,6 +1398,7 @@ test("bedrock runtime compatible Anthropic endpoint routes through managed infer await prepareSourceCliAndOpenShell(host, home); await mapBedrockHostToLoopback(host, home, hostsBackup, skip); await installBedrockTlsRedirect(host, home); + tlsRedirectInstalled = true; const tls = createBedrockTlsFixture(home); mock = await startFakeBedrockRuntimeMock({ port: BEDROCK_MOCK_PORT, diff --git a/test/e2e/live/brave-search-helpers.ts b/test/e2e/live/brave-search-helpers.ts index a17a38e1d91..c636b6d5a03 100644 --- a/test/e2e/live/brave-search-helpers.ts +++ b/test/e2e/live/brave-search-helpers.ts @@ -6,6 +6,7 @@ import os from "node:os"; import path from "node:path"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import type { CleanupRegistry } from "../fixtures/cleanup.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; import { resultText } from "../fixtures/clients/index.ts"; import { @@ -35,7 +36,7 @@ export function commandEnv(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { }; } -export async function bestEffort(run: () => Promise): Promise { +export async function bestEffortPreclean(run: () => Promise): Promise { try { await run(); } catch { @@ -65,14 +66,8 @@ export async function cleanupBraveState( host: HostCliClient, sandbox: SandboxClient, ): Promise { - await bestEffort(() => - host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { - artifactName: "cleanup-nemoclaw-destroy-brave-search", - env: commandEnv(), - timeoutMs: 120_000, - }), - ); - await bestEffort(() => + await bestEffortPreclean(() => cleanupBraveNemoClawSandbox(host)); + await bestEffortPreclean(() => sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { artifactName: "cleanup-openshell-delete-brave-search", env: commandEnv(), @@ -81,6 +76,22 @@ export async function cleanupBraveState( ); } +export async function cleanupBraveNemoClawSandbox(host: HostCliClient): Promise { + const result = await host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { + artifactName: "cleanup-nemoclaw-destroy-brave-search", + env: commandEnv(), + timeoutMs: 120_000, + }); + const output = resultText(result); + expect( + result.exitCode === 0 || + /Sandbox '.+' does not exist|Run 'nemoclaw onboard' to create one|sandbox .* not found|no such sandbox/iu.test( + output, + ), + `cleanup Brave sandbox ${SANDBOX_NAME}: ${output}`, + ).toBe(true); +} + function parsePlaceholder(configText: string): string | undefined { const parsed = JSON.parse(configText) as { tools?: { web?: { search?: { apiKey?: unknown } } }; @@ -203,7 +214,7 @@ export async function onboardBrave( export async function uploadSecretForLeakCheck( sandbox: SandboxClient, - cleanup: { add(name: string, run: () => Promise | void): void }, + cleanup: Pick, braveKey: string, redactionValues: string[], ): Promise { @@ -211,15 +222,18 @@ export async function uploadSecretForLeakCheck( const secretFile = path.join(secretDir, "brave-key"); fs.writeFileSync(secretFile, braveKey, { mode: 0o600 }); const remoteSecretFile = "/tmp/nemoclaw-brave-key-leak-check"; - cleanup.add("remove temporary Brave leak-check secret", async () => { + cleanup.trackDisposable("remove temporary Brave leak-check secret", async () => { fs.rmSync(secretDir, { recursive: true, force: true }); - await bestEffort(() => - sandbox.execShell(SANDBOX_NAME, trustedSandboxShellScript(`rm -f ${remoteSecretFile}`), { + const result = await sandbox.execShell( + SANDBOX_NAME, + trustedSandboxShellScript(`rm -f ${remoteSecretFile}`), + { artifactName: "cleanup-brave-leak-secret", env: commandEnv(), timeoutMs: 30_000, - }), + }, ); + expect(result.exitCode, resultText(result)).toBe(0); }); const uploadSecret = await sandbox.upload(SANDBOX_NAME, secretFile, remoteSecretFile, { artifactName: "phase-3-upload-brave-leak-secret", diff --git a/test/e2e/live/brave-search.test.ts b/test/e2e/live/brave-search.test.ts index 8e11e17231a..1f02cd58975 100644 --- a/test/e2e/live/brave-search.test.ts +++ b/test/e2e/live/brave-search.test.ts @@ -10,6 +10,7 @@ import { assertDockerAvailable, assertOptionalBraveEnv, assertRawConfigHasNoSecret, + cleanupBraveNemoClawSandbox, cleanupBraveState, commandEnv, extractOpenClawAgentText, @@ -49,8 +50,15 @@ test("Brave search preset wires policy/config, hides the real key, and performs }); assertDockerAvailable(dockerInfo, skip); - cleanup.add(`destroy brave search sandbox ${SANDBOX_NAME}`, () => - cleanupBraveState(host, sandbox), + cleanup.trackDisposable(`delete Brave search OpenShell sandbox ${SANDBOX_NAME}`, () => + sandbox.cleanupSandbox(SANDBOX_NAME, { + artifactName: "cleanup-openshell-delete-brave-search", + env: commandEnv(), + timeoutMs: 60_000, + }), + ); + cleanup.trackDisposable(`destroy Brave search sandbox ${SANDBOX_NAME}`, () => + cleanupBraveNemoClawSandbox(host), ); await cleanupBraveState(host, sandbox); diff --git a/test/e2e/live/channels-add-remove.test.ts b/test/e2e/live/channels-add-remove.test.ts index b72d652f5fd..e68fdc361d8 100644 --- a/test/e2e/live/channels-add-remove.test.ts +++ b/test/e2e/live/channels-add-remove.test.ts @@ -93,7 +93,7 @@ function redactionValues(apiKey: string): string[] { return [apiKey, TELEGRAM_TOKEN].filter((value) => value.length > 0); } -async function bestEffort(run: () => Promise): Promise { +async function bestEffortPreclean(run: () => Promise): Promise { try { await run(); } catch { @@ -408,33 +408,36 @@ test( ], }); - cleanup.add(`destroy ${SANDBOX_NAME} after channels add/remove live test`, async () => { - await bestEffort(() => onboard.destroySandbox(SANDBOX_NAME, "cleanup-nemoclaw-destroy")); - await bestEffort(() => - sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { - artifactName: "cleanup-openshell-sandbox-delete", - env: sandboxAccessEnv(), - timeoutMs: COMMAND_TIMEOUT_MS, - }), - ); - await bestEffort(() => - host.command("openshell", ["gateway", "destroy", "-g", "nemoclaw"], { - artifactName: "cleanup-openshell-gateway-destroy", - env: baseEnv(), - timeoutMs: COMMAND_TIMEOUT_MS, - }), - ); + cleanup.trackGateway(host, "nemoclaw", { + artifactName: "cleanup-openshell-gateway-destroy", + env: baseEnv(), + timeoutMs: COMMAND_TIMEOUT_MS, }); + cleanup.trackDisposable(`delete ${SANDBOX_NAME} OpenShell sandbox`, () => + sandbox.cleanupSandbox(SANDBOX_NAME, { + artifactName: "cleanup-openshell-sandbox-delete", + env: sandboxAccessEnv(), + timeoutMs: COMMAND_TIMEOUT_MS, + }), + ); + cleanup.trackDisposable( + `destroy ${SANDBOX_NAME} after channels add/remove live test`, + async () => { + await onboard.destroySandbox(SANDBOX_NAME, "cleanup-nemoclaw-destroy"); + }, + ); - await bestEffort(() => onboard.destroySandbox(SANDBOX_NAME, "pre-cleanup-nemoclaw-destroy")); - await bestEffort(() => + await bestEffortPreclean(() => + onboard.destroySandbox(SANDBOX_NAME, "pre-cleanup-nemoclaw-destroy"), + ); + await bestEffortPreclean(() => sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { artifactName: "pre-cleanup-openshell-sandbox-delete", env: sandboxAccessEnv(), timeoutMs: COMMAND_TIMEOUT_MS, }), ); - await bestEffort(() => + await bestEffortPreclean(() => host.command("openshell", ["gateway", "destroy", "-g", "nemoclaw"], { artifactName: "pre-cleanup-openshell-gateway-destroy", env: baseEnv(), diff --git a/test/e2e/live/channels-stop-start-helpers.ts b/test/e2e/live/channels-stop-start-helpers.ts index abb593abd38..5fe1a4d36ae 100644 --- a/test/e2e/live/channels-stop-start-helpers.ts +++ b/test/e2e/live/channels-stop-start-helpers.ts @@ -9,17 +9,18 @@ import { expect } from "../fixtures/e2e-test.ts"; import { assertChannelsStopStartSandboxName } from "./channels-stop-start-safety.ts"; import { type AgentKind, - bestEffort, + runSecondaryCleanup as bestEffortPreclean, CLI, - cleanupSandbox, dockerInfo, expectExitZero, expectSandboxReady, installSandboxOrSkipOnRateLimit, phase6Env, + precleanSandbox, resultText, sandboxSh, shellQuote, + trackSandboxCleanup, } from "./phase6-messaging-helpers.ts"; import { parsePolicyPresetState } from "./policy-list-state.ts"; @@ -334,13 +335,13 @@ async function precleanProviders( } } -async function destroyNemoclawGateway( +async function precleanNemoclawGateway( host: import("../fixtures/clients/host.ts").HostCliClient, env: NodeJS.ProcessEnv, redactions: string[], artifactName: string, ): Promise { - await bestEffort(() => + await bestEffortPreclean(() => host.command("openshell", ["gateway", "destroy", "-g", "nemoclaw"], { artifactName, env, @@ -438,29 +439,29 @@ export async function runChannelsStopStartTarget({ channels: CHANNELS, }); - cleanup.add(`destroy channels stop/start sandbox ${SANDBOX_NAME}`, async () => { - await cleanupSandbox( - host, - SANDBOX_NAME, - env, - redactions, - `cleanup-channels-stop-start-${AGENT}`, - ); - await destroyNemoclawGateway( - host, - env, - redactions, - `cleanup-openshell-gateway-destroy-${AGENT}`, - ); + cleanup.trackGateway(host, "nemoclaw", { + artifactName: `cleanup-openshell-gateway-destroy-${AGENT}`, + env, + redactionValues: redactions, + timeoutMs: 60_000, }); - await cleanupSandbox( + trackSandboxCleanup( + cleanup, + host, + sandbox, + SANDBOX_NAME, + env, + redactions, + `cleanup-channels-stop-start-${AGENT}`, + ); + await precleanSandbox( host, SANDBOX_NAME, env, redactions, `preclean-channels-stop-start-${AGENT}`, ); - await destroyNemoclawGateway( + await precleanNemoclawGateway( host, env, redactions, diff --git a/test/e2e/live/cloud-inference.test.ts b/test/e2e/live/cloud-inference.test.ts index 9fea1895c7f..bf83c7ae2c6 100644 --- a/test/e2e/live/cloud-inference.test.ts +++ b/test/e2e/live/cloud-inference.test.ts @@ -14,6 +14,7 @@ import os from "node:os"; import path from "node:path"; import type { ArtifactSink } from "../fixtures/artifacts.ts"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { assertCleanupSucceededOrAbsent } from "../fixtures/cleanup-resources.ts"; import { resultText } from "../fixtures/clients/command.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; import { type SandboxClient, validateSandboxName } from "../fixtures/clients/sandbox.ts"; @@ -85,7 +86,7 @@ function testEnv(home: string, extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv return testHomeEnvironment(home, extra, { ...process.env, OPENSHELL_GATEWAY: "nemoclaw" }); } -async function bestEffort(run: () => Promise): Promise { +async function bestEffortPreclean(run: () => Promise): Promise { try { await run(); } catch { @@ -100,14 +101,8 @@ async function cleanupCloudInferenceState( home: string, ): Promise { const env = testEnv(home); - await bestEffort(() => - host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { - artifactName: "cleanup-nemoclaw-destroy-cloud-inference", - env, - timeoutMs: 120_000, - }), - ); - await bestEffort(() => + await bestEffortPreclean(() => cleanupCloudInferenceNemoClawSandbox(host, home)); + await bestEffortPreclean(() => sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { artifactName: "cleanup-openshell-sandbox-delete-cloud-inference", env, @@ -116,6 +111,24 @@ async function cleanupCloudInferenceState( ); } +async function cleanupCloudInferenceNemoClawSandbox( + host: HostCliClient, + home: string, +): Promise { + const result = await host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { + artifactName: "cleanup-nemoclaw-destroy-cloud-inference", + env: testEnv(home), + timeoutMs: 120_000, + }); + assertCleanupSucceededOrAbsent( + result, + /Sandbox '.+' does not exist|Run 'nemoclaw onboard' to create one|sandbox .* not found|no such sandbox/iu.test( + resultText(result), + ), + `cleanup cloud inference sandbox ${SANDBOX_NAME}`, + ); +} + function openAiChatContent(raw: string): string { const parsed = JSON.parse(raw) as { choices?: Array<{ @@ -262,10 +275,19 @@ test( } const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cloud-inference-home-")); - cleanup.add(`remove cloud inference state for ${SANDBOX_NAME}`, async () => { - await cleanupCloudInferenceState(host, sandbox, home); - fs.rmSync(home, { recursive: true, force: true }); - }); + cleanup.trackDisposable(`remove cloud inference test home ${home}`, () => + fs.rmSync(home, { recursive: true, force: true }), + ); + cleanup.trackDisposable(`delete cloud inference OpenShell sandbox ${SANDBOX_NAME}`, () => + sandbox.cleanupSandbox(SANDBOX_NAME, { + artifactName: "cleanup-openshell-sandbox-delete-cloud-inference", + env: testEnv(home), + timeoutMs: 60_000, + }), + ); + cleanup.trackDisposable(`destroy cloud inference sandbox ${SANDBOX_NAME}`, () => + cleanupCloudInferenceNemoClawSandbox(host, home), + ); await cleanupCloudInferenceState(host, sandbox, home); const install = await host.command( diff --git a/test/e2e/live/cloud-onboard.test.ts b/test/e2e/live/cloud-onboard.test.ts index 1e8f7c9f27a..d8617e6fab4 100644 --- a/test/e2e/live/cloud-onboard.test.ts +++ b/test/e2e/live/cloud-onboard.test.ts @@ -81,7 +81,12 @@ test("cloud onboard: public installer creates healthy sandbox with security chec `https://raw.githubusercontent.com/NVIDIA/NemoClaw/${ref}/install.sh`; const installCwd = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-public-install-")); const corporateCa = createCorporateCaFixture("explicit", "nemoclaw-cloud-corporate-ca-"); - cleanupRegistry.add("remove corporate CA fixture", () => cleanupCorporateCaFixture(corporateCa)); + cleanupRegistry.trackDisposable("remove public installer workspace", () => + fs.rmSync(installCwd, { recursive: true, force: true }), + ); + cleanupRegistry.trackDisposable("remove corporate CA fixture", () => + cleanupCorporateCaFixture(corporateCa), + ); const redactionValues = [hosted.apiKey]; await artifacts.target.declare({ @@ -110,7 +115,7 @@ test("cloud onboard: public installer creates healthy sandbox with security chec skip(`Docker is required: ${resultText(docker)}`); } - cleanupRegistry.add("remove cloud-onboard sandbox", () => + cleanupRegistry.trackDisposable("remove cloud-onboard sandbox", () => cleanup(host, sandbox, { label: "cleanup", verify: true }), ); await cleanup(host, sandbox, { label: "pre-cleanup", verify: false }); diff --git a/test/e2e/live/common-egress-agent.test.ts b/test/e2e/live/common-egress-agent.test.ts index fdf46a91809..4c8dc2dd245 100644 --- a/test/e2e/live/common-egress-agent.test.ts +++ b/test/e2e/live/common-egress-agent.test.ts @@ -11,6 +11,7 @@ import { shellQuote } from "../../../src/lib/core/shell-quote.ts"; import type { ArtifactSink } from "../fixtures/artifacts.ts"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import type { CleanupRegistry } from "../fixtures/cleanup.ts"; +import { assertCleanupSucceededOrAbsent } from "../fixtures/cleanup-resources.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; import { type SandboxClient, @@ -70,6 +71,12 @@ interface CleanupAttempt { outputTail: string; } +interface CleanupSummary { + nemoclawDestroy?: CleanupAttempt; + openshellDelete?: CleanupAttempt; + errors: string[]; +} + interface ActivePolicyPreset { name: string; provenance: string; @@ -182,23 +189,13 @@ async function assertPrerequisites( return hosted; } -async function bestEffortDestroySandbox( +async function bestEffortPrecleanSandbox( host: HostCliClient, sandbox: SandboxClient, sandboxName: string, artifactPrefix: string, -): Promise<{ - nemoclawDestroy?: CleanupAttempt; - openshellDelete?: CleanupAttempt; - errors: string[]; -}> { - const result: { - nemoclawDestroy?: CleanupAttempt; - openshellDelete?: CleanupAttempt; - errors: string[]; - } = { - errors: [], - }; +): Promise { + const result: CleanupSummary = { errors: [] }; try { const destroy = await host.command("node", [CLI_ENTRYPOINT, sandboxName, "destroy", "--yes"], { artifactName: `${artifactPrefix}-nemoclaw-destroy-${sandboxName}`, @@ -238,12 +235,53 @@ async function registerSandboxCleanup( }); return; } - cleanup.add(`destroy common-egress sandbox ${sandboxName}`, async () => { - const summary = await bestEffortDestroySandbox(host, sandbox, sandboxName, "cleanup"); + const summary: CleanupSummary = { errors: [] }; + cleanup.trackDisposable(`write common-egress cleanup summary for ${sandboxName}`, async () => { await artifacts.writeJson(`cleanup-common-egress-${sandboxName}.json`, summary); }); - const summary = await bestEffortDestroySandbox(host, sandbox, sandboxName, "pre-cleanup"); - await artifacts.writeJson(`pre-cleanup-common-egress-${sandboxName}.json`, summary); + cleanup.trackDisposable(`delete common-egress OpenShell sandbox ${sandboxName}`, async () => { + try { + await sandbox.cleanupSandbox(sandboxName, { + artifactName: `cleanup-openshell-delete-${sandboxName}`, + env: commandEnv(), + timeoutMs: 60_000, + }); + summary.openshellDelete = { + exitCode: 0, + missingSandboxTolerated: false, + outputTail: "", + }; + } catch (error) { + summary.errors.push(error instanceof Error ? error.message : String(error)); + throw error; + } + }); + cleanup.trackDisposable(`destroy common-egress sandbox ${sandboxName}`, async () => { + const destroy = await host.command("node", [CLI_ENTRYPOINT, sandboxName, "destroy", "--yes"], { + artifactName: `cleanup-nemoclaw-destroy-${sandboxName}`, + env: commandEnv(), + timeoutMs: 120_000, + }); + const attempt = cleanupAttempt(destroy); + summary.nemoclawDestroy = attempt; + try { + assertCleanupSucceededOrAbsent( + destroy, + attempt.missingSandboxTolerated, + `cleanup destroy sandbox ${sandboxName}`, + ); + } catch (error) { + summary.errors.push(error instanceof Error ? error.message : String(error)); + throw error; + } + }); + const precleanSummary = await bestEffortPrecleanSandbox( + host, + sandbox, + sandboxName, + "pre-cleanup", + ); + await artifacts.writeJson(`pre-cleanup-common-egress-${sandboxName}.json`, precleanSummary); } async function runOnboard( diff --git a/test/e2e/live/concurrent-gateway-ports.test.ts b/test/e2e/live/concurrent-gateway-ports.test.ts index 068bd719537..e5e3f39ac81 100644 --- a/test/e2e/live/concurrent-gateway-ports.test.ts +++ b/test/e2e/live/concurrent-gateway-ports.test.ts @@ -204,7 +204,7 @@ async function prerequisiteOrSkip( skip(message); } -async function bestEffortCleanup( +async function bestEffortPreclean( host: HostCliClient, sandbox: SandboxClient, gatewayA: string, @@ -269,6 +269,26 @@ async function bestEffortCleanup( } } +async function cleanupNemoClawSandbox( + host: HostCliClient, + name: string, + port: string, +): Promise { + const result = await command(host, [name, "destroy", "--yes"], { + artifactName: `cleanup-destroy-${name}`, + env: commandEnv({ NEMOCLAW_GATEWAY_PORT: port }), + timeoutMs: 5 * 60_000, + }); + const output = resultText(result); + expect( + result.exitCode === 0 || + /Sandbox '.+' does not exist|Run 'nemoclaw onboard' to create one|sandbox .* not found|no such sandbox/iu.test( + output, + ), + `cleanup concurrent gateway sandbox ${name}: ${output}`, + ).toBe(true); +} + test("concurrent gateway ports: onboards two sandboxes on isolated gateways and dashboards", { timeout: TEST_TIMEOUT_MS, }, async ({ artifacts, cleanup, host, sandbox, skip }) => { @@ -278,7 +298,13 @@ test("concurrent gateway ports: onboards two sandboxes on isolated gateways and ).toBe(true); await prerequisiteOrSkip(host, skip, "docker", ["info"], "prereq-docker-info"); - await prerequisiteOrSkip(host, skip, "bash", ["-lc", "command -v openshell"], "prereq-openshell"); + await prerequisiteOrSkip( + host, + skip, + "bash", + ["-lc", 'command -v "$1"', "prereq-openshell", host.openshellCommandPath], + "prereq-openshell", + ); await prerequisiteOrSkip( host, skip, @@ -309,11 +335,39 @@ test("concurrent gateway ports: onboards two sandboxes on isolated gateways and await artifacts.writeJson("fake-openai-requests.json", fake.requests()); await fake.close(); }); - cleanup.add("remove concurrent gateway sandboxes and gateways", async () => { - await bestEffortCleanup(host, sandbox, gatewayA, gatewayB); - }); + for (const gateway of [gatewayA, gatewayB]) { + cleanup.trackGateway(host, gateway, { + artifactName: `cleanup-gateway-destroy-${gateway}`, + env: openshellEnvForGateway(gateway), + timeoutMs: 60_000, + }); + } + for (const port of [ + 18799, 18798, 18797, 18796, 18795, 18794, 18793, 18792, 18791, 18790, 18789, + ]) { + cleanup.trackForward(host, port, { + artifactName: `cleanup-forward-stop-${port}`, + env: commandEnv(), + timeoutMs: 15_000, + }); + } + for (const [name, gateway, port] of [ + [SANDBOX_A, gatewayA, GATEWAY_PORT_A], + [SANDBOX_B, gatewayB, GATEWAY_PORT_B], + ] as const) { + cleanup.trackDisposable(`delete concurrent gateway OpenShell sandbox ${name}`, () => + sandbox.cleanupSandbox(name, { + artifactName: `cleanup-openshell-delete-${name}`, + env: openshellEnvForGateway(gateway), + timeoutMs: 60_000, + }), + ); + cleanup.trackDisposable(`destroy concurrent gateway sandbox ${name}`, () => + cleanupNemoClawSandbox(host, name, port), + ); + } - await bestEffortCleanup(host, sandbox, gatewayA, gatewayB); + await bestEffortPreclean(host, sandbox, gatewayA, gatewayB); const onboardA = await runOnboard( host, diff --git a/test/e2e/live/credential-migration.test.ts b/test/e2e/live/credential-migration.test.ts index 1f72a63a9ba..1089cca1dff 100644 --- a/test/e2e/live/credential-migration.test.ts +++ b/test/e2e/live/credential-migration.test.ts @@ -5,6 +5,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { assertCleanupSucceededOrAbsent } from "../fixtures/cleanup-resources.ts"; import { resultText } from "../fixtures/clients/command.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; import { validateSandboxName } from "../fixtures/clients/sandbox.ts"; @@ -36,7 +37,7 @@ function testEnv(home: string, extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv return testHomeEnvironment(home, extra); } -async function bestEffort(run: () => Promise): Promise { +async function bestEffortPreclean(run: () => Promise): Promise { try { await run(); } catch { @@ -86,34 +87,47 @@ async function ensureOpenshellAvailable(host: HostCliClient, home: string): Prom return afterInstall; } -async function cleanupCredentialMigrationState(host: HostCliClient, home: string): Promise { +async function cleanupCredentialMigrationNemoClawSandbox( + host: HostCliClient, + home: string, +): Promise { const env = testEnv(home); - await bestEffort(() => - host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { - artifactName: "cleanup-nemoclaw-destroy", - env, - redactionValues: [ - process.env.NVIDIA_INFERENCE_API_KEY ?? "", - process.env.COMPATIBLE_API_KEY ?? "", - ], - timeoutMs: 120_000, - }), + const result = await host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { + artifactName: "cleanup-nemoclaw-destroy", + env, + redactionValues: [ + process.env.NVIDIA_INFERENCE_API_KEY ?? "", + process.env.COMPATIBLE_API_KEY ?? "", + ], + timeoutMs: 120_000, + }); + assertCleanupSucceededOrAbsent( + result, + /Sandbox '.+' does not exist|Run 'nemoclaw onboard' to create one|sandbox .* not found|no such sandbox/iu.test( + resultText(result), + ), + `cleanup credential migration sandbox ${SANDBOX_NAME}`, ); - await bestEffort(() => +} + +async function cleanupCredentialMigrationState(host: HostCliClient, home: string): Promise { + const env = testEnv(home); + await bestEffortPreclean(() => cleanupCredentialMigrationNemoClawSandbox(host, home)); + await bestEffortPreclean(() => host.command("openshell", ["sandbox", "delete", SANDBOX_NAME], { artifactName: "cleanup-openshell-sandbox-delete", env, timeoutMs: 60_000, }), ); - await bestEffort(() => + await bestEffortPreclean(() => host.command("openshell", ["forward", "stop", "18789"], { artifactName: "cleanup-openshell-forward-stop", env, timeoutMs: 30_000, }), ); - await bestEffort(() => + await bestEffortPreclean(() => host.command("openshell", ["gateway", "destroy", "-g", "nemoclaw"], { artifactName: "cleanup-openshell-gateway-destroy", env, @@ -124,7 +138,7 @@ async function cleanupCredentialMigrationState(host: HostCliClient, home: string test("credential migration stages legacy file into gateway and removes plaintext safely", { timeout: ONBOARD_TIMEOUT_MS + INSTALL_TIMEOUT_MS + 5 * 60_000, -}, async ({ artifacts, cleanup, host, secrets, skip }) => { +}, async ({ artifacts, cleanup, host, sandbox, secrets, skip }) => { // Use the existing nightly secret as the legacy provider credential. The // onboard child env below deliberately does not receive that credential, so // the only source is ~/.nemoclaw/credentials.json — matching the retired @@ -160,10 +174,30 @@ test("credential migration stages legacy file into gateway and removes plaintext const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cred-migration-")); const nemoclawDir = path.join(home, ".nemoclaw"); const legacyFile = path.join(nemoclawDir, "credentials.json"); - cleanup.add(`remove credential migration state for ${SANDBOX_NAME}`, async () => { - await cleanupCredentialMigrationState(host, home); - fs.rmSync(home, { recursive: true, force: true }); + const cleanupEnv = testEnv(home); + cleanup.trackDisposable(`remove credential migration test home ${home}`, () => + fs.rmSync(home, { recursive: true, force: true }), + ); + cleanup.trackGateway(host, "nemoclaw", { + artifactName: "cleanup-openshell-gateway-destroy", + env: cleanupEnv, + timeoutMs: 120_000, }); + cleanup.trackForward(host, 18789, { + artifactName: "cleanup-openshell-forward-stop", + env: cleanupEnv, + timeoutMs: 30_000, + }); + cleanup.trackDisposable(`delete credential migration OpenShell sandbox ${SANDBOX_NAME}`, () => + sandbox.cleanupSandbox(SANDBOX_NAME, { + artifactName: "cleanup-openshell-sandbox-delete", + env: cleanupEnv, + timeoutMs: 60_000, + }), + ); + cleanup.trackDisposable(`destroy credential migration sandbox ${SANDBOX_NAME}`, () => + cleanupCredentialMigrationNemoClawSandbox(host, home), + ); await artifacts.target.declare({ id: "credential-migration", diff --git a/test/e2e/live/credential-sanitization.test.ts b/test/e2e/live/credential-sanitization.test.ts index 992f560d89d..1bd2c5039b5 100644 --- a/test/e2e/live/credential-sanitization.test.ts +++ b/test/e2e/live/credential-sanitization.test.ts @@ -20,6 +20,7 @@ import { stripCredentials, } from "../../../src/lib/security/credential-filter.ts"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { assertCleanupSucceededOrAbsent } from "../fixtures/cleanup-resources.ts"; import { resultText } from "../fixtures/clients/command.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; import { type SandboxClient, validateSandboxName } from "../fixtures/clients/sandbox.ts"; @@ -47,7 +48,7 @@ function testEnv(home: string, extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv return testHomeEnvironment(home, extra); } -async function bestEffort(run: () => Promise): Promise { +async function bestEffortPreclean(run: () => Promise): Promise { try { await run(); } catch { @@ -61,21 +62,15 @@ async function cleanupCredentialSanitizationState( home: string, ): Promise { const env = testEnv(home); - await bestEffort(() => - host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { - artifactName: "cleanup-nemoclaw-destroy-credential-sanitization", - env, - timeoutMs: 120_000, - }), - ); - await bestEffort(() => + await bestEffortPreclean(() => cleanupCredentialSanitizationNemoClawSandbox(host, home)); + await bestEffortPreclean(() => host.command("openshell", ["sandbox", "delete", SANDBOX_NAME], { artifactName: "cleanup-openshell-sandbox-delete-credential-sanitization", env, timeoutMs: 60_000, }), ); - await bestEffort(() => + await bestEffortPreclean(() => host.command("openshell", ["gateway", "destroy", "-g", "nemoclaw"], { artifactName: "cleanup-openshell-gateway-destroy-credential-sanitization", env, @@ -84,6 +79,24 @@ async function cleanupCredentialSanitizationState( ); } +async function cleanupCredentialSanitizationNemoClawSandbox( + host: HostCliClient, + home: string, +): Promise { + const result = await host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { + artifactName: "cleanup-nemoclaw-destroy-credential-sanitization", + env: testEnv(home), + timeoutMs: 120_000, + }); + assertCleanupSucceededOrAbsent( + result, + /Sandbox '.+' does not exist|Run 'nemoclaw onboard' to create one|sandbox .* not found|no such sandbox/iu.test( + resultText(result), + ), + `cleanup credential sanitization sandbox ${SANDBOX_NAME}`, + ); +} + function removeSensitiveFiles(dir: string): void { for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { const fullPath = path.join(dir, entry.name); @@ -317,10 +330,27 @@ runCredentialSanitizationTest( } const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cred-sanitization-home-")); - cleanup.add(`remove credential sanitization state for ${SANDBOX_NAME}`, async () => { - await cleanupCredentialSanitizationState(host, home); - fs.rmSync(home, { recursive: true, force: true }); + const cleanupEnv = testEnv(home); + cleanup.trackDisposable(`remove credential sanitization test home ${home}`, () => + fs.rmSync(home, { recursive: true, force: true }), + ); + cleanup.trackGateway(host, "nemoclaw", { + artifactName: "cleanup-openshell-gateway-destroy-credential-sanitization", + env: cleanupEnv, + timeoutMs: 120_000, }); + cleanup.trackDisposable( + `delete credential sanitization OpenShell sandbox ${SANDBOX_NAME}`, + () => + sandbox.cleanupSandbox(SANDBOX_NAME, { + artifactName: "cleanup-openshell-sandbox-delete-credential-sanitization", + env: cleanupEnv, + timeoutMs: 60_000, + }), + ); + cleanup.trackDisposable(`destroy credential sanitization sandbox ${SANDBOX_NAME}`, () => + cleanupCredentialSanitizationNemoClawSandbox(host, home), + ); await cleanupCredentialSanitizationState(host, home); diff --git a/test/e2e/live/cron-preflight-inference-local.test.ts b/test/e2e/live/cron-preflight-inference-local.test.ts index 126624a86db..1a7e6343abb 100644 --- a/test/e2e/live/cron-preflight-inference-local.test.ts +++ b/test/e2e/live/cron-preflight-inference-local.test.ts @@ -166,7 +166,7 @@ function commandEnv(hostedEnv: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { }; } -async function bestEffort(run: () => Promise): Promise { +async function preCleanBestEffort(run: () => Promise): Promise { try { await run(); } catch { @@ -194,9 +194,9 @@ function probeShell(): string { ); } -async function cleanupCronSandbox(sandbox: SandboxClient): Promise { - await bestEffort(() => - sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { +async function preCleanCronSandbox(sandbox: SandboxClient): Promise { + await preCleanBestEffort(() => + sandbox.cleanupSandbox(SANDBOX_NAME, { artifactName: "cleanup-openshell-delete-cron-preflight", env: commandEnv(), timeoutMs: 60_000, @@ -236,25 +236,28 @@ test("cron preflight reaches managed inference.local provider without EAI_AGAIN" skip(`Docker is required for cron preflight E2E: ${resultText(dockerInfo)}`); } - cleanup.add(`destroy cron preflight sandbox ${SANDBOX_NAME}`, async () => { - await bestEffort(() => - host.nemoclaw([SANDBOX_NAME, "destroy", "--yes"], { - artifactName: "cleanup-nemoclaw-destroy-cron-preflight", - env: commandEnv(), - timeoutMs: 120_000, - }), - ); - await cleanupCronSandbox(sandbox); + const cleanupEnv = commandEnv(); + cleanup.trackDisposable(`delete OpenShell sandbox ${SANDBOX_NAME}`, () => + sandbox.cleanupSandbox(SANDBOX_NAME, { + artifactName: "cleanup-openshell-delete-cron-preflight", + env: cleanupEnv, + timeoutMs: 60_000, + }), + ); + cleanup.trackSandbox(host, SANDBOX_NAME, { + artifactName: "cleanup-nemoclaw-destroy-cron-preflight", + env: cleanupEnv, + timeoutMs: 120_000, }); - await bestEffort(() => + await preCleanBestEffort(() => host.nemoclaw([SANDBOX_NAME, "destroy", "--yes"], { artifactName: "pre-cleanup-nemoclaw-destroy-cron-preflight", env: commandEnv(), timeoutMs: 120_000, }), ); - await cleanupCronSandbox(sandbox); + await preCleanCronSandbox(sandbox); let install: ShellProbeResult | undefined; for (let attempt = 1; attempt <= INSTALL_ATTEMPTS; attempt += 1) { diff --git a/test/e2e/live/device-auth-health-helpers.ts b/test/e2e/live/device-auth-health-helpers.ts index 0a8f5b980b3..7455f891955 100644 --- a/test/e2e/live/device-auth-health-helpers.ts +++ b/test/e2e/live/device-auth-health-helpers.ts @@ -49,7 +49,7 @@ export function commandEnv(inference?: DeviceAuthInferenceFixture): NodeJS.Proce return env; } -export async function bestEffort(run: () => Promise): Promise { +export async function preCleanBestEffort(run: () => Promise): Promise { try { await run(); } catch { @@ -93,15 +93,15 @@ export async function cleanupDeviceAuthSandbox( host: HostCliClient, sandbox: SandboxClient, ): Promise { - await bestEffort(() => + await preCleanBestEffort(() => host.nemoclaw([SANDBOX_NAME, "destroy", "--yes"], { artifactName: "cleanup-nemoclaw-destroy-device-auth-health", env: commandEnv(), timeoutMs: 120_000, }), ); - await bestEffort(() => - sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { + await preCleanBestEffort(() => + sandbox.cleanupSandbox(SANDBOX_NAME, { artifactName: "cleanup-openshell-delete-device-auth-health", env: commandEnv(), timeoutMs: 60_000, diff --git a/test/e2e/live/device-auth-health.test.ts b/test/e2e/live/device-auth-health.test.ts index 6b06c938580..41b134660ad 100644 --- a/test/e2e/live/device-auth-health.test.ts +++ b/test/e2e/live/device-auth-health.test.ts @@ -15,7 +15,6 @@ import { expect, test } from "../fixtures/e2e-test.ts"; import { startFakeOpenAiCompatibleServer } from "../fixtures/fake-openai-compatible.ts"; import { assertDockerAvailable, - bestEffort, cleanupDeviceAuthSandbox, commandEnv, DASHBOARD_PORT, @@ -49,7 +48,7 @@ test("device auth health probes treat 401 as live instead of offline (#2342)", { publicHost: "host.openshell.internal", requireAuth: true, }); - cleanup.add("close device-auth compatible inference fixture", async () => { + cleanup.trackDisposable("close device-auth compatible inference fixture", async () => { await artifacts.writeJson("compatible-inference-requests.json", inference.requests()); await inference.close(); }); @@ -81,10 +80,20 @@ test("device auth health probes treat 401 as live instead of offline (#2342)", { }); assertDockerAvailable(dockerInfo, skip); - cleanup.add(`destroy device-auth sandbox ${SANDBOX_NAME}`, () => - cleanupDeviceAuthSandbox(host, sandbox), + const cleanupEnv = commandEnv(); + cleanup.trackDisposable(`delete OpenShell sandbox ${SANDBOX_NAME}`, () => + sandbox.cleanupSandbox(SANDBOX_NAME, { + artifactName: "cleanup-openshell-delete-device-auth-health", + env: cleanupEnv, + timeoutMs: 60_000, + }), ); - await bestEffort(() => cleanupDeviceAuthSandbox(host, sandbox)); + cleanup.trackSandbox(host, SANDBOX_NAME, { + artifactName: "cleanup-nemoclaw-destroy-device-auth-health", + env: cleanupEnv, + timeoutMs: 120_000, + }); + await cleanupDeviceAuthSandbox(host, sandbox); const install = await installDeviceAuthSandbox(host, inferenceConfig, installLog); expect(install.exitCode, resultText(install)).toBe(0); diff --git a/test/e2e/live/diagnostics.test.ts b/test/e2e/live/diagnostics.test.ts index 896ea8ab465..fb56ba017e4 100644 --- a/test/e2e/live/diagnostics.test.ts +++ b/test/e2e/live/diagnostics.test.ts @@ -76,7 +76,7 @@ function testEnv(home: string, extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv }); } -async function bestEffort(run: () => Promise): Promise { +async function preCleanBestEffort(run: () => Promise): Promise { try { await run(); } catch { @@ -149,35 +149,33 @@ test("diagnostics CLI creates sanitized archives and validates sandbox/credentia } const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-diagnostics-home-")); - cleanup.add(`remove diagnostics state for ${SANDBOX_NAME}`, async () => { - const env = testEnv(home); - await bestEffort(() => - host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { - artifactName: "cleanup-nemoclaw-destroy-diagnostics", - env, - redactionValues: [apiKey], - timeoutMs: 120_000, - }), - ); - await bestEffort(() => - host.command("openshell", ["sandbox", "delete", SANDBOX_NAME], { - artifactName: "cleanup-openshell-sandbox-delete-diagnostics", - env, - timeoutMs: 60_000, - }), - ); - await bestEffort(() => - host.command("openshell", ["gateway", "destroy", "-g", "nemoclaw"], { - artifactName: "cleanup-openshell-gateway-destroy-diagnostics", - env, - timeoutMs: 120_000, - }), - ); + const cleanupEnv = testEnv(home); + cleanup.trackDisposable("remove diagnostics home", () => { fs.rmSync(home, { recursive: true, force: true }); }); + cleanup.trackGateway(host, "nemoclaw", { + artifactName: "cleanup-openshell-gateway-destroy-diagnostics", + env: cleanupEnv, + redactionValues: [apiKey], + timeoutMs: 120_000, + }); + cleanup.trackDisposable(`delete OpenShell sandbox ${SANDBOX_NAME}`, () => + sandbox.cleanupSandbox(SANDBOX_NAME, { + artifactName: "cleanup-openshell-sandbox-delete-diagnostics", + env: cleanupEnv, + redactionValues: [apiKey], + timeoutMs: 60_000, + }), + ); + cleanup.trackSandbox(host, SANDBOX_NAME, { + artifactName: "cleanup-nemoclaw-destroy-diagnostics", + env: cleanupEnv, + redactionValues: [apiKey], + timeoutMs: 120_000, + }); const env = testEnv(home, hosted.env); - await bestEffort(() => + await preCleanBestEffort(() => host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { artifactName: "pre-cleanup-nemoclaw-destroy-diagnostics", env, @@ -185,8 +183,8 @@ test("diagnostics CLI creates sanitized archives and validates sandbox/credentia timeoutMs: 120_000, }), ); - await bestEffort(() => - host.command("openshell", ["sandbox", "delete", SANDBOX_NAME], { + await preCleanBestEffort(() => + sandbox.cleanupSandbox(SANDBOX_NAME, { artifactName: "pre-cleanup-openshell-sandbox-delete-diagnostics", env, timeoutMs: 60_000, diff --git a/test/e2e/live/double-onboard.test.ts b/test/e2e/live/double-onboard.test.ts index 05c14b5dbe2..c0b7a6021c5 100644 --- a/test/e2e/live/double-onboard.test.ts +++ b/test/e2e/live/double-onboard.test.ts @@ -238,11 +238,12 @@ cid="$(docker ps -qf "name=openshell-cluster-nemoclaw" 2>/dev/null | head -1)" [ -n "$cid" ] && docker stop "$cid" >/dev/null 2>&1 || true exit 0 `; - await host.command("bash", ["-lc", script], { + const stop = await host.command("bash", ["-lc", script], { artifactName, env: commandEnv(), timeoutMs: 60_000, }); + expect(stop.exitCode, resultText(stop)).toBe(0); } function gatewayAliasEndpoint(): string { @@ -461,13 +462,43 @@ test("double-onboard: reuses gateway, preserves sibling sandbox, and recovers st port: Number(process.env.NEMOCLAW_FAKE_PORT ?? 0), }); await artifacts.writeJson("fake-openai.json", { baseUrl: fake.baseUrl }); - cleanup.add("close fake OpenAI-compatible endpoint", async () => { + cleanup.trackDisposable("close fake OpenAI-compatible endpoint", async () => { await artifacts.writeJson("fake-openai-requests.json", fake.requests()); await fake.close(); }); - cleanup.add("remove double-onboard sandboxes and gateways", async () => { - await cleanupDoubleOnboardState(host, sandbox); + cleanup.trackGateway(host, ALT_GATEWAY_NAME, { + artifactName: `cleanup-openshell-gateway-destroy-${ALT_GATEWAY_NAME}`, + env: commandEnv(), + timeoutMs: 60_000, + }); + cleanup.trackGateway(host, "nemoclaw", { + artifactName: "cleanup-openshell-gateway-destroy-nemoclaw", + env: commandEnv(), + timeoutMs: 60_000, + }); + cleanup.trackDisposable("stop double-onboard gateway runtime", () => + stopGatewayRuntime(host, "cleanup-stop-gateway-runtime"), + ); + cleanup.trackForward(host, 18789, { + artifactName: "cleanup-openshell-forward-stop-18789", + env: commandEnv(), + timeoutMs: 30_000, }); + const cleanupSandboxNames = [INSTALL_SANDBOX_NAME, SANDBOX_A, SANDBOX_B].filter(Boolean); + for (const name of [...cleanupSandboxNames].reverse()) { + cleanup.trackDisposable(`delete OpenShell sandbox ${name}`, () => + sandbox.cleanupSandbox(name, { + artifactName: `cleanup-openshell-sandbox-delete-${name}`, + env: commandEnv(), + timeoutMs: 60_000, + }), + ); + cleanup.trackSandbox(host, name, { + artifactName: `cleanup-nemoclaw-destroy-${name}`, + env: commandEnv(), + timeoutMs: RECOVERY_PROBE_TIMEOUT_MS, + }); + } await artifacts.target.declare({ id: "double-onboard", diff --git a/test/e2e/live/full-e2e.test.ts b/test/e2e/live/full-e2e.test.ts index 1c0c7191da5..c177fa43fd4 100644 --- a/test/e2e/live/full-e2e.test.ts +++ b/test/e2e/live/full-e2e.test.ts @@ -256,12 +256,31 @@ test("full e2e: install, onboard, inference, cli operations, and cleanup", { skip(`Docker is required: ${resultText(docker)}`); } - cleanupRegistry.add("remove full-e2e sandbox", () => cleanup(host, sandbox)); + cleanupRegistry.trackGateway(host, "nemoclaw", { + artifactName: "cleanup-openshell-gateway-destroy", + env: env(), + redactionValues: [hosted.apiKey], + timeoutMs: 60_000, + }); + cleanupRegistry.trackDisposable(`delete OpenShell sandbox ${SANDBOX_NAME}`, () => + sandbox.cleanupSandbox(SANDBOX_NAME, { + artifactName: "cleanup-openshell-sandbox-delete", + env: env(), + redactionValues: [hosted.apiKey], + timeoutMs: 60_000, + }), + ); + cleanupRegistry.trackSandbox(host, SANDBOX_NAME, { + artifactName: "cleanup-nemoclaw-destroy", + env: env(), + redactionValues: [hosted.apiKey], + timeoutMs: 120_000, + }); await cleanup(host, sandbox); const coldOnboard = createColdOnboardCapture(); coldOnboard && - cleanupRegistry.add("remove raw full-e2e trace", async () => { + cleanupRegistry.trackDisposable("remove raw full-e2e trace", async () => { fs.rmSync(coldOnboard.traceDirectory, { recursive: true, force: true }); }); diff --git a/test/e2e/live/gateway-health-honest.test.ts b/test/e2e/live/gateway-health-honest.test.ts index e4f795fae50..2c4c52377e6 100644 --- a/test/e2e/live/gateway-health-honest.test.ts +++ b/test/e2e/live/gateway-health-honest.test.ts @@ -16,6 +16,7 @@ import os from "node:os"; import path from "node:path"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { terminateProcessIfRunning } from "../fixtures/cleanup-resources.ts"; import { resultText } from "../fixtures/clients/index.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; import { REPO_ROOT } from "../fixtures/paths.ts"; @@ -83,30 +84,17 @@ test("onboard surfaces crashed Docker-driver gateway instead of reporting health }, ); - cleanup.add("remove sabotaged OpenShell gateway metadata", async () => { - await host.command( - "sh", - [ - "-lc", - `command -v openshell >/dev/null 2>&1 && openshell gateway remove ${GATEWAY_NAME} || true`, - ], - { - artifactName: "cleanup-openshell-gateway-remove-gateway-health-honest", - env: buildAvailabilityProbeEnv(), - timeoutMs: 30_000, - }, - ); + cleanup.trackGateway(host, GATEWAY_NAME, { + artifactName: "cleanup-openshell-gateway-health-honest", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, }); - cleanup.add("remove sabotaged gateway runtime files", () => { + cleanup.trackDisposable("remove sabotaged gateway runtime files", () => { const pid = fs.existsSync(gatewayPidFile) ? Number.parseInt(fs.readFileSync(gatewayPidFile, "utf8"), 10) : Number.NaN; if (Number.isInteger(pid) && pid > 0) { - try { - process.kill(pid, "SIGTERM"); - } catch { - // Best-effort: the expected child has already exited. - } + terminateProcessIfRunning(pid); } fs.rmSync(gatewayPidFile, { force: true }); fs.rmSync(path.join(stateDir, "runtime-marker.json"), { force: true }); diff --git a/test/e2e/live/gpu-double-onboard.test.ts b/test/e2e/live/gpu-double-onboard.test.ts index 21d16f56067..cffc56f77bb 100644 --- a/test/e2e/live/gpu-double-onboard.test.ts +++ b/test/e2e/live/gpu-double-onboard.test.ts @@ -6,6 +6,10 @@ import os from "node:os"; import path from "node:path"; import { containsInteger42Answer } from "../../helpers/e2e-answer-assertions.ts"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { + cleanupWhenCommandAvailable, + cleanupWhenOpenShellAvailable, +} from "../fixtures/cleanup-resources.ts"; import { resultText } from "../fixtures/clients/command.ts"; import { type HostCliClient } from "../fixtures/clients/host.ts"; import { type SandboxClient, validateSandboxName } from "../fixtures/clients/sandbox.ts"; @@ -73,7 +77,17 @@ async function cleanup(host: HostCliClient, sandbox: SandboxClient): Promise/dev/null || true; pkill -f 'ollama-auth-proxy' 2>/dev/null || true", + String.raw`set +e +status=0 +for pattern in '[o]llama serve' '[o]llama-auth-proxy'; do + pkill -f "$pattern" + rc=$? + case "$rc" in + 0|1) ;; + *) status="$rc" ;; + esac +done +exit "$status"`, ], { artifactName: "cleanup-ollama-processes", @@ -176,7 +190,90 @@ test("gpu double onboard keeps Ollama auth proxy token consistent after re-onboa skip(`NVIDIA GPU is required: ${resultText(smi)}`); } - cleanupRegistry.add("remove gpu double-onboard state", () => cleanup(host, sandbox)); + cleanupRegistry.trackDisposable("stop gpu double-onboard Ollama processes", async () => { + const stop = await host.command( + "bash", + [ + "-lc", + String.raw`set +e +status=0 +for pattern in '[o]llama serve' '[o]llama-auth-proxy'; do + pkill -f "$pattern" + rc=$? + case "$rc" in + 0|1) ;; + *) status="$rc" ;; + esac +done +exit "$status"`, + ], + { + artifactName: "cleanup-ollama-processes", + env: env(), + timeoutMs: 30_000, + }, + ); + expect(stop.exitCode, resultText(stop)).toBe(0); + }); + const gatewayCleanupOptions = { + artifactName: "cleanup-openshell-gateway-destroy", + env: env(), + timeoutMs: 60_000, + }; + cleanupRegistry.trackGateway( + { + cleanupGatewayRegistration: (name: string) => + cleanupWhenOpenShellAvailable( + host, + { + artifactName: "cleanup-probe-openshell-gateway", + env: gatewayCleanupOptions.env, + timeoutMs: 30_000, + }, + () => host.cleanupGatewayRegistration(name, gatewayCleanupOptions), + ), + }, + "nemoclaw", + gatewayCleanupOptions, + ); + const openshellSandboxCleanupOptions = { + artifactName: "cleanup-openshell-sandbox-delete", + env: env(), + timeoutMs: 60_000, + }; + cleanupRegistry.trackDisposable(`delete OpenShell sandbox ${SANDBOX_NAME}`, () => + cleanupWhenOpenShellAvailable( + host, + { + artifactName: "cleanup-probe-openshell-sandbox", + env: openshellSandboxCleanupOptions.env, + timeoutMs: 30_000, + }, + () => sandbox.cleanupSandbox(SANDBOX_NAME, openshellSandboxCleanupOptions), + ), + ); + const nemoclawSandboxCleanupOptions = { + artifactName: "cleanup-nemoclaw-destroy", + env: env(), + timeoutMs: 120_000, + }; + cleanupRegistry.trackSandbox( + { + cleanupSandbox: (name: string) => + cleanupWhenCommandAvailable( + host, + host.commandPath, + { + artifactName: "cleanup-probe-nemoclaw-sandbox", + env: nemoclawSandboxCleanupOptions.env, + timeoutMs: 30_000, + }, + () => host.cleanupSandbox(name, nemoclawSandboxCleanupOptions), + ), + }, + SANDBOX_NAME, + nemoclawSandboxCleanupOptions, + ); await cleanup(host, sandbox); const installOllama = await host.command( diff --git a/test/e2e/live/gpu-e2e-helpers.ts b/test/e2e/live/gpu-e2e-helpers.ts index 01488967eb3..9daf570680f 100644 --- a/test/e2e/live/gpu-e2e-helpers.ts +++ b/test/e2e/live/gpu-e2e-helpers.ts @@ -54,7 +54,10 @@ function isShellProbeResult(value: unknown): value is ShellProbeResult { ); } -export async function bestEffort(label: string, run: () => Promise): Promise { +export async function preCleanBestEffort( + label: string, + run: () => Promise, +): Promise { try { const result = await run(); if (isShellProbeResult(result) && result.exitCode !== 0) { @@ -113,21 +116,21 @@ export function chatContent(raw: string): string { } export async function cleanupGpu(host: HostCliClient, sandbox: SandboxClient): Promise { - await bestEffort("destroy GPU sandbox", () => + await preCleanBestEffort("destroy GPU sandbox", () => host.command("node", [CLI, SANDBOX_NAME, "destroy", "--yes"], { artifactName: "cleanup-destroy-gpu", env: env(), timeoutMs: 120_000, }), ); - await bestEffort("delete OpenShell sandbox", () => - sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { + await preCleanBestEffort("delete OpenShell sandbox", () => + sandbox.cleanupSandbox(SANDBOX_NAME, { artifactName: "cleanup-delete-gpu", env: env(), timeoutMs: 60_000, }), ); - await bestEffort("destroy OpenShell gateway", () => + await preCleanBestEffort("destroy OpenShell gateway", () => sandbox.openshell(["gateway", "destroy", "-g", "nemoclaw"], { artifactName: "cleanup-gateway-destroy-gpu", env: env(), diff --git a/test/e2e/live/gpu-e2e.test.ts b/test/e2e/live/gpu-e2e.test.ts index e3856919803..8c629771c1c 100644 --- a/test/e2e/live/gpu-e2e.test.ts +++ b/test/e2e/live/gpu-e2e.test.ts @@ -116,7 +116,28 @@ test("GPU Ollama onboard enables CUDA, auth proxy, and sandbox inference", { ], }); - cleanup.add("destroy GPU Ollama sandbox", () => cleanupGpu(host, sandbox)); + const cleanupEnv = env(); + cleanup.trackDisposable("stop GPU Ollama processes", async () => { + const result = await cleanupOllama(host, "cleanup-ollama-processes"); + expect(result.exitCode, resultText(result)).toBe(0); + }); + cleanup.trackGateway(host, "nemoclaw", { + artifactName: "cleanup-gateway-destroy-gpu", + env: cleanupEnv, + timeoutMs: 60_000, + }); + cleanup.trackDisposable(`delete OpenShell sandbox ${SANDBOX_NAME}`, () => + sandbox.cleanupSandbox(SANDBOX_NAME, { + artifactName: "cleanup-delete-gpu", + env: cleanupEnv, + timeoutMs: 60_000, + }), + ); + cleanup.trackSandbox(host, SANDBOX_NAME, { + artifactName: "cleanup-destroy-gpu", + env: cleanupEnv, + timeoutMs: 120_000, + }); await cleanupGpu(host, sandbox); const docker = await host.command("docker", ["info"], { diff --git a/test/e2e/live/hermes-discord.test.ts b/test/e2e/live/hermes-discord.test.ts index 4c0e7de9ead..11d9c4255c2 100644 --- a/test/e2e/live/hermes-discord.test.ts +++ b/test/e2e/live/hermes-discord.test.ts @@ -6,6 +6,7 @@ import path from "node:path"; import { setTimeout as sleep } from "node:timers/promises"; import type { CleanupRegistry } from "../fixtures/cleanup.ts"; +import { cleanupWhenOpenShellAvailable } from "../fixtures/cleanup-resources.ts"; import type { HostCliClient, SandboxClient } from "../fixtures/clients/index.ts"; import { sandboxAccessEnv, validateSandboxName } from "../fixtures/clients/sandbox.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; @@ -14,7 +15,7 @@ import { buildProcessTokenProbe } from "../fixtures/process-token-probe.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; import { type FakeDockerApi, startFakeDockerApi } from "./messaging-providers-helpers.ts"; import { - bestEffort, + runSecondaryCleanup as bestEffortLifecycleCleanup, dockerInfo, expectExitZero, phase6Env, @@ -23,6 +24,7 @@ import { sandboxNode, sandboxSh, shellQuote, + trackPreinstallSandboxCleanup, } from "./phase6-messaging-helpers.ts"; const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-hermes-discord"; @@ -69,14 +71,14 @@ function normalizedCsv(value: string): string { return value.replace(/\s+/g, ""); } -async function cleanupHermesDiscord( +async function precleanHermesDiscord( host: HostCliClient, sandboxName: string, env: NodeJS.ProcessEnv, redactionValues: string[], prefix: string, ): Promise { - await bestEffort(() => + await bestEffortLifecycleCleanup(() => host.command("nemoclaw", [sandboxName, "destroy", "--yes"], { artifactName: `${prefix}-nemoclaw-destroy`, env, @@ -84,16 +86,16 @@ async function cleanupHermesDiscord( timeoutMs: 15 * 60_000, }), ); - await bestEffort(() => - host.command("openshell", ["sandbox", "delete", sandboxName], { + await bestEffortLifecycleCleanup(() => + host.command(host.openshellCommandPath, ["sandbox", "delete", sandboxName], { artifactName: `${prefix}-openshell-sandbox-delete`, env, redactionValues, timeoutMs: 120_000, }), ); - await bestEffort(() => - host.command("openshell", ["gateway", "destroy", "-g", "nemoclaw"], { + await bestEffortLifecycleCleanup(() => + host.command(host.openshellCommandPath, ["gateway", "destroy", "-g", "nemoclaw"], { artifactName: `${prefix}-openshell-gateway-destroy`, env, redactionValues, @@ -109,7 +111,7 @@ async function startHermesFakeDiscordGateway( token: string, redactionValues: string[], ): Promise { - return startFakeDockerApi(host, cleanup.add.bind(cleanup), { + return startFakeDockerApi(host, cleanup.trackDisposable.bind(cleanup), { kind: "discord-gateway", imageScript: "fake-discord-gateway.cjs", containerPrefix: "nemoclaw-fake-discord-hermes", @@ -130,7 +132,7 @@ async function applyHermesFakeDiscordPolicy(options: { redactions: string[]; }): Promise { const result = await options.host.command( - "openshell", + options.host.openshellCommandPath, [ "policy", "update", @@ -371,11 +373,40 @@ test("hermes-discord: Hermes Discord schema, credential isolation, native gatewa discordRequireMention: DISCORD_REQUIRE_MENTION, }); - cleanup.add(`destroy Hermes Discord sandbox ${SANDBOX_NAME}`, () => - cleanupHermesDiscord(host, SANDBOX_NAME, env, redactionValues, "cleanup-hermes-discord"), + const gatewayCleanupOptions = { + artifactName: "cleanup-hermes-discord-openshell-gateway-destroy", + env, + redactionValues, + timeoutMs: 120_000, + }; + cleanup.trackGateway( + { + cleanupGatewayRegistration: (name: string) => + cleanupWhenOpenShellAvailable( + host, + { + artifactName: "cleanup-hermes-discord-probe-openshell-gateway", + env, + redactionValues, + timeoutMs: 30_000, + }, + () => host.cleanupGatewayRegistration(name, gatewayCleanupOptions), + ), + }, + "nemoclaw", + gatewayCleanupOptions, + ); + trackPreinstallSandboxCleanup( + cleanup, + host, + sandbox, + SANDBOX_NAME, + env, + redactionValues, + "cleanup-hermes-discord", ); - await cleanupHermesDiscord(host, SANDBOX_NAME, env, redactionValues, "preclean-hermes-discord"); + await precleanHermesDiscord(host, SANDBOX_NAME, env, redactionValues, "preclean-hermes-discord"); const docker = await dockerInfo(host, env); expectExitZero(docker, "Docker is running"); @@ -395,7 +426,12 @@ test("hermes-discord: Hermes Discord schema, credential isolation, native gatewa const cliProbe = await host.command( "bash", - ["-lc", "command -v nemoclaw && openshell --version"], + [ + "-lc", + 'command -v nemoclaw && command -v "$1" && "$1" --version', + "cli-probe-hermes-discord", + host.openshellCommandPath, + ], { artifactName: "phase-1-cli-probe", env, @@ -416,7 +452,7 @@ test("hermes-discord: Hermes Discord schema, credential isolation, native gatewa expect(resultText(list)).toContain(SANDBOX_NAME); const provider = await host.command( - "openshell", + host.openshellCommandPath, ["provider", "get", `${SANDBOX_NAME}-discord-bridge`], { artifactName: "phase-2-discord-provider-get", @@ -670,7 +706,7 @@ done`, expectExitZero(bridgeResidue, "no local Discord bridge residue probe"); expect(bridgeResidue.stdout.trim()).toBe(""); - await bestEffort(() => + await bestEffortLifecycleCleanup(() => host.command("docker", ["rm", "-f", fakeGateway.container], { artifactName: "phase-8-remove-fake-discord-container-before-rebuild", env, @@ -679,7 +715,7 @@ done`, }), ); fs.rmSync(fakeGateway.dir, { recursive: true, force: true }); - await bestEffort(() => + await bestEffortLifecycleCleanup(() => host.command( "bash", [ @@ -722,8 +758,8 @@ done`, timeoutMs: 15 * 60_000, }); expectExitZero(destroy, "destroy Hermes Discord sandbox"); - await bestEffort(() => - host.command("openshell", ["gateway", "destroy", "-g", "nemoclaw"], { + await bestEffortLifecycleCleanup(() => + host.command(host.openshellCommandPath, ["gateway", "destroy", "-g", "nemoclaw"], { artifactName: "phase-9-openshell-gateway-destroy", env, redactionValues, diff --git a/test/e2e/live/hermes-e2e.test.ts b/test/e2e/live/hermes-e2e.test.ts index a24e56b9379..80193990830 100644 --- a/test/e2e/live/hermes-e2e.test.ts +++ b/test/e2e/live/hermes-e2e.test.ts @@ -208,15 +208,31 @@ function parseGatewayProcess(output: string): { owner: string; pid: string; ppid return { owner, pid, ppid }; } -async function bestEffort(run: () => Promise): Promise { +async function preCleanBestEffort(run: () => Promise): Promise { try { await run(); } catch { - // Cleanup is best-effort because the pre-install path may not have + // Pre-cleanup is best-effort because the pre-install path may not have // nemoclaw/openshell available yet. } } +async function captureDiagnosticsBestEffort(run: () => Promise): Promise { + try { + await run(); + } catch { + // Failure diagnostics must not mask the install failure. + } +} + +async function postDestroyGatewayBestEffort(run: () => Promise): Promise { + try { + await run(); + } catch { + // The explicit sandbox-destroy assertion remains the primary phase-9 contract. + } +} + async function retryHostedInference( label: string, run: (attempt: number) => Promise, @@ -256,21 +272,21 @@ test("hermes-e2e: install.sh onboards Hermes and proves health plus live inferen const redactionValues = [apiKey]; const cleanupHermes = async (label: string) => { - await bestEffort(() => + await preCleanBestEffort(() => host.command("nemoclaw", [SANDBOX_NAME, "destroy", "--yes"], { artifactName: `${label}-nemoclaw-destroy`, env: commandEnv(), timeoutMs: 120_000, }), ); - await bestEffort(() => + await preCleanBestEffort(() => sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { artifactName: `${label}-openshell-sandbox-delete`, env: commandEnv(), timeoutMs: 60_000, }), ); - await bestEffort(() => + await preCleanBestEffort(() => sandbox.openshell(["gateway", "destroy", "-g", "nemoclaw"], { artifactName: `${label}-openshell-gateway-destroy`, env: commandEnv(), @@ -279,8 +295,23 @@ test("hermes-e2e: install.sh onboards Hermes and proves health plus live inferen ); }; - cleanup.add(`destroy Hermes sandbox ${SANDBOX_NAME}`, async () => { - await cleanupHermes("cleanup"); + const cleanupEnv = commandEnv(); + cleanup.trackGateway(host, "nemoclaw", { + artifactName: "cleanup-openshell-gateway-destroy", + env: cleanupEnv, + timeoutMs: 60_000, + }); + cleanup.trackDisposable(`delete OpenShell sandbox ${SANDBOX_NAME}`, () => + sandbox.cleanupSandbox(SANDBOX_NAME, { + artifactName: "cleanup-openshell-sandbox-delete", + env: cleanupEnv, + timeoutMs: 60_000, + }), + ); + cleanup.trackSandbox(host, SANDBOX_NAME, { + artifactName: "cleanup-nemoclaw-destroy", + env: cleanupEnv, + timeoutMs: 120_000, }); // Phase 0: pre-cleanup, after the secret gate so local skipped runs do not @@ -321,7 +352,7 @@ test("hermes-e2e: install.sh onboards Hermes and proves health plus live inferen }); await (install.exitCode === 0 ? Promise.resolve() - : bestEffort(() => + : captureDiagnosticsBestEffort(() => sandbox.execShell( SANDBOX_NAME, trustedSandboxShellScript( @@ -1441,7 +1472,7 @@ test("hermes-e2e: install.sh onboards Hermes and proves health plus live inferen timeoutMs: 120_000, }); expect(destroy.exitCode, resultText(destroy)).toBe(0); - await bestEffort(() => + await postDestroyGatewayBestEffort(() => sandbox.openshell(["gateway", "destroy", "-g", "nemoclaw"], { artifactName: "phase-9-openshell-gateway-destroy", env: commandEnv(), diff --git a/test/e2e/live/hermes-gpu-startup.test.ts b/test/e2e/live/hermes-gpu-startup.test.ts index aa9d1a6e3ca..eec644833f5 100644 --- a/test/e2e/live/hermes-gpu-startup.test.ts +++ b/test/e2e/live/hermes-gpu-startup.test.ts @@ -5,6 +5,7 @@ import fs from "node:fs"; import path from "node:path"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { cleanupUnlessVerified } from "../fixtures/cleanup-resources.ts"; import { type HostCliClient, outputContainsSandbox, @@ -72,33 +73,23 @@ function commandEnv(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { }; } -async function bestEffort(run: () => Promise): Promise { +async function preCleanBestEffort(run: () => Promise): Promise { try { await run(); } catch { - // Cleanup and failure diagnostics must not mask the primary live-test result. + // Pre-cleanup must not mask the primary live-test result. } } -async function cleanupHermes( - host: HostCliClient, - sandbox: SandboxClient, - label: string, -): Promise { - await bestEffort(() => - host.nemoclaw([SANDBOX_NAME, "destroy", "--yes"], { - artifactName: `${label}-nemoclaw-destroy`, - env: commandEnv(), - timeoutMs: 120_000, - }), - ); - await bestEffort(() => - sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { - artifactName: `${label}-openshell-sandbox-delete`, - env: commandEnv(), - timeoutMs: 60_000, - }), - ); +async function captureDiagnosticsBestEffort(run: () => Promise): Promise { + try { + await run(); + } catch { + // Failure diagnostics must not mask the primary live-test result. + } +} + +async function expectSandboxAbsent(host: HostCliClient, label: string): Promise { // Check the OpenShell control-plane view before intentionally removing its // gateway. A clean runner can have the CLI installed but no active gateway; // that explicit state is also valid absence evidence. @@ -119,7 +110,9 @@ async function cleanupHermes( ? !outputContainsSandbox(sandboxList, SANDBOX_NAME) : GATEWAY_ALREADY_ABSENT.test(resultText(sandboxList)); expect(sandboxAbsenceProven, resultText(sandboxList)).toBe(true); +} +async function cleanupOwnedGatewayRuntime(host: HostCliClient, label: string): Promise { const runtimeCleanup = await host.command( "bash", ["-c", GATEWAY_CLEANUP_SCRIPT, "gateway-runtime-cleanup", GATEWAY_CLEANUP_MODULE, "nemoclaw"], @@ -133,6 +126,12 @@ async function cleanupHermes( runtimeCleanup.exitCode, `owned gateway runtime cleanup failed: ${resultText(runtimeCleanup)}`, ).toBe(0); +} + +async function cleanupGatewayRegistrationBeforeTest( + host: HostCliClient, + label: string, +): Promise { await host .cleanupGatewayRegistration("nemoclaw", { artifactName: `${label}-openshell-gateway`, @@ -142,6 +141,9 @@ async function cleanupHermes( .catch((error: unknown) => { expect(error).toMatchObject({ message: "spawn openshell ENOENT" }); }); +} + +async function expectGatewayPortAvailable(host: HostCliClient, label: string): Promise { const gatewayPort = process.env.NEMOCLAW_GATEWAY_PORT ?? "8080"; const portAvailable = await host.command( "node", @@ -190,6 +192,56 @@ async function cleanupHermes( ).toEqual([]); } +async function cleanupHermes( + host: HostCliClient, + sandbox: SandboxClient, + label: string, +): Promise { + await host.cleanupSandbox(SANDBOX_NAME, { + artifactName: `${label}-nemoclaw-destroy`, + env: commandEnv(), + timeoutMs: 120_000, + }); + await sandbox.cleanupSandbox(SANDBOX_NAME, { + artifactName: `${label}-openshell-sandbox-delete`, + env: commandEnv(), + timeoutMs: 60_000, + }); + await expectSandboxAbsent(host, label); + await cleanupOwnedGatewayRuntime(host, label); + await host.cleanupGatewayRegistration("nemoclaw", { + artifactName: `${label}-openshell-gateway`, + env: commandEnv(), + timeoutMs: 60_000, + }); + await expectGatewayPortAvailable(host, label); +} + +async function preCleanHermes( + host: HostCliClient, + sandbox: SandboxClient, + label: string, +): Promise { + await preCleanBestEffort(() => + host.nemoclaw([SANDBOX_NAME, "destroy", "--yes", "--cleanup-gateway"], { + artifactName: `${label}-nemoclaw-destroy`, + env: commandEnv(), + timeoutMs: 120_000, + }), + ); + await preCleanBestEffort(() => + sandbox.cleanupSandbox(SANDBOX_NAME, { + artifactName: `${label}-openshell-sandbox-delete`, + env: commandEnv(), + timeoutMs: 60_000, + }), + ); + await expectSandboxAbsent(host, label); + await cleanupOwnedGatewayRuntime(host, label); + await cleanupGatewayRegistrationBeforeTest(host, label); + await expectGatewayPortAvailable(host, label); +} + async function captureFailedGpuContainer( host: HostCliClient, preRollbackDiagnosticsDir: string, @@ -226,7 +278,7 @@ for id in $ids; do printf '%s\n' "== container $id logs ==" docker logs --tail 300 "$id" 2>&1 || true done`; - await bestEffort(() => + await captureDiagnosticsBestEffort(() => host.command( "bash", ["-lc", script, "hermes-gpu-failure-diagnostics", sandboxFilter, preRollbackDiagnosticsDir], @@ -252,7 +304,7 @@ test(`hermes-gpu-startup: ${GPU_STARTUP_SCENARIO} OpenShell GPU route reaches st scenario: GPU_STARTUP_SCENARIO, }); - await cleanupHermes(host, sandbox, "pre-cleanup"); + await preCleanHermes(host, sandbox, "pre-cleanup"); const dockerInfo = await host.command("docker", ["info"], { artifactName: "phase-1-docker-info", @@ -271,14 +323,51 @@ test(`hermes-gpu-startup: ${GPU_STARTUP_SCENARIO} OpenShell GPU route reaches st publicHost: hostAddress, requireAuth: true, }); - cleanup.add("close fake OpenAI-compatible endpoint", async () => { + cleanup.trackDisposable("close fake OpenAI-compatible endpoint", async () => { await artifacts.writeJson("fake-openai-compatible-requests.json", fake.requests()); await fake.close(); }); let cleanTeardownVerified = false; - cleanup.add(`destroy Hermes sandbox ${SANDBOX_NAME}`, () => - cleanTeardownVerified ? Promise.resolve() : cleanupHermes(host, sandbox, "cleanup"), + const cleanupEnv = commandEnv(); + const cleanupHost: Pick = { + cleanupGatewayRegistration: (name, options) => + cleanupUnlessVerified(cleanTeardownVerified, () => + host.cleanupGatewayRegistration(name, options), + ), + cleanupSandbox: (name, options) => + cleanupUnlessVerified(cleanTeardownVerified, () => host.cleanupSandbox(name, options)), + }; + // Phase 5 runs this ordered teardown explicitly so the target can record a + // clean-teardown proof. Once that succeeds, fixture teardown must not retry + // resource operations after their OpenShell gateway has been removed. + cleanup.trackDisposable("verify Hermes GPU gateway port is available", () => + cleanupUnlessVerified(cleanTeardownVerified, () => expectGatewayPortAvailable(host, "cleanup")), + ); + cleanup.trackGateway(cleanupHost, "nemoclaw", { + artifactName: "cleanup-openshell-gateway", + env: cleanupEnv, + timeoutMs: 60_000, + }); + cleanup.trackDisposable("clean up owned Hermes GPU gateway runtime", () => + cleanupUnlessVerified(cleanTeardownVerified, () => cleanupOwnedGatewayRuntime(host, "cleanup")), ); + cleanup.trackDisposable("verify Hermes GPU sandbox is absent", () => + cleanupUnlessVerified(cleanTeardownVerified, () => expectSandboxAbsent(host, "cleanup")), + ); + cleanup.trackDisposable(`delete OpenShell sandbox ${SANDBOX_NAME}`, () => + cleanupUnlessVerified(cleanTeardownVerified, () => + sandbox.cleanupSandbox(SANDBOX_NAME, { + artifactName: "cleanup-openshell-sandbox-delete", + env: cleanupEnv, + timeoutMs: 60_000, + }), + ), + ); + cleanup.trackSandbox(cleanupHost, SANDBOX_NAME, { + artifactName: "cleanup-nemoclaw-destroy", + env: cleanupEnv, + timeoutMs: 120_000, + }); await artifacts.writeJson("fake-openai-compatible.json", { baseUrl: fake.baseUrl, model: FAKE_MODEL, @@ -306,7 +395,7 @@ test(`hermes-gpu-startup: ${GPU_STARTUP_SCENARIO} OpenShell GPU route reaches st const wrapper = createHermesGpuFallbackWrapper(realOpenshell.stdout.trim()); // Security-scoped #6110 fault injection: always remove the private wrapper root through the // E2E cleanup stack. Do not generalize PATH interception to other tests without review. - cleanup.add("remove Hermes GPU fallback wrapper", () => + cleanup.trackDisposable("remove Hermes GPU fallback wrapper", () => fs.rmSync(wrapper.rootDir, { recursive: true, force: true }), ); await artifacts.writeJson("gpu-fallback-wrapper.json", { @@ -437,7 +526,7 @@ test(`hermes-gpu-startup: ${GPU_STARTUP_SCENARIO} OpenShell GPU route reaches st supervisorTopologyValid: true, authenticatedInferenceRequestVerified: true, placeholderTokensAbsentFromInference: true, - cleanTeardownVerified: true, + cleanTeardownVerified, }, }); }); diff --git a/test/e2e/live/hermes-inference-switch-helpers.ts b/test/e2e/live/hermes-inference-switch-helpers.ts index b15b50e0ca5..d0e45a86ee3 100644 --- a/test/e2e/live/hermes-inference-switch-helpers.ts +++ b/test/e2e/live/hermes-inference-switch-helpers.ts @@ -133,7 +133,7 @@ export function env(apiKey?: string, extra: NodeJS.ProcessEnv = {}): NodeJS.Proc return { ...out, ...extra }; } -export async function bestEffort(run: () => Promise): Promise { +export async function preCleanBestEffort(run: () => Promise): Promise { try { await run(); } catch {} @@ -225,14 +225,14 @@ export async function cleanupHermesSwitch( host: HostCliClient, sandbox: SandboxClient, ): Promise { - await bestEffort(() => + await preCleanBestEffort(() => host.command("node", [CLI, SANDBOX_NAME, "destroy", "--yes", "--cleanup-gateway"], { artifactName: "cleanup-nemoclaw-destroy", env: env(), timeoutMs: 120_000, }), ); - await bestEffort(() => + await preCleanBestEffort(() => sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { artifactName: "cleanup-openshell-delete", env: env(), diff --git a/test/e2e/live/hermes-inference-switch.test.ts b/test/e2e/live/hermes-inference-switch.test.ts index f231d0f0a7e..2636c5ba551 100644 --- a/test/e2e/live/hermes-inference-switch.test.ts +++ b/test/e2e/live/hermes-inference-switch.test.ts @@ -91,7 +91,24 @@ test("Hermes inference set updates route/config and preserves live runtime", { runtimeSwitchApi: RUNTIME_SWITCH_API, }); - cleanup.add("destroy Hermes inference switch sandbox", () => cleanupHermesSwitch(host, sandbox)); + const cleanupEnv = env(); + cleanup.trackGateway(host, "nemoclaw", { + artifactName: "cleanup-openshell-gateway", + env: cleanupEnv, + timeoutMs: 60_000, + }); + cleanup.trackDisposable(`delete OpenShell sandbox ${SANDBOX_NAME}`, () => + sandbox.cleanupSandbox(SANDBOX_NAME, { + artifactName: "cleanup-openshell-delete", + env: cleanupEnv, + timeoutMs: 60_000, + }), + ); + cleanup.trackSandbox(host, SANDBOX_NAME, { + artifactName: "cleanup-nemoclaw-destroy", + env: cleanupEnv, + timeoutMs: 120_000, + }); await cleanupHermesSwitch(host, sandbox); const docker = await host.command("docker", ["info"], { @@ -108,7 +125,7 @@ test("Hermes inference set updates route/config and preserves live runtime", { requireAuth: true, }) : undefined; - cleanup.add("close Hermes inference switch baseline fixture", async () => { + cleanup.trackDisposable("close Hermes inference switch baseline fixture", async () => { await artifacts.writeJson( "baseline-openai-compatible-requests.json", mockBaseline?.requests() ?? [], diff --git a/test/e2e/live/hermes-shields-config.test.ts b/test/e2e/live/hermes-shields-config.test.ts index ab275811d59..3fec7053517 100644 --- a/test/e2e/live/hermes-shields-config.test.ts +++ b/test/e2e/live/hermes-shields-config.test.ts @@ -2,6 +2,10 @@ // SPDX-License-Identifier: Apache-2.0 import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { + cleanupWhenCommandAvailable, + cleanupWhenOpenShellAvailable, +} from "../fixtures/cleanup-resources.ts"; import { assertExitZero, resultText } from "../fixtures/clients/command.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; import { @@ -47,15 +51,15 @@ function commandEnv(endpointUrl?: string): NodeJS.ProcessEnv { }; } -async function cleanup(host: HostCliClient, label: string): Promise { +async function preClean(host: HostCliClient): Promise { await host.bestEffortCleanupSandbox(SANDBOX_NAME, { - artifactName: `${label}-destroy-sandbox`, + artifactName: "pre-cleanup-destroy-sandbox", env: commandEnv(), timeoutMs: 15 * 60_000, }); await host .cleanupGatewayRegistration(GATEWAY_NAME, { - artifactName: `${label}-destroy-gateway`, + artifactName: "pre-cleanup-destroy-gateway", env: commandEnv(), timeoutMs: 60_000, }) @@ -176,14 +180,61 @@ test("hermes-shields-config: fresh non-root Hermes sandbox completes two shields publicHost: "host.openshell.internal", requireAuth: true, }); - cleanupRegistry.add("close Hermes shields fake inference endpoint", async () => { - await artifacts.writeJson("fake-openai-compatible-requests.json", fake.requests()); - await fake.close(); - }); - cleanupRegistry.add(`destroy Hermes shields sandbox ${SANDBOX_NAME}`, async () => { - await cleanup(host, "cleanup"); + cleanupRegistry.trackDisposable("close Hermes shields fake inference endpoint", async () => { + try { + await artifacts.writeJson("fake-openai-compatible-requests.json", fake.requests()); + } finally { + await fake.close(); + } }); - await cleanup(host, "pre-cleanup"); + const gatewayCleanupOptions = { + artifactName: "cleanup-destroy-gateway", + env: commandEnv(), + redactionValues: [COMPATIBLE_API_KEY], + timeoutMs: 60_000, + }; + cleanupRegistry.trackGateway( + { + cleanupGatewayRegistration: (name: string) => + cleanupWhenOpenShellAvailable( + host, + { + artifactName: "cleanup-probe-openshell-gateway", + env: gatewayCleanupOptions.env, + redactionValues: gatewayCleanupOptions.redactionValues, + timeoutMs: 30_000, + }, + () => host.cleanupGatewayRegistration(name, gatewayCleanupOptions), + ), + }, + GATEWAY_NAME, + gatewayCleanupOptions, + ); + const sandboxCleanupOptions = { + artifactName: "cleanup-destroy-sandbox", + env: commandEnv(), + redactionValues: [COMPATIBLE_API_KEY], + timeoutMs: 15 * 60_000, + }; + cleanupRegistry.trackSandbox( + { + cleanupSandbox: (name: string) => + cleanupWhenCommandAvailable( + host, + host.commandPath, + { + artifactName: "cleanup-probe-nemoclaw-sandbox", + env: sandboxCleanupOptions.env, + redactionValues: sandboxCleanupOptions.redactionValues, + timeoutMs: 30_000, + }, + () => host.cleanupSandbox(name, sandboxCleanupOptions), + ), + }, + SANDBOX_NAME, + sandboxCleanupOptions, + ); + await preClean(host); const env = commandEnv(fake.baseUrl); const install = await host.command("bash", ["install.sh", "--non-interactive", "--fresh"], { diff --git a/test/e2e/live/hermes-slack-e2e-helpers.ts b/test/e2e/live/hermes-slack-e2e-helpers.ts index df23454dcb8..0996763dfec 100644 --- a/test/e2e/live/hermes-slack-e2e-helpers.ts +++ b/test/e2e/live/hermes-slack-e2e-helpers.ts @@ -2,12 +2,13 @@ // SPDX-License-Identifier: Apache-2.0 import { setTimeout as sleep } from "node:timers/promises"; +import { cleanupWhenOpenShellAvailable } from "../fixtures/cleanup-resources.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; import type { SandboxClient } from "../fixtures/clients/sandbox.ts"; import { type E2ETargetFixtures, expect } from "../fixtures/e2e-test.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; import { - bestEffort, + runSecondaryCleanup as bestEffortLifecycleCleanup, CLI, dockerInfo, expectExitZero, @@ -18,6 +19,7 @@ import { sandboxEncodedSh, sandboxSh, shellQuote, + trackPreinstallSandboxCleanup, } from "./phase6-messaging-helpers.ts"; const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-hermes-slack"; @@ -61,14 +63,14 @@ function redactions(apiKey?: string): string[] { ); } -async function cleanupHermesSlack(options: { +async function precleanHermesSlack(options: { host: HostCliClient; apiKey?: string; artifactPrefix: string; }): Promise { const env = hermesSlackEnv(options.apiKey); const redactionValues = redactions(options.apiKey); - await bestEffort(() => + await bestEffortLifecycleCleanup(() => options.host.command("node", [CLI, SANDBOX_NAME, "destroy", "--yes"], { artifactName: `${options.artifactPrefix}-nemoclaw-destroy`, env, @@ -76,8 +78,8 @@ async function cleanupHermesSlack(options: { timeoutMs: 15 * 60_000, }), ); - await bestEffort(() => - options.host.command("openshell", ["sandbox", "delete", SANDBOX_NAME], { + await bestEffortLifecycleCleanup(() => + options.host.command(options.host.openshellCommandPath, ["sandbox", "delete", SANDBOX_NAME], { artifactName: `${options.artifactPrefix}-openshell-sandbox-delete`, env, redactionValues, @@ -85,8 +87,8 @@ async function cleanupHermesSlack(options: { }), ); for (const provider of [`${SANDBOX_NAME}-slack-bridge`, `${SANDBOX_NAME}-slack-app`]) { - await bestEffort(() => - options.host.command("openshell", ["provider", "delete", provider], { + await bestEffortLifecycleCleanup(() => + options.host.command(options.host.openshellCommandPath, ["provider", "delete", provider], { artifactName: `${options.artifactPrefix}-openshell-provider-delete-${provider}`, env, redactionValues, @@ -94,14 +96,44 @@ async function cleanupHermesSlack(options: { }), ); } - await bestEffort(() => - options.host.command("openshell", ["gateway", "destroy", "-g", "nemoclaw"], { - artifactName: `${options.artifactPrefix}-openshell-gateway-destroy`, - env, - redactionValues, - timeoutMs: 120_000, - }), + await bestEffortLifecycleCleanup(() => + options.host.command( + options.host.openshellCommandPath, + ["gateway", "destroy", "-g", "nemoclaw"], + { + artifactName: `${options.artifactPrefix}-openshell-gateway-destroy`, + env, + redactionValues, + timeoutMs: 120_000, + }, + ), + ); +} + +async function cleanupHermesSlackProvider(options: { + host: HostCliClient; + apiKey?: string; + provider: string; +}): Promise { + const result = await options.host.command( + options.host.openshellCommandPath, + ["provider", "delete", options.provider], + { + artifactName: `cleanup-hermes-slack-openshell-provider-delete-${options.provider}`, + env: hermesSlackEnv(options.apiKey), + redactionValues: redactions(options.apiKey), + timeoutMs: 60_000, + }, ); + if ( + result.exitCode === 0 || + /\bNotFound\b|provider[^\n]*(?:not found|does not exist)|no such provider/i.test( + resultText(result), + ) + ) { + return; + } + expectExitZero(result, `cleanup OpenShell provider ${options.provider}`); } async function hostSlackTokenStdin(options: { @@ -115,7 +147,7 @@ async function hostSlackTokenStdin(options: { "set -euo pipefail", "ssh_config=$(mktemp)", "trap 'rm -f \"$ssh_config\"' EXIT", - `openshell sandbox ssh-config ${shellQuote(SANDBOX_NAME)} >"$ssh_config"`, + `${shellQuote(options.host.openshellCommandPath)} sandbox ssh-config ${shellQuote(SANDBOX_NAME)} >"$ssh_config"`, [ 'printf "%s\\n%s\\n" "$SLACK_BOT_TOKEN" "$SLACK_APP_TOKEN"', "|", @@ -145,7 +177,7 @@ async function expectProvider(options: { artifactName: string; }): Promise { const result = await options.host.command( - "openshell", + options.host.openshellCommandPath, ["provider", "get", options.providerName], { artifactName: options.artifactName, @@ -189,7 +221,7 @@ async function providerExists(options: { artifactName: string; }): Promise { const result = await options.host.command( - "openshell", + options.host.openshellCommandPath, ["provider", "get", options.providerName], { artifactName: options.artifactName, @@ -217,9 +249,52 @@ export async function runHermesSlackE2E({ const env = hermesSlackEnv(apiKey); const redactionValues = redactions(apiKey); - cleanup.add(`destroy Hermes Slack sandbox ${SANDBOX_NAME}`, async () => { - await cleanupHermesSlack({ host, apiKey, artifactPrefix: "cleanup-hermes-slack" }); - }); + const gatewayCleanupOptions = { + artifactName: "cleanup-hermes-slack-openshell-gateway-destroy", + env, + redactionValues, + timeoutMs: 120_000, + }; + cleanup.trackGateway( + { + cleanupGatewayRegistration: (name: string) => + cleanupWhenOpenShellAvailable( + host, + { + artifactName: "cleanup-hermes-slack-probe-openshell-gateway", + env, + redactionValues, + timeoutMs: 30_000, + }, + () => host.cleanupGatewayRegistration(name, gatewayCleanupOptions), + ), + }, + "nemoclaw", + gatewayCleanupOptions, + ); + for (const provider of [`${SANDBOX_NAME}-slack-app`, `${SANDBOX_NAME}-slack-bridge`]) { + cleanup.trackDisposable(`delete OpenShell provider ${provider}`, () => + cleanupWhenOpenShellAvailable( + host, + { + artifactName: `cleanup-hermes-slack-probe-openshell-provider-${provider}`, + env, + redactionValues, + timeoutMs: 30_000, + }, + () => cleanupHermesSlackProvider({ host, apiKey, provider }), + ), + ); + } + trackPreinstallSandboxCleanup( + cleanup, + host, + sandbox, + SANDBOX_NAME, + env, + redactionValues, + "cleanup-hermes-slack", + ); await artifacts.target.declare({ id: "hermes-slack-e2e", @@ -237,7 +312,7 @@ export async function runHermesSlackE2E({ return; } - await cleanupHermesSlack({ host, apiKey, artifactPrefix: "preclean-hermes-slack" }); + await precleanHermesSlack({ host, apiKey, artifactPrefix: "preclean-hermes-slack" }); await precleanSandbox(host, SANDBOX_NAME, env, redactionValues, "preclean-hermes-slack-cli"); const install = await installSandboxOrSkipOnRateLimit( @@ -252,7 +327,12 @@ export async function runHermesSlackE2E({ const cliProbe = await host.command( "bash", - ["-lc", "command -v nemoclaw && command -v openshell && openshell --version"], + [ + "-lc", + 'command -v nemoclaw && command -v "$1" && "$1" --version', + "cli-probe-hermes-slack", + host.openshellCommandPath, + ], { artifactName: "phase-1-cli-probe-hermes-slack", env, @@ -496,12 +576,16 @@ else: expectExitZero(processScan, "raw Slack token process scan"); if (processScan.stdout.trim() !== "EMPTY") expect(processScan.stdout.trim()).toBe("OK"); - const policy = await host.command("openshell", ["policy", "get", "--full", SANDBOX_NAME], { - artifactName: "phase-5-policy-get", - env, - redactionValues, - timeoutMs: 60_000, - }); + const policy = await host.command( + host.openshellCommandPath, + ["policy", "get", "--full", SANDBOX_NAME], + { + artifactName: "phase-5-policy-get", + env, + redactionValues, + timeoutMs: 60_000, + }, + ); expectExitZero(policy, "openshell policy get --full"); const policyText = resultText(policy); const slackBlockMatch = policyText.match(/^ slack:[\s\S]*?(?=^ [A-Za-z0-9_-]+:|$(?![\s\S]))/m); @@ -642,8 +726,8 @@ PY`, timeoutMs: 15 * 60_000, }); expectExitZero(destroy, "nemoclaw destroy Hermes Slack sandbox"); - await bestEffort(() => - host.command("openshell", ["gateway", "destroy", "-g", "nemoclaw"], { + await bestEffortLifecycleCleanup(() => + host.command(host.openshellCommandPath, ["gateway", "destroy", "-g", "nemoclaw"], { artifactName: "phase-7-openshell-gateway-destroy", env, redactionValues, diff --git a/test/e2e/live/issue-2478-crash-loop-recovery.test.ts b/test/e2e/live/issue-2478-crash-loop-recovery.test.ts index 22b3639aca2..486502aada1 100644 --- a/test/e2e/live/issue-2478-crash-loop-recovery.test.ts +++ b/test/e2e/live/issue-2478-crash-loop-recovery.test.ts @@ -192,7 +192,11 @@ async function onboardWithCompatibleEndpoint( result.exitCode, `compatible OpenClaw onboard failed\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`, ).toBe(0); - cleanup.add(`destroy NemoClaw sandbox ${sandboxName}`, () => cleanupSandbox(host, sandboxName)); + cleanup.trackSandbox(host, sandboxName, { + artifactName: `cleanup-destroy-${sandboxName}`, + env: probeEnv(), + timeoutMs: 15 * 60_000, + }); return { onboarding: "cloud-openclaw", diff --git a/test/e2e/live/issue-4462-scope-upgrade-approval.test.ts b/test/e2e/live/issue-4462-scope-upgrade-approval.test.ts index d9c1931faff..40ea53798a1 100644 --- a/test/e2e/live/issue-4462-scope-upgrade-approval.test.ts +++ b/test/e2e/live/issue-4462-scope-upgrade-approval.test.ts @@ -1233,7 +1233,26 @@ test("keeps issue 4462 scope-upgrade approval on the gateway path without an adm skip(`Docker is required: ${resultText(docker)}`); } - cleanupRegistry.add("remove issue-4462 sandbox", () => cleanup(host, sandbox)); + cleanupRegistry.trackGateway(host, "nemoclaw", { + artifactName: "cleanup-openshell-gateway-destroy", + env: env(), + redactionValues: [apiKey], + timeoutMs: 60_000, + }); + cleanupRegistry.trackDisposable(`delete OpenShell sandbox ${SANDBOX_NAME}`, () => + sandbox.cleanupSandbox(SANDBOX_NAME, { + artifactName: "cleanup-openshell-sandbox-delete", + env: env(), + redactionValues: [apiKey], + timeoutMs: 60_000, + }), + ); + cleanupRegistry.trackSandbox(host, SANDBOX_NAME, { + artifactName: "cleanup-nemoclaw-destroy", + env: env({ NEMOCLAW_CLEANUP_GATEWAY: "1" }), + redactionValues: [apiKey], + timeoutMs: 120_000, + }); await cleanup(host, sandbox); const install = await host.command( diff --git a/test/e2e/live/jetson-nvmap-gpu.test.ts b/test/e2e/live/jetson-nvmap-gpu.test.ts index a57d5a2c830..d7837314988 100644 --- a/test/e2e/live/jetson-nvmap-gpu.test.ts +++ b/test/e2e/live/jetson-nvmap-gpu.test.ts @@ -4,6 +4,10 @@ import path from "node:path"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { + cleanupWhenCommandAvailable, + cleanupWhenOpenShellAvailable, +} from "../fixtures/cleanup-resources.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; import { resultText } from "../fixtures/clients/index.ts"; import { type SandboxClient, trustedSandboxShellScript } from "../fixtures/clients/sandbox.ts"; @@ -97,7 +101,84 @@ fi`, "Not a Jetson/Tegra host (/dev/nvmap absent) — reporter workflow requires Jetson hardware; hermetic #4231 coverage remains in src/lib/onboard/docker-gpu-patch.test.ts.", ); - cleanup.add("destroy Jetson nvmap sandbox", () => cleanupJetsonSandbox(host)); + cleanup.trackDisposable("stop Jetson Ollama processes", async () => { + const stop = await hostShell( + host, + String.raw`set +e +status=0 +for pattern in '[o]llama serve' '[o]llama-auth-proxy'; do + pkill -f "$pattern" + rc=$? + case "$rc" in + 0|1) ;; + *) status="$rc" ;; + esac +done +exit "$status"`, + "cleanup-jetson-ollama-processes", + 120_000, + ); + expect(stop.exitCode, resultText(stop)).toBe(0); + }); + const gatewayCleanupOptions = { + artifactName: "cleanup-jetson-openshell-gateway", + env: env(), + timeoutMs: 120_000, + }; + cleanup.trackGateway( + { + cleanupGatewayRegistration: (name: string) => + cleanupWhenOpenShellAvailable( + host, + { + artifactName: "cleanup-probe-jetson-openshell-gateway", + env: gatewayCleanupOptions.env, + timeoutMs: 30_000, + }, + () => host.cleanupGatewayRegistration(name, gatewayCleanupOptions), + ), + }, + "nemoclaw", + gatewayCleanupOptions, + ); + const openshellSandboxCleanupOptions = { + artifactName: "cleanup-jetson-openshell-sandbox", + env: env(), + timeoutMs: 120_000, + }; + cleanup.trackDisposable(`delete OpenShell sandbox ${SANDBOX_NAME}`, () => + cleanupWhenOpenShellAvailable( + host, + { + artifactName: "cleanup-probe-jetson-openshell-sandbox", + env: openshellSandboxCleanupOptions.env, + timeoutMs: 30_000, + }, + () => sandbox.cleanupSandbox(SANDBOX_NAME, openshellSandboxCleanupOptions), + ), + ); + const nemoclawSandboxCleanupOptions = { + artifactName: "cleanup-jetson-nemoclaw-sandbox", + env: env(), + timeoutMs: 120_000, + }; + cleanup.trackSandbox( + { + cleanupSandbox: (name: string) => + cleanupWhenCommandAvailable( + host, + host.commandPath, + { + artifactName: "cleanup-probe-jetson-nemoclaw-sandbox", + env: nemoclawSandboxCleanupOptions.env, + timeoutMs: 30_000, + }, + () => host.cleanupSandbox(name, nemoclawSandboxCleanupOptions), + ), + }, + SANDBOX_NAME, + nemoclawSandboxCleanupOptions, + ); await cleanupJetsonSandbox(host); const hostNvmap = await hostShell( diff --git a/test/e2e/live/kimi-inference-compat-helpers.ts b/test/e2e/live/kimi-inference-compat-helpers.ts index 448b7a4ed10..c950d82d83c 100644 --- a/test/e2e/live/kimi-inference-compat-helpers.ts +++ b/test/e2e/live/kimi-inference-compat-helpers.ts @@ -104,7 +104,7 @@ export function env( }; } -export async function bestEffort(run: () => Promise): Promise { +export async function preCleanBestEffort(run: () => Promise): Promise { try { await run(); } catch {} @@ -373,15 +373,15 @@ function streamTextChunks(id: string, content: string): unknown[] { } export async function cleanupKimi(host: HostCliClient, sandbox: SandboxClient): Promise { - await bestEffort(() => + await preCleanBestEffort(() => host.command("node", [CLI, SANDBOX_NAME, "destroy", "--yes"], { artifactName: "cleanup-destroy-kimi", env: env(), timeoutMs: 120_000, }), ); - await bestEffort(() => - sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { + await preCleanBestEffort(() => + sandbox.cleanupSandbox(SANDBOX_NAME, { artifactName: "cleanup-delete-kimi", env: env(), timeoutMs: 60_000, diff --git a/test/e2e/live/kimi-inference-compat.test.ts b/test/e2e/live/kimi-inference-compat.test.ts index 6fc58037335..83ccefbda08 100644 --- a/test/e2e/live/kimi-inference-compat.test.ts +++ b/test/e2e/live/kimi-inference-compat.test.ts @@ -36,7 +36,19 @@ test("Kimi-compatible endpoint config enables plugin wiring and managed inferenc : undefined; const fake = await startKimiUpstream(mode); maybeRegisterKimiMockCleanup(cleanup, fake); - cleanup.add("destroy Kimi sandbox", () => cleanupKimi(host, sandbox)); + const cleanupEnv = env(); + cleanup.trackDisposable(`delete OpenShell sandbox ${SANDBOX_NAME}`, () => + sandbox.cleanupSandbox(SANDBOX_NAME, { + artifactName: "cleanup-delete-kimi", + env: cleanupEnv, + timeoutMs: 60_000, + }), + ); + cleanup.trackSandbox(host, SANDBOX_NAME, { + artifactName: "cleanup-destroy-kimi", + env: cleanupEnv, + timeoutMs: 120_000, + }); await artifacts.target.declare({ id: "kimi-inference-compat", diff --git a/test/e2e/live/mcp-bridge.test.ts b/test/e2e/live/mcp-bridge.test.ts index e5a6fc08b72..f21a6540947 100644 --- a/test/e2e/live/mcp-bridge.test.ts +++ b/test/e2e/live/mcp-bridge.test.ts @@ -15,6 +15,7 @@ import { parseOpenShellPolicy } from "../../../src/lib/policy/merge"; import type { McpBridgeEntry } from "../../../src/lib/state/registry"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import type { CleanupRegistry } from "../fixtures/cleanup.ts"; +import { assertCleanupSucceededOrAbsent } from "../fixtures/cleanup-resources.ts"; import { assertExitZero as expectExitZero, resultText } from "../fixtures/clients/command.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; import { type SandboxClient, trustedSandboxShellScript } from "../fixtures/clients/sandbox.ts"; @@ -70,6 +71,8 @@ const MCP_MUTATION_TIMEOUT_MS: Record = { "hermes-config": 12 * 60_000, mcporter: 3 * 60_000, }; +const MCP_BRIDGE_ALREADY_ABSENT = + /No MCP servers are registered|No MCP server '.+' is registered|MCP server '.+' not found/iu; function expectExitNonZero(result: ShellProbeResult, label: string, pattern: RegExp): void { expect( @@ -83,24 +86,22 @@ function parseCurrentPolicy(raw: string): string { return parseOpenShellPolicy(raw).yamlBody; } -async function bestEffortRemoveBridge( +async function cleanupMcpBridge( host: HostCliClient, sandboxName: string, server: string, adapter: McpAdapter, ): Promise { - await host.nemoclaw([sandboxName, "mcp", "remove", server, "--force"], { + const result = await host.nemoclaw([sandboxName, "mcp", "remove", server, "--force"], { artifactName: `cleanup-mcp-remove-${server}`, env: buildAvailabilityProbeEnv(), timeoutMs: MCP_MUTATION_TIMEOUT_MS[adapter], }); -} - -async function cleanupSandbox(host: HostCliClient, sandboxName: string): Promise { - await host.bestEffortCleanupSandbox(sandboxName, { - artifactName: "cleanup-destroy-sandbox", - timeoutMs: 15 * 60_000, - }); + assertCleanupSucceededOrAbsent( + result, + MCP_BRIDGE_ALREADY_ABSENT, + `cleanup MCP bridge ${server} on sandbox ${sandboxName}`, + ); } async function onboardAgent( @@ -109,9 +110,10 @@ async function onboardAgent( endpointUrl: string, options: { agent: McpAgent; sandboxName: string; artifactName: string }, ): Promise { - cleanup.add(`destroy MCP bridge ${options.agent} sandbox`, () => - cleanupSandbox(host, options.sandboxName), - ); + cleanup.trackSandbox(host, options.sandboxName, { + artifactName: "cleanup-destroy-sandbox", + timeoutMs: 15 * 60_000, + }); await host.cleanupSandbox(options.sandboxName, { artifactName: "precleanup-destroy-sandbox", timeoutMs: 15 * 60_000, @@ -180,7 +182,7 @@ async function assertAdapterDnsRebindingDenied( rebindMcp.close(), ); cleanup.add(`remove ${options.artifactPrefix} DNS rebinding MCP bridge`, () => - bestEffortRemoveBridge(host, options.sandboxName, REBIND_SERVER_NAME, options.adapter), + cleanupMcpBridge(host, options.sandboxName, REBIND_SERVER_NAME, options.adapter), ); const rebindMcpUrl = `https://${REBIND_HOSTNAME}:${rebindMcp.port}/mcp`; const hostsFixture = await setupDnsRebindingHostsFixture( @@ -410,12 +412,7 @@ async function assertConcurrentAddSerialized( }, ): Promise { cleanup.add(`remove ${options.artifactPrefix} concurrent MCP bridge`, () => - bestEffortRemoveBridge( - host, - options.sandboxName, - CONCURRENT_SERVER_NAME, - options.expectedAdapter, - ), + cleanupMcpBridge(host, options.sandboxName, CONCURRENT_SERVER_NAME, options.expectedAdapter), ); const args = [ options.sandboxName, @@ -870,10 +867,10 @@ test("mcp-bridge", { timeout: 45 * 60_000 }, async ({ artifacts, cleanup, host, }); cleanup.add("remove MCP bridge", () => - bestEffortRemoveBridge(host, OPENCLAW_SANDBOX_NAME, SERVER_NAME, "mcporter"), + cleanupMcpBridge(host, OPENCLAW_SANDBOX_NAME, SERVER_NAME, "mcporter"), ); cleanup.add("remove unexpected missing-secret MCP state", () => - bestEffortRemoveBridge(host, OPENCLAW_SANDBOX_NAME, "missingsecret", "mcporter"), + cleanupMcpBridge(host, OPENCLAW_SANDBOX_NAME, "missingsecret", "mcporter"), ); await expectMcpCliFailure( @@ -1227,7 +1224,7 @@ liveAgentMatrixTest( artifactName: "onboard-hermes-mcp-bridge", }); cleanup.add("remove Hermes MCP bridge", () => - bestEffortRemoveBridge(host, HERMES_SANDBOX_NAME, SERVER_NAME, "hermes-config"), + cleanupMcpBridge(host, HERMES_SANDBOX_NAME, SERVER_NAME, "hermes-config"), ); await assertConcurrentAddSerialized(host, cleanup, { @@ -1386,7 +1383,7 @@ liveAgentMatrixTest( artifactName: "onboard-deepagents-mcp-bridge", }); cleanup.add("remove Deep Agents MCP bridge", () => - bestEffortRemoveBridge(host, DEEPAGENTS_SANDBOX_NAME, SERVER_NAME, "deepagents-config"), + cleanupMcpBridge(host, DEEPAGENTS_SANDBOX_NAME, SERVER_NAME, "deepagents-config"), ); await assertConcurrentAddSerialized(host, cleanup, { diff --git a/test/e2e/live/messaging-compatible-endpoint-helpers.ts b/test/e2e/live/messaging-compatible-endpoint-helpers.ts index d3863ffd55c..6daebdcc147 100644 --- a/test/e2e/live/messaging-compatible-endpoint-helpers.ts +++ b/test/e2e/live/messaging-compatible-endpoint-helpers.ts @@ -1,9 +1,17 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import path from "node:path"; +import { execFileSync } from "node:child_process"; +import fs from "node:fs"; +import { setTimeout as sleep } from "node:timers/promises"; +import { resolveGatewayPortFromName } from "../../../src/lib/onboard/gateway-binding.ts"; +import { + hostGatewayCmdlineMatches, + resolveDockerDriverGatewayPidFile, +} from "../../../src/lib/onboard/host-gateway-process.ts"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { assertExitZero } from "../fixtures/clients/command.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; import { CLI_ENTRYPOINT } from "../fixtures/paths.ts"; @@ -17,7 +25,7 @@ export function commandEnv(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { }; } -async function bestEffort(run: () => Promise): Promise { +async function preCleanBestEffort(run: () => Promise): Promise { try { await run(); } catch { @@ -27,34 +35,202 @@ async function bestEffort(run: () => Promise): Promise { } } +const GATEWAY_NAME = "nemoclaw"; +const GATEWAY_PORT = resolveGatewayPortFromName(GATEWAY_NAME); + +type GatewayPidState = + | { kind: "absent" } + | { kind: "unverified" } + | { kind: "owned"; startTime: string }; + +type GatewayPidFileState = { kind: "absent" } | { kind: "invalid" } | { kind: "pid"; pid: number }; + +function readGatewayPid(): GatewayPidFileState { + try { + const raw = fs.readFileSync(resolveDockerDriverGatewayPidFile(), "utf8").trim(); + if (!/^\d+$/u.test(raw)) return { kind: "invalid" }; + const pid = Number(raw); + return Number.isSafeInteger(pid) && pid > 0 ? { kind: "pid", pid } : { kind: "invalid" }; + } catch (error) { + return (error as NodeJS.ErrnoException).code === "ENOENT" + ? { kind: "absent" } + : { kind: "invalid" }; + } +} + +function processExists(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ESRCH") return false; + return true; + } +} + +function readProcessSnapshot(pid: number): { cmdline: string; startTime: string } | null { + if (process.platform === "linux") { + const stat = fs.readFileSync(`/proc/${pid}/stat`, "utf8"); + const commandEnd = stat.lastIndexOf(")"); + const fields = + commandEnd >= 0 + ? stat + .slice(commandEnd + 2) + .trim() + .split(/\s+/u) + : []; + const startTime = fields[19]; + const cmdline = fs.readFileSync(`/proc/${pid}/cmdline`, "utf8").replaceAll("\0", " ").trim(); + return startTime ? { cmdline, startTime } : null; + } + if (process.platform === "darwin") { + const line = execFileSync("ps", ["-p", String(pid), "-o", "lstart=", "-o", "command="], { + encoding: "utf8", + }).trim(); + if (line.length <= 24) return null; + return { cmdline: line.slice(24).trim(), startTime: line.slice(0, 24) }; + } + return null; +} + +function inspectGatewayPid(pid: number): GatewayPidState { + if (!processExists(pid)) return { kind: "absent" }; + try { + const snapshot = readProcessSnapshot(pid); + if ( + !snapshot || + !hostGatewayCmdlineMatches(snapshot.cmdline, undefined, { + name: GATEWAY_NAME, + port: GATEWAY_PORT, + }) + ) { + return { kind: "unverified" }; + } + return { kind: "owned", startTime: snapshot.startTime }; + } catch { + return processExists(pid) ? { kind: "unverified" } : { kind: "absent" }; + } +} + +async function waitForOriginalGatewayExit( + pid: number, + startTime: string, + timeoutMs: number, +): Promise { + const deadline = Date.now() + timeoutMs; + let state = inspectGatewayPid(pid); + while (Date.now() < deadline) { + if (state.kind === "absent") return state; + if (state.kind === "owned" && state.startTime !== startTime) return state; + await sleep(100); + state = inspectGatewayPid(pid); + } + return state; +} + +function signalOriginalGateway( + pid: number, + startTime: string, + signal: NodeJS.Signals, + strict: boolean, +): boolean { + const current = inspectGatewayPid(pid); + if (current.kind === "absent") return false; + if (current.kind === "owned" && current.startTime !== startTime) return false; + if (current.kind === "unverified") { + if (strict) + throw new Error(`Refusing ${signal} because gateway process ${pid} is no longer owned`); + return false; + } + try { + process.kill(pid, signal); + return true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ESRCH") return false; + if (strict) throw error; + return false; + } +} + +async function stopOwnedGatewayPid(strict: boolean): Promise { + const pidFile = readGatewayPid(); + if (pidFile.kind === "absent") return; + if (pidFile.kind === "invalid") { + if (strict) + throw new Error("Refusing cleanup because the gateway PID file is invalid or unreadable"); + return; + } + const { pid } = pidFile; + const initial = inspectGatewayPid(pid); + if (initial.kind === "absent") return; + if (initial.kind === "unverified") { + if (strict) { + throw new Error( + `Refusing cleanup because PID-file process ${pid} does not prove ownership of gateway '${GATEWAY_NAME}'`, + ); + } + return; + } + + if (!signalOriginalGateway(pid, initial.startTime, "SIGTERM", strict)) return; + const afterTerm = await waitForOriginalGatewayExit(pid, initial.startTime, 10_000); + if (afterTerm.kind === "absent") return; + if (afterTerm.kind === "owned" && afterTerm.startTime !== initial.startTime) return; + if (afterTerm.kind === "unverified") { + if (strict) throw new Error(`Could not verify gateway process ${pid} after SIGTERM`); + return; + } + + if (!signalOriginalGateway(pid, initial.startTime, "SIGKILL", strict)) return; + const afterKill = await waitForOriginalGatewayExit(pid, initial.startTime, 5_000); + if (afterKill.kind === "absent") return; + if (afterKill.kind === "owned" && afterKill.startTime !== initial.startTime) return; + if (strict) throw new Error(`Owned gateway process ${pid} did not stop`); +} + +export async function cleanupOwnedGatewayRuntimeStrict( + host: HostCliClient, + artifactName: string, +): Promise { + await stopOwnedGatewayPid(true); + const result = await host.command( + "bash", + [ + "-lc", + [ + "set -uo pipefail", + 'cid="$(docker ps -qf "name=openshell-cluster-nemoclaw" | head -1)" || exit $?', + 'if [ -n "$cid" ]; then docker stop "$cid" >/dev/null; fi', + ].join("\n"), + ], + { + artifactName, + env: commandEnv(), + timeoutMs: 90_000, + }, + ); + assertExitZero(result, "cleanup messaging-compatible owned gateway runtime"); +} + export async function stopGatewayRuntime(host: HostCliClient, artifactName: string): Promise { - await bestEffort(() => + await preCleanBestEffort(() => host.command( "bash", [ "-lc", [ "set +e", - "openshell forward stop 18789 >/dev/null 2>&1", - "openshell gateway stop -g nemoclaw >/dev/null 2>&1", - 'pid_file="$HOME/.local/state/nemoclaw/openshell-docker-gateway/openshell-gateway.pid"', - 'if [ -f "$pid_file" ]; then', - ' pid="$(tr -d "[:space:]" <"$pid_file" 2>/dev/null || true)"', - ' if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then', - ' kill "$pid" 2>/dev/null || true', - " for _ in $(seq 1 10); do", - ' kill -0 "$pid" 2>/dev/null || break', - " sleep 1", - " done", - ' kill -0 "$pid" 2>/dev/null && kill -9 "$pid" 2>/dev/null || true', - " fi", - "fi", + 'openshell_bin="$1"', + '"$openshell_bin" forward stop 18789 >/dev/null 2>&1', + '"$openshell_bin" gateway stop -g nemoclaw >/dev/null 2>&1', 'cid="$(docker ps -qf "name=openshell-cluster-nemoclaw" 2>/dev/null | head -1)"', 'if [ -n "$cid" ]; then docker stop "$cid" >/dev/null 2>&1 || true; fi', - "openshell gateway remove nemoclaw >/dev/null 2>&1", - "openshell gateway destroy -g nemoclaw >/dev/null 2>&1", + '"$openshell_bin" gateway remove nemoclaw >/dev/null 2>&1', + '"$openshell_bin" gateway destroy -g nemoclaw >/dev/null 2>&1', "exit 0", ].join("\n"), + "gateway-runtime-preclean", + host.openshellCommandPath, ], { artifactName, @@ -63,6 +239,7 @@ export async function stopGatewayRuntime(host: HostCliClient, artifactName: stri }, ), ); + await stopOwnedGatewayPid(false); } export async function cleanupMessagingState( @@ -72,15 +249,15 @@ export async function cleanupMessagingState( // Endpoint-validation skips can happen before the sandbox exists. Keep // teardown non-throwing so "Sandbox ... does not exist" stays a normal // pre-contract cleanup outcome instead of masking the original evidence. - await bestEffort(() => + await preCleanBestEffort(() => host.command("node", [CLI_ENTRYPOINT, sandboxName, "destroy", "--yes"], { artifactName: `cleanup-nemoclaw-destroy-${sandboxName}`, env: commandEnv(), timeoutMs: 120_000, }), ); - await bestEffort(() => - host.command("openshell", ["sandbox", "delete", sandboxName], { + await preCleanBestEffort(() => + host.command(host.openshellCommandPath, ["sandbox", "delete", sandboxName], { artifactName: `cleanup-openshell-sandbox-delete-${sandboxName}`, env: commandEnv(), timeoutMs: 60_000, diff --git a/test/e2e/live/messaging-compatible-endpoint.test.ts b/test/e2e/live/messaging-compatible-endpoint.test.ts index 2046618dd91..16434df7d01 100644 --- a/test/e2e/live/messaging-compatible-endpoint.test.ts +++ b/test/e2e/live/messaging-compatible-endpoint.test.ts @@ -33,6 +33,7 @@ import { } from "../support/messaging-endpoint-classifiers.ts"; import { cleanupMessagingState, + cleanupOwnedGatewayRuntimeStrict, commandEnv, parseOpenClawAgentText, stopGatewayRuntime, @@ -566,13 +567,36 @@ test("messaging compatible endpoint routes Telegram-enabled OpenClaw through inf ], }); - cleanup.add(`destroy messaging compatible endpoint state ${SANDBOX_NAME}`, () => - cleanupMessagingState(host, SANDBOX_NAME), + const cleanupEnv = commandEnv(); + cleanup.trackDisposable("clean up messaging-compatible owned gateway runtime", () => + cleanupOwnedGatewayRuntimeStrict(host, "cleanup-owned-gateway-runtime-nemoclaw"), ); + cleanup.trackGateway(host, "nemoclaw", { + artifactName: "cleanup-openshell-gateway-runtime-nemoclaw", + env: cleanupEnv, + timeoutMs: 90_000, + }); + cleanup.trackForward(host, 18789, { + artifactName: "cleanup-openshell-forward-stop-18789", + env: cleanupEnv, + timeoutMs: 30_000, + }); + cleanup.trackDisposable(`delete OpenShell sandbox ${SANDBOX_NAME}`, () => + sandbox.cleanupSandbox(SANDBOX_NAME, { + artifactName: `cleanup-openshell-sandbox-delete-${SANDBOX_NAME}`, + env: cleanupEnv, + timeoutMs: 60_000, + }), + ); + cleanup.trackSandbox(host, SANDBOX_NAME, { + artifactName: `cleanup-nemoclaw-destroy-${SANDBOX_NAME}`, + env: cleanupEnv, + timeoutMs: 120_000, + }); await cleanupMessagingState(host, SANDBOX_NAME); const compatibleMock = await startCompatibleMock(MOCK_PORT, COMPAT_MODEL, COMPATIBLE_KEY); - cleanup.add("stop compatible endpoint mock", async () => { + cleanup.trackDisposable("stop compatible endpoint mock", async () => { await artifacts.writeJson("compatible-endpoint-mock-requests.json", compatibleMock.requests); await compatibleMock.close(); }); diff --git a/test/e2e/live/messaging-providers-helpers.ts b/test/e2e/live/messaging-providers-helpers.ts index 254b91c0c3c..2bf2b4691db 100644 --- a/test/e2e/live/messaging-providers-helpers.ts +++ b/test/e2e/live/messaging-providers-helpers.ts @@ -8,7 +8,11 @@ import path from "node:path"; import type { ArtifactSink } from "../fixtures/artifacts.ts"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; -import { assertExitZero as expectExitZero, shellQuote } from "../fixtures/clients/command.ts"; +import { + assertExitZero as expectExitZero, + resultText, + shellQuote, +} from "../fixtures/clients/command.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; import { type SandboxClient, @@ -299,7 +303,7 @@ export function messagingEnv(): MessagingEnv { return { env, tokens, telegramIds, telegramAllowlistKey, slackIds, wechatAccount }; } -export async function bestEffort(run: () => Promise): Promise { +export async function runSecondaryCleanup(run: () => Promise): Promise { try { await run(); } catch { @@ -612,15 +616,19 @@ export async function startFakeDockerApi( ); cleanup(`remove ${container}`, async () => { - await bestEffort(() => - runHost(host, "docker", ["rm", "-f", container], { + try { + const remove = await runHost(host, "docker", ["rm", "-f", container], { artifactName: `cleanup-${container}`, env: options.env, redactionValues: options.redactionValues, timeoutMs: 60_000, - }), - ); - fs.rmSync(dir, { recursive: true, force: true }); + }); + if (remove.exitCode !== 0 && !/No such container:/iu.test(resultText(remove))) { + expectExitZero(remove, `remove fake ${options.kind} API container ${container}`); + } + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } }); const start = await runHost(host, "docker", dockerArgs, { diff --git a/test/e2e/live/messaging-providers.test.ts b/test/e2e/live/messaging-providers.test.ts index 21534067704..6e13572c3ab 100644 --- a/test/e2e/live/messaging-providers.test.ts +++ b/test/e2e/live/messaging-providers.test.ts @@ -18,7 +18,6 @@ import { accountString, applyRestRewritePolicy, applyWebSocketRewritePolicy, - bestEffort, CLI_ENTRYPOINT, channelAccount, channelEnabled, @@ -42,6 +41,7 @@ import { runDiscordGatewayClient, runHost, runSandboxShell, + runSecondaryCleanup, runSlackApiRequest, SANDBOX_NAME, sandboxOutput, @@ -54,6 +54,8 @@ import { import { runInstalledSlackRuntimeProof } from "./messaging-providers-slack-runtime-proof.ts"; import { runInstalledTelegramRuntimeProof } from "./messaging-providers-telegram-runtime-proof.ts"; +process.env.NEMOCLAW_CLI_BIN ??= CLI_ENTRYPOINT; + test( "messaging providers preserve placeholder, policy, runtime, and send contracts", testTimeoutOptions(LIVE_TIMEOUT_MS), @@ -81,29 +83,25 @@ test( wechatAccount: state.wechatAccount, }); - cleanup.add(`destroy messaging providers sandbox ${SANDBOX_NAME}`, async () => { - await bestEffort(() => - runHost(host, "node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { - artifactName: "cleanup-nemoclaw-destroy-messaging-providers", - env: state.env, - redactionValues, - timeoutMs: 15 * 60_000, - }), - ); - await bestEffort(() => - runHost(host, "openshell", ["sandbox", "delete", SANDBOX_NAME], { - artifactName: "cleanup-openshell-sandbox-delete-messaging-providers", - env: state.env, - redactionValues, - timeoutMs: 120_000, - }), - ); + cleanup.trackDisposable(`delete OpenShell sandbox ${SANDBOX_NAME}`, () => + sandbox.cleanupSandbox(SANDBOX_NAME, { + artifactName: "cleanup-openshell-sandbox-delete-messaging-providers", + env: state.env, + redactionValues, + timeoutMs: 120_000, + }), + ); + cleanup.trackSandbox(host, SANDBOX_NAME, { + artifactName: "cleanup-nemoclaw-destroy-messaging-providers", + env: state.env, + redactionValues, + timeoutMs: 15 * 60_000, }); const restoreSlackPolicy = await premergeSlackPolicyIfNeeded(); cleanup.add("restore messaging E2E Slack policy pre-merge", restoreSlackPolicy); - await bestEffort(() => + await runSecondaryCleanup(() => runHost(host, "node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { artifactName: "preclean-nemoclaw-destroy-messaging-providers", env: state.env, @@ -111,7 +109,7 @@ test( timeoutMs: 15 * 60_000, }), ); - await bestEffort(() => + await runSecondaryCleanup(() => runHost(host, "openshell", ["sandbox", "delete", SANDBOX_NAME], { artifactName: "preclean-openshell-sandbox-delete-messaging-providers", env: state.env, @@ -119,7 +117,7 @@ test( timeoutMs: 120_000, }), ); - await bestEffort(() => + await runSecondaryCleanup(() => runHost(host, "openshell", ["gateway", "destroy", "-g", "nemoclaw"], { artifactName: "preclean-openshell-gateway-destroy-messaging-providers", env: state.env, diff --git a/test/e2e/live/model-router-provider-routed-inference.test.ts b/test/e2e/live/model-router-provider-routed-inference.test.ts index c4567a99aef..d605288257f 100644 --- a/test/e2e/live/model-router-provider-routed-inference.test.ts +++ b/test/e2e/live/model-router-provider-routed-inference.test.ts @@ -21,6 +21,8 @@ const ONBOARD_TIMEOUT_MS = 25 * 60_000; const HEALTH_ATTEMPTS = 20; const COMPLETION_ATTEMPTS = 3; +process.env.NEMOCLAW_CLI_BIN ??= CLI_ENTRYPOINT; + interface ModelRouterHealth { healthy_count?: unknown; } @@ -116,12 +118,11 @@ test("model-router provider-routed onboard returns routed inference.local PONG", timeoutMs: 120_000, }); - cleanup.add(`destroy sandbox ${SANDBOX_NAME}`, async () => { - await host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { - artifactName: "cleanup-nemoclaw-destroy-model-router-provider-routed", - env: buildAvailabilityProbeEnv(), - timeoutMs: 120_000, - }); + cleanup.trackSandbox(host, SANDBOX_NAME, { + artifactName: "cleanup-nemoclaw-destroy-model-router-provider-routed", + env: buildAvailabilityProbeEnv(), + redactionValues: [apiKey], + timeoutMs: 120_000, }); const onboard = await host.command( diff --git a/test/e2e/live/network-policy.test.ts b/test/e2e/live/network-policy.test.ts index 22e556ffbb6..19bb5cf18c9 100644 --- a/test/e2e/live/network-policy.test.ts +++ b/test/e2e/live/network-policy.test.ts @@ -57,6 +57,8 @@ const ENCODED_SLASH_DENIED_REASON = "request-target contains an encoded '/' (%2F) which is not allowed on this endpoint"; type NemoEnv = NodeJS.ProcessEnv; +process.env.NEMOCLAW_CLI_BIN ??= CLI_ENTRYPOINT; + function text(result: Pick): string { return [result.stdout, result.stderr].filter(Boolean).join("\n"); } @@ -550,17 +552,19 @@ test("network-policy: restricted sandbox enforces live allow/deny policy probes" expect(text(openshellVersion)).toContain("0.0.72"); const apiKey = secrets.required("NVIDIA_INFERENCE_API_KEY"); - cleanup.add(`destroy network-policy sandbox ${SANDBOX_NAME}`, async () => { - await runNemoclaw(host, [SANDBOX_NAME, "destroy", "--yes"], { - artifactName: "cleanup-nemoclaw-destroy-network-policy", - env: baseEnv(), - timeoutMs: 120_000, - }); - await sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { + cleanup.trackDisposable(`delete OpenShell sandbox ${SANDBOX_NAME}`, () => + sandbox.cleanupSandbox(SANDBOX_NAME, { artifactName: "cleanup-openshell-delete-network-policy", env: baseEnv(), + redactionValues: [apiKey], timeoutMs: 60_000, - }); + }), + ); + cleanup.trackSandbox(host, SANDBOX_NAME, { + artifactName: "cleanup-nemoclaw-destroy-network-policy", + env: baseEnv(), + redactionValues: [apiKey], + timeoutMs: 120_000, }); await runNemoclaw(host, [SANDBOX_NAME, "destroy", "--yes"], { @@ -1120,17 +1124,19 @@ test("network-policy: default restricted OpenClaw onboard leaves policy-list wit // credential through this historical env name. The real onboard below is // the authoritative credential validation boundary, regardless of prefix. - cleanup.add(`destroy restricted-zero-presets sandbox ${SUPPRESSION_SANDBOX_NAME}`, async () => { - await runNemoclaw(host, [SUPPRESSION_SANDBOX_NAME, "destroy", "--yes"], { - artifactName: "cleanup-nemoclaw-destroy-restricted-zero-presets", - env: baseEnv(), - timeoutMs: 120_000, - }); - await sandbox.openshell(["sandbox", "delete", SUPPRESSION_SANDBOX_NAME], { + cleanup.trackDisposable(`delete OpenShell sandbox ${SUPPRESSION_SANDBOX_NAME}`, () => + sandbox.cleanupSandbox(SUPPRESSION_SANDBOX_NAME, { artifactName: "cleanup-openshell-delete-restricted-zero-presets", env: baseEnv(), + redactionValues: [apiKey], timeoutMs: 60_000, - }); + }), + ); + cleanup.trackSandbox(host, SUPPRESSION_SANDBOX_NAME, { + artifactName: "cleanup-nemoclaw-destroy-restricted-zero-presets", + env: baseEnv(), + redactionValues: [apiKey], + timeoutMs: 120_000, }); await runNemoclaw(host, [SUPPRESSION_SANDBOX_NAME, "destroy", "--yes"], { diff --git a/test/e2e/live/ollama-auth-proxy.test.ts b/test/e2e/live/ollama-auth-proxy.test.ts index 1c6a10f4c91..4cc43263132 100644 --- a/test/e2e/live/ollama-auth-proxy.test.ts +++ b/test/e2e/live/ollama-auth-proxy.test.ts @@ -48,14 +48,6 @@ function token(): string { return randomBytes(24).toString("hex"); } -async function bestEffort(run: () => Promise | unknown): Promise { - try { - await run(); - } catch { - // Cleanup only. - } -} - async function terminate(child: ChildProcess | undefined): Promise { if (!child || child.killed || child.exitCode !== null) return; child.kill("SIGTERM"); @@ -186,10 +178,10 @@ test("Ollama auth proxy enforces tokens, proxies inference, persists tokens, and const tokenFile = path.join(tokenRoot, ".nemoclaw", "ollama-proxy-token"); let ollama: ChildProcess | undefined; let proxy: ChildProcess | undefined; - cleanup.add("stop Ollama auth proxy test processes", async () => { + cleanup.trackDisposable("stop Ollama auth proxy test processes", async () => { await terminate(proxy); await terminate(ollama); - await bestEffort(() => rm(tokenRoot, { force: true, recursive: true })); + await rm(tokenRoot, { force: true, recursive: true }); }); const nodeVersion = await host.command("node", ["--version"], { diff --git a/test/e2e/live/onboard-repair.test.ts b/test/e2e/live/onboard-repair.test.ts index 94b809934b4..23afff9030d 100644 --- a/test/e2e/live/onboard-repair.test.ts +++ b/test/e2e/live/onboard-repair.test.ts @@ -5,6 +5,10 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { + assertCleanupSucceededOrAbsent, + cleanupWhenOpenShellAvailable, +} from "../fixtures/cleanup-resources.ts"; import { resultText } from "../fixtures/clients/command.ts"; import { type HostCliClient } from "../fixtures/clients/host.ts"; import { type SandboxClient, validateSandboxName } from "../fixtures/clients/sandbox.ts"; @@ -130,7 +134,9 @@ test("onboard repair resumes missing sandbox and rejects conflicting resume inpu timeout: LIVE_TIMEOUT_MS, }, async ({ artifacts, cleanup: cleanupRegistry, host, sandbox, skip }) => { const corporateCa = createCorporateCaFixture("requests", "nemoclaw-repair-corporate-ca-"); - cleanupRegistry.add("remove corporate CA fixture", () => cleanupCorporateCaFixture(corporateCa)); + cleanupRegistry.trackDisposable("remove corporate CA fixture", () => + cleanupCorporateCaFixture(corporateCa), + ); await artifacts.target.declare({ id: "onboard-repair", sandboxName: SANDBOX_NAME, @@ -161,8 +167,106 @@ test("onboard repair resumes missing sandbox and rejects conflicting resume inpu host: "0.0.0.0", publicHost: "host.openshell.internal", }); - cleanupRegistry.add("close fake OpenAI-compatible endpoint", async () => fake.close()); - cleanupRegistry.add("remove repair sandboxes", () => cleanup(host, sandbox)); + cleanupRegistry.trackDisposable("close fake OpenAI-compatible endpoint", async () => + fake.close(), + ); + cleanupRegistry.trackDisposable("remove onboard-repair local state", () => { + updateExtraProviders((providers) => { + providers.delete(STALE_EXTRA_PROVIDER); + providers.delete(LIVE_EXTRA_PROVIDER); + }); + fs.rmSync(SESSION_FILE, { force: true }); + }); + const cleanupWhenInstalled = (artifactName: string, run: () => Promise): Promise => + cleanupWhenOpenShellAvailable( + host, + { + artifactName, + env: env(), + redactionValues: [EXTRA_PROVIDER_TOKEN], + timeoutMs: 30_000, + }, + run, + ); + const gatewayCleanupOptions = { + artifactName: "cleanup-gateway-destroy", + env: env(), + redactionValues: [EXTRA_PROVIDER_TOKEN], + timeoutMs: 60_000, + }; + cleanupRegistry.trackGateway( + { + cleanupGatewayRegistration: (name: string) => + cleanupWhenInstalled("cleanup-probe-openshell-gateway", () => + host.cleanupGatewayRegistration(name, gatewayCleanupOptions), + ), + }, + "nemoclaw", + gatewayCleanupOptions, + ); + cleanupRegistry.trackDisposable(`remove provider ${LIVE_EXTRA_PROVIDER}`, () => + cleanupWhenInstalled("cleanup-probe-openshell-provider", async () => { + const remove = await sandbox.openshell( + ["provider", "delete", "-g", "nemoclaw", LIVE_EXTRA_PROVIDER], + { + artifactName: "cleanup-live-extra-provider-delete", + env: env({ [EXTRA_PROVIDER_TOKEN_ENV]: EXTRA_PROVIDER_TOKEN }), + redactionValues: [EXTRA_PROVIDER_TOKEN], + timeoutMs: 60_000, + }, + ); + assertCleanupSucceededOrAbsent( + remove, + /\bNotFound\b|provider[^\n]*(?:not found|does not exist)|no such provider/i, + `cleanup provider ${LIVE_EXTRA_PROVIDER}`, + ); + }), + ); + const forwardCleanupOptions = { + artifactName: "cleanup-forward-stop-18789", + env: env(), + redactionValues: [EXTRA_PROVIDER_TOKEN], + timeoutMs: 30_000, + }; + cleanupRegistry.trackForward( + { + cleanupForward: (port: number) => + cleanupWhenInstalled("cleanup-probe-openshell-forward", () => + host.cleanupForward(port, forwardCleanupOptions), + ), + }, + 18789, + forwardCleanupOptions, + ); + const repairSandboxNames = [SANDBOX_NAME, OTHER_SANDBOX_NAME]; + for (const name of [...repairSandboxNames].reverse()) { + cleanupRegistry.trackDisposable(`delete OpenShell sandbox ${name}`, () => + cleanupWhenInstalled(`cleanup-probe-openshell-sandbox-${name}`, () => + sandbox.cleanupSandbox(name, { + artifactName: `cleanup-openshell-delete-${name}`, + env: env(), + redactionValues: [EXTRA_PROVIDER_TOKEN], + timeoutMs: 60_000, + }), + ), + ); + const sandboxCleanupOptions = { + artifactName: `cleanup-destroy-${name}`, + env: env(), + redactionValues: [EXTRA_PROVIDER_TOKEN], + timeoutMs: 20 * 60_000, + }; + cleanupRegistry.trackSandbox( + { + cleanupSandbox: (sandboxName: string) => + cleanupWhenInstalled(`cleanup-probe-openshell-nemoclaw-${sandboxName}`, () => + host.cleanupSandbox(sandboxName, sandboxCleanupOptions), + ), + }, + name, + sandboxCleanupOptions, + ); + } await cleanup(host, sandbox); const first = await nemoclaw( diff --git a/test/e2e/live/onboard-resume.test.ts b/test/e2e/live/onboard-resume.test.ts index 3a8da8892f0..632a4da6449 100644 --- a/test/e2e/live/onboard-resume.test.ts +++ b/test/e2e/live/onboard-resume.test.ts @@ -6,6 +6,7 @@ import os from "node:os"; import path from "node:path"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { assertCleanupSucceededOrAbsent } from "../fixtures/cleanup-resources.ts"; import { resultText } from "../fixtures/clients/command.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; import { trustedSandboxShellScript, validateSandboxName } from "../fixtures/clients/sandbox.ts"; @@ -16,8 +17,8 @@ import { } from "../fixtures/corporate-ca.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; import { - readExtraProviders, REGISTRY_FILE, + readExtraProviders, updateExtraProviders, } from "../fixtures/extra-providers-registry.ts"; import { @@ -52,6 +53,7 @@ const LIVE_EXTRA_PROVIDER = "e2e-resume-live-extra-provider"; const EXTRA_PROVIDER_TOKEN_ENV = "NEMOCLAW_E2E_EXTRA_PROVIDER_TOKEN"; const EXTRA_PROVIDER_TOKEN = "e2e-resume-extra-provider-token"; validateSandboxName(SANDBOX_NAME); +process.env.NEMOCLAW_CLI_BIN ??= CLI_ENTRYPOINT; // 15 minutes per onboard run; matches NEMOCLAW_E2E_DEFAULT_TIMEOUT in the // former shell test (`export NEMOCLAW_E2E_DEFAULT_TIMEOUT=600` is per-step; @@ -152,7 +154,9 @@ test("onboard-resume: interrupted onboard then --resume can recreate with cached sandbox, }) => { const corporateCa = createCorporateCaFixture("host-anchor", "nemoclaw-resume-corporate-ca-"); - cleanup.add("remove corporate CA fixture", () => cleanupCorporateCaFixture(corporateCa)); + cleanup.trackDisposable("remove corporate CA fixture", () => + cleanupCorporateCaFixture(corporateCa), + ); await artifacts.writeJson("corporate-ca-source.json", { mode: corporateCa.mode, source: corporateCa.sourceLabel, @@ -216,7 +220,7 @@ test("onboard-resume: interrupted onboard then --resume can recreate with cached requireAuth: true, requireAuthModels: true, }); - cleanup.add("close fake OpenAI-compatible endpoint", async () => { + cleanup.trackDisposable("close fake OpenAI-compatible endpoint", async () => { await artifacts.writeJson("fake-openai-compatible-requests.json", fake.requests()); await fake.close(); }); @@ -266,44 +270,16 @@ test("onboard-resume: interrupted onboard then --resume can recreate with cached }); fs.rmSync(SESSION_FILE, { force: true }); - // Register cleanup for the sandbox we are about to create. The cleanup - // fixture runs these in LIFO at end-of-test regardless of pass/fail. - cleanup.add(`destroy sandbox ${SANDBOX_NAME}`, async () => { - const cleanupEnv = buildAvailabilityProbeEnv(); - await host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { - artifactName: "cleanup-nemoclaw-destroy", - env: cleanupEnv, - timeoutMs: 120_000, - }); - await sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { - artifactName: "cleanup-openshell-sandbox-delete", - env: cleanupEnv, - timeoutMs: 60_000, - }); - await sandbox.openshell(["forward", "stop", "18789"], { - artifactName: "cleanup-openshell-forward-stop", - env: cleanupEnv, - timeoutMs: 30_000, - }); - await sandbox.openshell(["provider", "delete", "-g", "nemoclaw", LIVE_EXTRA_PROVIDER], { - artifactName: "cleanup-live-extra-provider-delete", - env: { ...cleanupEnv, [EXTRA_PROVIDER_TOKEN_ENV]: EXTRA_PROVIDER_TOKEN }, - timeoutMs: 60_000, - }); - await sandbox.openshell(["gateway", "destroy", "-g", "nemoclaw"], { - artifactName: "cleanup-openshell-gateway-destroy", - env: cleanupEnv, - timeoutMs: 60_000, - }); - fs.rmSync(SESSION_FILE, { force: true }); - updateExtraProviders((providers) => { - providers.delete(STALE_EXTRA_PROVIDER); - providers.delete(LIVE_EXTRA_PROVIDER); - }); - + // Register resources in reverse dependency order. CleanupRegistry runs them + // LIFO, so the sandbox is destroyed before its forward, provider, gateway, + // and local resume state are removed. + const cleanupEnv = buildAvailabilityProbeEnv(); + const cleanupRedactions = [FAKE_COMPATIBLE_AUTH_VALUE, EXTRA_PROVIDER_TOKEN]; + cleanup.trackDisposable("verify onboard-resume cleanup", async () => { const sandboxAfterCleanup = await sandbox.openshell(["sandbox", "get", SANDBOX_NAME], { artifactName: "cleanup-openshell-sandbox-get-after-delete", env: cleanupEnv, + redactionValues: cleanupRedactions, timeoutMs: 30_000, }); expect( @@ -312,6 +288,55 @@ test("onboard-resume: interrupted onboard then --resume can recreate with cached ).not.toBe(0); expect(fs.existsSync(SESSION_FILE), `${SESSION_FILE} still exists after cleanup`).toBe(false); }); + cleanup.trackDisposable("remove onboard-resume local state", () => { + fs.rmSync(SESSION_FILE, { force: true }); + updateExtraProviders((providers) => { + providers.delete(STALE_EXTRA_PROVIDER); + providers.delete(LIVE_EXTRA_PROVIDER); + }); + }); + cleanup.trackGateway(host, "nemoclaw", { + artifactName: "cleanup-openshell-gateway-destroy", + env: cleanupEnv, + redactionValues: cleanupRedactions, + timeoutMs: 60_000, + }); + cleanup.trackDisposable(`remove provider ${LIVE_EXTRA_PROVIDER}`, async () => { + const remove = await sandbox.openshell( + ["provider", "delete", "-g", "nemoclaw", LIVE_EXTRA_PROVIDER], + { + artifactName: "cleanup-live-extra-provider-delete", + env: { ...cleanupEnv, [EXTRA_PROVIDER_TOKEN_ENV]: EXTRA_PROVIDER_TOKEN }, + redactionValues: cleanupRedactions, + timeoutMs: 60_000, + }, + ); + assertCleanupSucceededOrAbsent( + remove, + /\bNotFound\b|provider[^\n]*(?:not found|does not exist)|no such provider/i, + `cleanup provider ${LIVE_EXTRA_PROVIDER}`, + ); + }); + cleanup.trackForward(host, 18789, { + artifactName: "cleanup-openshell-forward-stop", + env: cleanupEnv, + redactionValues: cleanupRedactions, + timeoutMs: 30_000, + }); + cleanup.trackDisposable(`delete OpenShell sandbox ${SANDBOX_NAME}`, () => + sandbox.cleanupSandbox(SANDBOX_NAME, { + artifactName: "cleanup-openshell-sandbox-delete", + env: cleanupEnv, + redactionValues: cleanupRedactions, + timeoutMs: 60_000, + }), + ); + cleanup.trackSandbox(host, SANDBOX_NAME, { + artifactName: "cleanup-nemoclaw-destroy", + env: cleanupEnv, + redactionValues: cleanupRedactions, + timeoutMs: 120_000, + }); // ────────────────────────────────────────────────────────────────── // Phase 2: first onboard (forced failure at the policies step) diff --git a/test/e2e/live/openclaw-discord-pairing.test.ts b/test/e2e/live/openclaw-discord-pairing.test.ts index a1d302f65ad..01a13ae1f5e 100644 --- a/test/e2e/live/openclaw-discord-pairing.test.ts +++ b/test/e2e/live/openclaw-discord-pairing.test.ts @@ -26,6 +26,7 @@ import { resultText, sandboxSh, shellQuote, + trackSandboxCleanup, } from "./phase6-messaging-helpers.ts"; const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-openclaw-discord-pairing"; @@ -53,8 +54,20 @@ test("OpenClaw Discord pairing request is shared with connect-shell approval", { dmChannel: DISCORD_DM_CHANNEL, }); - cleanup.add(`destroy Discord pairing sandbox ${SANDBOX_NAME}`, () => - cleanupPairingSandbox(host, SANDBOX_NAME, env, redactions, "cleanup-discord-pairing"), + cleanup.trackGateway(host, "nemoclaw", { + artifactName: "cleanup-discord-pairing-openshell-gateway-destroy", + env, + redactionValues: redactions, + timeoutMs: 120_000, + }); + trackSandboxCleanup( + cleanup, + host, + sandbox, + SANDBOX_NAME, + env, + redactions, + "cleanup-discord-pairing", ); await cleanupPairingSandbox(host, SANDBOX_NAME, env, redactions, "preclean-discord-pairing"); diff --git a/test/e2e/live/openclaw-inference-switch.test.ts b/test/e2e/live/openclaw-inference-switch.test.ts index 5d18cdebae8..7b0ffb8ef10 100644 --- a/test/e2e/live/openclaw-inference-switch.test.ts +++ b/test/e2e/live/openclaw-inference-switch.test.ts @@ -181,7 +181,7 @@ function commandEnv(home: string, extra: NodeJS.ProcessEnv = {}): NodeJS.Process }; } -async function bestEffort(run: () => Promise): Promise { +async function bestEffortStateReset(run: () => Promise): Promise { try { await run(); } catch { @@ -229,26 +229,26 @@ async function sandboxShell( ); } -async function cleanupOpenClawInferenceSwitchState( +async function resetOpenClawInferenceSwitchState( host: HostCliClient, sandbox: SandboxClient, home: string, artifactPrefix: string, ): Promise { - await bestEffort(() => + await bestEffortStateReset(() => runNemoclaw(host, home, [SANDBOX_NAME, "destroy", "--yes"], { artifactName: `${artifactPrefix}-nemoclaw-destroy-openclaw-inference-switch`, timeoutMs: 120_000, }), ); - await bestEffort(() => + await bestEffortStateReset(() => sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { artifactName: `${artifactPrefix}-openshell-sandbox-delete-openclaw-inference-switch`, env: commandEnv(home), timeoutMs: 60_000, }), ); - await bestEffort(() => + await bestEffortStateReset(() => sandbox.openshell(["gateway", "destroy", "-g", "nemoclaw"], { artifactName: `${artifactPrefix}-openshell-gateway-destroy-openclaw-inference-switch`, env: commandEnv(home), @@ -955,14 +955,34 @@ test("openclaw-inference-switch: switches route and preserves live OpenClaw beha const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-openclaw-switch-home-")); let mockProvider: MockAnthropicProvider | undefined; - cleanup.add(`destroy OpenClaw inference switch sandbox ${SANDBOX_NAME}`, async () => { - await cleanupOpenClawInferenceSwitchState(host, sandbox, home, "cleanup"); - await baselineProvider?.close(); - if (mockProvider) await mockProvider.close(); + cleanup.trackDisposable(`remove OpenClaw inference switch test home for ${SANDBOX_NAME}`, () => { fs.rmSync(home, { recursive: true, force: true }); }); + cleanup.trackDisposable("close switched Anthropic provider", async () => { + await mockProvider?.close(); + }); + cleanup.trackDisposable("close baseline inference provider", async () => { + await baselineProvider?.close(); + }); + cleanup.trackGateway(host, "nemoclaw", { + artifactName: "cleanup-openshell-gateway-destroy-openclaw-inference-switch", + env: commandEnv(home), + timeoutMs: 120_000, + }); + cleanup.trackDisposable(`delete OpenShell sandbox ${SANDBOX_NAME}`, () => + sandbox.cleanupSandbox(SANDBOX_NAME, { + artifactName: "cleanup-openshell-sandbox-delete-openclaw-inference-switch", + env: commandEnv(home), + timeoutMs: 60_000, + }), + ); + cleanup.trackSandbox(host, SANDBOX_NAME, { + artifactName: "cleanup-nemoclaw-destroy-openclaw-inference-switch", + env: commandEnv(home), + timeoutMs: 120_000, + }); - await cleanupOpenClawInferenceSwitchState(host, sandbox, home, "pre-cleanup"); + await resetOpenClawInferenceSwitchState(host, sandbox, home, "pre-cleanup"); const install = await host.command( "bash", @@ -1068,7 +1088,7 @@ test("openclaw-inference-switch: switches route and preserves live OpenClaw beha } if (process.env.NEMOCLAW_E2E_KEEP_SANDBOX !== "1") { - await cleanupOpenClawInferenceSwitchState(host, sandbox, home, "final"); + await resetOpenClawInferenceSwitchState(host, sandbox, home, "final"); const registryPath = path.join(home, ".nemoclaw", "sandboxes.json"); const registryText = fs.existsSync(registryPath) ? fs.readFileSync(registryPath, "utf8") : ""; expect(registryText).not.toContain(`"${SANDBOX_NAME}"`); diff --git a/test/e2e/live/openclaw-pairing-helpers.ts b/test/e2e/live/openclaw-pairing-helpers.ts index d577513e1c1..b7a46d4571f 100644 --- a/test/e2e/live/openclaw-pairing-helpers.ts +++ b/test/e2e/live/openclaw-pairing-helpers.ts @@ -11,11 +11,11 @@ import { expect } from "../fixtures/e2e-test.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; import { type FakeDockerApi, startFakeDockerApi } from "./messaging-providers-helpers.ts"; import { - bestEffort, cleanupSandbox, expectExitZero, phase6Env, resultText, + runSecondaryCleanup, sandboxEncodedSh, sandboxNode, sandboxSh, @@ -93,7 +93,7 @@ export async function cleanupPairingSandbox( prefix: string, ): Promise { await cleanupSandbox(host, sandboxName, env, redactions, prefix); - await bestEffort(() => + await runSecondaryCleanup(() => host.command("openshell", ["gateway", "destroy", "-g", "nemoclaw"], { artifactName: `${prefix}-openshell-gateway-destroy`, env, diff --git a/test/e2e/live/openclaw-plugin-runtime-exdev.test.ts b/test/e2e/live/openclaw-plugin-runtime-exdev.test.ts index 049fc00c7b3..56a2562d14f 100644 --- a/test/e2e/live/openclaw-plugin-runtime-exdev.test.ts +++ b/test/e2e/live/openclaw-plugin-runtime-exdev.test.ts @@ -94,6 +94,7 @@ const STOCK_OPENCLAW_POLICY_PATHS = [ path.join(REPO_ROOT, "nemoclaw-blueprint", "policies", "openclaw-sandbox-permissive.yaml"), ] as const; validateSandboxName(SANDBOX_NAME); +process.env.NEMOCLAW_CLI_BIN ??= CLI_ENTRYPOINT; const EXDEV_PATTERNS = [ /EXDEV: cross-device link not permitted/i, @@ -936,22 +937,17 @@ test("a custom OpenClaw plugin survives restart, recreation, and rebuild without "bin/nemoclaw.js missing — run npm run build:cli before this live target", ).toBe(true); - cleanup.add(`destroy sandbox ${SANDBOX_NAME}`, async () => { - const cleanupEnv = liveEnv(); - await ignoreCleanupError(() => - host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { - artifactName: "cleanup-nemoclaw-destroy-openclaw-plugin-exdev", - env: cleanupEnv, - timeoutMs: 120_000, - }), - ); - await ignoreCleanupError(() => - sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { - artifactName: "cleanup-openshell-delete-openclaw-plugin-exdev", - env: cleanupEnv, - timeoutMs: 60_000, - }), - ); + cleanup.trackDisposable(`delete OpenShell sandbox ${SANDBOX_NAME}`, () => + sandbox.cleanupSandbox(SANDBOX_NAME, { + artifactName: "cleanup-openshell-delete-openclaw-plugin-exdev", + env: liveEnv(), + timeoutMs: 60_000, + }), + ); + cleanup.trackSandbox(host, SANDBOX_NAME, { + artifactName: "cleanup-nemoclaw-destroy-openclaw-plugin-exdev", + env: liveEnv(), + timeoutMs: 120_000, }); await ignoreCleanupError(() => diff --git a/test/e2e/live/openclaw-skill-cli.test.ts b/test/e2e/live/openclaw-skill-cli.test.ts index b18a27b70b8..59534b16c11 100644 --- a/test/e2e/live/openclaw-skill-cli.test.ts +++ b/test/e2e/live/openclaw-skill-cli.test.ts @@ -48,7 +48,7 @@ function testEnv(home: string, extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv return testHomeEnvironment(home, extra); } -async function bestEffort(run: () => Promise): Promise { +async function bestEffortPreclean(run: () => Promise): Promise { try { await run(); } catch { @@ -57,27 +57,27 @@ async function bestEffort(run: () => Promise): Promise { } } -async function cleanupOpenClawSkillCliState( +async function precleanOpenClawSkillCliState( host: HostCliClient, sandbox: SandboxClient, home: string, ): Promise { const env = testEnv(home); - await bestEffort(() => + await bestEffortPreclean(() => host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { artifactName: "cleanup-nemoclaw-destroy-openclaw-skill-cli", env, timeoutMs: 120_000, }), ); - await bestEffort(() => + await bestEffortPreclean(() => sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { artifactName: "cleanup-openshell-sandbox-delete-openclaw-skill-cli", env, timeoutMs: 60_000, }), ); - await bestEffort(() => + await bestEffortPreclean(() => sandbox.openshell(["gateway", "destroy", "-g", "nemoclaw"], { artifactName: "cleanup-openshell-gateway-destroy-openclaw-skill-cli", env, @@ -164,11 +164,27 @@ test("openclaw-skill-cli: direct OpenClaw skills install/list/info/check roundtr const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-openclaw-skill-cli-home-")); const env = testEnv(home); - cleanup.add(`remove openclaw-skill-cli state for ${SANDBOX_NAME}`, async () => { - await cleanupOpenClawSkillCliState(host, sandbox, home); + cleanup.trackDisposable(`remove openclaw-skill-cli test home for ${SANDBOX_NAME}`, () => { fs.rmSync(home, { recursive: true, force: true }); }); - await cleanupOpenClawSkillCliState(host, sandbox, home); + cleanup.trackGateway(host, "nemoclaw", { + artifactName: "cleanup-openshell-gateway-destroy-openclaw-skill-cli", + env, + timeoutMs: 120_000, + }); + cleanup.trackDisposable(`delete OpenShell sandbox ${SANDBOX_NAME}`, () => + sandbox.cleanupSandbox(SANDBOX_NAME, { + artifactName: "cleanup-openshell-sandbox-delete-openclaw-skill-cli", + env, + timeoutMs: 60_000, + }), + ); + cleanup.trackSandbox(host, SANDBOX_NAME, { + artifactName: "cleanup-nemoclaw-destroy-openclaw-skill-cli", + env, + timeoutMs: 120_000, + }); + await precleanOpenClawSkillCliState(host, sandbox, home); const install = await host.command( "bash", diff --git a/test/e2e/live/openclaw-slack-pairing.test.ts b/test/e2e/live/openclaw-slack-pairing.test.ts index 748f431c7d4..9cad877d5d9 100644 --- a/test/e2e/live/openclaw-slack-pairing.test.ts +++ b/test/e2e/live/openclaw-slack-pairing.test.ts @@ -24,6 +24,7 @@ import { expectSandboxReady, installSandboxOrSkipOnRateLimit, resultText, + trackSandboxCleanup, } from "./phase6-messaging-helpers.ts"; const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-openclaw-slack-pairing"; @@ -91,8 +92,20 @@ test("OpenClaw Slack Socket Mode pairing request is shared with connect-shell ap pairingUser: PAIRING_USER.slack, }); - cleanup.add(`destroy Slack pairing sandbox ${SANDBOX_NAME}`, () => - cleanupPairingSandbox(host, SANDBOX_NAME, env, redactions, "cleanup-slack-pairing"), + cleanup.trackGateway(host, "nemoclaw", { + artifactName: "cleanup-slack-pairing-openshell-gateway-destroy", + env, + redactionValues: redactions, + timeoutMs: 120_000, + }); + trackSandboxCleanup( + cleanup, + host, + sandbox, + SANDBOX_NAME, + env, + redactions, + "cleanup-slack-pairing", ); await cleanupPairingSandbox(host, SANDBOX_NAME, env, redactions, "preclean-slack-pairing"); diff --git a/test/e2e/live/openshell-gateway-upgrade-helpers.ts b/test/e2e/live/openshell-gateway-upgrade-helpers.ts index 54cee2ef568..54b50102c36 100644 --- a/test/e2e/live/openshell-gateway-upgrade-helpers.ts +++ b/test/e2e/live/openshell-gateway-upgrade-helpers.ts @@ -54,13 +54,8 @@ export function currentGatewayUpgradeInstallerArgs(installer: string): string[] return [installer, ...COMMON_INSTALLER_ARGS]; } -export function upgradeGatewayCleanupScript(pidFile: string): string { - return `if command -v openshell >/dev/null 2>&1; then - openshell gateway remove nemoclaw >/dev/null 2>&1 \\ - || openshell gateway destroy -g nemoclaw >/dev/null 2>&1 \\ - || openshell gateway destroy >/dev/null 2>&1 \\ - || true -fi +export function upgradeGatewayStateCleanupScript(pidFile: string): string { + return `set -e volume_prefix=${GATEWAY_VOLUME_PREFIX} gateway_volumes="$(docker volume ls -q --filter "name=\${volume_prefix}")" while IFS= read -r volume; do @@ -74,3 +69,13 @@ while IFS= read -r volume; do done <<<"$gateway_volumes" rm -f ${shellQuote(pidFile)}`; } + +export function upgradeGatewayCleanupScript(pidFile: string): string { + return `if command -v openshell >/dev/null 2>&1; then + openshell gateway remove nemoclaw >/dev/null 2>&1 \\ + || openshell gateway destroy -g nemoclaw >/dev/null 2>&1 \\ + || openshell gateway destroy >/dev/null 2>&1 \\ + || true +fi +${upgradeGatewayStateCleanupScript(pidFile)}`; +} diff --git a/test/e2e/live/openshell-gateway-upgrade.test.ts b/test/e2e/live/openshell-gateway-upgrade.test.ts index 9d041899a52..accdc7e8032 100644 --- a/test/e2e/live/openshell-gateway-upgrade.test.ts +++ b/test/e2e/live/openshell-gateway-upgrade.test.ts @@ -36,6 +36,7 @@ import { currentGatewayUpgradeInstallerArgs, oldGatewayUpgradeInstallerArgs, upgradeGatewayCleanupScript, + upgradeGatewayStateCleanupScript, validateLegacyGatewayUpgradeFixture, } from "./openshell-gateway-upgrade-helpers.ts"; @@ -388,8 +389,12 @@ bash ${quotedInstallerArgs} >${shellQuote(logFile)} 2>&1`, return result; } -async function removeUpgradeGateway(host: HostCliClient, artifactName: string): Promise { - await bash(host, upgradeGatewayCleanupScript(PID_FILE), { artifactName, timeoutMs: 120_000 }); +async function preCleanUpgradeGateway(host: HostCliClient, artifactName: string): Promise { + const result = await bash(host, upgradeGatewayCleanupScript(PID_FILE), { + artifactName, + timeoutMs: 120_000, + }); + expectExitZero(result, "pre-clean OpenShell gateway upgrade state"); } async function installOldNemoclawAndClaw( @@ -716,7 +721,7 @@ const runLinuxOpenShellGatewayUpgrade = test.skipIf(process.platform !== "linux" runLinuxOpenShellGatewayUpgrade( "openshell-gateway-upgrade: upgrades old working OpenClaw claw and restores survivor state", { timeout: TEST_TIMEOUT_MS }, - async ({ artifacts, cleanup, host }) => { + async ({ artifacts, cleanup, host, sandbox }) => { await artifacts.writeJson("live-upgrade-target.json", { id: "openshell-gateway-upgrade", runner: "vitest", @@ -738,21 +743,30 @@ runLinuxOpenShellGatewayUpgrade( survivorSandbox: SURVIVOR_SANDBOX, }); - cleanup.add("remove openshell gateway upgrade gateway", async () => { - await removeUpgradeGateway(host, "cleanup-gateway"); + cleanup.trackDisposable("remove openshell gateway upgrade state", async () => { + const result = await bash(host, upgradeGatewayStateCleanupScript(PID_FILE), { + artifactName: "cleanup-gateway-state", + timeoutMs: 120_000, + }); + expectExitZero(result, "cleanup OpenShell gateway upgrade state"); }); - cleanup.add("remove openshell gateway upgrade survivor sandbox", async () => { - await bash( - host, - `command -v openshell >/dev/null 2>&1 && openshell sandbox delete ${shellQuote(SURVIVOR_SANDBOX)} >/dev/null 2>&1 || true`, - { artifactName: "cleanup-survivor-sandbox", timeoutMs: 120_000 }, - ); + cleanup.trackGateway(host, "nemoclaw", { + artifactName: "cleanup-gateway", + env: liveEnv(), + timeoutMs: 120_000, }); + cleanup.trackDisposable("remove openshell gateway upgrade survivor sandbox", () => + sandbox.cleanupSandbox(SURVIVOR_SANDBOX, { + artifactName: "cleanup-survivor-sandbox", + env: liveEnv(), + timeoutMs: 120_000, + }), + ); // Vitest retries execute in the same runner process. Tear down any failed // legacy gateway before each attempt so partial containerd layers from a // transient image-import failure cannot consume the next attempt's disk. - await removeUpgradeGateway(host, "pre-cleanup-gateway"); + await preCleanUpgradeGateway(host, "pre-cleanup-gateway"); const fake = await startFakeOpenAiCompatibleServer({ apiKey: "dummy", diff --git a/test/e2e/live/overlayfs-autofix-cleanup.ts b/test/e2e/live/overlayfs-autofix-cleanup.ts new file mode 100644 index 00000000000..e86fa95e28d --- /dev/null +++ b/test/e2e/live/overlayfs-autofix-cleanup.ts @@ -0,0 +1,112 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import type { CleanupRegistry } from "../fixtures/cleanup.ts"; +import { assertCleanupSucceededOrAbsent } from "../fixtures/cleanup-resources.ts"; +import { assertExitZero } from "../fixtures/clients/command.ts"; +import type { HostCliClient } from "../fixtures/clients/host.ts"; +import type { SandboxClient } from "../fixtures/clients/sandbox.ts"; + +interface OverlayfsAutofixCleanupOptions { + cleanup: CleanupRegistry; + cleanupEnv: NodeJS.ProcessEnv; + gatewayContainer: string; + host: Pick; + preserveSandbox: boolean; + redactionValues: string[]; + sandbox: Pick; + sandboxName: string; +} + +async function removeOverlayGatewayContainer({ + cleanupEnv, + gatewayContainer, + host, + redactionValues, +}: Pick< + OverlayfsAutofixCleanupOptions, + "cleanupEnv" | "gatewayContainer" | "host" | "redactionValues" +>): Promise { + const removeContainer = await host.command("docker", ["rm", "-f", gatewayContainer], { + artifactName: "cleanup-overlayfs-gateway-container", + env: cleanupEnv, + redactionValues, + timeoutMs: 5 * 60_000, + }); + assertCleanupSucceededOrAbsent( + removeContainer, + /No such container/i, + "remove overlayfs gateway container", + ); +} + +async function removeOverlayPatchedImages({ + cleanupEnv, + host, + redactionValues, +}: Pick): Promise { + const imageList = await host.command( + "docker", + ["image", "ls", "--format", "{{.Repository}}:{{.Tag}}"], + { + artifactName: "cleanup-overlayfs-patched-image-list", + env: cleanupEnv, + redactionValues, + timeoutMs: 5 * 60_000, + }, + ); + assertExitZero(imageList, "list overlayfs patched images during cleanup"); + const patchedImages = imageList.stdout + .split(/\r?\n/) + .map((image) => image.trim()) + .filter((image) => image.startsWith("nemoclaw-cluster:")); + if (patchedImages.length > 0) { + const removeImages = await host.command("docker", ["rmi", "-f", ...patchedImages], { + artifactName: "cleanup-overlayfs-patched-images", + env: cleanupEnv, + redactionValues, + timeoutMs: 5 * 60_000, + }); + assertExitZero(removeImages, "remove overlayfs patched images"); + } +} + +export function trackOverlayfsAutofixCleanup(options: OverlayfsAutofixCleanupOptions): void { + if (options.preserveSandbox) return; + + const { cleanup, cleanupEnv, gatewayContainer, host, redactionValues, sandbox, sandboxName } = + options; + cleanup.trackDisposable("remove overlayfs onboard lock", () => { + fs.rmSync(path.join(os.homedir(), ".nemoclaw", "onboard.lock"), { force: true }); + }); + cleanup.trackDisposable("remove overlayfs patched images", () => + removeOverlayPatchedImages({ cleanupEnv, host, redactionValues }), + ); + cleanup.trackDisposable("remove overlayfs gateway container", () => + removeOverlayGatewayContainer({ cleanupEnv, gatewayContainer, host, redactionValues }), + ); + cleanup.trackGateway(host, "nemoclaw", { + artifactName: "cleanup-overlayfs-openshell-gateway", + env: cleanupEnv, + redactionValues, + timeoutMs: 5 * 60_000, + }); + cleanup.trackDisposable(`delete overlayfs OpenShell sandbox ${sandboxName}`, () => + sandbox.cleanupSandbox(sandboxName, { + artifactName: "cleanup-overlayfs-openshell-sandbox-delete", + env: cleanupEnv, + redactionValues, + timeoutMs: 5 * 60_000, + }), + ); + cleanup.trackSandbox(host, sandboxName, { + artifactName: "cleanup-overlayfs-sandbox", + env: cleanupEnv, + redactionValues, + timeoutMs: 5 * 60_000, + }); +} diff --git a/test/e2e/live/overlayfs-autofix.test.ts b/test/e2e/live/overlayfs-autofix.test.ts index f595cd77308..89f450e8a2f 100644 --- a/test/e2e/live/overlayfs-autofix.test.ts +++ b/test/e2e/live/overlayfs-autofix.test.ts @@ -11,6 +11,7 @@ import type { HostCliClient } from "../fixtures/clients/host.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; import { requireHostedInferenceConfig } from "../fixtures/hosted-inference.ts"; import type { ShellProbeResult, ShellProbeRunOptions } from "../fixtures/shell-probe.ts"; +import { trackOverlayfsAutofixCleanup } from "./overlayfs-autofix-cleanup.ts"; import { negativeOverlayOutcome } from "./overlayfs-autofix-outcome.ts"; // Keep this direct: the contract mutates the host Docker daemon into @@ -69,14 +70,6 @@ async function bash( return host.command("bash", ["-lc", script], options); } -async function bestEffort(run: () => Promise): Promise { - try { - await run(); - } catch { - // Cleanup stays best-effort so the primary setup/assertion failure is visible. - } -} - async function preCleanup( host: HostCliClient, apiKey: string, @@ -143,7 +136,7 @@ async function waitForDocker(host: HostCliClient): Promise { test.skipIf(overlayfsAutofixNotInRuntimePath())( "overlayfs-autofix: patched cluster image handles Docker containerd overlayfs", - async ({ artifacts, cleanup, host, secrets, skip }) => { + async ({ artifacts, cleanup, host, sandbox, secrets, skip }) => { assertTestOwnedSandboxName(); await artifacts.writeJson("contract.json", { @@ -239,9 +232,15 @@ test.skipIf(overlayfsAutofixNotInRuntimePath())( expect(await waitForDocker(host), "Docker must come back after daemon restore").toBe(true); fs.rmSync(stateDir, { recursive: true, force: true }); }); - cleanup.add(`destroy overlayfs-autofix sandbox ${SANDBOX_NAME}`, async () => { - process.env.NEMOCLAW_E2E_KEEP_SANDBOX !== "1" && - (await bestEffort(() => preCleanup(host, apiKey, "cleanup-overlayfs-sandbox"))); + trackOverlayfsAutofixCleanup({ + cleanup, + cleanupEnv: overlayEnv(apiKey, { GATEWAY_CONTAINER }), + gatewayContainer: GATEWAY_CONTAINER, + host, + preserveSandbox: process.env.NEMOCLAW_E2E_KEEP_SANDBOX === "1", + redactionValues, + sandbox, + sandboxName: SANDBOX_NAME, }); const backup = await bash( diff --git a/test/e2e/live/phase6-messaging-helpers.ts b/test/e2e/live/phase6-messaging-helpers.ts index 96bf6b56887..3358af802e0 100644 --- a/test/e2e/live/phase6-messaging-helpers.ts +++ b/test/e2e/live/phase6-messaging-helpers.ts @@ -2,6 +2,11 @@ // SPDX-License-Identifier: Apache-2.0 import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import type { CleanupRegistry } from "../fixtures/cleanup.ts"; +import { + cleanupWhenCommandAvailable, + cleanupWhenOpenShellAvailable, +} from "../fixtures/cleanup-resources.ts"; import { assertExitZero as expectExitZero, resultText, @@ -64,7 +69,7 @@ export function redactionValues(apiKey: string | undefined): string[] { return [apiKey].filter((value): value is string => typeof value === "string" && value.length > 0); } -export async function bestEffort(run: () => Promise): Promise { +export async function runSecondaryCleanup(run: () => Promise): Promise { try { await run(); } catch { @@ -79,7 +84,7 @@ export async function precleanSandbox( redactions: string[], prefix: string, ): Promise { - await bestEffort(() => + await runSecondaryCleanup(() => host.command("node", [CLI, sandboxName, "destroy", "--yes"], { artifactName: `${prefix}-nemoclaw-destroy`, env, @@ -87,8 +92,8 @@ export async function precleanSandbox( timeoutMs: 15 * 60_000, }), ); - await bestEffort(() => - host.command("openshell", ["sandbox", "delete", sandboxName], { + await runSecondaryCleanup(() => + host.command(host.openshellCommandPath, ["sandbox", "delete", sandboxName], { artifactName: `${prefix}-openshell-sandbox-delete`, env, redactionValues: redactions, @@ -107,6 +112,94 @@ export async function cleanupSandbox( await precleanSandbox(host, sandboxName, env, redactions, prefix); } +export function trackSandboxCleanup( + cleanup: CleanupRegistry, + host: HostCliClient, + sandbox: SandboxClient, + sandboxName: string, + env: NodeJS.ProcessEnv, + redactions: string[], + prefix: string, +): void { + cleanup.trackDisposable(`delete OpenShell sandbox ${sandboxName}`, () => + sandbox.cleanupSandbox(sandboxName, { + artifactName: `${prefix}-openshell-sandbox-delete`, + env, + redactionValues: redactions, + timeoutMs: 120_000, + }), + ); + cleanup.trackSandbox(host, sandboxName, { + artifactName: `${prefix}-nemoclaw-destroy`, + env, + redactionValues: redactions, + timeoutMs: 15 * 60_000, + }); +} + +export function trackPreinstallSandboxCleanup( + cleanup: CleanupRegistry, + host: HostCliClient, + sandbox: SandboxClient, + sandboxName: string, + env: NodeJS.ProcessEnv, + redactions: string[], + prefix: string, +): void { + const openshellOptions = { + artifactName: `${prefix}-openshell-sandbox-delete`, + env, + redactionValues: redactions, + timeoutMs: 120_000, + }; + cleanup.trackDisposable(`delete OpenShell sandbox ${sandboxName}`, () => + cleanupWhenOpenShellAvailable( + host, + { + artifactName: `${prefix}-probe-openshell-sandbox-delete`, + env, + redactionValues: redactions, + timeoutMs: 30_000, + }, + () => sandbox.cleanupSandbox(sandboxName, openshellOptions), + ), + ); + const nemoclawOptions = { + artifactName: `${prefix}-nemoclaw-destroy`, + env, + redactionValues: redactions, + timeoutMs: 15 * 60_000, + }; + cleanup.trackSandbox( + { + cleanupSandbox: (name: string) => + cleanupWhenCommandAvailable( + host, + host.commandPath, + { + artifactName: `${prefix}-probe-nemoclaw-destroy`, + env, + redactionValues: redactions, + timeoutMs: 30_000, + }, + () => + cleanupWhenOpenShellAvailable( + host, + { + artifactName: `${prefix}-probe-openshell-nemoclaw-destroy`, + env, + redactionValues: redactions, + timeoutMs: 30_000, + }, + () => host.cleanupSandbox(name, nemoclawOptions), + ), + ), + }, + sandboxName, + nemoclawOptions, + ); +} + export async function installSandbox( host: HostCliClient, env: NodeJS.ProcessEnv, @@ -151,7 +244,7 @@ export async function expectSandboxReady( redactions: string[], artifactName: string, ): Promise { - const list = await host.command("openshell", ["sandbox", "list"], { + const list = await host.command(host.openshellCommandPath, ["sandbox", "list"], { artifactName, env, redactionValues: redactions, diff --git a/test/e2e/live/rebuild-hermes.test.ts b/test/e2e/live/rebuild-hermes.test.ts index f56929f2157..df5fccf886d 100644 --- a/test/e2e/live/rebuild-hermes.test.ts +++ b/test/e2e/live/rebuild-hermes.test.ts @@ -8,6 +8,7 @@ import path from "node:path"; import { setTimeout as sleep } from "node:timers/promises"; import { shellQuote } from "../../../src/lib/core/shell-quote"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { assertCleanupSucceededOrAbsent } from "../fixtures/cleanup-resources.ts"; import { assertExitZero as expectExitZero } from "../fixtures/clients/command.ts"; import { type HostCliClient, resultText } from "../fixtures/clients/index.ts"; import { validateSandboxName } from "../fixtures/clients/sandbox.ts"; @@ -144,7 +145,7 @@ function expectEqual(actual: string | undefined, expected: string, message: stri } } -async function cleanupHermesResources( +async function bestEffortPrecleanHermesResources( host: HostCliClient, apiKey: string | undefined, artifactName: string, @@ -175,6 +176,69 @@ async function cleanupHermesResources( ); } +function hermesCleanupEnv(apiKey: string | undefined): NodeJS.ProcessEnv { + return testEnv(apiKey, { + DISCORD_PROVIDER: `${SANDBOX_NAME}-discord-bridge`, + OLD_BASE_TAG, + }); +} + +function hermesCleanupRedactions(apiKey: string | undefined): string[] { + return [apiKey ?? "", DISCORD_FAKE_TOKEN]; +} + +async function cleanupHermesNemoClawSandbox( + host: HostCliClient, + apiKey: string | undefined, +): Promise { + const result = await host.command("nemoclaw", [SANDBOX_NAME, "destroy", "--yes"], { + artifactName: "cleanup-hermes-rebuild-resources-nemoclaw-destroy", + env: hermesCleanupEnv(apiKey), + redactionValues: hermesCleanupRedactions(apiKey), + timeoutMs: 3 * 60_000, + }); + assertCleanupSucceededOrAbsent( + result, + /Sandbox '.+' does not exist|Run 'nemoclaw onboard' to create one|sandbox .* not found|no such sandbox/iu, + `cleanup Hermes rebuild sandbox ${SANDBOX_NAME}`, + ); +} + +async function cleanupHermesDiscordProvider( + host: HostCliClient, + apiKey: string | undefined, +): Promise { + const provider = `${SANDBOX_NAME}-discord-bridge`; + const result = await host.command("openshell", ["provider", "delete", provider], { + artifactName: "cleanup-hermes-rebuild-resources-provider-delete", + env: hermesCleanupEnv(apiKey), + redactionValues: hermesCleanupRedactions(apiKey), + timeoutMs: 3 * 60_000, + }); + assertCleanupSucceededOrAbsent( + result, + /\bNotFound\b|provider[^\n]*(?:not found|does not exist)|No provider|No active gateway|No gateway metadata/iu, + `cleanup Hermes Discord provider ${provider}`, + ); +} + +async function cleanupOldHermesBaseImage( + host: HostCliClient, + apiKey: string | undefined, +): Promise { + const result = await host.command("docker", ["rmi", OLD_BASE_TAG], { + artifactName: "cleanup-hermes-rebuild-resources-docker-rmi-old-base", + env: hermesCleanupEnv(apiKey), + redactionValues: hermesCleanupRedactions(apiKey), + timeoutMs: 3 * 60_000, + }); + assertCleanupSucceededOrAbsent( + result, + /No such image|image .* not found/iu, + `cleanup old Hermes base image ${OLD_BASE_TAG}`, + ); +} + function oldHermesDockerfile(): string { return [ `FROM ${OLD_BASE_TAG}`, @@ -343,7 +407,7 @@ test(STALE_BASE_REBUILD ? "rebuild-hermes: stale base cache is refreshed while Hermes state survives rebuild" : "rebuild-hermes: old Hermes sandbox rebuild preserves messaging state and upgrades runtime", { timeout: LIVE_TIMEOUT_MS, -}, async ({ artifacts, cleanup, host, secrets, skip }) => { +}, async ({ artifacts, cleanup, host, sandbox, secrets, skip }) => { const apiKey = secrets.required("NVIDIA_INFERENCE_API_KEY"); const redactionValues = [apiKey, DISCORD_FAKE_TOKEN]; const expectedVersion = expectedHermesVersion(); @@ -351,15 +415,11 @@ test(STALE_BASE_REBUILD const registrySnapshot = snapshotFile(REGISTRY_FILE); const sessionSnapshot = snapshotFile(SESSION_FILE); const sandboxBackupRoot = path.join(BACKUP_ROOT, SANDBOX_NAME); - cleanup.add(`restore NemoClaw state files for ${SANDBOX_NAME}`, () => { + cleanup.trackDisposable(`restore NemoClaw state files for ${SANDBOX_NAME}`, () => { restoreFile(REGISTRY_FILE, registrySnapshot); restoreFile(SESSION_FILE, sessionSnapshot); fs.rmSync(sandboxBackupRoot, { recursive: true, force: true }); }); - cleanup.add(`destroy Hermes rebuild resources for ${SANDBOX_NAME}`, async () => { - await cleanupHermesResources(host, apiKey, "cleanup-hermes-rebuild-resources"); - }); - await artifacts.writeJson("contract.json", { staleBaseMode: STALE_BASE_REBUILD, sandboxName: SANDBOX_NAME, @@ -398,7 +458,37 @@ test(STALE_BASE_REBUILD } } - await cleanupHermesResources(host, apiKey, "pre-cleanup-hermes-rebuild-resources"); + await bestEffortPrecleanHermesResources(host, apiKey, "pre-cleanup-hermes-rebuild-resources"); + + cleanup.trackDisposable(`remove old Hermes base image ${OLD_BASE_TAG}`, () => + cleanupOldHermesBaseImage(host, apiKey), + ); + cleanup.trackGateway(host, "nemoclaw", { + artifactName: "cleanup-hermes-rebuild-resources-gateway", + env: hermesCleanupEnv(apiKey), + redactionValues: hermesCleanupRedactions(apiKey), + timeoutMs: 3 * 60_000, + }); + cleanup.trackDisposable(`remove Hermes Discord provider for ${SANDBOX_NAME}`, () => + cleanupHermesDiscordProvider(host, apiKey), + ); + cleanup.trackForward(host, 8642, { + artifactName: "cleanup-hermes-rebuild-resources-forward-stop", + env: hermesCleanupEnv(apiKey), + redactionValues: hermesCleanupRedactions(apiKey), + timeoutMs: 3 * 60_000, + }); + cleanup.trackDisposable(`delete Hermes rebuild OpenShell sandbox ${SANDBOX_NAME}`, () => + sandbox.cleanupSandbox(SANDBOX_NAME, { + artifactName: "cleanup-hermes-rebuild-resources-openshell-sandbox-delete", + env: hermesCleanupEnv(apiKey), + redactionValues: hermesCleanupRedactions(apiKey), + timeoutMs: 3 * 60_000, + }), + ); + cleanup.trackDisposable(`destroy Hermes rebuild sandbox ${SANDBOX_NAME}`, () => + cleanupHermesNemoClawSandbox(host, apiKey), + ); const install = await host.command("bash", ["install.sh", "--non-interactive"], { artifactName: "phase-1-install-hermes", diff --git a/test/e2e/live/rebuild-openclaw.test.ts b/test/e2e/live/rebuild-openclaw.test.ts index bb9933c8e76..0359528cbb9 100644 --- a/test/e2e/live/rebuild-openclaw.test.ts +++ b/test/e2e/live/rebuild-openclaw.test.ts @@ -7,6 +7,7 @@ import os from "node:os"; import path from "node:path"; import { shellQuote } from "../../../src/lib/core/shell-quote"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { assertCleanupSucceededOrAbsent } from "../fixtures/cleanup-resources.ts"; import { assertExitZero as expectExitZero, resultText } from "../fixtures/clients/command.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; import { validateSandboxName } from "../fixtures/clients/sandbox.ts"; @@ -137,6 +138,33 @@ function openshellBestEffort( ); } +async function cleanupRebuiltNemoClawSandbox(host: HostCliClient, apiKey: string): Promise { + const result = await host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { + artifactName: "cleanup-nemoclaw-destroy", + env: cliEnv(apiKey), + redactionValues: [apiKey], + timeoutMs: 2 * 60_000, + }); + assertCleanupSucceededOrAbsent( + result, + /Sandbox '.+' does not exist|Run 'nemoclaw onboard' to create one|sandbox .* not found|no such sandbox/iu, + `cleanup rebuilt sandbox ${SANDBOX_NAME}`, + ); +} + +async function cleanupOldOpenClawBaseImage(host: HostCliClient): Promise { + const result = await host.command("docker", ["rmi", OLD_BASE_TAG], { + artifactName: "cleanup-docker-rmi-old-base", + env: dockerContextEnv(), + timeoutMs: OPENSHELL_TIMEOUT_MS, + }); + assertCleanupSucceededOrAbsent( + result, + /No such image|image .* not found/iu, + `cleanup old OpenClaw base image ${OLD_BASE_TAG}`, + ); +} + function pythonExecArgs(script: string): string[] { const encoded = Buffer.from(script, "utf8").toString("base64"); return ["python3", "-c", `import base64; exec(base64.b64decode('${encoded}'))`]; @@ -379,39 +407,29 @@ test( const registrySnapshot = snapshotFile(REGISTRY_FILE); const sessionSnapshot = snapshotFile(SESSION_FILE); const sandboxBackupRoot = path.join(BACKUP_ROOT, SANDBOX_NAME); - cleanup.add(`restore NemoClaw state files for ${SANDBOX_NAME}`, () => { + cleanup.trackDisposable(`restore NemoClaw state files for ${SANDBOX_NAME}`, () => { restoreFile(REGISTRY_FILE, registrySnapshot); restoreFile(SESSION_FILE, sessionSnapshot); fs.rmSync(sandboxBackupRoot, { recursive: true, force: true }); }); - - cleanup.add(`destroy rebuilt sandbox ${SANDBOX_NAME}`, async () => { - await host.command( - "node", - [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes", "--cleanup-gateway"], - { - artifactName: "cleanup-nemoclaw-destroy", - env: cliEnv(apiKey), - redactionValues: [apiKey], - timeoutMs: 2 * 60_000, - }, - ); - await openshellBestEffort( - host, - ["sandbox", "delete", SANDBOX_NAME], - "cleanup-openshell-sandbox-delete", - ); - await openshellBestEffort( - host, - ["gateway", "destroy", "-g", "nemoclaw"], - "cleanup-openshell-gateway-destroy", - ); - await host.command("docker", ["rmi", OLD_BASE_TAG], { - artifactName: "cleanup-docker-rmi-old-base", + cleanup.trackDisposable(`remove old OpenClaw base image ${OLD_BASE_TAG}`, () => + cleanupOldOpenClawBaseImage(host), + ); + cleanup.trackGateway(host, "nemoclaw", { + artifactName: "cleanup-openshell-gateway-destroy", + env: dockerContextEnv(), + timeoutMs: OPENSHELL_TIMEOUT_MS, + }); + cleanup.trackDisposable(`delete rebuilt OpenShell sandbox ${SANDBOX_NAME}`, () => + sandbox.cleanupSandbox(SANDBOX_NAME, { + artifactName: "cleanup-openshell-sandbox-delete", env: dockerContextEnv(), timeoutMs: OPENSHELL_TIMEOUT_MS, - }); - }); + }), + ); + cleanup.trackDisposable(`destroy rebuilt sandbox ${SANDBOX_NAME}`, () => + cleanupRebuiltNemoClawSandbox(host, apiKey), + ); // Phase 1: create a normal current sandbox first so the real gateway and // session/credential scaffolding exist, matching the legacy install/onboard diff --git a/test/e2e/live/sandbox-rebuild.test.ts b/test/e2e/live/sandbox-rebuild.test.ts index 62f1baef3d4..63ef58f10d9 100644 --- a/test/e2e/live/sandbox-rebuild.test.ts +++ b/test/e2e/live/sandbox-rebuild.test.ts @@ -56,7 +56,7 @@ function assertTestOwnedSandboxName(): void { } } -async function bestEffort(run: () => Promise): Promise { +async function bestEffortPreclean(run: () => Promise): Promise { try { await run(); } catch { @@ -121,24 +121,29 @@ test( "rebuild-backups", SANDBOX_NAME, ); - cleanup.add(`restore NemoClaw state files for ${SANDBOX_NAME}`, () => { + cleanup.trackDisposable(`restore NemoClaw state files for ${SANDBOX_NAME}`, () => { restoreRegistryAndSession(stateSnapshot); fs.rmSync(backupRoot, { recursive: true, force: true }); }); - cleanup.add(`destroy sandbox ${SANDBOX_NAME}`, async () => { - if (process.env.NEMOCLAW_E2E_KEEP_SANDBOX === "1") return; - await bestEffort(() => onboard.destroySandbox(SANDBOX_NAME, "cleanup-nemoclaw-destroy")); - await bestEffort(() => - sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { + if (process.env.NEMOCLAW_E2E_KEEP_SANDBOX !== "1") { + cleanup.trackDisposable(`delete OpenShell sandbox ${SANDBOX_NAME}`, () => + sandbox.cleanupSandbox(SANDBOX_NAME, { artifactName: "cleanup-openshell-sandbox-delete", env: buildAvailabilityProbeEnv(), timeoutMs: 60_000, }), ); - }); + cleanup.trackSandbox(host, SANDBOX_NAME, { + artifactName: "cleanup-nemoclaw-destroy", + env: buildAvailabilityProbeEnv(), + timeoutMs: 15 * 60_000, + }); + } - await bestEffort(() => onboard.destroySandbox(SANDBOX_NAME, "pre-cleanup-nemoclaw-destroy")); - await bestEffort(() => + await bestEffortPreclean(() => + onboard.destroySandbox(SANDBOX_NAME, "pre-cleanup-nemoclaw-destroy"), + ); + await bestEffortPreclean(() => sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { artifactName: "pre-cleanup-openshell-sandbox-delete", env: buildAvailabilityProbeEnv(), diff --git a/test/e2e/live/sandbox-rlimits-connect.test.ts b/test/e2e/live/sandbox-rlimits-connect.test.ts index d18a17e4b82..0a4cb1e1600 100644 --- a/test/e2e/live/sandbox-rlimits-connect.test.ts +++ b/test/e2e/live/sandbox-rlimits-connect.test.ts @@ -77,12 +77,10 @@ runConnectRlimitTest( }); expect(docker.exitCode, resultText(docker)).toBe(0); - cleanup.add("remove rlimit acceptance sandbox", () => - host.bestEffortCleanupSandbox(SANDBOX_NAME, { - env: buildAvailabilityProbeEnv(), - timeoutMs: 15 * 60_000, - }), - ); + cleanup.trackSandbox(host, SANDBOX_NAME, { + env: buildAvailabilityProbeEnv(), + timeoutMs: 15 * 60_000, + }); await host.bestEffortCleanupSandbox(SANDBOX_NAME, { env: buildAvailabilityProbeEnv(), timeoutMs: 15 * 60_000, diff --git a/test/e2e/live/sandbox-survival.test.ts b/test/e2e/live/sandbox-survival.test.ts index 7a455593c7c..730cde03c29 100644 --- a/test/e2e/live/sandbox-survival.test.ts +++ b/test/e2e/live/sandbox-survival.test.ts @@ -13,6 +13,10 @@ import fs from "node:fs"; import path from "node:path"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { + cleanupWhenCommandAvailable, + cleanupWhenOpenShellAvailable, +} from "../fixtures/cleanup-resources.ts"; import { assertExitZero, resultText, sandboxAccessEnv } from "../fixtures/clients/index.ts"; import { trustedProviderEndpoint } from "../fixtures/clients/provider.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; @@ -160,25 +164,51 @@ test( force: true, }); - cleanup.add(`destroy sandbox ${SANDBOX_NAME}`, async () => { - await host.bestEffortCleanupSandbox(SANDBOX_NAME, { - artifactName: "cleanup-nemoclaw-destroy-sandbox-survival", - }); - }); - cleanup.add("destroy shared NemoClaw gateway", async () => { - await host.command( - "sh", - [ - "-lc", - "command -v openshell >/dev/null 2>&1 && openshell gateway destroy -g nemoclaw || true", - ], - { - artifactName: "cleanup-openshell-gateway-destroy", - env: buildAvailabilityProbeEnv(), - timeoutMs: 120_000, - }, - ); - }); + const gatewayCleanupOptions = { + artifactName: "cleanup-openshell-gateway-destroy", + env: buildAvailabilityProbeEnv(), + redactionValues: [apiKey], + timeoutMs: 120_000, + }; + cleanup.trackGateway( + { + cleanupGatewayRegistration: (name: string) => + cleanupWhenOpenShellAvailable( + host, + { + artifactName: "cleanup-probe-openshell-gateway-sandbox-survival", + env: gatewayCleanupOptions.env, + redactionValues: gatewayCleanupOptions.redactionValues, + timeoutMs: 30_000, + }, + () => host.cleanupGatewayRegistration(name, gatewayCleanupOptions), + ), + }, + "nemoclaw", + gatewayCleanupOptions, + ); + const sandboxCleanupOptions = { + artifactName: "cleanup-nemoclaw-destroy-sandbox-survival", + redactionValues: [apiKey], + }; + cleanup.trackSandbox( + { + cleanupSandbox: (name: string) => + cleanupWhenCommandAvailable( + host, + host.commandPath, + { + artifactName: "cleanup-probe-nemoclaw-sandbox-survival", + env: buildAvailabilityProbeEnv(), + redactionValues: sandboxCleanupOptions.redactionValues, + timeoutMs: 30_000, + }, + () => host.cleanupSandbox(name, sandboxCleanupOptions), + ), + }, + SANDBOX_NAME, + sandboxCleanupOptions, + ); const install = await host.command("bash", ["install.sh", "--non-interactive"], { artifactName: "install-sh-sandbox-survival", diff --git a/test/e2e/live/sessions-agents-cli.test.ts b/test/e2e/live/sessions-agents-cli.test.ts index 295fc62d52e..e4e9d4f4a15 100644 --- a/test/e2e/live/sessions-agents-cli.test.ts +++ b/test/e2e/live/sessions-agents-cli.test.ts @@ -14,6 +14,7 @@ import fs from "node:fs"; import path from "node:path"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { registerSandboxCleanupUnlessKept } from "../fixtures/cleanup-resources.ts"; import { resultText } from "../fixtures/clients/command.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; import { validateSandboxName } from "../fixtures/clients/sandbox.ts"; @@ -67,14 +68,6 @@ function commandEnv(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { }; } -async function bestEffort(run: () => Promise): Promise { - try { - await run(); - } catch { - // Cleanup is best effort so an early setup failure keeps the original error. - } -} - async function runNemoclaw( host: HostCliClient, args: string[], @@ -131,7 +124,7 @@ async function ensureOpenshellAvailable(host: HostCliClient): Promise { ).toBe(0); } -async function cleanupSandbox(host: HostCliClient, hosted: HostedInferenceConfig): Promise { +async function precleanSandbox(host: HostCliClient, hosted: HostedInferenceConfig): Promise { if (process.env.NEMOCLAW_E2E_KEEP_SANDBOX === "1") return; const destroy = await runNemoclaw(host, [SANDBOX_NAME, "destroy", "--yes"], hosted, { @@ -307,7 +300,7 @@ async function expectJsonCommand( test("sessions/agents host CLI routes to OpenClaw and preserves JSON envelopes", { timeout: TEST_TIMEOUT_MS, -}, async ({ artifacts, cleanup, host, secrets, skip }) => { +}, async ({ artifacts, cleanup, host, sandbox, secrets, skip }) => { expect(fs.existsSync(CLI_ENTRYPOINT), "bin/nemoclaw.js missing").toBe(true); expect( fs.existsSync(CLI_DIST_ENTRYPOINT), @@ -340,10 +333,22 @@ test("sessions/agents host CLI routes to OpenClaw and preserves JSON envelopes", const hosted = requireHostedInferenceConfig(secrets); await ensureOpenshellAvailable(host); - cleanup.add(`destroy sessions/agents sandbox ${SANDBOX_NAME}`, async () => - bestEffort(() => cleanupSandbox(host, hosted)), - ); - await cleanupSandbox(host, hosted); + registerSandboxCleanupUnlessKept(process.env.NEMOCLAW_E2E_KEEP_SANDBOX === "1", () => { + cleanup.trackDisposable(`delete OpenShell sandbox ${SANDBOX_NAME}`, () => + sandbox.cleanupSandbox(SANDBOX_NAME, { + artifactName: "cleanup-openshell-sandbox-delete-sessions-agents-cli", + env: commandEnv(), + timeoutMs: 60_000, + }), + ); + cleanup.trackSandbox(host, SANDBOX_NAME, { + artifactName: "cleanup-nemoclaw-destroy-sessions-agents-cli", + env: commandEnv(hosted.env), + redactionValues: [hosted.apiKey], + timeoutMs: 5 * 60_000, + }); + }); + await precleanSandbox(host, hosted); fs.rmSync(path.join(process.env.HOME ?? "", ".nemoclaw", "onboard.lock"), { force: true }); const onboard = await runNemoclaw( diff --git a/test/e2e/live/shields-config.test.ts b/test/e2e/live/shields-config.test.ts index 99087e07b77..85b4df8a19d 100644 --- a/test/e2e/live/shields-config.test.ts +++ b/test/e2e/live/shields-config.test.ts @@ -14,6 +14,10 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { + cleanupWhenCommandAvailable, + cleanupWhenOpenShellAvailable, +} from "../fixtures/cleanup-resources.ts"; import { resultText } from "../fixtures/clients/command.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; import { @@ -139,7 +143,7 @@ async function statPath( return { ...parsed, raw: result.stdout.trim() }; } -async function cleanupSandbox( +async function preCleanSandbox( host: HostCliClient, sandbox: SandboxClient, artifactPrefix: string, @@ -250,10 +254,80 @@ test("shields-config: live shields up/down locks config and detects drift", { const hosted = requireHostedInferenceConfig(secrets); const apiKey = hosted.apiKey; - await cleanupSandbox(host, sandbox, "pre-cleanup"); - cleanup.add(`destroy shields-config sandbox ${SANDBOX_NAME}`, async () => { - await cleanupSandbox(host, sandbox, "cleanup"); + await preCleanSandbox(host, sandbox, "pre-cleanup"); + cleanup.trackDisposable(`remove shields state for ${SANDBOX_NAME}`, () => { + for (const file of [STATE_FILE(SANDBOX_NAME), TIMER_FILE(SANDBOX_NAME), AUDIT_FILE]) { + fs.rmSync(file, { force: true }); + } + fs.rmSync(path.join(os.homedir(), ".nemoclaw", "onboard.lock"), { + force: true, + }); }); + const gatewayCleanupOptions = { + artifactName: "cleanup-openshell-gateway-destroy", + env: commandEnv(), + redactionValues: [apiKey], + timeoutMs: 60_000, + }; + cleanup.trackGateway( + { + cleanupGatewayRegistration: (name: string) => + cleanupWhenOpenShellAvailable( + host, + { + artifactName: "cleanup-probe-openshell-gateway", + env: gatewayCleanupOptions.env, + redactionValues: gatewayCleanupOptions.redactionValues, + timeoutMs: 30_000, + }, + () => host.cleanupGatewayRegistration(name, gatewayCleanupOptions), + ), + }, + "nemoclaw", + gatewayCleanupOptions, + ); + const openshellSandboxCleanupOptions = { + artifactName: "cleanup-openshell-sandbox-delete", + env: commandEnv(), + redactionValues: [apiKey], + timeoutMs: 60_000, + }; + cleanup.trackDisposable(`delete OpenShell sandbox ${SANDBOX_NAME}`, () => + cleanupWhenOpenShellAvailable( + host, + { + artifactName: "cleanup-probe-openshell-sandbox", + env: openshellSandboxCleanupOptions.env, + redactionValues: openshellSandboxCleanupOptions.redactionValues, + timeoutMs: 30_000, + }, + () => sandbox.cleanupSandbox(SANDBOX_NAME, openshellSandboxCleanupOptions), + ), + ); + const nemoclawSandboxCleanupOptions = { + artifactName: "cleanup-nemoclaw-destroy", + env: commandEnv(), + redactionValues: [apiKey], + timeoutMs: 120_000, + }; + cleanup.trackSandbox( + { + cleanupSandbox: (name: string) => + cleanupWhenCommandAvailable( + host, + host.commandPath, + { + artifactName: "cleanup-probe-nemoclaw-sandbox", + env: nemoclawSandboxCleanupOptions.env, + redactionValues: nemoclawSandboxCleanupOptions.redactionValues, + timeoutMs: 30_000, + }, + () => host.cleanupSandbox(name, nemoclawSandboxCleanupOptions), + ), + }, + SANDBOX_NAME, + nemoclawSandboxCleanupOptions, + ); const install = await installedShellCommand( host, diff --git a/test/e2e/live/skill-agent.test.ts b/test/e2e/live/skill-agent.test.ts index 40a1cf82a5a..52d081c01da 100644 --- a/test/e2e/live/skill-agent.test.ts +++ b/test/e2e/live/skill-agent.test.ts @@ -13,6 +13,7 @@ import { } from "../fixtures/clients/sandbox.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; import { requireHostedInferenceConfig } from "../fixtures/hosted-inference.ts"; +import { CLI_ENTRYPOINT, REPO_ROOT } from "../fixtures/paths.ts"; import { agentSectionContainsToken, isAgentVerificationFailClosed, @@ -20,7 +21,6 @@ import { shouldSkipExternalAgentVerificationFailure, VERIFY_PHRASE, } from "../support/skill-agent-classifiers.ts"; -import { CLI_ENTRYPOINT, REPO_ROOT } from "../fixtures/paths.ts"; // Keep this as a direct live test: the contract is skill fixture // injection into a real OpenClaw sandbox plus an agent turn that must read @@ -147,6 +147,41 @@ test( let sandboxProvisioned = false; const cleanupEnv = buildAvailabilityProbeEnv(); + type CleanupAttemptState = { + outcome: "pending" | "passed" | "failed"; + error?: string; + }; + const cleanupAttempts: { + nemoclaw: CleanupAttemptState; + openshell: CleanupAttemptState; + } = { + nemoclaw: { outcome: "pending" }, + openshell: { outcome: "pending" }, + }; + const cleanupResultEvidence = (artifactName: string) => { + const resultArtifact = `shell/${artifactName}.result.json`; + try { + const result = JSON.parse(fs.readFileSync(artifacts.pathFor(resultArtifact), "utf8")) as { + exitCode: number | null; + signal: NodeJS.Signals | null; + timedOut: boolean; + }; + return { + exitCode: result.exitCode, + resultArtifact, + signal: result.signal, + timedOut: result.timedOut, + }; + } catch (error) { + return { + evidenceError: error instanceof Error ? error.message : String(error), + exitCode: null, + resultArtifact, + signal: null, + timedOut: null, + }; + } + }; await ignoreCleanupError(() => host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { artifactName: "pre-cleanup-nemoclaw-destroy-skill-agent", @@ -162,32 +197,57 @@ test( }), ); - cleanup.add(`destroy skill-agent sandbox ${SANDBOX_NAME}`, async () => { - const destroy = await host.command( - "node", - [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], - { - artifactName: "cleanup-nemoclaw-destroy-skill-agent", - env: buildAvailabilityProbeEnv(), - timeoutMs: 120_000, - }, - ); - const deleteSandbox = await sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { - artifactName: "cleanup-openshell-sandbox-delete-skill-agent", - env: buildAvailabilityProbeEnv(), - timeoutMs: 60_000, - }); + cleanup.trackDisposable("write skill-agent cleanup summary", async () => { + const destroyEvidence = cleanupResultEvidence("cleanup-nemoclaw-destroy-skill-agent"); + const deleteEvidence = cleanupResultEvidence("cleanup-openshell-sandbox-delete-skill-agent"); await artifacts.writeJson("cleanup-skill-agent-summary.json", { sandboxProvisioned, - destroyExitCode: destroy.exitCode, - deleteExitCode: deleteSandbox.exitCode, + destroyExitCode: destroyEvidence.exitCode, + deleteExitCode: deleteEvidence.exitCode, + resources: ["nemoclaw sandbox", "OpenShell sandbox fallback"], + attempts: { + nemoclaw: { ...cleanupAttempts.nemoclaw, ...destroyEvidence }, + openshell: { ...cleanupAttempts.openshell, ...deleteEvidence }, + }, }); - if (sandboxProvisioned && destroy.exitCode !== 0 && deleteSandbox.exitCode !== 0) { - throw new Error( - `skill-agent cleanup failed\n${resultText(destroy)}\n${resultText(deleteSandbox)}`, - ); + }); + const openshellCleanupOptions = { + artifactName: "cleanup-openshell-sandbox-delete-skill-agent", + env: buildAvailabilityProbeEnv(), + timeoutMs: 60_000, + }; + cleanup.trackDisposable(`delete OpenShell sandbox ${SANDBOX_NAME}`, async () => { + try { + await sandbox.cleanupSandbox(SANDBOX_NAME, openshellCleanupOptions); + cleanupAttempts.openshell.outcome = "passed"; + } catch (error) { + cleanupAttempts.openshell.outcome = "failed"; + cleanupAttempts.openshell.error = error instanceof Error ? error.message : String(error); + throw error; } }); + const nemoclawCleanupOptions = { + artifactName: "cleanup-nemoclaw-destroy-skill-agent", + env: buildAvailabilityProbeEnv(), + redactionValues: [apiKey], + timeoutMs: 120_000, + }; + cleanup.trackSandbox( + { + cleanupSandbox: async (name: string) => { + try { + await host.cleanupSandbox(name, nemoclawCleanupOptions); + cleanupAttempts.nemoclaw.outcome = "passed"; + } catch (error) { + cleanupAttempts.nemoclaw.outcome = "failed"; + cleanupAttempts.nemoclaw.error = error instanceof Error ? error.message : String(error); + throw error; + } + }, + }, + SANDBOX_NAME, + nemoclawCleanupOptions, + ); const onboard = await host.command( "node", diff --git a/test/e2e/live/snapshot-commands.test.ts b/test/e2e/live/snapshot-commands.test.ts index 0452a31beef..ad1ce148899 100644 --- a/test/e2e/live/snapshot-commands.test.ts +++ b/test/e2e/live/snapshot-commands.test.ts @@ -46,7 +46,7 @@ function commandEnv(inference?: SnapshotInferenceFixture): NodeJS.ProcessEnv { return buildSnapshotCommandEnv(SANDBOX_NAME, inference); } -async function bestEffort(run: () => Promise): Promise { +async function bestEffortPreclean(run: () => Promise): Promise { try { await run(); } catch { @@ -54,26 +54,24 @@ async function bestEffort(run: () => Promise): Promise { } } -async function cleanupSnapshotSandbox( +async function precleanSnapshotSandbox( host: HostCliClient, sandbox: SandboxClient, label: string, ): Promise { - await bestEffort(() => - host.command("nemoclaw", [SANDBOX_NAME, "destroy", "--yes"], { - artifactName: `${label}-nemoclaw-destroy`, - env: commandEnv(), - timeoutMs: 120_000, - }), - ); - await bestEffort(() => + await host.bestEffortCleanupSandbox(SANDBOX_NAME, { + artifactName: `${label}-nemoclaw-destroy`, + env: commandEnv(), + timeoutMs: 120_000, + }); + await bestEffortPreclean(() => sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { artifactName: `${label}-openshell-sandbox-delete`, env: commandEnv(), timeoutMs: 60_000, }), ); - await bestEffort(() => + await bestEffortPreclean(() => sandbox.openshell(["gateway", "destroy", "-g", "nemoclaw"], { artifactName: `${label}-openshell-gateway-destroy`, env: commandEnv(), @@ -144,7 +142,7 @@ test("snapshot commands preserve create/list/latest restore/targeted restore/no- requireAuth: true, requireAuthModels: true, }); - cleanup.add("close snapshot commands compatible inference fixture", async () => { + cleanup.trackDisposable("close snapshot commands compatible inference fixture", async () => { await artifacts.writeJson("compatible-inference-requests.json", inference.requests()); await inference.close(); }); @@ -154,11 +152,25 @@ test("snapshot commands preserve create/list/latest restore/targeted restore/no- model: INFERENCE_MODEL, }; - cleanup.add(`destroy snapshot sandbox ${SANDBOX_NAME}`, () => - cleanupSnapshotSandbox(host, sandbox, "cleanup"), + cleanup.trackGateway(host, "nemoclaw", { + artifactName: "cleanup-openshell-gateway-destroy", + env: commandEnv(), + timeoutMs: 60_000, + }); + cleanup.trackDisposable(`delete OpenShell sandbox ${SANDBOX_NAME}`, () => + sandbox.cleanupSandbox(SANDBOX_NAME, { + artifactName: "cleanup-openshell-sandbox-delete", + env: commandEnv(), + timeoutMs: 60_000, + }), ); + cleanup.trackSandbox(host, SANDBOX_NAME, { + artifactName: "cleanup-nemoclaw-destroy", + env: commandEnv(), + timeoutMs: 120_000, + }); - await cleanupSnapshotSandbox(host, sandbox, "pre-cleanup"); + await precleanSnapshotSandbox(host, sandbox, "pre-cleanup"); fs.rmSync(BACKUP_DIR, { recursive: true, force: true }); const install = await host.command("bash", ["install.sh", "--non-interactive", "--fresh"], { diff --git a/test/e2e/live/spark-install.test.ts b/test/e2e/live/spark-install.test.ts index 1adcaffa4d1..87f5dcfb07f 100644 --- a/test/e2e/live/spark-install.test.ts +++ b/test/e2e/live/spark-install.test.ts @@ -54,7 +54,7 @@ function sourceInstalledPathProbe(): string { ].join("; "); } -async function bestEffortCleanup(host: HostCliClient): Promise { +async function bestEffortPreclean(host: HostCliClient): Promise { const cleanup = [ `if command -v nemoclaw >/dev/null 2>&1; then nemoclaw ${shellQuote(SANDBOX_NAME)} destroy --yes >/dev/null 2>&1 || true; fi`, `if command -v openshell >/dev/null 2>&1; then openshell sandbox delete ${shellQuote(SANDBOX_NAME)} >/dev/null 2>&1 || true; fi`, @@ -70,7 +70,7 @@ async function bestEffortCleanup(host: HostCliClient): Promise { liveTest( "spark install path: standard non-interactive install leaves NemoClaw and OpenShell usable", { timeout: LIVE_TIMEOUT_MS }, - async ({ artifacts, cleanup, host, secrets }) => { + async ({ artifacts, cleanup, host, sandbox, secrets }) => { await artifacts.target.declare({ id: "spark-install", sandboxName: SANDBOX_NAME, @@ -102,8 +102,27 @@ liveTest( const hosted = requireHostedInferenceConfig(secrets); const redactionValues = [hosted.apiKey]; - cleanup.add(`remove ${SANDBOX_NAME} after Spark install smoke`, () => bestEffortCleanup(host)); - await bestEffortCleanup(host); + cleanup.trackGateway(host, "nemoclaw", { + artifactName: "cleanup-spark-install-state-openshell-gateway-destroy", + env: env(), + redactionValues, + timeoutMs: 120_000, + }); + cleanup.trackDisposable(`delete OpenShell sandbox ${SANDBOX_NAME}`, () => + sandbox.cleanupSandbox(SANDBOX_NAME, { + artifactName: "cleanup-spark-install-state-openshell-sandbox-delete", + env: env(), + redactionValues, + timeoutMs: 120_000, + }), + ); + cleanup.trackSandbox(host, SANDBOX_NAME, { + artifactName: "cleanup-spark-install-state-nemoclaw-destroy", + env: env(), + redactionValues, + timeoutMs: 120_000, + }); + await bestEffortPreclean(host); const installLog = process.env.INSTALL_LOG ?? artifacts.pathFor("logs/install.log"); fs.mkdirSync(path.dirname(installLog), { recursive: true }); diff --git a/test/e2e/live/state-backup-restore.test.ts b/test/e2e/live/state-backup-restore.test.ts index fec46097c95..33bd2c365f1 100644 --- a/test/e2e/live/state-backup-restore.test.ts +++ b/test/e2e/live/state-backup-restore.test.ts @@ -6,7 +6,7 @@ import os from "node:os"; import path from "node:path"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; -import { resultText } from "../fixtures/clients/index.ts"; +import { assertExitZero, resultText } from "../fixtures/clients/index.ts"; import { sandboxAccessEnv, validateSandboxName } from "../fixtures/clients/sandbox.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; import { REPO_ROOT } from "../fixtures/paths.ts"; @@ -95,7 +95,7 @@ function isNvidiaEndpointValidationUnavailable(text: string): boolean { ); } -async function bestEffort(run: () => Promise): Promise { +async function bestEffortLifecycleCleanup(run: () => Promise): Promise { try { await run(); } catch { @@ -114,7 +114,7 @@ async function destroySandboxUntilAbsent( ): Promise { let lastList = ""; for (let attempt = 1; attempt <= DESTROY_ATTEMPTS; attempt += 1) { - await bestEffort(() => destroy(`phase-3-destroy-attempt-${attempt}`)); + await bestEffortLifecycleCleanup(() => destroy(`phase-3-destroy-attempt-${attempt}`)); const listResult = await list(`phase-3-list-after-destroy-${attempt}`); lastList = resultText(listResult); if (listResult.exitCode === 0 && !lastList.includes(sandboxName)) return; @@ -172,10 +172,10 @@ test("state-backup-restore: backup-workspace.sh restores workspace files and mem const stateSnapshot = snapshotRegistryAndSession(); let createdBackupDir: string | undefined; - cleanup.add(`restore NemoClaw state files for ${SANDBOX_NAME}`, () => { + cleanup.trackDisposable(`restore NemoClaw state files for ${SANDBOX_NAME}`, () => { restoreRegistryAndSession(stateSnapshot); }); - cleanup.add("remove generated backup-workspace.sh backup", () => { + cleanup.trackDisposable("remove generated backup-workspace.sh backup", () => { if (!createdBackupDir) return; const root = backupRoot(); const resolved = path.resolve(createdBackupDir); @@ -183,29 +183,33 @@ test("state-backup-restore: backup-workspace.sh restores workspace files and mem fs.rmSync(resolved, { recursive: true, force: true }); } }); - cleanup.add(`destroy sandbox ${SANDBOX_NAME}`, async () => { - if (process.env.NEMOCLAW_E2E_KEEP_SANDBOX === "1") return; - await bestEffort(() => onboard.destroySandbox(SANDBOX_NAME, "cleanup-nemoclaw-destroy")); - await bestEffort(() => - sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { + if (process.env.NEMOCLAW_E2E_KEEP_SANDBOX !== "1") { + cleanup.trackDisposable(`delete OpenShell sandbox ${SANDBOX_NAME}`, () => + sandbox.cleanupSandbox(SANDBOX_NAME, { artifactName: "cleanup-openshell-sandbox-delete", env: sandboxAccessEnv(), timeoutMs: 60_000, }), ); - }); - cleanup.add("stop NemoClaw gateway", async () => { - await bestEffort(() => - host.nemoclaw(["stop"], { - artifactName: "cleanup-nemoclaw-stop", - env: buildAvailabilityProbeEnv(), - timeoutMs: 60_000, - }), - ); + cleanup.trackSandbox(host, SANDBOX_NAME, { + artifactName: "cleanup-nemoclaw-destroy", + env: buildAvailabilityProbeEnv(), + timeoutMs: 15 * 60_000, + }); + } + cleanup.trackDisposable("stop NemoClaw gateway", async () => { + const result = await host.nemoclaw(["stop"], { + artifactName: "cleanup-nemoclaw-stop", + env: buildAvailabilityProbeEnv(), + timeoutMs: 60_000, + }); + assertExitZero(result, "cleanup nemoclaw stop"); }); - await bestEffort(() => onboard.destroySandbox(SANDBOX_NAME, "pre-cleanup-nemoclaw-destroy")); - await bestEffort(() => + await bestEffortLifecycleCleanup(() => + onboard.destroySandbox(SANDBOX_NAME, "pre-cleanup-nemoclaw-destroy"), + ); + await bestEffortLifecycleCleanup(() => sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { artifactName: "pre-cleanup-openshell-sandbox-delete", env: sandboxAccessEnv(), diff --git a/test/e2e/live/telegram-injection.test.ts b/test/e2e/live/telegram-injection.test.ts index 02f16c99cb3..7355ec50b0c 100644 --- a/test/e2e/live/telegram-injection.test.ts +++ b/test/e2e/live/telegram-injection.test.ts @@ -6,20 +6,21 @@ import path from "node:path"; import { expect, test } from "../fixtures/e2e-test.ts"; import { base64, - bestEffort, + runSecondaryCleanup as bestEffortDiagnostic, CLI, COMMAND_TIMEOUT_MS, - cleanupSandbox, dockerInfo, expectExitZero, expectSandboxReady, installSandboxOrSkipOnRateLimit, phase6Env, + precleanSandbox, REPO_ROOT, redactionValues, resultText, sandboxSh, shellQuote, + trackSandboxCleanup, } from "./phase6-messaging-helpers.ts"; const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-telegram-injection"; @@ -216,10 +217,16 @@ test("Telegram bridge-style message handling treats shell metacharacters as data ], }); - cleanup.add(`destroy telegram injection sandbox ${SANDBOX_NAME}`, () => - cleanupSandbox(host, SANDBOX_NAME, env, redactions, "cleanup-telegram-injection"), + trackSandboxCleanup( + cleanup, + host, + sandbox, + SANDBOX_NAME, + env, + redactions, + "cleanup-telegram-injection", ); - await cleanupSandbox(host, SANDBOX_NAME, env, redactions, "preclean-telegram-injection"); + await precleanSandbox(host, SANDBOX_NAME, env, redactions, "preclean-telegram-injection"); const docker = await dockerInfo(host, env); expect(docker.exitCode, resultText(docker)).toBe(0); @@ -354,7 +361,7 @@ test("Telegram bridge-style message handling treats shell metacharacters as data expect(special.exitCode, resultText(special)).toBe(0); expect(resultText(special).trim()).not.toBe(""); - await bestEffort(() => + await bestEffortDiagnostic(() => host.command("node", [CLI, SANDBOX_NAME, "status"], { artifactName: "post-assert-status-telegram-injection", env, diff --git a/test/e2e/live/token-rotation.test.ts b/test/e2e/live/token-rotation.test.ts index bdfce0a6122..d109632bdf4 100644 --- a/test/e2e/live/token-rotation.test.ts +++ b/test/e2e/live/token-rotation.test.ts @@ -5,6 +5,10 @@ import fs from "node:fs"; import path from "node:path"; import { testTimeoutOptions } from "../../helpers/timeouts"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { + cleanupWhenCommandAvailable, + cleanupWhenOpenShellAvailable, +} from "../fixtures/cleanup-resources.ts"; import { resultText } from "../fixtures/clients/command.ts"; import { validateSandboxName } from "../fixtures/clients/sandbox.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; @@ -24,6 +28,8 @@ validateSandboxName(SANDBOX_NAME); const ONBOARD_TIMEOUT_MS = 25 * 60_000; const PHASE_TIMEOUT_MS = 40 * 60_000; +process.env.NEMOCLAW_CLI_BIN ??= CLI_ENTRYPOINT; + interface TokenSet { telegram: string; discord: string; @@ -261,7 +267,7 @@ async function destroyGatewayIfOpenshellExists( test( "messaging token rotation rebuilds only the changed provider and reuses unchanged credentials", testTimeoutOptions(PHASE_TIMEOUT_MS), - async ({ artifacts, cleanup, host, skip }) => { + async ({ artifacts, cleanup, host, sandbox, skip }) => { expect( fs.existsSync(CLI_ENTRYPOINT), "run `npm run build:cli` before live repo CLI targets", @@ -287,7 +293,7 @@ test( publicHost: "host.openshell.internal", responseText: "OK", }); - cleanup.add("stop fake OpenAI-compatible endpoint for token rotation", async () => { + cleanup.trackDisposable("stop fake OpenAI-compatible endpoint for token rotation", async () => { await artifacts.writeJson("fake-openai-compatible-requests.json", fakeOpenAI.requests()); await fakeOpenAI.close(); }); @@ -318,18 +324,71 @@ test( }); const cleanupEnv = buildAvailabilityProbeEnv(); - cleanup.add(`destroy token-rotation sandbox ${SANDBOX_NAME}`, async () => { - await host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { - artifactName: "cleanup-nemoclaw-destroy-token-rotation", - env: cleanupEnv, - timeoutMs: 120_000, - }); - await deleteSandboxIfOpenshellExists(host, "cleanup-openshell-sandbox-delete-token-rotation"); - await destroyGatewayIfOpenshellExists( + const gatewayCleanupOptions = { + artifactName: "cleanup-openshell-gateway-destroy-token-rotation", + env: cleanupEnv, + redactionValues: redactionValues(), + timeoutMs: 60_000, + }; + cleanup.trackGateway( + { + cleanupGatewayRegistration: (name: string) => + cleanupWhenOpenShellAvailable( + host, + { + artifactName: "cleanup-probe-openshell-gateway-token-rotation", + env: gatewayCleanupOptions.env, + redactionValues: gatewayCleanupOptions.redactionValues, + timeoutMs: 30_000, + }, + () => host.cleanupGatewayRegistration(name, gatewayCleanupOptions), + ), + }, + "nemoclaw", + gatewayCleanupOptions, + ); + const openshellSandboxCleanupOptions = { + artifactName: "cleanup-openshell-sandbox-delete-token-rotation", + env: cleanupEnv, + redactionValues: redactionValues(), + timeoutMs: 60_000, + }; + cleanup.trackDisposable(`delete OpenShell sandbox ${SANDBOX_NAME}`, () => + cleanupWhenOpenShellAvailable( host, - "cleanup-openshell-gateway-destroy-token-rotation", - ); - }); + { + artifactName: "cleanup-probe-openshell-sandbox-token-rotation", + env: openshellSandboxCleanupOptions.env, + redactionValues: openshellSandboxCleanupOptions.redactionValues, + timeoutMs: 30_000, + }, + () => sandbox.cleanupSandbox(SANDBOX_NAME, openshellSandboxCleanupOptions), + ), + ); + const nemoclawSandboxCleanupOptions = { + artifactName: "cleanup-nemoclaw-destroy-token-rotation", + env: cleanupEnv, + redactionValues: redactionValues(), + timeoutMs: 120_000, + }; + cleanup.trackSandbox( + { + cleanupSandbox: (name: string) => + cleanupWhenCommandAvailable( + host, + host.commandPath, + { + artifactName: "cleanup-probe-nemoclaw-sandbox-token-rotation", + env: nemoclawSandboxCleanupOptions.env, + redactionValues: nemoclawSandboxCleanupOptions.redactionValues, + timeoutMs: 30_000, + }, + () => host.cleanupSandbox(name, nemoclawSandboxCleanupOptions), + ), + }, + SANDBOX_NAME, + nemoclawSandboxCleanupOptions, + ); await host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { artifactName: "pre-cleanup-nemoclaw-destroy-token-rotation", @@ -365,12 +424,13 @@ test( }, ); expect(retainBuildCache.exitCode, resultText(retainBuildCache)).toBe(0); - cleanup.add("remove token-rotation build cache tag", async () => { - await host.command("docker", ["image", "rm", cacheImageTag], { + cleanup.trackDisposable("remove token-rotation build cache tag", async () => { + const remove = await host.command("docker", ["image", "rm", cacheImageTag], { artifactName: "cleanup-token-rotation-build-cache", env: buildAvailabilityProbeEnv(), timeoutMs: 30_000, }); + expect(remove.exitCode, resultText(remove)).toBe(0); }); const openshellVersion = await host.command("openshell", ["--version"], { diff --git a/test/e2e/live/tunnel-lifecycle-helpers.ts b/test/e2e/live/tunnel-lifecycle-helpers.ts index 0bb9d5d48e0..68e4f62544b 100644 --- a/test/e2e/live/tunnel-lifecycle-helpers.ts +++ b/test/e2e/live/tunnel-lifecycle-helpers.ts @@ -155,7 +155,7 @@ function parseCurlProbe(result: ShellProbeResult): CurlProbe { return { httpCode, body, result }; } -async function bestEffort(run: () => Promise): Promise { +async function bestEffortRecovery(run: () => Promise): Promise { try { await run(); } catch { @@ -180,7 +180,7 @@ type TunnelLifecycleFixtures = Pick< type TunnelLifecycleCleanupHost = Pick; -type TunnelLifecycleCleanupRegistry = Pick; +type TunnelLifecycleCleanupRegistry = Pick; export function registerTunnelLifecycleCleanup( cleanup: TunnelLifecycleCleanupRegistry, @@ -195,13 +195,12 @@ export function registerTunnelLifecycleCleanup( // condition: replace this ordering guard once NemoClaw exposes one atomic // machine-readable lifecycle cleanup that stops tunnels before destroying the // sandbox. - cleanup.add(`destroy sandbox ${SANDBOX_NAME}`, async () => { - if (process.env.NEMOCLAW_E2E_KEEP_SANDBOX === "1") return; - await host.cleanupSandbox(SANDBOX_NAME, { + if (process.env.NEMOCLAW_E2E_KEEP_SANDBOX !== "1") { + cleanup.trackSandbox(host, SANDBOX_NAME, { artifactName: "cleanup-nemoclaw-destroy-tunnel-lifecycle", timeoutMs: 15 * 60_000, }); - }); + } cleanup.add("stop cloudflared quick tunnel", async () => { const stop = await host.nemoclaw(["tunnel", "stop"], { artifactName: "cleanup-tunnel-stop", @@ -331,7 +330,7 @@ export async function runTunnelLifecycleContract({ if (start.exitCode !== 0) { await artifacts.writeText("cloudflared-log-after-start-failure.txt", cloudflaredLogTail()); if (isCloudflareTransientText(resultText(start)) || classifyCloudflaredLog() === "cloudflare") { - await bestEffort(() => + await bestEffortRecovery(() => host.nemoclaw(["tunnel", "stop"], { artifactName: "tunnel-stop-after-cloudflare-start-failure", env: commandEnv(), @@ -364,7 +363,7 @@ export async function runTunnelLifecycleContract({ if (!tunnelUrl) { await artifacts.writeText("cloudflared-log-without-status-url.txt", cloudflaredLogTail()); const cfClass = classifyCloudflaredLog(); - await bestEffort(() => + await bestEffortRecovery(() => host.nemoclaw(["tunnel", "stop"], { artifactName: "tunnel-stop-after-missing-url", env: commandEnv(), diff --git a/test/e2e/live/upgrade-stale-sandbox-helpers.ts b/test/e2e/live/upgrade-stale-sandbox-helpers.ts index 541fb36378a..7b2da7fa60d 100644 --- a/test/e2e/live/upgrade-stale-sandbox-helpers.ts +++ b/test/e2e/live/upgrade-stale-sandbox-helpers.ts @@ -6,8 +6,9 @@ import os from "node:os"; import path from "node:path"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import type { CleanupRegistry } from "../fixtures/cleanup.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; -import { resultText } from "../fixtures/clients/index.ts"; +import { assertExitZero, resultText } from "../fixtures/clients/index.ts"; import { type SandboxClient, validateSandboxName } from "../fixtures/clients/sandbox.ts"; import { expect } from "../fixtures/e2e-test.ts"; import { @@ -64,7 +65,7 @@ export function commandEnv(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { }; } -export async function bestEffort(run: () => Promise): Promise { +async function bestEffortPreclean(run: () => Promise): Promise { try { await run(); } catch { @@ -161,29 +162,27 @@ export function assertDockerAvailable( })(); } -export function registerStateRestore(cleanup: { - add(name: string, run: () => Promise | void): void; -}): void { +export function registerStateRestore(cleanup: Pick): void { const registrySnapshot = snapshotFile(REGISTRY_FILE); const sessionSnapshot = snapshotFile(SESSION_FILE); - cleanup.add(`restore NemoClaw state files for ${SANDBOX_NAME}`, () => { + cleanup.trackDisposable(`restore NemoClaw state files for ${SANDBOX_NAME}`, () => { restoreFile(REGISTRY_FILE, registrySnapshot); restoreFile(SESSION_FILE, sessionSnapshot); }); } -export async function cleanupStaleSandbox( +export async function precleanStaleSandbox( host: HostCliClient, sandbox: SandboxClient, ): Promise { - await bestEffort(() => + await bestEffortPreclean(() => host.nemoclaw([SANDBOX_NAME, "destroy", "--yes"], { artifactName: "cleanup-nemoclaw-destroy-upgrade-stale", env: commandEnv(), timeoutMs: 120_000, }), ); - await bestEffort(() => + await bestEffortPreclean(() => sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { artifactName: "cleanup-openshell-delete-upgrade-stale", env: commandEnv(), @@ -193,13 +192,14 @@ export async function cleanupStaleSandbox( } export async function cleanupOldImage(host: HostCliClient): Promise { - await bestEffort(() => - host.command("docker", ["image", "rm", "-f", OLD_BASE_TAG], { - artifactName: "cleanup-docker-image-upgrade-stale", - env: buildAvailabilityProbeEnv(), - timeoutMs: 60_000, - }), - ); + const result = await host.command("docker", ["image", "rm", "-f", OLD_BASE_TAG], { + artifactName: "cleanup-docker-image-upgrade-stale", + env: buildAvailabilityProbeEnv(), + timeoutMs: 60_000, + }); + if (result.exitCode === 0 || /No such image|image[^\n]*not found/i.test(resultText(result))) + return; + assertExitZero(result, `cleanup Docker image ${OLD_BASE_TAG}`); } export async function installCurrentNemoclaw( @@ -264,11 +264,9 @@ export async function buildOldOpenClawBase(host: HostCliClient): Promise Promise | void): void; -}): string { +export function createFixtureDockerfile(cleanup: Pick): string { const fixtureDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-old-openclaw-")); - cleanup.add("remove stale sandbox fixture Dockerfile", () => + cleanup.trackDisposable("remove stale sandbox fixture Dockerfile", () => fs.rmSync(fixtureDir, { recursive: true, force: true }), ); const fixtureDockerfile = path.join(fixtureDir, "Dockerfile"); diff --git a/test/e2e/live/upgrade-stale-sandbox.test.ts b/test/e2e/live/upgrade-stale-sandbox.test.ts index 86923e04307..1f72350a897 100644 --- a/test/e2e/live/upgrade-stale-sandbox.test.ts +++ b/test/e2e/live/upgrade-stale-sandbox.test.ts @@ -17,11 +17,11 @@ import { assertDockerAvailable, buildOldOpenClawBase, cleanupOldImage, - cleanupStaleSandbox, commandEnv, createFixtureDockerfile, installCurrentNemoclaw, OLD_OPENCLAW_VERSION, + precleanStaleSandbox, registeredStaleSandboxJson, registerStateRestore, SANDBOX_NAME, @@ -58,9 +58,20 @@ test("upgrade-sandboxes detects and rebuilds stale OpenClaw sandboxes (#1904)", assertDockerAvailable(dockerInfo, skip); registerStateRestore(cleanup); - cleanup.add(`destroy stale sandbox ${SANDBOX_NAME}`, () => cleanupStaleSandbox(host, sandbox)); - cleanup.add("remove stale OpenClaw test image", () => cleanupOldImage(host)); - await cleanupStaleSandbox(host, sandbox); + cleanup.trackDisposable("remove stale OpenClaw test image", () => cleanupOldImage(host)); + cleanup.trackDisposable(`delete OpenShell sandbox ${SANDBOX_NAME}`, () => + sandbox.cleanupSandbox(SANDBOX_NAME, { + artifactName: "cleanup-openshell-delete-upgrade-stale", + env: commandEnv(), + timeoutMs: 60_000, + }), + ); + cleanup.trackSandbox(host, SANDBOX_NAME, { + artifactName: "cleanup-nemoclaw-destroy-upgrade-stale", + env: commandEnv(), + timeoutMs: 120_000, + }); + await precleanStaleSandbox(host, sandbox); const install = await installCurrentNemoclaw(host, hosted); expect(install.exitCode, resultText(install)).toBe(0); diff --git a/test/e2e/mock-parity.json b/test/e2e/mock-parity.json index 2e24128df0b..0e9ad117ed2 100644 --- a/test/e2e/mock-parity.json +++ b/test/e2e/mock-parity.json @@ -7,7 +7,9 @@ "fast": [ "test/e2e/support/hermes-gpu-startup-fallback.test.ts", "test/e2e/support/hermes-gpu-startup-integrity.test.ts", - "test/e2e/support/hermes-workflow-boundary.test.ts" + "test/e2e/support/hermes-workflow-boundary.test.ts", + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" ] }, { @@ -24,7 +26,11 @@ }, { "live": "test/e2e/live/full-e2e.test.ts", - "fast": ["test/e2e/support/onboard-performance.test.ts"] + "fast": [ + "test/e2e/support/onboard-performance.test.ts", + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] }, { "live": "test/e2e/live/gateway-guard-recovery.test.ts", @@ -39,7 +45,11 @@ }, { "live": "test/e2e/live/network-policy.test.ts", - "fast": ["test/validate-blueprint.test.ts"] + "fast": [ + "test/validate-blueprint.test.ts", + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] }, { "live": "test/e2e/live/openshell-gateway-auth-source-contract.test.ts", @@ -70,7 +80,9 @@ "src/lib/onboard/extra-provider-reconciliation.test.ts", "src/lib/onboard/machine/handlers/sandbox-resume.test.ts", "src/lib/onboard/sandbox-create-plan-extra-providers.test.ts", - "test/onboard-extra-provider-reconciliation.test.ts" + "test/onboard-extra-provider-reconciliation.test.ts", + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" ] }, { @@ -82,7 +94,9 @@ "src/lib/onboard/machine/handlers/sandbox-recreate-resume.test.ts", "src/lib/onboard/machine/handlers/sandbox-resume.test.ts", "src/lib/onboard/sandbox-create-plan-extra-providers.test.ts", - "test/onboard-extra-provider-reconciliation.test.ts" + "test/onboard-extra-provider-reconciliation.test.ts", + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" ] }, { @@ -90,14 +104,362 @@ "fast": [ "test/cli/dispatch-basics.test.ts", "test/e2e/support/e2e-workflow.test.ts", - "test/sandbox-rlimit-hooks.test.ts" + "test/sandbox-rlimit-hooks.test.ts", + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" ] }, { "live": "test/e2e/live/snapshot-commands.test.ts", "fast": [ "test/e2e/support/e2e-workflow.test.ts", - "test/e2e/support/snapshot-commands-helpers.test.ts" + "test/e2e/support/snapshot-commands-helpers.test.ts", + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] + }, + { + "live": "test/e2e/live/agent-turn-latency.test.ts", + "fast": [ + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] + }, + { + "live": "test/e2e/live/bedrock-runtime-compatible-anthropic.test.ts", + "fast": [ + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] + }, + { + "live": "test/e2e/live/brave-search.test.ts", + "fast": [ + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] + }, + { + "live": "test/e2e/live/channels-add-remove.test.ts", + "fast": [ + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] + }, + { + "live": "test/e2e/live/cloud-inference.test.ts", + "fast": [ + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] + }, + { + "live": "test/e2e/live/cloud-onboard.test.ts", + "fast": [ + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] + }, + { + "live": "test/e2e/live/common-egress-agent.test.ts", + "fast": [ + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] + }, + { + "live": "test/e2e/live/concurrent-gateway-ports.test.ts", + "fast": [ + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] + }, + { + "live": "test/e2e/live/credential-migration.test.ts", + "fast": [ + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] + }, + { + "live": "test/e2e/live/credential-sanitization.test.ts", + "fast": [ + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] + }, + { + "live": "test/e2e/live/cron-preflight-inference-local.test.ts", + "fast": [ + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] + }, + { + "live": "test/e2e/live/device-auth-health.test.ts", + "fast": [ + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] + }, + { + "live": "test/e2e/live/diagnostics.test.ts", + "fast": [ + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] + }, + { + "live": "test/e2e/live/double-onboard.test.ts", + "fast": [ + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] + }, + { + "live": "test/e2e/live/gateway-health-honest.test.ts", + "fast": [ + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] + }, + { + "live": "test/e2e/live/gpu-double-onboard.test.ts", + "fast": [ + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] + }, + { + "live": "test/e2e/live/gpu-e2e.test.ts", + "fast": [ + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] + }, + { + "live": "test/e2e/live/hermes-discord.test.ts", + "fast": [ + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] + }, + { + "live": "test/e2e/live/hermes-e2e.test.ts", + "fast": [ + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] + }, + { + "live": "test/e2e/live/hermes-inference-switch.test.ts", + "fast": [ + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] + }, + { + "live": "test/e2e/live/hermes-shields-config.test.ts", + "fast": [ + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] + }, + { + "live": "test/e2e/live/issue-2478-crash-loop-recovery.test.ts", + "fast": [ + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] + }, + { + "live": "test/e2e/live/issue-4462-scope-upgrade-approval.test.ts", + "fast": [ + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] + }, + { + "live": "test/e2e/live/jetson-nvmap-gpu.test.ts", + "fast": [ + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] + }, + { + "live": "test/e2e/live/kimi-inference-compat.test.ts", + "fast": [ + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] + }, + { + "live": "test/e2e/live/mcp-bridge.test.ts", + "fast": [ + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] + }, + { + "live": "test/e2e/live/messaging-compatible-endpoint.test.ts", + "fast": [ + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] + }, + { + "live": "test/e2e/live/messaging-providers.test.ts", + "fast": [ + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] + }, + { + "live": "test/e2e/live/model-router-provider-routed-inference.test.ts", + "fast": [ + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] + }, + { + "live": "test/e2e/live/ollama-auth-proxy.test.ts", + "fast": [ + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] + }, + { + "live": "test/e2e/live/openclaw-discord-pairing.test.ts", + "fast": [ + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] + }, + { + "live": "test/e2e/live/openclaw-inference-switch.test.ts", + "fast": [ + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] + }, + { + "live": "test/e2e/live/openclaw-plugin-runtime-exdev.test.ts", + "fast": [ + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] + }, + { + "live": "test/e2e/live/openclaw-skill-cli.test.ts", + "fast": [ + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] + }, + { + "live": "test/e2e/live/openclaw-slack-pairing.test.ts", + "fast": [ + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] + }, + { + "live": "test/e2e/live/openshell-gateway-upgrade.test.ts", + "fast": [ + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] + }, + { + "live": "test/e2e/live/overlayfs-autofix.test.ts", + "fast": [ + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts", + "test/e2e/support/overlayfs-autofix-outcome.test.ts" + ] + }, + { + "live": "test/e2e/live/rebuild-hermes.test.ts", + "fast": [ + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] + }, + { + "live": "test/e2e/live/rebuild-openclaw.test.ts", + "fast": [ + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] + }, + { + "live": "test/e2e/live/sandbox-rebuild.test.ts", + "fast": [ + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] + }, + { + "live": "test/e2e/live/sandbox-survival.test.ts", + "fast": [ + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] + }, + { + "live": "test/e2e/live/sessions-agents-cli.test.ts", + "fast": [ + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] + }, + { + "live": "test/e2e/live/shields-config.test.ts", + "fast": [ + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] + }, + { + "live": "test/e2e/live/skill-agent.test.ts", + "fast": [ + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] + }, + { + "live": "test/e2e/live/spark-install.test.ts", + "fast": [ + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] + }, + { + "live": "test/e2e/live/state-backup-restore.test.ts", + "fast": [ + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] + }, + { + "live": "test/e2e/live/telegram-injection.test.ts", + "fast": [ + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] + }, + { + "live": "test/e2e/live/token-rotation.test.ts", + "fast": [ + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] + }, + { + "live": "test/e2e/live/upgrade-stale-sandbox.test.ts", + "fast": [ + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" ] } ] diff --git a/test/e2e/support/e2e-cleanup-resources.test.ts b/test/e2e/support/e2e-cleanup-resources.test.ts index acb2a19f155..689e1c84e16 100644 --- a/test/e2e/support/e2e-cleanup-resources.test.ts +++ b/test/e2e/support/e2e-cleanup-resources.test.ts @@ -1,9 +1,23 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it } from "vitest"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it, vi } from "vitest"; import { type CleanupHost, CleanupRegistry } from "../fixtures/cleanup.ts"; +import { + assertCleanupSucceededOrAbsent, + cleanupAcquiredResource, + cleanupExistingPath, + cleanupUnlessVerified, + cleanupWhenCommandAvailable, + cleanupWhenOpenShellAvailable, + registerSandboxCleanupUnlessKept, + terminateProcessIfRunning, +} from "../fixtures/cleanup-resources.ts"; describe("cleanup resources", () => { it("tears down acquired resources in reverse order", async () => { @@ -30,41 +44,98 @@ describe("cleanup resources", () => { }); it("passes cleanup run options through tracked resources", async () => { - const calls: unknown[] = []; + const calls: Array<{ resource: string; options: unknown }> = []; const host: CleanupHost = { cleanupSandbox: async (_name, options) => { - calls.push(options); + calls.push({ resource: "sandbox", options }); }, cleanupGatewayRegistration: async (_name, options) => { - calls.push(options); + calls.push({ resource: "gateway", options }); }, cleanupForward: async (_port, options) => { - calls.push(options); + calls.push({ resource: "forward", options }); }, }; const cleanup = new CleanupRegistry(); - const options = { artifactName: "cleanup-resource", timeoutMs: 1000 }; + const gatewayOptions = { + artifactName: "cleanup-gateway", + env: { OPENSHELL_GATEWAY: "nemoclaw" }, + redactionValues: ["gateway-secret"], + timeoutMs: 1_000, + }; + const sandboxOptions = { + artifactName: "cleanup-sandbox", + env: { NEMOCLAW_GATEWAY_PORT: "18080" }, + redactionValues: ["sandbox-secret"], + timeoutMs: 2_000, + }; + const forwardOptions = { + artifactName: "cleanup-forward", + env: { OPENSHELL_GATEWAY: "nemoclaw-18080" }, + redactionValues: ["forward-secret"], + timeoutMs: 3_000, + }; - cleanup.trackGateway(host, "nemoclaw", options); - cleanup.trackSandbox(host, "e2e-resource", options); - cleanup.trackForward(host, 18789, options); + cleanup.trackGateway(host, "nemoclaw", gatewayOptions); + cleanup.trackSandbox(host, "e2e-resource", sandboxOptions); + cleanup.trackForward(host, 18789, forwardOptions); await cleanup.runAll(); - expect(calls).toEqual([options, options, options]); + expect(calls).toEqual([ + { resource: "forward", options: forwardOptions }, + { resource: "sandbox", options: sandboxOptions }, + { resource: "gateway", options: gatewayOptions }, + ]); }); - it("supports partial setup and runs each registration only once", async () => { + it("supports partial typed setup and runs each registration only once", async () => { let calls = 0; + const host: CleanupHost = { + cleanupSandbox: async () => { + calls += 1; + }, + cleanupGatewayRegistration: async () => { + throw new Error("unregistered gateway cleanup must not run"); + }, + cleanupForward: async () => { + throw new Error("unregistered forward cleanup must not run"); + }, + }; const cleanup = new CleanupRegistry(); - cleanup.trackDisposable("close acquired server", () => { - calls += 1; - }); + cleanup.trackSandbox(host, "partially-created"); - expect((await cleanup.runAll()).passed).toEqual(["close acquired server"]); + expect((await cleanup.runAll()).passed).toEqual(["destroy sandbox partially-created"]); expect(await cleanup.runAll()).toEqual({ passed: [], failures: [] }); expect(calls).toBe(1); }); + it("continues typed cleanup after a resource failure", async () => { + const calls: string[] = []; + const host: CleanupHost = { + cleanupSandbox: async () => { + calls.push("sandbox"); + throw new Error("sandbox cleanup denied"); + }, + cleanupGatewayRegistration: async () => { + calls.push("gateway"); + }, + cleanupForward: async () => { + calls.push("forward"); + }, + }; + const cleanup = new CleanupRegistry(); + cleanup.trackGateway(host, "nemoclaw"); + cleanup.trackSandbox(host, "e2e-resource"); + cleanup.trackForward(host, 18789); + + const result = await cleanup.runAll(); + expect(calls).toEqual(["forward", "sandbox", "gateway"]); + expect(result).toEqual({ + passed: ["stop forward 18789", "remove gateway nemoclaw"], + failures: [{ name: "destroy sandbox e2e-resource", message: "sandbox cleanup denied" }], + }); + }); + it("redacts failures and continues cleanup", async () => { const calls: string[] = []; const cleanup = new CleanupRegistry((text) => text.replaceAll("secret", "[REDACTED]")); @@ -82,4 +153,123 @@ describe("cleanup resources", () => { failures: [{ name: "failing [REDACTED] cleanup", message: "[REDACTED] failure" }], }); }); + + it("accepts successful or explicitly absent cleanup results and rejects other failures (#6352)", () => { + expect(() => + assertCleanupSucceededOrAbsent( + { exitCode: 0, stderr: "", stdout: "removed" }, + false, + "remove resource", + ), + ).not.toThrow(); + expect(() => + assertCleanupSucceededOrAbsent( + { exitCode: 1, stderr: "resource not found", stdout: "" }, + /not found/i, + "remove resource", + ), + ).not.toThrow(); + expect(() => + assertCleanupSucceededOrAbsent( + { exitCode: 1, stderr: "already absent", stdout: "" }, + true, + "remove resource", + ), + ).not.toThrow(); + expect(() => + assertCleanupSucceededOrAbsent( + { exitCode: 1, stderr: "permission denied", stdout: "" }, + /not found/i, + "remove resource", + ), + ).toThrow("remove resource failed: permission denied"); + }); + + it("runs acquired and existing-path cleanup only after acquisition (#6352)", async () => { + const cleanup = vi.fn(async () => {}); + await cleanupAcquiredResource(false, cleanup); + await cleanupAcquiredResource(true, cleanup); + + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cleanup-resource-")); + const marker = path.join(directory, "acquired"); + fs.writeFileSync(marker, "acquired", "utf8"); + await cleanupExistingPath(marker, () => fs.rmSync(marker)); + await cleanupExistingPath(marker, cleanup); + fs.rmSync(directory, { recursive: true, force: true }); + + expect(cleanup).toHaveBeenCalledTimes(1); + }); + + it("does not repeat cleanup after explicit teardown has been verified (#6352)", async () => { + const cleanup = vi.fn(async () => {}); + await cleanupUnlessVerified(true, cleanup); + await cleanupUnlessVerified(false, cleanup); + expect(cleanup).toHaveBeenCalledTimes(1); + }); + + it("runs command-owned cleanup only when the managing command is available (#6352)", async () => { + const isCommandAvailable = vi.fn().mockResolvedValueOnce(false).mockResolvedValueOnce(true); + const cleanup = vi.fn(async () => {}); + const probeOptions = { artifactName: "cleanup-command-probe", timeoutMs: 1_000 }; + + await cleanupWhenCommandAvailable({ isCommandAvailable }, "openshell", probeOptions, cleanup); + await cleanupWhenCommandAvailable({ isCommandAvailable }, "openshell", probeOptions, cleanup); + + expect(isCommandAvailable).toHaveBeenCalledTimes(2); + expect(isCommandAvailable).toHaveBeenNthCalledWith(1, "openshell", probeOptions); + expect(cleanup).toHaveBeenCalledTimes(1); + }); + + it("uses the configured OpenShell command and preserves strict cleanup failures (#6352)", async () => { + const isCommandAvailable = vi.fn().mockResolvedValueOnce(false).mockResolvedValueOnce(true); + const host = { + isCommandAvailable, + openshellCommandPath: "/opt/openshell/bin/openshell", + }; + const cleanup = vi + .fn<() => Promise>() + .mockRejectedValueOnce(new Error("permission denied")); + const probeOptions = { artifactName: "cleanup-openshell-probe", timeoutMs: 1_000 }; + + await cleanupWhenOpenShellAvailable(host, probeOptions, cleanup); + await expect(cleanupWhenOpenShellAvailable(host, probeOptions, cleanup)).rejects.toThrow( + "permission denied", + ); + + expect(isCommandAvailable).toHaveBeenNthCalledWith( + 1, + "/opt/openshell/bin/openshell", + probeOptions, + ); + expect(isCommandAvailable).toHaveBeenNthCalledWith( + 2, + "/opt/openshell/bin/openshell", + probeOptions, + ); + expect(cleanup).toHaveBeenCalledTimes(1); + }); + + it("does not register destructive sandbox cleanup when retention is requested (#6352)", () => { + const register = vi.fn(); + registerSandboxCleanupUnlessKept(true, register); + registerSandboxCleanupUnlessKept(false, register); + expect(register).toHaveBeenCalledTimes(1); + }); + + it("tolerates only an already-exited process during termination (#6352)", () => { + const kill = vi.spyOn(process, "kill"); + kill.mockImplementationOnce(() => { + throw Object.assign(new Error("already exited"), { code: "ESRCH" }); + }); + expect(() => terminateProcessIfRunning(1234)).not.toThrow(); + + kill.mockImplementationOnce(() => { + throw Object.assign(new Error("not permitted"), { code: "EPERM" }); + }); + expect(() => terminateProcessIfRunning(1234)).toThrow("not permitted"); + + kill.mockImplementationOnce(() => true); + terminateProcessIfRunning(1234, "SIGKILL"); + expect(kill).toHaveBeenLastCalledWith(1234, "SIGKILL"); + }); }); diff --git a/test/e2e/support/e2e-clients.test.ts b/test/e2e/support/e2e-clients.test.ts index 44d7d7dd419..230a9370751 100644 --- a/test/e2e/support/e2e-clients.test.ts +++ b/test/e2e/support/e2e-clients.test.ts @@ -97,6 +97,40 @@ describe("E2E fixture clients", () => { ]); }); + it.each([ + { exitCode: 0, expected: true, label: "available" }, + { exitCode: 1, expected: false, label: "missing" }, + ])("host client reports a command as $label", async ({ exitCode, expected }) => { + const runner = new FakeRunner(); + runner.enqueue({ exitCode }); + const host = new HostCliClient(runner, { cliPath: "nemoclaw" }); + const options = { + artifactName: "cleanup-command-probe", + env: { PATH: "/test/bin" }, + redactionValues: ["cleanup-secret"], + timeoutMs: 123_000, + }; + + await expect(host.isCommandAvailable("openshell", options)).resolves.toBe(expected); + expect(runner.calls).toEqual([ + { + command: "bash", + args: ["-lc", 'command -v "$1" >/dev/null 2>&1', "command-availability-probe", "openshell"], + options, + }, + ]); + }); + + it("host client surfaces an unexpected command availability failure", async () => { + const runner = new FakeRunner(); + runner.enqueue({ exitCode: 2, stderr: "shell probe failed" }); + const host = new HostCliClient(runner, { cliPath: "nemoclaw" }); + + await expect(host.isCommandAvailable("openshell")).rejects.toThrow( + "probe command availability for openshell failed: shell probe failed", + ); + }); + it("host client validates list/status and cleans up sandbox destroys", async () => { const runner = new FakeRunner(); runner.stdout = "NAME\nassistant\n"; @@ -233,6 +267,51 @@ describe("E2E fixture clients", () => { ); }); + it("host cleanup resources preserve caller probe options and gateway artifact suffixes", async () => { + const runner = new FakeRunner(); + runner.enqueue({ exitCode: 0 }); + runner.enqueue({ exitCode: 2, stderr: "unrecognized subcommand 'remove'" }); + runner.enqueue({ exitCode: 0 }); + runner.enqueue({ exitCode: 0 }); + const host = new HostCliClient(runner, { + cliPath: "nemoclaw", + openshellPath: "/opt/openshell/bin/openshell", + }); + const options = { + artifactName: "resource-cleanup", + env: { OPENSHELL_GATEWAY: "nemoclaw-18080" }, + redactionValues: ["cleanup-secret"], + timeoutMs: 123_000, + }; + + await host.cleanupSandbox("assistant", options); + await host.cleanupGatewayRegistration("nemoclaw-18080", options); + await host.cleanupForward(18789, options); + + expect(runner.calls).toEqual([ + { + command: "nemoclaw", + args: ["assistant", "destroy", "--yes"], + options, + }, + { + command: "/opt/openshell/bin/openshell", + args: ["gateway", "remove", "nemoclaw-18080"], + options: { ...options, artifactName: "resource-cleanup-remove" }, + }, + { + command: "/opt/openshell/bin/openshell", + args: ["gateway", "destroy", "-g", "nemoclaw-18080"], + options: { ...options, artifactName: "resource-cleanup-legacy-destroy" }, + }, + { + command: "/opt/openshell/bin/openshell", + args: ["forward", "stop", "18789"], + options, + }, + ]); + }); + it("host client propagates cwd, env, and timeout options", async () => { const runner = new FakeRunner(); const host = new HostCliClient(runner, { @@ -341,6 +420,49 @@ describe("E2E fixture clients", () => { }); }); + it("sandbox client removes an OpenShell sandbox with caller cleanup options", async () => { + const runner = new FakeRunner(); + const sandbox = new SandboxClient(runner); + const options = { + artifactName: "cleanup-partial-sandbox", + env: { OPENSHELL_GATEWAY: "nemoclaw-18080" }, + redactionValues: ["secret-cleanup-value"], + timeoutMs: 60_000, + }; + + await sandbox.cleanupSandbox("assistant", options); + + expect(runner.calls).toEqual([ + { + command: "openshell", + args: ["sandbox", "delete", "assistant"], + options, + }, + ]); + }); + + it.each([ + "NotFound: sandbox assistant", + "sandbox assistant not present", + "no such sandbox: assistant", + ])("sandbox client accepts canonical already-absent cleanup output: %s", async (stderr) => { + const runner = new FakeRunner(); + runner.enqueue({ exitCode: 1, stderr }); + const sandbox = new SandboxClient(runner); + + await expect(sandbox.cleanupSandbox("assistant")).resolves.toBeUndefined(); + }); + + it("sandbox client surfaces unexpected cleanup failures", async () => { + const runner = new FakeRunner(); + runner.enqueue({ exitCode: 1, stderr: "permission denied" }); + const sandbox = new SandboxClient(runner); + + await expect(sandbox.cleanupSandbox("assistant")).rejects.toThrow( + "cleanup OpenShell sandbox assistant failed: permission denied", + ); + }); + it("sandbox client validates list output using the OpenShell gateway env", async () => { const runner = new FakeRunner(); runner.stdout = "NAME\nassistant\n"; diff --git a/test/e2e/support/messaging-compatible-endpoint-helpers.test.ts b/test/e2e/support/messaging-compatible-endpoint-helpers.test.ts index 536d8e43ca5..faa1072a067 100644 --- a/test/e2e/support/messaging-compatible-endpoint-helpers.test.ts +++ b/test/e2e/support/messaging-compatible-endpoint-helpers.test.ts @@ -1,12 +1,20 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it } from "vitest"; +import { spawn } from "node:child_process"; +import { once } from "node:events"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it, vi } from "vitest"; import type { HostCliClient } from "../fixtures/clients/host.ts"; import { cleanupMessagingState, + cleanupOwnedGatewayRuntimeStrict, parseOpenClawAgentText, + stopGatewayRuntime, } from "../live/messaging-compatible-endpoint-helpers.ts"; const COMPAT_AGENT_REPLY = "COMPAT_MOCK_ROUTE_5098_OK"; @@ -14,9 +22,83 @@ const COMPAT_AGENT_PROMPT = "Call the configured model and report the compatible endpoint route token."; describe("messaging compatible endpoint helper coverage", () => { + it.runIf(process.platform === "linux")( + "never signals an unrelated process from a stale gateway PID file (#6352)", + async () => { + const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-gateway-pid-")); + const child = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { + stdio: "ignore", + }); + await once(child, "spawn"); + const pid = child.pid; + expect(pid).toBeTypeOf("number"); + fs.writeFileSync(path.join(stateDir, "openshell-gateway.pid"), String(pid), "utf8"); + vi.stubEnv("NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR", stateDir); + const command = vi.fn(async () => ({ exitCode: 0 })); + const host = { + command, + openshellCommandPath: "/configured/openshell", + } as unknown as HostCliClient; + + try { + await expect( + stopGatewayRuntime(host, "preclean-unrelated-gateway-pid"), + ).resolves.toBeUndefined(); + expect(() => process.kill(pid!, 0)).not.toThrow(); + await expect( + cleanupOwnedGatewayRuntimeStrict(host, "strict-unrelated-gateway-pid"), + ).rejects.toThrow(/does not prove ownership/u); + expect(() => process.kill(pid!, 0)).not.toThrow(); + fs.writeFileSync(path.join(stateDir, "openshell-gateway.pid"), "not-a-pid", "utf8"); + await expect( + cleanupOwnedGatewayRuntimeStrict(host, "strict-invalid-gateway-pid"), + ).rejects.toThrow(/PID file is invalid or unreadable/u); + expect(command).toHaveBeenCalledTimes(1); + } finally { + vi.unstubAllEnvs(); + process.kill(pid!, "SIGKILL"); + await once(child, "exit"); + fs.rmSync(stateDir, { force: true, recursive: true }); + } + }, + ); + + it.runIf(process.platform === "linux")( + "signals only a start-time-matched owned gateway process (#6352)", + async () => { + const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-owned-gateway-pid-")); + const child = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { + argv0: "openshell-gateway[nemoclaw=nemoclaw;port=8080]", + stdio: "ignore", + }); + await once(child, "spawn"); + const childExit = once(child, "exit"); + const pid = child.pid; + expect(pid).toBeTypeOf("number"); + fs.writeFileSync(path.join(stateDir, "openshell-gateway.pid"), String(pid), "utf8"); + vi.stubEnv("NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR", stateDir); + const command = vi.fn(async () => ({ exitCode: 0, stderr: "", stdout: "" })); + const host = { command } as unknown as HostCliClient; + + try { + await expect( + cleanupOwnedGatewayRuntimeStrict(host, "strict-owned-gateway-pid"), + ).resolves.toBeUndefined(); + await childExit; + expect(command).toHaveBeenCalledTimes(1); + } finally { + vi.unstubAllEnvs(); + child.kill("SIGKILL"); + await childExit; + fs.rmSync(stateDir, { force: true, recursive: true }); + } + }, + ); + it("keeps missing-sandbox cleanup from masking endpoint validation evidence", async () => { const calls: Array<{ command: string; args: string[] }> = []; const host = { + openshellCommandPath: "openshell", command: async (command: string, args: string[]) => { calls.push({ command, args }); throw new Error("Sandbox e2e-msg-compat-missing does not exist"); @@ -44,7 +126,8 @@ describe("messaging compatible endpoint helper coverage", () => { }); expect(calls[2]?.command).toBe("bash"); expect(calls[2]?.args[0]).toBe("-lc"); - expect(calls[2]?.args[1]).toContain("openshell gateway destroy -g nemoclaw"); + expect(calls[2]?.args[1]).toContain('"$openshell_bin" gateway destroy -g nemoclaw'); + expect(calls[2]?.args.at(-1)).toBe("openshell"); }); it("extracts noisy OpenClaw JSON while rejecting prompt echo text", () => { diff --git a/test/e2e/support/overlayfs-autofix-outcome.test.ts b/test/e2e/support/overlayfs-autofix-outcome.test.ts index 26de71cc54d..7d815e14784 100644 --- a/test/e2e/support/overlayfs-autofix-outcome.test.ts +++ b/test/e2e/support/overlayfs-autofix-outcome.test.ts @@ -1,10 +1,20 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it } from "vitest"; +import fs from "node:fs"; + +import { afterEach, describe, expect, it, vi } from "vitest"; +import { CleanupRegistry } from "../fixtures/cleanup.ts"; +import type { HostCliClient } from "../fixtures/clients/host.ts"; +import type { SandboxClient } from "../fixtures/clients/sandbox.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; +import { trackOverlayfsAutofixCleanup } from "../live/overlayfs-autofix-cleanup.ts"; import { negativeOverlayOutcome } from "../live/overlayfs-autofix-outcome.ts"; +afterEach(() => { + vi.restoreAllMocks(); +}); + function shellProbeResult(overrides: Partial): ShellProbeResult { return { command: ["timeout", "300", "bash", "install.sh", "--non-interactive"], @@ -18,6 +28,40 @@ function shellProbeResult(overrides: Partial): ShellProbeResul }; } +function cleanupClients(calls: string[]): { + command: ReturnType; + host: Pick; + sandbox: Pick; +} { + const command = vi + .fn() + .mockImplementationOnce(async () => { + calls.push("remove gateway container"); + return shellProbeResult({ exitCode: 0 }); + }) + .mockImplementationOnce(async () => { + calls.push("list patched images"); + return shellProbeResult({ exitCode: 0, stdout: "" }); + }); + return { + command, + host: { + command, + cleanupGatewayRegistration: async () => { + calls.push("remove gateway registration"); + }, + cleanupSandbox: async () => { + calls.push("destroy NemoClaw sandbox"); + }, + }, + sandbox: { + cleanupSandbox: async () => { + calls.push("delete OpenShell sandbox"); + }, + }, + }; +} + describe("overlayfs autofix negative outcome classifier", () => { it("treats nested-overlay evidence as a reproduced failure", () => { expect( @@ -43,3 +87,98 @@ describe("overlayfs autofix negative outcome classifier", () => { ).toBe("unrelated"); }); }); + +describe("overlayfs autofix cleanup resources", () => { + it("does not register destructive cleanup when the sandbox is preserved (#6352)", async () => { + const calls: string[] = []; + const { command, host, sandbox } = cleanupClients(calls); + const cleanup = new CleanupRegistry(); + trackOverlayfsAutofixCleanup({ + cleanup, + cleanupEnv: {}, + gatewayContainer: "nemoclaw-gateway", + host, + preserveSandbox: true, + redactionValues: [], + sandbox, + sandboxName: "e2e-overlayfs-autofix", + }); + + expect(await cleanup.runAll()).toEqual({ failures: [], passed: [] }); + expect(command).not.toHaveBeenCalled(); + expect(calls).toEqual([]); + }); + + it("tears down sandbox resources before gateway artifacts in LIFO order (#6352)", async () => { + const calls: string[] = []; + const { host, sandbox } = cleanupClients(calls); + const cleanup = new CleanupRegistry(); + const remove = vi.spyOn(fs, "rmSync").mockImplementation(() => {}); + trackOverlayfsAutofixCleanup({ + cleanup, + cleanupEnv: { OPENSHELL_GATEWAY: "nemoclaw" }, + gatewayContainer: "nemoclaw-gateway", + host, + preserveSandbox: false, + redactionValues: ["cleanup-secret"], + sandbox, + sandboxName: "e2e-overlayfs-autofix", + }); + + const result = await cleanup.runAll(); + expect(result.failures).toEqual([]); + expect(calls).toEqual([ + "destroy NemoClaw sandbox", + "delete OpenShell sandbox", + "remove gateway registration", + "remove gateway container", + "list patched images", + ]); + expect(remove).toHaveBeenCalledWith(expect.stringContaining(".nemoclaw/onboard.lock"), { + force: true, + }); + }); + + it("continues image and lock cleanup after gateway container removal fails (#6352)", async () => { + const calls: string[] = []; + const { command, host, sandbox } = cleanupClients(calls); + command.mockReset(); + command + .mockImplementationOnce(async () => { + calls.push("remove gateway container"); + return shellProbeResult({ exitCode: 1, stderr: "permission denied" }); + }) + .mockImplementationOnce(async () => { + calls.push("list patched images"); + return shellProbeResult({ exitCode: 0, stdout: "" }); + }); + const cleanup = new CleanupRegistry(); + const remove = vi.spyOn(fs, "rmSync").mockImplementation(() => {}); + trackOverlayfsAutofixCleanup({ + cleanup, + cleanupEnv: {}, + gatewayContainer: "nemoclaw-gateway", + host, + preserveSandbox: false, + redactionValues: [], + sandbox, + sandboxName: "e2e-overlayfs-autofix", + }); + + const result = await cleanup.runAll(); + expect(result.failures).toEqual([ + { + message: "remove overlayfs gateway container failed: permission denied", + name: "remove overlayfs gateway container", + }, + ]); + expect(calls).toEqual([ + "destroy NemoClaw sandbox", + "delete OpenShell sandbox", + "remove gateway registration", + "remove gateway container", + "list patched images", + ]); + expect(remove).toHaveBeenCalledOnce(); + }); +});