refactor(e2e): complete cleanup resource tracking

test/e2e/fixtures/cleanup.ts

@@ -42,15 +42,27 @@ export class CleanupRegistry {
     this.entries.push({ name, run });
   }
 
-  trackSandbox(host: CleanupHost, name: string, options: ShellProbeRunOptions = {}): void {
+  trackSandbox(
+    host: Pick<CleanupHost, "cleanupSandbox">,
+    name: string,
+    options: ShellProbeRunOptions = {},
+  ): void {
     this.add(`destroy sandbox ${name}`, () => host.cleanupSandbox(name, options));
   }
 
-  trackGateway(host: CleanupHost, name: string, options: ShellProbeRunOptions = {}): void {
+  trackGateway(
+    host: Pick<CleanupHost, "cleanupGatewayRegistration">,
+    name: string,
+    options: ShellProbeRunOptions = {},
+  ): void {
     this.add(`remove gateway ${name}`, () => host.cleanupGatewayRegistration(name, options));
   }
 
-  trackForward(host: CleanupHost, port: number, options: ShellProbeRunOptions = {}): void {
+  trackForward(
+    host: Pick<CleanupHost, "cleanupForward">,
+    port: number,
+    options: ShellProbeRunOptions = {},
+  ): void {
     this.add(`stop forward ${port}`, () => host.cleanupForward(port, options));
   }
 

test/e2e/fixtures/clients/host.ts

@@ -58,6 +58,23 @@ export class HostCliClient {
     );
   }
 
+  async isCommandAvailable(command: string, options: ShellProbeRunOptions = {}): Promise<boolean> {
+    const result = await this.command(
+      "bash",
+      ["-lc", 'command -v "$1" >/dev/null 2>&1', "command-availability-probe", command],
+      {
+        artifactName: `command-available-${artifactLabel(command)}`,
+        env: buildAvailabilityProbeEnv(),
+        timeoutMs: 30_000,
+        ...options,
+      },
+    );
+    if (result.exitCode === 0) return true;
+    if (result.exitCode === 1) return false;
+    assertExitZero(result, `probe command availability for ${command}`);
+    return false;
+  }
+
   nemoclaw(args: string[] = [], options: ShellProbeRunOptions = {}): Promise<ShellProbeResult> {
     return this.command(this.cliPath, args, {
       artifactName: `nemoclaw-${artifactLabel(args.join("-") || "default")}`,

test/e2e/fixtures/clients/sandbox.ts

@@ -10,8 +10,12 @@ import {
   assertExitZero,
   type CommandRunner,
   outputContainsSandbox,
+  resultText,
 } from "./command.ts";
 
+const SANDBOX_ALREADY_ABSENT =
+  /\bNotFound\b|\bNot Found\b|sandbox[^\n]*(?:not found|not present|does not exist)|no such sandbox/i;
+
 /**
  * Default env for openshell-targeted spawns. ShellProbe filters env via
  * the framework allowlist (HOME, PATH, …) which excludes OPENSHELL_GATEWAY,
@@ -99,6 +103,16 @@ export class SandboxClient {
     });
   }
 
+  async cleanupSandbox(name: string, options: ShellProbeRunOptions = {}): Promise<void> {
+    validateSandboxName(name);
+    const result = await this.openshell(["sandbox", "delete", name], {
+      artifactName: `cleanup-openshell-sandbox-${name}`,
+      ...options,
+    });
+    if (result.exitCode === 0 || SANDBOX_ALREADY_ABSENT.test(resultText(result))) return;
+    assertExitZero(result, `cleanup OpenShell sandbox ${name}`);
+  }
+
   exec(
     name: string,
     command: string[],

test/e2e/live/agent-turn-latency-helpers.ts

@@ -4,6 +4,7 @@
 import path from "node:path";
 
 import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts";
+import { resultText } from "../fixtures/clients/command.ts";
 import type { HostCliClient } from "../fixtures/clients/host.ts";
 import {
   type SandboxClient,
@@ -73,7 +74,10 @@ export function env(
   return out;
 }
 
-export async function bestEffort(label: string, run: () => Promise<unknown>): Promise<void> {
+export async function bestEffortPreclean(
+  label: string,
+  run: () => Promise<unknown>,
+): Promise<void> {
   try {
     await run();
   } catch (error) {
@@ -283,29 +287,25 @@ export async function cleanupTurnSandboxes(
     [OPENCLAW_SANDBOX, "openclaw"],
     [HERMES_SANDBOX, "hermes"],
   ] as const) {
-    await bestEffort(`destroy ${agent} sandbox`, () =>
-      host.command("node", [CLI, name, "destroy", "--yes"], {
-        artifactName: `cleanup-${agent}-destroy`,
-        env: env(name, agent),
-        timeoutMs: 120_000,
-      }),
+    await bestEffortPreclean(`destroy ${agent} sandbox`, () =>
+      cleanupTurnSandbox(host, name, agent),
     );
-    await bestEffort(`delete ${agent} sandbox`, () =>
+    await bestEffortPreclean(`delete ${agent} sandbox`, () =>
       sandbox.openshell(["sandbox", "delete", name], {
         artifactName: `cleanup-${agent}-delete`,
         env: env(name, agent),
         timeoutMs: 60_000,
       }),
     );
   }
-  await bestEffort("stop Hermes API forward", () =>
+  await bestEffortPreclean("stop Hermes API forward", () =>
     sandbox.openshell(["forward", "stop", "8642"], {
       artifactName: "cleanup-forward-stop-hermes-api",
       env: buildAvailabilityProbeEnv(),
       timeoutMs: 30_000,
     }),
   );
-  await bestEffort("destroy OpenShell gateway", () =>
+  await bestEffortPreclean("destroy OpenShell gateway", () =>
     sandbox.openshell(["gateway", "destroy", "-g", "nemoclaw"], {
       artifactName: "cleanup-gateway-destroy-turn-latency",
       env: buildAvailabilityProbeEnv(),
@@ -314,6 +314,26 @@ export async function cleanupTurnSandboxes(
   );
 }
 
+export async function cleanupTurnSandbox(
+  host: HostCliClient,
+  name: string,
+  agent: "openclaw" | "hermes",
+): Promise<void> {
+  const result = await host.command("node", [CLI, name, "destroy", "--yes"], {
+    artifactName: `cleanup-${agent}-destroy`,
+    env: env(name, agent),
+    timeoutMs: 120_000,
+  });
+  const output = resultText(result);
+  expect(
+    result.exitCode === 0 ||
+      /Sandbox '.+' does not exist|Run 'nemoclaw onboard' to create one|sandbox .* not found|no such sandbox/iu.test(
+        output,
+      ),
+    `cleanup ${agent} sandbox ${name}: ${output}`,
+  ).toBe(true);
+}
+
 export async function route(
   sandbox: SandboxClient,
   sandboxName: string,

test/e2e/live/agent-turn-latency.test.ts

@@ -14,6 +14,7 @@ import {
   assertOpenClawConfig,
   CLI,
   chatContent,
+  cleanupTurnSandbox,
   cleanupTurnSandboxes,
   EXPECTED_ROUTE_PROVIDER,
   env,
@@ -45,7 +46,36 @@ test("OpenClaw and Hermes complete real hosted inference turns within the latenc
     hermesSandbox: HERMES_SANDBOX,
   });
 
-  cleanup.add("destroy turn latency sandboxes", () => cleanupTurnSandboxes(host, sandbox));
+  cleanup.trackGateway(host, "nemoclaw", {
+    artifactName: "cleanup-gateway-destroy-turn-latency",
+    env: buildAvailabilityProbeEnv(),
+    timeoutMs: 60_000,
+  });
+  cleanup.trackForward(host, 8642, {
+    artifactName: "cleanup-forward-stop-hermes-api",
+    env: buildAvailabilityProbeEnv(),
+    timeoutMs: 30_000,
+  });
+  cleanup.trackDisposable("delete Hermes OpenShell sandbox", () =>
+    sandbox.cleanupSandbox(HERMES_SANDBOX, {
+      artifactName: "cleanup-hermes-delete",
+      env: env(HERMES_SANDBOX, "hermes"),
+      timeoutMs: 60_000,
+    }),
+  );
+  cleanup.trackDisposable("destroy Hermes sandbox", () =>
+    cleanupTurnSandbox(host, HERMES_SANDBOX, "hermes"),
+  );
+  cleanup.trackDisposable("delete OpenClaw OpenShell sandbox", () =>
+    sandbox.cleanupSandbox(OPENCLAW_SANDBOX, {
+      artifactName: "cleanup-openclaw-delete",
+      env: env(OPENCLAW_SANDBOX, "openclaw"),
+      timeoutMs: 60_000,
+    }),
+  );
+  cleanup.trackDisposable("destroy OpenClaw sandbox", () =>
+    cleanupTurnSandbox(host, OPENCLAW_SANDBOX, "openclaw"),
+  );
 
   const docker = await host.command("docker", ["info"], {
     artifactName: "docker-info",

test/e2e/live/bedrock-runtime-compatible-anthropic.test.ts

@@ -496,7 +496,7 @@ async function startFakeBedrockRuntimeMock(options: {
   };
 }
 
-async function bestEffort(run: () => Promise<unknown> | unknown): Promise<void> {
+async function bestEffortPreclean(run: () => Promise<unknown> | unknown): Promise<void> {
   try {
     await run();
   } catch {
@@ -549,12 +549,12 @@ async function cleanupOpenShellGateway(host: HostCliClient, home: string): Promi
 }
 
 async function cleanupSandboxState(host: HostCliClient, home: string): Promise<void> {
-  await bestEffort(() => cleanupNemoClawSandbox(host, home));
-  await bestEffort(() => cleanupOpenShellSandbox(host, home));
-  await bestEffort(() => cleanupOpenShellGateway(host, home));
+  await bestEffortPreclean(() => cleanupNemoClawSandbox(host, home));
+  await bestEffortPreclean(() => cleanupOpenShellSandbox(host, home));
+  await bestEffortPreclean(() => cleanupOpenShellGateway(host, home));
 }
 
-function stopBedrockAdapterBestEffort(home: string): void {
+function stopBedrockAdapter(home: string): void {
   const stateFile = path.join(home, ".nemoclaw", "bedrock-runtime-adapter.json");
   const pidFile = path.join(home, ".nemoclaw", "bedrock-runtime-adapter.pid");
   const tokenFile = path.join(home, ".nemoclaw", "bedrock-runtime-adapter-token");
@@ -568,8 +568,8 @@ function stopBedrockAdapterBestEffort(home: string): void {
       if (Number.isInteger(pid) && pid > 0 && isBedrockAdapterProcess(pid)) {
         try {
           process.kill(pid, "SIGTERM");
-        } catch {
-          // Already stopped.
+        } catch (error) {
+          if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error;
         }
       }
     }
@@ -602,21 +602,23 @@ async function restoreHostsFile(
   backupDir: string,
   home: string,
 ): Promise<void> {
-  await bestEffort(() =>
-    host.command("sudo", ["cp", backupPath, "/etc/hosts"], {
+  expectExitZero(
+    await host.command("sudo", ["cp", backupPath, "/etc/hosts"], {
       artifactName: "restore-etc-hosts-bedrock-runtime",
       env: testEnv(home),
       timeoutMs: 30_000,
     }),
+    "restore /etc/hosts after Bedrock Runtime mapping",
   );
-  await bestEffort(() =>
-    host.command("sudo", ["rm", "-rf", backupDir], {
+  expectExitZero(
+    await host.command("sudo", ["rm", "-rf", backupDir], {
       artifactName: "remove-etc-hosts-backup-bedrock-runtime",
       env: testEnv(home),
       timeoutMs: 30_000,
     }),
+    "remove Bedrock Runtime hosts backup",
   );
-  await bestEffort(() => fs.rmSync(backupDir, { recursive: true, force: true }));
+  fs.rmSync(backupDir, { recursive: true, force: true });
 }
 
 async function mapBedrockHostToLoopback(
@@ -715,8 +717,8 @@ async function installBedrockTlsRedirect(host: HostCliClient, home: string): Pro
 }
 
 async function removeBedrockTlsRedirect(host: HostCliClient, home: string): Promise<void> {
-  await bestEffort(() =>
-    host.command(
+  expectExitZero(
+    await host.command(
       "sudo",
       [
         "-n",
@@ -731,6 +733,7 @@ async function removeBedrockTlsRedirect(host: HostCliClient, home: string): Prom
         timeoutMs: 30_000,
       },
     ),
+    "remove Bedrock Runtime canonical TLS port redirect",
   );
 }
 
@@ -1310,24 +1313,39 @@ test("bedrock runtime compatible Anthropic endpoint routes through managed infer
   const hostsBackup = path.join(hostsBackupDir, "hosts");
   let mock: MockBedrockRuntime | undefined;
   let onboarding: RawRunResult | undefined;
+  let tlsRedirectInstalled = false;
 
-  cleanup.add(`remove Bedrock Runtime test home ${home}`, () =>
+  cleanup.trackDisposable(`remove Bedrock Runtime test home ${home}`, () =>
     fs.rmSync(home, { recursive: true, force: true }),
   );
-  cleanup.add(`destroy Bedrock Runtime sandbox ${SANDBOX_NAME}`, () =>
-    cleanupSandboxState(host, home),
-  );
-  cleanup.add("restore /etc/hosts after Bedrock Runtime mapping", () =>
-    restoreHostsFile(host, hostsBackup, hostsBackupDir, home),
+  cleanup.trackGateway(host, "nemoclaw", {
+    artifactName: "cleanup-openshell-gateway-destroy-bedrock-runtime",
+    env: testEnv(home),
+    timeoutMs: 120_000,
+  });
+  cleanup.trackDisposable(`delete Bedrock Runtime OpenShell sandbox ${SANDBOX_NAME}`, () =>
+    sandbox.cleanupSandbox(SANDBOX_NAME, {
+      artifactName: "cleanup-openshell-sandbox-delete-bedrock-runtime",
+      env: testEnv(home),
+      timeoutMs: 60_000,
+    }),
   );
-  cleanup.add("remove Bedrock Runtime canonical TLS port redirect", () =>
-    removeBedrockTlsRedirect(host, home),
+  cleanup.trackDisposable(`destroy Bedrock Runtime sandbox ${SANDBOX_NAME}`, () =>
+    cleanupNemoClawSandbox(host, home),
   );
-  cleanup.add("stop Bedrock Runtime adapter", () => stopBedrockAdapterBestEffort(home));
-  cleanup.add("stop fake Bedrock Runtime endpoint", async () => {
+  cleanup.trackDisposable("restore /etc/hosts after Bedrock Runtime mapping", async () => {
+    if (fs.existsSync(hostsBackup)) {
+      await restoreHostsFile(host, hostsBackup, hostsBackupDir, home);
+    }
+  });
+  cleanup.trackDisposable("remove Bedrock Runtime canonical TLS port redirect", async () => {
+    if (tlsRedirectInstalled) await removeBedrockTlsRedirect(host, home);
+  });
+  cleanup.trackDisposable("stop Bedrock Runtime adapter", () => stopBedrockAdapter(home));
+  cleanup.trackDisposable("stop fake Bedrock Runtime endpoint", async () => {
     if (mock) await mock.close();
   });
-  cleanup.add("write fake Bedrock Runtime log", async () => {
+  cleanup.trackDisposable("write fake Bedrock Runtime log", async () => {
     if (mock) {
       await artifacts.writeText(
         "fake-bedrock-runtime.log",
@@ -1379,6 +1397,7 @@ test("bedrock runtime compatible Anthropic endpoint routes through managed infer
   await prepareSourceCliAndOpenShell(host, home);
   await mapBedrockHostToLoopback(host, home, hostsBackup, skip);
   await installBedrockTlsRedirect(host, home);
+  tlsRedirectInstalled = true;
   const tls = createBedrockTlsFixture(home);
   mock = await startFakeBedrockRuntimeMock({
     port: BEDROCK_MOCK_PORT,