From ecedafc179e4c9a2af6f8fccce557a3c9b43a628 Mon Sep 17 00:00:00 2001
From: Julie Yaunches <jyaunches@nvidia.com>
Date: Thu, 11 Jun 2026 09:17:05 -0400
Subject: [PATCH 1/9] test(e2e): add Hermes live Vitest migration

---
 .github/workflows/e2e-vitest-scenarios.yaml |  73 ++-
 test/e2e-scenario/live/hermes-e2e.test.ts   | 549 ++++++++++++++++++++
 2 files changed, 620 insertions(+), 2 deletions(-)
 create mode 100644 test/e2e-scenario/live/hermes-e2e.test.ts

diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml
index 07bb8c54f54..999f71cdc39 100644
--- a/.github/workflows/e2e-vitest-scenarios.yaml
+++ b/.github/workflows/e2e-vitest-scenarios.yaml
@@ -45,14 +45,34 @@ jobs:
         run: |
           set -euo pipefail
           args=(--emit-live-matrix)
+          matrix=""
           if [ -n "${SCENARIOS}" ]; then
             if [[ ! "${SCENARIOS}" =~ ^[A-Za-z0-9_-]+(,[A-Za-z0-9_-]+)*$ ]]; then
               echo "::error::Invalid scenario input: ${SCENARIOS}" >&2
               exit 1
             fi
-            args+=(--scenarios "${SCENARIOS}")
+            registry_scenarios=()
+            IFS=, read -r -a selected_scenarios <<< "${SCENARIOS}"
+            for scenario in "${selected_scenarios[@]}"; do
+              case "${scenario}" in
+                hermes-e2e)
+                  # Free-standing live Vitest job below; not registry-driven.
+                  ;;
+                *)
+                  registry_scenarios+=("${scenario}")
+                  ;;
+              esac
+            done
+            if [ "${#registry_scenarios[@]}" -gt 0 ]; then
+              registry_csv=$(IFS=,; echo "${registry_scenarios[*]}")
+              args+=(--scenarios "${registry_csv}")
+              matrix="$(npx tsx test/e2e-scenario/scenarios/run.ts "${args[@]}")"
+            else
+              matrix="[]"
+            fi
+          else
+            matrix="$(npx tsx test/e2e-scenario/scenarios/run.ts "${args[@]}")"
           fi
-          matrix="$(npx tsx test/e2e-scenario/scenarios/run.ts "${args[@]}")"
           echo "matrix=${matrix}" >> "$GITHUB_OUTPUT"
           MATRIX_JSON="${matrix}" python - <<'PY' >> "$GITHUB_STEP_SUMMARY"
           import json
@@ -250,6 +270,55 @@ jobs:
           if-no-files-found: ignore
           retention-days: 14
 
+  hermes-e2e-vitest:
+    if: ${{ inputs.scenarios == '' || contains(format(',{0},', inputs.scenarios), ',hermes-e2e,') }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 75
+    env:
+      E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/hermes-e2e
+      NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
+      NEMOCLAW_RUN_E2E_SCENARIOS: "1"
+      NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
+      NEMOCLAW_AGENT: hermes
+      NEMOCLAW_NON_INTERACTIVE: "1"
+      NEMOCLAW_RECREATE_SANDBOX: "1"
+      NEMOCLAW_SANDBOX_NAME: e2e-hermes
+    steps:
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+        with:
+          persist-credentials: false
+
+      - name: Set up Node
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0
+        with:
+          node-version: 22
+          cache: npm
+
+      - name: Install root dependencies
+        run: npm ci --ignore-scripts
+
+      - name: Build CLI
+        run: npm run build:cli
+
+      - name: Run Hermes live Vitest test
+        env:
+          NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
+        run: |
+          set -euo pipefail
+          npx vitest run --project e2e-scenarios-live \
+            test/e2e-scenario/live/hermes-e2e.test.ts \
+            --silent=false --reporter=default
+
+      - name: Upload Hermes live Vitest artifacts
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: e2e-vitest-scenarios-hermes-e2e
+          path: e2e-artifacts/vitest/hermes-e2e/
+          include-hidden-files: false
+          if-no-files-found: ignore
+          retention-days: 14
+
   # Focused coverage slice for the #2603/#3145 OpenClaw websocket
   # protocol/history contract. The retained legacy bash lane remains the
   # source for full closeout until a later PR proves replacement and deletes it.
diff --git a/test/e2e-scenario/live/hermes-e2e.test.ts b/test/e2e-scenario/live/hermes-e2e.test.ts
new file mode 100644
index 00000000000..2080a8b97a5
--- /dev/null
+++ b/test/e2e-scenario/live/hermes-e2e.test.ts
@@ -0,0 +1,549 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { setTimeout as sleep } from "node:timers/promises";
+
+import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts";
+import { trustedProviderEndpoint } from "../fixtures/clients/provider.ts";
+import { trustedSandboxShellScript, validateSandboxName } from "../fixtures/clients/sandbox.ts";
+import { expect, test } from "../fixtures/e2e-test.ts";
+import { shouldRunLiveE2EScenarios } from "../fixtures/live-project-gate.ts";
+import type { ShellProbeResult } from "../fixtures/shell-probe.ts";
+
+// Migrated from test/e2e/test-hermes-e2e.sh.
+//
+// This is intentionally a direct live Vitest test, not a new registry layer:
+// the legacy contract is the real installer/onboard/runtime boundary for Hermes.
+// Vitest owns artifacts, cleanup, redaction, and timeouts while still spawning
+// `bash install.sh --non-interactive`, `nemoclaw`, `openshell`, sandbox exec,
+// direct NVIDIA Endpoints curl, and inference.local probes.
+
+const REPO_ROOT = path.resolve(import.meta.dirname, "../../..");
+const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-hermes";
+validateSandboxName(SANDBOX_NAME);
+const HERMES_HEALTH_URL = "http://localhost:8642/health";
+const HERMES_HOST_HEALTH_URL = "http://127.0.0.1:8642/health";
+const HERMES_DASHBOARD_PORT = process.env.NEMOCLAW_DASHBOARD_PORT ?? "18789";
+const HERMES_DASHBOARD_INTERNAL_PORT =
+  process.env.NEMOCLAW_HERMES_DASHBOARD_INTERNAL_PORT ?? "19119";
+const SESSION_FILE = path.join(os.homedir(), ".nemoclaw", "onboard-session.json");
+const REGISTRY_FILE = path.join(os.homedir(), ".nemoclaw", "sandboxes.json");
+const LIVE_TIMEOUT_MS = 70 * 60_000;
+const CHAT_MODEL = "nvidia/nemotron-3-super-120b-a12b";
+
+interface OpenAiChoiceLike {
+  message?: {
+    content?: unknown;
+    reasoning_content?: unknown;
+  };
+  text?: unknown;
+}
+
+interface OpenAiChatLike {
+  choices?: OpenAiChoiceLike[];
+}
+
+function resultText(result: Pick<ShellProbeResult, "stdout" | "stderr">): string {
+  return [result.stdout, result.stderr].filter(Boolean).join("\n");
+}
+
+function truthyEnv(value: string | undefined): boolean {
+  return ["1", "true", "yes", "on"].includes(value?.trim().toLowerCase() ?? "");
+}
+
+function hermesDashboardE2eEnabled(): boolean {
+  return (
+    truthyEnv(process.env.NEMOCLAW_E2E_HERMES_DASHBOARD) ||
+    truthyEnv(process.env.NEMOCLAW_HERMES_DASHBOARD)
+  );
+}
+
+function commandEnv(apiKey?: string): NodeJS.ProcessEnv {
+  const env: NodeJS.ProcessEnv = {
+    ...buildAvailabilityProbeEnv(),
+    NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1",
+    NEMOCLAW_AGENT: "hermes",
+    NEMOCLAW_NON_INTERACTIVE: "1",
+    NEMOCLAW_RECREATE_SANDBOX: "1",
+    NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME,
+  };
+  if (apiKey) env.NVIDIA_API_KEY = apiKey;
+  if (process.env.NEMOCLAW_E2E_HERMES_DASHBOARD) {
+    env.NEMOCLAW_E2E_HERMES_DASHBOARD = process.env.NEMOCLAW_E2E_HERMES_DASHBOARD;
+  }
+  if (process.env.NEMOCLAW_HERMES_DASHBOARD) {
+    env.NEMOCLAW_HERMES_DASHBOARD = process.env.NEMOCLAW_HERMES_DASHBOARD;
+  }
+  if (process.env.NEMOCLAW_HERMES_DASHBOARD_TUI) {
+    env.NEMOCLAW_HERMES_DASHBOARD_TUI = process.env.NEMOCLAW_HERMES_DASHBOARD_TUI;
+  }
+  if (process.env.NEMOCLAW_DASHBOARD_PORT) {
+    env.NEMOCLAW_DASHBOARD_PORT = process.env.NEMOCLAW_DASHBOARD_PORT;
+  }
+  if (process.env.NEMOCLAW_HERMES_DASHBOARD_INTERNAL_PORT) {
+    env.NEMOCLAW_HERMES_DASHBOARD_INTERNAL_PORT =
+      process.env.NEMOCLAW_HERMES_DASHBOARD_INTERNAL_PORT;
+  }
+  return env;
+}
+
+function chatPayload(prompt: string): string {
+  return JSON.stringify({
+    model: CHAT_MODEL,
+    messages: [{ role: "user", content: prompt }],
+    max_tokens: 100,
+  });
+}
+
+function chatContent(response: unknown): string {
+  if (!response || typeof response !== "object") return "";
+  const choices = (response as OpenAiChatLike).choices;
+  if (!Array.isArray(choices)) return "";
+  for (const choice of choices) {
+    const message = choice?.message;
+    if (message) {
+      if (typeof message.content === "string" && message.content.trim()) {
+        return message.content.trim();
+      }
+      if (typeof message.reasoning_content === "string" && message.reasoning_content.trim()) {
+        return message.reasoning_content.trim();
+      }
+    }
+    if (typeof choice?.text === "string" && choice.text.trim()) return choice.text.trim();
+  }
+  return "";
+}
+
+function expectPong(label: string, response: unknown): void {
+  const content = chatContent(response);
+  expect(
+    content,
+    `${label} expected PONG; response=${JSON.stringify(response).slice(0, 500)}`,
+  ).toMatch(/PONG/i);
+}
+
+function readJsonFile(filePath: string): unknown {
+  return JSON.parse(fs.readFileSync(filePath, "utf8"));
+}
+
+function registryEntry(name: string): Record<string, unknown> | undefined {
+  if (!fs.existsSync(REGISTRY_FILE)) return undefined;
+  const registry = readJsonFile(REGISTRY_FILE);
+  if (!registry || typeof registry !== "object") return undefined;
+  const sandboxes = (registry as { sandboxes?: unknown }).sandboxes;
+  if (!sandboxes || typeof sandboxes !== "object") return undefined;
+  const entry = (sandboxes as Record<string, unknown>)[name];
+  return entry && typeof entry === "object" ? (entry as Record<string, unknown>) : undefined;
+}
+
+function httpStatusOk(status: string): boolean {
+  return /^[23][0-9][0-9]$/.test(status.trim());
+}
+
+function stripAnsi(value: string): string {
+  return value.replace(/\x1B(?:\[[0-?]*[ -/]*[@-~]|\][^\x07]*(?:\x07|\x1B\\)|[@-_])/g, "");
+}
+
+function forwardListHasRunningPort(output: string, sandboxName: string, port: string): boolean {
+  return output
+    .split("\n")
+    .map(stripAnsi)
+    .some((line) => {
+      const parts = line.trim().split(/\s+/);
+      return (
+        parts.length >= 5 &&
+        parts[0] === sandboxName &&
+        parts[2] === port &&
+        ["running", "active"].includes(parts.at(-1)?.toLowerCase() ?? "")
+      );
+    });
+}
+
+async function bestEffort(run: () => Promise<unknown>): Promise<void> {
+  try {
+    await run();
+  } catch {
+    // Cleanup is best-effort because the pre-install path may not have
+    // nemoclaw/openshell available yet.
+  }
+}
+
+test.skipIf(!shouldRunLiveE2EScenarios())(
+  "hermes-e2e: install.sh onboards Hermes and proves health plus live inference",
+  { timeout: LIVE_TIMEOUT_MS },
+  async ({ artifacts, cleanup, host, provider, sandbox, secrets }) => {
+    const apiKey = secrets.required("NVIDIA_API_KEY");
+    expect(apiKey.startsWith("nvapi-"), "NVIDIA_API_KEY must start with nvapi-").toBe(true);
+
+    await artifacts.writeJson("scenario.json", {
+      id: "hermes-e2e",
+      runner: "vitest",
+      migratedFrom: "test/e2e/test-hermes-e2e.sh",
+      boundary: "install.sh --non-interactive + Hermes sandbox runtime",
+      sandboxName: SANDBOX_NAME,
+      dashboardEnabled: hermesDashboardE2eEnabled(),
+    });
+
+    const env = commandEnv(apiKey);
+    const redactionValues = [apiKey];
+
+    const cleanupHermes = async (label: string) => {
+      await bestEffort(() =>
+        host.command("nemoclaw", [SANDBOX_NAME, "destroy", "--yes"], {
+          artifactName: `${label}-nemoclaw-destroy`,
+          env: commandEnv(),
+          timeoutMs: 120_000,
+        }),
+      );
+      await bestEffort(() =>
+        sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], {
+          artifactName: `${label}-openshell-sandbox-delete`,
+          env: commandEnv(),
+          timeoutMs: 60_000,
+        }),
+      );
+      await bestEffort(() =>
+        sandbox.openshell(["gateway", "destroy", "-g", "nemoclaw"], {
+          artifactName: `${label}-openshell-gateway-destroy`,
+          env: commandEnv(),
+          timeoutMs: 60_000,
+        }),
+      );
+    };
+
+    cleanup.add(`destroy Hermes sandbox ${SANDBOX_NAME}`, async () => {
+      await cleanupHermes("cleanup");
+    });
+
+    // Phase 0: pre-cleanup, after the secret gate so local skipped runs do not
+    // mutate host state.
+    await cleanupHermes("pre-cleanup");
+
+    // Phase 1: prerequisites.
+    const dockerInfo = await host.command("docker", ["info"], {
+      artifactName: "phase-1-docker-info",
+      env: buildAvailabilityProbeEnv(),
+      timeoutMs: 30_000,
+    });
+    expect(dockerInfo.exitCode, resultText(dockerInfo)).toBe(0);
+
+    expect(fs.existsSync(path.join(REPO_ROOT, "agents", "hermes", "manifest.yaml"))).toBe(true);
+
+    const providerModels = await provider.requestJson(
+      trustedProviderEndpoint("https://integrate.api.nvidia.com/v1/models", {
+        allowedHosts: ["integrate.api.nvidia.com"],
+      }),
+      {
+        artifactName: "phase-1-integrate-models",
+        curlMaxTimeSeconds: 15,
+        headers: [`Authorization: Bearer ${apiKey}`],
+        env: buildAvailabilityProbeEnv(),
+        redactionValues,
+        timeoutMs: 30_000,
+      },
+    );
+    expect(providerModels.json).toBeTruthy();
+
+    // Phase 2: real installer + non-interactive Hermes onboard.
+    const install = await host.command("bash", ["install.sh", "--non-interactive"], {
+      artifactName: "phase-2-install-hermes",
+      cwd: REPO_ROOT,
+      env,
+      redactionValues,
+      timeoutMs: 60 * 60_000,
+    });
+    expect(install.exitCode, resultText(install)).toBe(0);
+
+    const cliProbe = await host.command(
+      "bash",
+      ["-lc", "command -v nemoclaw && command -v openshell"],
+      {
+        artifactName: "phase-2-cli-probe",
+        env: commandEnv(),
+        timeoutMs: 30_000,
+      },
+    );
+    expect(cliProbe.exitCode, resultText(cliProbe)).toBe(0);
+    expect(cliProbe.stdout).toContain("nemoclaw");
+    expect(cliProbe.stdout).toContain("openshell");
+
+    const help = await host.command("nemoclaw", ["--help"], {
+      artifactName: "phase-2-nemoclaw-help",
+      env: commandEnv(),
+      timeoutMs: 30_000,
+    });
+    expect(help.exitCode, resultText(help)).toBe(0);
+
+    if (hermesDashboardE2eEnabled()) {
+      expect(resultText(install)).toContain(
+        "Deployment verified — gateway and dashboard are healthy.",
+      );
+      expect(resultText(install)).toContain("Hermes Agent Dashboard");
+      expect(resultText(install)).toContain(`http://127.0.0.1:${HERMES_DASHBOARD_PORT}/`);
+    }
+
+    // Phase 3: sandbox verification.
+    const list = await host.command("nemoclaw", ["list"], {
+      artifactName: "phase-3-nemoclaw-list",
+      env: commandEnv(),
+      timeoutMs: 30_000,
+    });
+    expect(list.exitCode, resultText(list)).toBe(0);
+    expect(resultText(list)).toContain(SANDBOX_NAME);
+
+    const status = await host.command("nemoclaw", [SANDBOX_NAME, "status"], {
+      artifactName: "phase-3-nemoclaw-status",
+      env: commandEnv(),
+      timeoutMs: 60_000,
+    });
+    expect(status.exitCode, resultText(status)).toBe(0);
+
+    expect(fs.existsSync(SESSION_FILE), `${SESSION_FILE} missing`).toBe(true);
+    expect(readJsonFile(SESSION_FILE)).toMatchObject({ agent: "hermes" });
+
+    const inference = await sandbox.openshell(["inference", "get"], {
+      artifactName: "phase-3-openshell-inference-get",
+      env: commandEnv(),
+      timeoutMs: 30_000,
+    });
+    expect(inference.exitCode, resultText(inference)).toBe(0);
+    expect(resultText(inference)).toMatch(/nvidia-prod/i);
+
+    const policy = await sandbox.openshell(["policy", "get", "--full", SANDBOX_NAME], {
+      artifactName: "phase-3-openshell-policy-get",
+      env: commandEnv(),
+      timeoutMs: 30_000,
+    });
+    expect(policy.exitCode, resultText(policy)).toBe(0);
+    expect(resultText(policy)).toMatch(/network_policies/i);
+
+    // Phase 4: Hermes health and sandbox state.
+    let health: ShellProbeResult | undefined;
+    for (let attempt = 1; attempt <= 15; attempt += 1) {
+      health = await sandbox.exec(SANDBOX_NAME, ["curl", "-sf", HERMES_HEALTH_URL], {
+        artifactName: `phase-4-hermes-health-attempt-${attempt}`,
+        env: commandEnv(),
+        timeoutMs: 20_000,
+      });
+      if (health.exitCode === 0 && /"ok"/i.test(resultText(health))) break;
+      await sleep(4_000);
+    }
+    expect(health, "Hermes health probe did not run").toBeTruthy();
+    expect(health?.exitCode, health ? resultText(health) : "missing health result").toBe(0);
+    expect(resultText(health!)).toMatch(/"ok"/i);
+
+    const hermesVersion = await sandbox.exec(SANDBOX_NAME, ["hermes", "--version"], {
+      artifactName: "phase-4-hermes-version",
+      env: commandEnv(),
+      timeoutMs: 30_000,
+    });
+    expect(hermesVersion.exitCode, resultText(hermesVersion)).toBe(0);
+    expect(resultText(hermesVersion)).not.toMatch(/MISSING|not found|No such file/i);
+
+    const configProbe = await sandbox.execShell(
+      SANDBOX_NAME,
+      trustedSandboxShellScript(
+        "test -f /sandbox/.hermes/config.yaml && test -d /sandbox/.hermes && touch /sandbox/.hermes/test-write && rm -f /sandbox/.hermes/test-write && echo OK",
+      ),
+      {
+        artifactName: "phase-4-hermes-config-state",
+        env: commandEnv(),
+        timeoutMs: 30_000,
+      },
+    );
+    expect(configProbe.exitCode, resultText(configProbe)).toBe(0);
+    expect(configProbe.stdout).toContain("OK");
+
+    if (hermesDashboardE2eEnabled()) {
+      const entry = registryEntry(SANDBOX_NAME);
+      expect(entry, `registry missing ${SANDBOX_NAME}`).toBeTruthy();
+      expect(entry).toMatchObject({
+        agent: "hermes",
+        dashboardPort: Number(HERMES_DASHBOARD_PORT),
+      });
+
+      const forwardList = await sandbox.openshell(["forward", "list"], {
+        artifactName: "phase-4-dashboard-forward-list",
+        env: commandEnv(),
+        timeoutMs: 30_000,
+      });
+      expect(forwardList.exitCode, resultText(forwardList)).toBe(0);
+      expect(forwardListHasRunningPort(forwardList.stdout, SANDBOX_NAME, "8642")).toBe(true);
+      expect(
+        forwardListHasRunningPort(forwardList.stdout, SANDBOX_NAME, HERMES_DASHBOARD_PORT),
+      ).toBe(true);
+
+      const hostDashboard = await host.command(
+        "curl",
+        [
+          "-sS",
+          "-L",
+          "--max-time",
+          "10",
+          "-o",
+          "/tmp/hermes-dashboard-vitest-body",
+          "-w",
+          "%{http_code}",
+          `http://127.0.0.1:${HERMES_DASHBOARD_PORT}/`,
+        ],
+        {
+          artifactName: "phase-4-dashboard-host-probe",
+          env: commandEnv(),
+          timeoutMs: 30_000,
+        },
+      );
+      expect(hostDashboard.exitCode, resultText(hostDashboard)).toBe(0);
+      expect(httpStatusOk(hostDashboard.stdout)).toBe(true);
+
+      const hostHealth = await host.command(
+        "curl",
+        ["-sf", "--max-time", "10", HERMES_HOST_HEALTH_URL],
+        {
+          artifactName: "phase-4-hermes-host-health",
+          env: commandEnv(),
+          timeoutMs: 30_000,
+        },
+      );
+      expect(hostHealth.exitCode, resultText(hostHealth)).toBe(0);
+      expect(resultText(hostHealth)).toMatch(/"ok"/i);
+
+      const dashboardInternal = await sandbox.exec(
+        SANDBOX_NAME,
+        [
+          "curl",
+          "-sS",
+          "-L",
+          "--max-time",
+          "10",
+          "-o",
+          "/tmp/hermes-dashboard-vitest-body",
+          "-w",
+          "%{http_code}",
+          `http://127.0.0.1:${HERMES_DASHBOARD_INTERNAL_PORT}/`,
+        ],
+        {
+          artifactName: "phase-4-dashboard-sandbox-probe",
+          env: commandEnv(),
+          timeoutMs: 30_000,
+        },
+      );
+      expect(dashboardInternal.exitCode, resultText(dashboardInternal)).toBe(0);
+      expect(httpStatusOk(dashboardInternal.stdout)).toBe(true);
+    }
+
+    // Phase 5: live inference through both the external provider and the
+    // sandbox's inference.local route.
+    const directChat = await provider.requestJson(
+      trustedProviderEndpoint("https://integrate.api.nvidia.com/v1/chat/completions", {
+        allowedHosts: ["integrate.api.nvidia.com"],
+      }),
+      {
+        artifactName: "phase-5-direct-nvidia-chat",
+        body: chatPayload("Reply with exactly one word: PONG"),
+        curlMaxTimeSeconds: 60,
+        headers: ["Content-Type: application/json", `Authorization: Bearer ${apiKey}`],
+        env: buildAvailabilityProbeEnv(),
+        redactionValues,
+        timeoutMs: 90_000,
+      },
+    );
+    expectPong("direct NVIDIA Endpoints chat", directChat.json);
+
+    const sandboxChat = await sandbox.exec(
+      SANDBOX_NAME,
+      [
+        "curl",
+        "-fsS",
+        "--max-time",
+        "60",
+        "-H",
+        "Content-Type: application/json",
+        "--data-raw",
+        chatPayload("Reply with exactly one word: PONG"),
+        "https://inference.local/v1/chat/completions",
+      ],
+      {
+        artifactName: "phase-5-inference-local-chat",
+        env: commandEnv(),
+        timeoutMs: 90_000,
+      },
+    );
+    expect(sandboxChat.exitCode, resultText(sandboxChat)).toBe(0);
+    let sandboxChatJson: unknown;
+    try {
+      sandboxChatJson = JSON.parse(sandboxChat.stdout) as unknown;
+    } catch (error) {
+      throw new Error(
+        `Hermes sandbox inference.local chat response was not JSON: ${
+          error instanceof Error ? error.message : String(error)
+        }; body=${sandboxChat.stdout.slice(0, 500)}`,
+      );
+    }
+    expectPong("Hermes sandbox inference.local chat", sandboxChatJson);
+
+    // Phase 6: CLI operations and agent manifest regression.
+    const logs = await host.command("nemoclaw", [SANDBOX_NAME, "logs"], {
+      artifactName: "phase-6-nemoclaw-logs",
+      env: commandEnv(),
+      timeoutMs: 60_000,
+    });
+    expect(logs.exitCode, resultText(logs)).toBe(0);
+    expect(resultText(logs).trim().length).toBeGreaterThan(0);
+
+    const manifestCheck = await host.command(
+      "node",
+      [
+        "-e",
+        `const { loadAgent, listAgents } = require(${JSON.stringify(path.join(REPO_ROOT, "bin", "lib", "agent-defs"))});\n` +
+          `const agents = listAgents();\n` +
+          `console.log('agents:', agents.join(', '));\n` +
+          `console.log('openclaw_display:', loadAgent('openclaw').displayName);\n` +
+          `console.log('hermes_display:', loadAgent('hermes').displayName);`,
+      ],
+      {
+        artifactName: "phase-6-agent-manifest-check",
+        env: commandEnv(),
+        timeoutMs: 30_000,
+      },
+    );
+    expect(manifestCheck.exitCode, resultText(manifestCheck)).toBe(0);
+    expect(manifestCheck.stdout).toMatch(/openclaw_display:.*OpenClaw/);
+    expect(manifestCheck.stdout).toMatch(/hermes_display:.*Hermes/);
+    expect(manifestCheck.stdout).toMatch(/agents:.*(openclaw.*hermes|hermes.*openclaw)/);
+
+    // Phase 8: explicit cleanup and post-destroy registry proof.
+    if (process.env.NEMOCLAW_E2E_KEEP_SANDBOX !== "1") {
+      const destroy = await host.command("nemoclaw", [SANDBOX_NAME, "destroy", "--yes"], {
+        artifactName: "phase-8-nemoclaw-destroy",
+        env: commandEnv(),
+        timeoutMs: 120_000,
+      });
+      expect(destroy.exitCode, resultText(destroy)).toBe(0);
+      await bestEffort(() =>
+        sandbox.openshell(["gateway", "destroy", "-g", "nemoclaw"], {
+          artifactName: "phase-8-openshell-gateway-destroy",
+          env: commandEnv(),
+          timeoutMs: 60_000,
+        }),
+      );
+      expect(
+        registryEntry(SANDBOX_NAME),
+        `${SANDBOX_NAME} still in ${REGISTRY_FILE}`,
+      ).toBeUndefined();
+    }
+
+    await artifacts.writeJson("scenario-result.json", {
+      id: "hermes-e2e",
+      assertions: {
+        installShNonInteractiveHermes: true,
+        sandboxListedAndHealthy: true,
+        directProviderInferencePong: true,
+        sandboxInferenceLocalPong: true,
+        dashboardChecked: hermesDashboardE2eEnabled(),
+      },
+    });
+  },
+);

From 3ec98c3bd42951b693e5bcbcfd7227e63cdc5f06 Mon Sep 17 00:00:00 2001
From: Julie Yaunches <jyaunches@nvidia.com>
Date: Thu, 11 Jun 2026 09:39:03 -0400
Subject: [PATCH 2/9] test(e2e): stabilize Hermes live Vitest run

---
 .github/workflows/e2e-vitest-scenarios.yaml | 1 +
 test/e2e-scenario/live/hermes-e2e.test.ts   | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml
index 999f71cdc39..4dad0644a72 100644
--- a/.github/workflows/e2e-vitest-scenarios.yaml
+++ b/.github/workflows/e2e-vitest-scenarios.yaml
@@ -283,6 +283,7 @@ jobs:
       NEMOCLAW_NON_INTERACTIVE: "1"
       NEMOCLAW_RECREATE_SANDBOX: "1"
       NEMOCLAW_SANDBOX_NAME: e2e-hermes
+      NEMOCLAW_MODEL: minimaxai/minimax-m2.7
     steps:
       - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
         with:
diff --git a/test/e2e-scenario/live/hermes-e2e.test.ts b/test/e2e-scenario/live/hermes-e2e.test.ts
index 2080a8b97a5..99ef2148ff6 100644
--- a/test/e2e-scenario/live/hermes-e2e.test.ts
+++ b/test/e2e-scenario/live/hermes-e2e.test.ts
@@ -32,7 +32,7 @@ const HERMES_DASHBOARD_INTERNAL_PORT =
 const SESSION_FILE = path.join(os.homedir(), ".nemoclaw", "onboard-session.json");
 const REGISTRY_FILE = path.join(os.homedir(), ".nemoclaw", "sandboxes.json");
 const LIVE_TIMEOUT_MS = 70 * 60_000;
-const CHAT_MODEL = "nvidia/nemotron-3-super-120b-a12b";
+const CHAT_MODEL = process.env.NEMOCLAW_MODEL ?? "nvidia/nemotron-3-super-120b-a12b";
 
 interface OpenAiChoiceLike {
   message?: {
@@ -68,6 +68,7 @@ function commandEnv(apiKey?: string): NodeJS.ProcessEnv {
     NEMOCLAW_AGENT: "hermes",
     NEMOCLAW_NON_INTERACTIVE: "1",
     NEMOCLAW_RECREATE_SANDBOX: "1",
+    NEMOCLAW_MODEL: CHAT_MODEL,
     NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME,
   };
   if (apiKey) env.NVIDIA_API_KEY = apiKey;

From 3fb1f1d0be55b83af491385ef8da62ee353e7717 Mon Sep 17 00:00:00 2001
From: Julie Yaunches <jyaunches@nvidia.com>
Date: Thu, 11 Jun 2026 10:27:50 -0400
Subject: [PATCH 3/9] ci(e2e): validate Hermes scenario selection

---
 .github/workflows/e2e-vitest-scenarios.yaml |   9 +-
 tools/e2e-scenarios/workflow-boundary.mts   | 111 ++++++++++++++++++++
 2 files changed, 119 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml
index 4dad0644a72..0dc0dbf3383 100644
--- a/.github/workflows/e2e-vitest-scenarios.yaml
+++ b/.github/workflows/e2e-vitest-scenarios.yaml
@@ -24,6 +24,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       matrix: ${{ steps.matrix.outputs.matrix }}
+      hermes_selected: ${{ steps.matrix.outputs.hermes_selected }}
     steps:
       - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
         with:
@@ -46,6 +47,7 @@ jobs:
           set -euo pipefail
           args=(--emit-live-matrix)
           matrix=""
+          hermes_selected=false
           if [ -n "${SCENARIOS}" ]; then
             if [[ ! "${SCENARIOS}" =~ ^[A-Za-z0-9_-]+(,[A-Za-z0-9_-]+)*$ ]]; then
               echo "::error::Invalid scenario input: ${SCENARIOS}" >&2
@@ -57,6 +59,7 @@ jobs:
               case "${scenario}" in
                 hermes-e2e)
                   # Free-standing live Vitest job below; not registry-driven.
+                  hermes_selected=true
                   ;;
                 *)
                   registry_scenarios+=("${scenario}")
@@ -71,9 +74,11 @@ jobs:
               matrix="[]"
             fi
           else
+            hermes_selected=true
             matrix="$(npx tsx test/e2e-scenario/scenarios/run.ts "${args[@]}")"
           fi
           echo "matrix=${matrix}" >> "$GITHUB_OUTPUT"
+          echo "hermes_selected=${hermes_selected}" >> "$GITHUB_OUTPUT"
           MATRIX_JSON="${matrix}" python - <<'PY' >> "$GITHUB_STEP_SUMMARY"
           import json
           import os
@@ -89,6 +94,7 @@ jobs:
 
   live-scenarios:
     needs: generate-matrix
+    if: ${{ needs.generate-matrix.outputs.matrix != '[]' }}
     runs-on: ${{ matrix.runner }}
     timeout-minutes: 45
     strategy:
@@ -271,7 +277,8 @@ jobs:
           retention-days: 14
 
   hermes-e2e-vitest:
-    if: ${{ inputs.scenarios == '' || contains(format(',{0},', inputs.scenarios), ',hermes-e2e,') }}
+    needs: generate-matrix
+    if: ${{ needs.generate-matrix.outputs.hermes_selected == 'true' }}
     runs-on: ubuntu-latest
     timeout-minutes: 75
     env:
diff --git a/tools/e2e-scenarios/workflow-boundary.mts b/tools/e2e-scenarios/workflow-boundary.mts
index 9b06e6d219d..543fbc30a17 100644
--- a/tools/e2e-scenarios/workflow-boundary.mts
+++ b/tools/e2e-scenarios/workflow-boundary.mts
@@ -300,6 +300,103 @@ function validateOnboardNegativePathsVitestJob(errors: string[], jobs: WorkflowR
   }
 }
 
+function validateHermesE2EVitestJob(errors: string[], jobs: WorkflowRecord): void {
+  const jobName = "hermes-e2e-vitest";
+  const job = asRecord(jobs[jobName]);
+  if (Object.keys(job).length === 0) {
+    errors.push("workflow missing hermes-e2e-vitest job");
+    return;
+  }
+
+  if (job["runs-on"] !== "ubuntu-latest") {
+    errors.push("hermes-e2e-vitest job must run on ubuntu-latest");
+  }
+  if (job.needs !== "generate-matrix") {
+    errors.push("hermes-e2e-vitest job must depend on generate-matrix validation");
+  }
+  if (job.if !== "${{ needs.generate-matrix.outputs.hermes_selected == 'true' }}") {
+    errors.push("hermes-e2e-vitest job must use validated hermes_selected output");
+  }
+  if (stringValue(job.if).includes("inputs.scenarios")) {
+    errors.push("hermes-e2e-vitest job must not inspect raw workflow dispatch scenarios");
+  }
+
+  const jobEnv = asRecord(job.env);
+  if (jobEnv.NEMOCLAW_RUN_E2E_SCENARIOS !== "1") {
+    errors.push("hermes-e2e-vitest job must set NEMOCLAW_RUN_E2E_SCENARIOS=1");
+  }
+  if (jobEnv.NEMOCLAW_CLI_BIN !== "${{ github.workspace }}/bin/nemoclaw.js") {
+    errors.push("hermes-e2e-vitest job must point NEMOCLAW_CLI_BIN at the repo CLI");
+  }
+  if (jobEnv.E2E_ARTIFACT_DIR !== "${{ github.workspace }}/e2e-artifacts/vitest/hermes-e2e") {
+    errors.push("hermes-e2e-vitest job must write artifacts under e2e-artifacts/vitest/hermes-e2e");
+  }
+  if (jobEnv.NEMOCLAW_AGENT !== "hermes") {
+    errors.push("hermes-e2e-vitest job must set NEMOCLAW_AGENT=hermes");
+  }
+  if (jobEnv.NEMOCLAW_MODEL !== "minimaxai/minimax-m2.7") {
+    errors.push("hermes-e2e-vitest job must pin the CI-safe Hermes model");
+  }
+  requireEnvDoesNotExposeSecret(errors, "hermes-e2e-vitest job", jobEnv, "NVIDIA_API_KEY");
+
+  const steps = asSteps(job.steps);
+  requireNoDispatchInputInterpolation(errors, steps);
+  for (const step of steps) {
+    if (step.name !== "Run Hermes live Vitest test") {
+      requireEnvDoesNotExposeSecret(
+        errors,
+        `hermes-e2e-vitest step '${step.name ?? step.uses ?? "<unnamed>"}'`,
+        asRecord(step.env),
+        "NVIDIA_API_KEY",
+      );
+    }
+  }
+
+  const checkout = steps.find((step) => stringValue(step.uses).startsWith("actions/checkout@"));
+  if (!checkout) errors.push("hermes-e2e-vitest job missing checkout step");
+  requireFullShaAction(errors, checkout, "hermes-e2e-vitest checkout");
+  if (asRecord(checkout?.with)["persist-credentials"] !== false) {
+    errors.push("hermes-e2e-vitest checkout step must set persist-credentials=false");
+  }
+
+  const setupNode = namedStep(steps, "Set up Node");
+  if (!setupNode) errors.push("hermes-e2e-vitest job missing step: Set up Node");
+  requireFullShaAction(errors, setupNode, "hermes-e2e-vitest setup-node");
+
+  const installRootDependencies = requireJobStep(errors, jobName, steps, "Install root dependencies");
+  requireRunContains(errors, installRootDependencies, "npm ci --ignore-scripts");
+
+  const buildCli = requireJobStep(errors, jobName, steps, "Build CLI");
+  requireRunContains(errors, buildCli, "npm run build:cli");
+
+  const runVitest = requireJobStep(errors, jobName, steps, "Run Hermes live Vitest test");
+  const runVitestEnv = asRecord(runVitest?.env);
+  if (runVitestEnv.NVIDIA_API_KEY !== "${{ secrets.NVIDIA_API_KEY }}") {
+    errors.push("hermes-e2e-vitest Vitest step must receive NVIDIA_API_KEY from secrets");
+  }
+  requireRunContains(errors, runVitest, "npx vitest run --project e2e-scenarios-live");
+  requireRunContains(errors, runVitest, "test/e2e-scenario/live/hermes-e2e.test.ts");
+  requireRunDoesNotContain(errors, runVitest, "${{ inputs.");
+
+  const upload = requireJobStep(errors, jobName, steps, "Upload Hermes live Vitest artifacts");
+  requireFullShaAction(errors, upload, "hermes-e2e-vitest upload-artifact");
+  const uploadWith = asRecord(upload?.with);
+  if (uploadWith.name !== "e2e-vitest-scenarios-hermes-e2e") {
+    errors.push("hermes-e2e-vitest artifact upload name must be stable");
+  }
+  const uploadPath = stringValue(uploadWith.path);
+  requireUploadPathContains(errors, uploadPath, "e2e-artifacts/vitest/hermes-e2e/");
+  if (uploadWith["include-hidden-files"] !== false) {
+    errors.push("hermes-e2e-vitest artifact upload must set include-hidden-files: false");
+  }
+  if (uploadWith["if-no-files-found"] !== "ignore") {
+    errors.push("hermes-e2e-vitest artifact upload must ignore missing fixture artifacts");
+  }
+  if (uploadWith["retention-days"] !== 14) {
+    errors.push("hermes-e2e-vitest artifact upload retention-days must be 14");
+  }
+}
+
 export function validateE2eVitestScenariosWorkflowBoundary(
   workflowPath = DEFAULT_VITEST_WORKFLOW_PATH,
 ): string[] {
@@ -325,6 +422,13 @@ export function validateE2eVitestScenariosWorkflowBoundary(
   if (generateMatrix["runs-on"] !== "ubuntu-latest") {
     errors.push("generate-matrix job must run on ubuntu-latest");
   }
+  const generateOutputs = asRecord(generateMatrix.outputs);
+  if (generateOutputs.matrix !== "${{ steps.matrix.outputs.matrix }}") {
+    errors.push("generate-matrix job must expose matrix output");
+  }
+  if (generateOutputs.hermes_selected !== "${{ steps.matrix.outputs.hermes_selected }}") {
+    errors.push("generate-matrix job must expose hermes_selected output");
+  }
   const generateSteps = asSteps(generateMatrix.steps);
   requireNoDispatchInputInterpolation(errors, generateSteps);
   const generateCheckout = generateSteps.find((step) => stringValue(step.uses).startsWith("actions/checkout@"));
@@ -346,11 +450,17 @@ export function validateE2eVitestScenariosWorkflowBoundary(
   requireRunContains(errors, generate, "--scenarios");
   requireRunContains(errors, generate, "^[A-Za-z0-9_-]+(,[A-Za-z0-9_-]+)*$");
   requireRunDoesNotContain(errors, generate, "^[A-Za-z0-9._-]+");
+  requireRunContains(errors, generate, "hermes_selected=false");
+  requireRunContains(errors, generate, "hermes_selected=true");
+  requireRunContains(errors, generate, 'echo "hermes_selected=${hermes_selected}" >> "$GITHUB_OUTPUT"');
   requireRunContains(errors, generate, "## Vitest E2E Scenario Matrix");
   requireRunContains(errors, generate, "| Scenario | Runner | Label |");
 
   const liveScenarios = asRecord(jobs["live-scenarios"]);
   if (Object.keys(liveScenarios).length === 0) errors.push("workflow missing live-scenarios job");
+  if (liveScenarios.if !== "${{ needs.generate-matrix.outputs.matrix != '[]' }}") {
+    errors.push("live-scenarios job must skip empty validated registry matrix");
+  }
   if (liveScenarios["runs-on"] !== "${{ matrix.runner }}") {
     errors.push("live-scenarios job must run on the matrix runner");
   }
@@ -482,6 +592,7 @@ export function validateE2eVitestScenariosWorkflowBoundary(
 
   validateOpenShellVersionPinVitestJob(errors, jobs);
   validateOnboardNegativePathsVitestJob(errors, jobs);
+  validateHermesE2EVitestJob(errors, jobs);
 
   return errors;
 }

From e8d5b4b2d8a063fe1867fc1f1e47d6e715b3347d Mon Sep 17 00:00:00 2001
From: Julie Yaunches <jyaunches@nvidia.com>
Date: Thu, 11 Jun 2026 10:50:05 -0400
Subject: [PATCH 4/9] test(e2e): extend Hermes endpoint validation budget

---
 .github/workflows/e2e-vitest-scenarios.yaml | 1 +
 test/e2e-scenario/live/hermes-e2e.test.ts   | 3 +++
 tools/e2e-scenarios/workflow-boundary.mts   | 3 +++
 3 files changed, 7 insertions(+)

diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml
index 0dc0dbf3383..d888a897ae6 100644
--- a/.github/workflows/e2e-vitest-scenarios.yaml
+++ b/.github/workflows/e2e-vitest-scenarios.yaml
@@ -291,6 +291,7 @@ jobs:
       NEMOCLAW_RECREATE_SANDBOX: "1"
       NEMOCLAW_SANDBOX_NAME: e2e-hermes
       NEMOCLAW_MODEL: minimaxai/minimax-m2.7
+      NEMOCLAW_ONBOARD_VALIDATION_TIMEOUT_SECONDS: "60"
     steps:
       - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
         with:
diff --git a/test/e2e-scenario/live/hermes-e2e.test.ts b/test/e2e-scenario/live/hermes-e2e.test.ts
index 99ef2148ff6..0e58bef48d9 100644
--- a/test/e2e-scenario/live/hermes-e2e.test.ts
+++ b/test/e2e-scenario/live/hermes-e2e.test.ts
@@ -33,6 +33,8 @@ const SESSION_FILE = path.join(os.homedir(), ".nemoclaw", "onboard-session.json"
 const REGISTRY_FILE = path.join(os.homedir(), ".nemoclaw", "sandboxes.json");
 const LIVE_TIMEOUT_MS = 70 * 60_000;
 const CHAT_MODEL = process.env.NEMOCLAW_MODEL ?? "nvidia/nemotron-3-super-120b-a12b";
+const ONBOARD_VALIDATION_TIMEOUT_SECONDS =
+  process.env.NEMOCLAW_ONBOARD_VALIDATION_TIMEOUT_SECONDS ?? "60";
 
 interface OpenAiChoiceLike {
   message?: {
@@ -69,6 +71,7 @@ function commandEnv(apiKey?: string): NodeJS.ProcessEnv {
     NEMOCLAW_NON_INTERACTIVE: "1",
     NEMOCLAW_RECREATE_SANDBOX: "1",
     NEMOCLAW_MODEL: CHAT_MODEL,
+    NEMOCLAW_ONBOARD_VALIDATION_TIMEOUT_SECONDS: ONBOARD_VALIDATION_TIMEOUT_SECONDS,
     NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME,
   };
   if (apiKey) env.NVIDIA_API_KEY = apiKey;
diff --git a/tools/e2e-scenarios/workflow-boundary.mts b/tools/e2e-scenarios/workflow-boundary.mts
index 543fbc30a17..177f09ab391 100644
--- a/tools/e2e-scenarios/workflow-boundary.mts
+++ b/tools/e2e-scenarios/workflow-boundary.mts
@@ -337,6 +337,9 @@ function validateHermesE2EVitestJob(errors: string[], jobs: WorkflowRecord): voi
   if (jobEnv.NEMOCLAW_MODEL !== "minimaxai/minimax-m2.7") {
     errors.push("hermes-e2e-vitest job must pin the CI-safe Hermes model");
   }
+  if (jobEnv.NEMOCLAW_ONBOARD_VALIDATION_TIMEOUT_SECONDS !== "60") {
+    errors.push("hermes-e2e-vitest job must give hosted endpoint validation a CI-safe timeout");
+  }
   requireEnvDoesNotExposeSecret(errors, "hermes-e2e-vitest job", jobEnv, "NVIDIA_API_KEY");
 
   const steps = asSteps(job.steps);

From 9545ea9268e5a1dca7dbe9c63f956a60fbaeaea0 Mon Sep 17 00:00:00 2001
From: Julie Yaunches <jyaunches@nvidia.com>
Date: Thu, 11 Jun 2026 11:01:46 -0400
Subject: [PATCH 5/9] test(e2e): retry Hermes hosted inference probes

---
 test/e2e-scenario/live/hermes-e2e.test.ts | 89 +++++++++++++++--------
 1 file changed, 59 insertions(+), 30 deletions(-)

diff --git a/test/e2e-scenario/live/hermes-e2e.test.ts b/test/e2e-scenario/live/hermes-e2e.test.ts
index 0e58bef48d9..8d31a5d5fdd 100644
--- a/test/e2e-scenario/live/hermes-e2e.test.ts
+++ b/test/e2e-scenario/live/hermes-e2e.test.ts
@@ -175,6 +175,26 @@ async function bestEffort(run: () => Promise<unknown>): Promise<void> {
   }
 }
 
+async function retryHostedInference<T>(
+  label: string,
+  run: (attempt: number) => Promise<T>,
+): Promise<T> {
+  let lastError: unknown;
+  for (let attempt = 1; attempt <= 3; attempt += 1) {
+    try {
+      return await run(attempt);
+    } catch (error) {
+      lastError = error;
+      if (attempt < 3) await sleep(5_000 * attempt);
+    }
+  }
+  throw new Error(
+    `${label} failed after retries: ${
+      lastError instanceof Error ? lastError.message : String(lastError)
+    }`,
+  );
+}
+
 test.skipIf(!shouldRunLiveE2EScenarios())(
   "hermes-e2e: install.sh onboards Hermes and proves health plus live inference",
   { timeout: LIVE_TIMEOUT_MS },
@@ -440,39 +460,48 @@ test.skipIf(!shouldRunLiveE2EScenarios())(
 
     // Phase 5: live inference through both the external provider and the
     // sandbox's inference.local route.
-    const directChat = await provider.requestJson(
-      trustedProviderEndpoint("https://integrate.api.nvidia.com/v1/chat/completions", {
-        allowedHosts: ["integrate.api.nvidia.com"],
-      }),
-      {
-        artifactName: "phase-5-direct-nvidia-chat",
-        body: chatPayload("Reply with exactly one word: PONG"),
-        curlMaxTimeSeconds: 60,
-        headers: ["Content-Type: application/json", `Authorization: Bearer ${apiKey}`],
-        env: buildAvailabilityProbeEnv(),
-        redactionValues,
-        timeoutMs: 90_000,
-      },
+    const directChat = await retryHostedInference("direct NVIDIA Endpoints chat", (attempt) =>
+      provider.requestJson(
+        trustedProviderEndpoint("https://integrate.api.nvidia.com/v1/chat/completions", {
+          allowedHosts: ["integrate.api.nvidia.com"],
+        }),
+        {
+          artifactName: `phase-5-direct-nvidia-chat-attempt-${attempt}`,
+          body: chatPayload("Reply with exactly one word: PONG"),
+          curlMaxTimeSeconds: 90,
+          headers: ["Content-Type: application/json", `Authorization: Bearer ${apiKey}`],
+          env: buildAvailabilityProbeEnv(),
+          redactionValues,
+          timeoutMs: 120_000,
+        },
+      ),
     );
     expectPong("direct NVIDIA Endpoints chat", directChat.json);
 
-    const sandboxChat = await sandbox.exec(
-      SANDBOX_NAME,
-      [
-        "curl",
-        "-fsS",
-        "--max-time",
-        "60",
-        "-H",
-        "Content-Type: application/json",
-        "--data-raw",
-        chatPayload("Reply with exactly one word: PONG"),
-        "https://inference.local/v1/chat/completions",
-      ],
-      {
-        artifactName: "phase-5-inference-local-chat",
-        env: commandEnv(),
-        timeoutMs: 90_000,
+    const sandboxChat = await retryHostedInference(
+      "Hermes sandbox inference.local chat",
+      async (attempt) => {
+        const result = await sandbox.exec(
+          SANDBOX_NAME,
+          [
+            "curl",
+            "-fsS",
+            "--max-time",
+            "90",
+            "-H",
+            "Content-Type: application/json",
+            "--data-raw",
+            chatPayload("Reply with exactly one word: PONG"),
+            "https://inference.local/v1/chat/completions",
+          ],
+          {
+            artifactName: `phase-5-inference-local-chat-attempt-${attempt}`,
+            env: commandEnv(),
+            timeoutMs: 120_000,
+          },
+        );
+        if (result.exitCode !== 0) throw new Error(resultText(result));
+        return result;
       },
     );
     expect(sandboxChat.exitCode, resultText(sandboxChat)).toBe(0);

From 75b63ac8745c7cf6b3527c1416d67a82de58c75e Mon Sep 17 00:00:00 2001
From: Julie Yaunches <jyaunches@nvidia.com>
Date: Thu, 11 Jun 2026 12:17:02 -0400
Subject: [PATCH 6/9] fix(e2e): gate Hermes jobs-only dispatch

---
 .github/workflows/e2e-vitest-scenarios.yaml   | 31 ++++++++-
 .../e2e-scenarios-workflow.test.ts            | 64 +++++++++++++++++++
 tools/e2e-scenarios/workflow-boundary.mts     | 14 +++-
 3 files changed, 104 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml
index 71fb4687a96..7b1879debb5 100644
--- a/.github/workflows/e2e-vitest-scenarios.yaml
+++ b/.github/workflows/e2e-vitest-scenarios.yaml
@@ -40,7 +40,7 @@ jobs:
           SCENARIOS: ${{ inputs.scenarios }}
         run: |
           set -euo pipefail
-          allowed_jobs="openshell-version-pin-vitest,onboard-negative-paths-vitest,openclaw-tui-chat-correlation-vitest,gateway-guard-recovery"
+          allowed_jobs="openshell-version-pin-vitest,onboard-negative-paths-vitest,hermes-e2e-vitest,openclaw-tui-chat-correlation-vitest,gateway-guard-recovery"
           if [ -n "${JOBS}" ] && [ -n "${SCENARIOS}" ]; then
             echo "::error::Use either scenarios or jobs, not both." >&2
             exit 1
@@ -85,13 +85,38 @@ jobs:
       - id: matrix
         name: Generate Vitest scenario matrix
         env:
+          JOBS: ${{ inputs.jobs }}
           SCENARIOS: ${{ inputs.scenarios }}
         run: |
           set -euo pipefail
+          allowed_jobs="openshell-version-pin-vitest,onboard-negative-paths-vitest,hermes-e2e-vitest,openclaw-tui-chat-correlation-vitest,gateway-guard-recovery"
           args=(--emit-live-matrix)
           matrix=""
           hermes_selected=false
-          if [ -n "${SCENARIOS}" ]; then
+          if [ -n "${JOBS}" ] && [ -n "${SCENARIOS}" ]; then
+            echo "::error::Use either scenarios or jobs, not both." >&2
+            exit 1
+          fi
+          if [ -n "${JOBS}" ]; then
+            if [[ ! "${JOBS}" =~ ^[A-Za-z0-9_-]+(,[A-Za-z0-9_-]+)*$ ]]; then
+              echo "::error::Invalid jobs input; use comma-separated job ids containing only letters, numbers, underscores, and hyphens." >&2
+              exit 1
+            fi
+            IFS=, read -r -a selected_jobs <<< "${JOBS}"
+            for job in "${selected_jobs[@]}"; do
+              if [[ ",${allowed_jobs}," != *",${job},"* ]]; then
+                echo "::error::Unknown free-standing Vitest job: ${job}" >&2
+                echo "::error::Allowed jobs: ${allowed_jobs}" >&2
+                exit 1
+              fi
+              case "${job}" in
+                hermes-e2e-vitest)
+                  hermes_selected=true
+                  ;;
+              esac
+            done
+            matrix="[]"
+          elif [ -n "${SCENARIOS}" ]; then
             if [[ ! "${SCENARIOS}" =~ ^[A-Za-z0-9_-]+(,[A-Za-z0-9_-]+)*$ ]]; then
               echo "::error::Invalid scenario input: ${SCENARIOS}" >&2
               exit 1
@@ -324,7 +349,7 @@ jobs:
           retention-days: 14
 
   hermes-e2e-vitest:
-    needs: generate-matrix
+    needs: [validate-jobs, generate-matrix]
     if: ${{ needs.generate-matrix.outputs.hermes_selected == 'true' }}
     runs-on: ubuntu-latest
     timeout-minutes: 75
diff --git a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts
index 056f8b1dc01..5cedcd91637 100644
--- a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts
+++ b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts
@@ -1,18 +1,82 @@
 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
 
+import { spawnSync } from "node:child_process";
 import fs from "node:fs";
 import os from "node:os";
 import path from "node:path";
 
 import { describe, expect, it } from "vitest";
+import YAML from "yaml";
 import { validateE2eVitestScenariosWorkflowBoundary } from "../../../tools/e2e-scenarios/workflow-boundary.mts";
 
+function readWorkflow(): Record<string, unknown> {
+  return YAML.parse(
+    fs.readFileSync(
+      path.join(process.cwd(), ".github/workflows/e2e-vitest-scenarios.yaml"),
+      "utf-8",
+    ),
+  ) as Record<string, unknown>;
+}
+
+function generateMatrixForDispatch(env: {
+  JOBS: string;
+  SCENARIOS: string;
+}): Record<string, string> {
+  const workflow = readWorkflow();
+  const jobs = workflow.jobs as Record<string, { steps?: Array<Record<string, unknown>> }>;
+  const generateStep = jobs["generate-matrix"]?.steps?.find(
+    (step) => step.name === "Generate Vitest scenario matrix",
+  );
+  expect(generateStep?.run).toEqual(expect.any(String));
+
+  const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-vitest-matrix-"));
+  const outputPath = path.join(tmp, "github-output");
+  const summaryPath = path.join(tmp, "github-summary");
+  try {
+    const result = spawnSync("bash", ["-c", generateStep?.run as string], {
+      cwd: process.cwd(),
+      encoding: "utf-8",
+      env: {
+        ...process.env,
+        GITHUB_OUTPUT: outputPath,
+        GITHUB_STEP_SUMMARY: summaryPath,
+        JOBS: env.JOBS,
+        SCENARIOS: env.SCENARIOS,
+      },
+    });
+    expect(result.stderr).toBe("");
+    expect(result.status).toBe(0);
+    return Object.fromEntries(
+      fs
+        .readFileSync(outputPath, "utf-8")
+        .trim()
+        .split("\n")
+        .map((line) => line.split(/=(.*)/s).slice(0, 2)),
+    );
+  } finally {
+    fs.rmSync(tmp, { recursive: true, force: true });
+  }
+}
+
 describe("e2e-vitest-scenarios workflow boundary", () => {
   it("keeps the live Vitest scenario workflow manual, pinned, and artifact-safe", () => {
     expect(validateE2eVitestScenariosWorkflowBoundary()).toEqual([]);
   });
 
+  it("keeps jobs-only dispatches from selecting the Hermes secret-bearing job", () => {
+    expect(
+      generateMatrixForDispatch({ JOBS: "openshell-version-pin-vitest", SCENARIOS: "" }),
+    ).toMatchObject({
+      hermes_selected: "false",
+      matrix: "[]",
+    });
+    expect(generateMatrixForDispatch({ JOBS: "hermes-e2e-vitest", SCENARIOS: "" })).toMatchObject({
+      hermes_selected: "true",
+      matrix: "[]",
+    });
+  });
+
   it("flags direct dispatch-input interpolation and unsafe artifact upload", () => {
     const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-vitest-workflow-"));
     const workflowPath = path.join(tmp, "workflow.yaml");
diff --git a/tools/e2e-scenarios/workflow-boundary.mts b/tools/e2e-scenarios/workflow-boundary.mts
index ae70c3efa53..884460290b4 100644
--- a/tools/e2e-scenarios/workflow-boundary.mts
+++ b/tools/e2e-scenarios/workflow-boundary.mts
@@ -163,6 +163,7 @@ function validateJobsSelector(errors: string[], jobs: WorkflowRecord): void {
   requireRunContains(errors, validate, "allowed_jobs=");
   requireRunContains(errors, validate, "openshell-version-pin-vitest");
   requireRunContains(errors, validate, "onboard-negative-paths-vitest");
+  requireRunContains(errors, validate, "hermes-e2e-vitest");
   requireRunContains(errors, validate, "openclaw-tui-chat-correlation-vitest");
   requireRunContains(errors, validate, "gateway-guard-recovery");
   requireRunContains(errors, validate, "^[A-Za-z0-9_-]+(,[A-Za-z0-9_-]+)*$");
@@ -346,8 +347,9 @@ function validateHermesE2EVitestJob(errors: string[], jobs: WorkflowRecord): voi
   if (job["runs-on"] !== "ubuntu-latest") {
     errors.push("hermes-e2e-vitest job must run on ubuntu-latest");
   }
-  if (job.needs !== "generate-matrix") {
-    errors.push("hermes-e2e-vitest job must depend on generate-matrix validation");
+  const needs = Array.isArray(job.needs) ? job.needs : [];
+  if (!needs.includes("validate-jobs") || !needs.includes("generate-matrix")) {
+    errors.push("hermes-e2e-vitest job must depend on validate-jobs and generate-matrix validation");
   }
   if (job.if !== "${{ needs.generate-matrix.outputs.hermes_selected == 'true' }}") {
     errors.push("hermes-e2e-vitest job must use validated hermes_selected output");
@@ -483,9 +485,17 @@ export function validateE2eVitestScenariosWorkflowBoundary(
   requireFullShaAction(errors, generateSetupNode, "generate-matrix setup-node");
   const generate = requireStep(errors, generateSteps, "Generate Vitest scenario matrix");
   const generateEnv = asRecord(generate?.env);
+  if (generateEnv.JOBS !== "${{ inputs.jobs }}") {
+    errors.push("matrix generation step must pass jobs through JOBS env");
+  }
   if (generateEnv.SCENARIOS !== "${{ inputs.scenarios }}") {
     errors.push("matrix generation step must pass scenarios through SCENARIOS env");
   }
+  requireRunContains(errors, generate, "allowed_jobs=");
+  requireRunContains(errors, generate, "Use either scenarios or jobs, not both");
+  requireRunContains(errors, generate, "Unknown free-standing Vitest job");
+  requireRunContains(errors, generate, "hermes-e2e-vitest");
+  requireRunContains(errors, generate, "matrix=\"[]\"");
   requireRunContains(errors, generate, "npx tsx test/e2e-scenario/scenarios/run.ts");
   requireRunContains(errors, generate, "--emit-live-matrix");
   requireRunContains(errors, generate, "--scenarios");

From 147d1bf664fec2d902ea3426a542b899e72e7a42 Mon Sep 17 00:00:00 2001
From: Julie Yaunches <jyaunches@nvidia.com>
Date: Thu, 11 Jun 2026 13:07:23 -0400
Subject: [PATCH 7/9] test(e2e): bound workflow matrix helper

---
 test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts
index 5cedcd91637..d9a7828eaea 100644
--- a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts
+++ b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts
@@ -37,6 +37,8 @@ function generateMatrixForDispatch(env: {
     const result = spawnSync("bash", ["-c", generateStep?.run as string], {
       cwd: process.cwd(),
       encoding: "utf-8",
+      timeout: 120_000,
+      killSignal: "SIGKILL",
       env: {
         ...process.env,
         GITHUB_OUTPUT: outputPath,
@@ -45,6 +47,7 @@ function generateMatrixForDispatch(env: {
         SCENARIOS: env.SCENARIOS,
       },
     });
+    expect(result.signal).toBeNull();
     expect(result.stderr).toBe("");
     expect(result.status).toBe(0);
     return Object.fromEntries(

From 02e8ddb7f907f155969dea52aeafc2ca06712666 Mon Sep 17 00:00:00 2001
From: Julie Yaunches <jyaunches@nvidia.com>
Date: Thu, 11 Jun 2026 14:01:53 -0400
Subject: [PATCH 8/9] test(e2e): harden Hermes inference probe

---
 test/e2e-scenario/live/hermes-e2e.test.ts | 91 +++++++++++++++--------
 1 file changed, 60 insertions(+), 31 deletions(-)

diff --git a/test/e2e-scenario/live/hermes-e2e.test.ts b/test/e2e-scenario/live/hermes-e2e.test.ts
index 8d31a5d5fdd..e086f52fc41 100644
--- a/test/e2e-scenario/live/hermes-e2e.test.ts
+++ b/test/e2e-scenario/live/hermes-e2e.test.ts
@@ -42,6 +42,7 @@ interface OpenAiChoiceLike {
     reasoning_content?: unknown;
   };
   text?: unknown;
+  finish_reason?: unknown;
 }
 
 interface OpenAiChatLike {
@@ -94,11 +95,11 @@ function commandEnv(apiKey?: string): NodeJS.ProcessEnv {
   return env;
 }
 
-function chatPayload(prompt: string): string {
+function chatPayload(prompt: string, maxTokens = 256): string {
   return JSON.stringify({
     model: CHAT_MODEL,
     messages: [{ role: "user", content: prompt }],
-    max_tokens: 100,
+    max_tokens: maxTokens,
   });
 }
 
@@ -121,6 +122,25 @@ function chatContent(response: unknown): string {
   return "";
 }
 
+function firstChoice(response: unknown): OpenAiChoiceLike | undefined {
+  if (!response || typeof response !== "object") return undefined;
+  const choices = (response as OpenAiChatLike).choices;
+  if (!Array.isArray(choices)) return undefined;
+  return choices.find((choice) => choice && typeof choice === "object");
+}
+
+function shouldRetryForReasoningBudget(response: unknown): boolean {
+  const content = chatContent(response);
+  if (/PONG/i.test(content)) return false;
+  const choice = firstChoice(response);
+  const message = choice?.message;
+  return (
+    choice?.finish_reason === "length" &&
+    typeof message?.reasoning_content === "string" &&
+    message.reasoning_content.trim().length > 0
+  );
+}
+
 function expectPong(label: string, response: unknown): void {
   const content = chatContent(response);
   expect(
@@ -460,25 +480,32 @@ test.skipIf(!shouldRunLiveE2EScenarios())(
 
     // Phase 5: live inference through both the external provider and the
     // sandbox's inference.local route.
-    const directChat = await retryHostedInference("direct NVIDIA Endpoints chat", (attempt) =>
-      provider.requestJson(
-        trustedProviderEndpoint("https://integrate.api.nvidia.com/v1/chat/completions", {
-          allowedHosts: ["integrate.api.nvidia.com"],
-        }),
-        {
-          artifactName: `phase-5-direct-nvidia-chat-attempt-${attempt}`,
-          body: chatPayload("Reply with exactly one word: PONG"),
-          curlMaxTimeSeconds: 90,
-          headers: ["Content-Type: application/json", `Authorization: Bearer ${apiKey}`],
-          env: buildAvailabilityProbeEnv(),
-          redactionValues,
-          timeoutMs: 120_000,
-        },
-      ),
+    const directChat = await retryHostedInference(
+      "direct NVIDIA Endpoints chat",
+      async (attempt) => {
+        const response = await provider.requestJson(
+          trustedProviderEndpoint("https://integrate.api.nvidia.com/v1/chat/completions", {
+            allowedHosts: ["integrate.api.nvidia.com"],
+          }),
+          {
+            artifactName: `phase-5-direct-nvidia-chat-attempt-${attempt}`,
+            body: chatPayload("Reply with exactly one word: PONG", attempt === 1 ? 256 : 1024),
+            curlMaxTimeSeconds: 90,
+            headers: ["Content-Type: application/json", `Authorization: Bearer ${apiKey}`],
+            env: buildAvailabilityProbeEnv(),
+            redactionValues,
+            timeoutMs: 120_000,
+          },
+        );
+        if (shouldRetryForReasoningBudget(response.json)) {
+          throw new Error("direct chat exhausted response budget while reasoning before PONG");
+        }
+        return response;
+      },
     );
     expectPong("direct NVIDIA Endpoints chat", directChat.json);
 
-    const sandboxChat = await retryHostedInference(
+    const sandboxChatJson = await retryHostedInference(
       "Hermes sandbox inference.local chat",
       async (attempt) => {
         const result = await sandbox.exec(
@@ -491,7 +518,7 @@ test.skipIf(!shouldRunLiveE2EScenarios())(
             "-H",
             "Content-Type: application/json",
             "--data-raw",
-            chatPayload("Reply with exactly one word: PONG"),
+            chatPayload("Reply with exactly one word: PONG", attempt === 1 ? 256 : 1024),
             "https://inference.local/v1/chat/completions",
           ],
           {
@@ -501,20 +528,22 @@ test.skipIf(!shouldRunLiveE2EScenarios())(
           },
         );
         if (result.exitCode !== 0) throw new Error(resultText(result));
-        return result;
+        let parsed: unknown;
+        try {
+          parsed = JSON.parse(result.stdout) as unknown;
+        } catch (error) {
+          throw new Error(
+            `Hermes sandbox inference.local chat response was not JSON: ${
+              error instanceof Error ? error.message : String(error)
+            }; body=${result.stdout.slice(0, 500)}`,
+          );
+        }
+        if (shouldRetryForReasoningBudget(parsed)) {
+          throw new Error("sandbox chat exhausted response budget while reasoning before PONG");
+        }
+        return parsed;
       },
     );
-    expect(sandboxChat.exitCode, resultText(sandboxChat)).toBe(0);
-    let sandboxChatJson: unknown;
-    try {
-      sandboxChatJson = JSON.parse(sandboxChat.stdout) as unknown;
-    } catch (error) {
-      throw new Error(
-        `Hermes sandbox inference.local chat response was not JSON: ${
-          error instanceof Error ? error.message : String(error)
-        }; body=${sandboxChat.stdout.slice(0, 500)}`,
-      );
-    }
     expectPong("Hermes sandbox inference.local chat", sandboxChatJson);
 
     // Phase 6: CLI operations and agent manifest regression.

From e8f1efc5b1d5ca1d3f8e54bde3de0d2e5cbc8f61 Mon Sep 17 00:00:00 2001
From: Julie Yaunches <jyaunches@nvidia.com>
Date: Thu, 11 Jun 2026 16:59:32 -0400
Subject: [PATCH 9/9] style(e2e): format Hermes workflow test

---
 .../support-tests/e2e-scenarios-workflow.test.ts           | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts
index 4e004245162..2a1385c2e60 100644
--- a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts
+++ b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts
@@ -122,8 +122,6 @@ describe("e2e-vitest-scenarios workflow boundary", () => {
     });
   });
 
-
-
   it("keeps jobs-only dispatches from selecting the Hermes secret-bearing job", () => {
     expect(
       generateMatrixForDispatch({ JOBS: "openshell-version-pin-vitest", SCENARIOS: "" }),
@@ -135,7 +133,9 @@ describe("e2e-vitest-scenarios workflow boundary", () => {
       hermes_selected: "true",
       matrix: "[]",
     });
-    expect(generateMatrixForDispatch({ JOBS: "network-policy-vitest", SCENARIOS: "" })).toMatchObject({
+    expect(
+      generateMatrixForDispatch({ JOBS: "network-policy-vitest", SCENARIOS: "" }),
+    ).toMatchObject({
       hermes_selected: "false",
       matrix: "[]",
     });
@@ -454,7 +454,6 @@ jobs:
     }
   });
 
-
   it("rejects raw jobs selector echo from matrix generation", () => {
     const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-vitest-workflow-"));
     const workflowPath = path.join(tmp, "workflow.yaml");