diff --git a/.github/workflows/nightly-e2e.yaml b/.github/workflows/nightly-e2e.yaml index efca3afc5da..b31840d18fc 100644 --- a/.github/workflows/nightly-e2e.yaml +++ b/.github/workflows/nightly-e2e.yaml @@ -33,6 +33,8 @@ # agent state, and the same agent type running. # hermes-e2e Hermes Agent E2E — install → onboard --agent hermes → health # probe → live inference. Validates the multi-agent architecture. +# hermes-dashboard-e2e Hermes Agent E2E with optional web dashboard enabled, +# validating API/dashboard forwards and host reachability. # hermes-root-entrypoint-smoke-e2e # Builds the real Hermes image and verifies root entrypoint startup, # gateway-user execution, v0.14 layout repair, and PID migration. @@ -94,6 +96,7 @@ on: token-rotation-e2e, sandbox-survival-e2e, openshell-gateway-upgrade-e2e, issue-2478-crash-loop-recovery-e2e, hermes-e2e, + hermes-dashboard-e2e, hermes-root-entrypoint-smoke-e2e, openclaw-onboard-security-posture-e2e, hermes-onboard-security-posture-e2e, @@ -685,6 +688,24 @@ jobs: secrets: NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }} + hermes-dashboard-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',hermes-dashboard-e2e,')) + uses: ./.github/workflows/e2e-script.yaml + with: + ref: ${{ inputs.target_ref || github.ref }} + script: test/e2e/test-hermes-e2e.sh + timeout_minutes: 60 + artifact_name: "hermes-dashboard-e2e-install-log" + artifact_path: "/tmp/nemoclaw-e2e-hermes-install.log" + env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_AGENT":"hermes","NEMOCLAW_E2E_HERMES_DASHBOARD":"1","NEMOCLAW_HERMES_DASHBOARD":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_RECREATE_SANDBOX":"1","NEMOCLAW_SANDBOX_NAME":"e2e-hermes-dashboard"}' + nvidia_api_key: true + github_token: true + secrets: + NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} + BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }} hermes-root-entrypoint-smoke-e2e: if: >- github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || @@ -1845,6 +1866,7 @@ jobs: sandbox-survival-e2e, issue-2478-crash-loop-recovery-e2e, hermes-e2e, + hermes-dashboard-e2e, hermes-root-entrypoint-smoke-e2e, openclaw-onboard-security-posture-e2e, hermes-onboard-security-posture-e2e, @@ -1949,6 +1971,7 @@ jobs: sandbox-survival-e2e, issue-2478-crash-loop-recovery-e2e, hermes-e2e, + hermes-dashboard-e2e, hermes-root-entrypoint-smoke-e2e, openclaw-onboard-security-posture-e2e, hermes-onboard-security-posture-e2e, @@ -2110,6 +2133,7 @@ jobs: sandbox-survival-e2e, issue-2478-crash-loop-recovery-e2e, hermes-e2e, + hermes-dashboard-e2e, hermes-root-entrypoint-smoke-e2e, openclaw-onboard-security-posture-e2e, hermes-onboard-security-posture-e2e, diff --git a/agents/hermes/Dockerfile.base b/agents/hermes/Dockerfile.base index 4cbfc40ca5b..26e6493a750 100644 --- a/agents/hermes/Dockerfile.base +++ b/agents/hermes/Dockerfile.base @@ -26,7 +26,7 @@ ENV DEBIAN_FRONTEND=noninteractive # Calver tag v2026.5.16 = Hermes Agent v0.14.0. ARG HERMES_VERSION=v2026.5.16 ARG HERMES_TARBALL_SHA256=c0a554050a50ee9a62f3fa5cd288a167ba5640c42d647d100cdea084b7294143 -ARG HERMES_UV_EXTRAS="messaging web" +ARG HERMES_UV_EXTRAS="messaging web pty" ARG UV_VERSION=0.11.8 RUN apt-get update && apt-get install -y --no-install-recommends \ @@ -154,8 +154,9 @@ RUN printf '%s\n' \ # Install Hermes Agent from the selected GitHub release. # The image prebakes only the extras mapped to NemoClaw-supported onboarding -# integrations: messaging (Telegram, Discord, Slack, WeChat, WhatsApp) and -# web (API health/UI runtime). New Hermes integrations should be installed +# integrations: messaging (Telegram, Discord, Slack, WeChat, WhatsApp), +# web (API health/UI runtime), and pty (optional browser TUI bridge). +# New Hermes integrations should be installed # by the agent workflow when they are enabled rather than shipped in the # base image by default. # Root Node dependencies provide Hermes browser tooling such as agent-browser. @@ -175,6 +176,10 @@ RUN set -eu; \ done; \ uv sync --frozen --no-dev "$@" --no-cache \ && npm ci --prefer-offline --no-audit --no-fund \ + && npm ci --prefix ui-tui --prefer-offline --no-audit --no-fund \ + && npm run build --prefix ui-tui \ + && npm ci --prefix web --prefer-offline --no-audit --no-fund \ + && npm run build --prefix web \ && rm -rf /tmp/camoufox-* \ && ln -sf /opt/hermes/.venv/bin/hermes /usr/local/bin/hermes \ && ln -sf /opt/hermes/.venv/bin/hermes-agent /usr/local/bin/hermes-agent \ diff --git a/agents/hermes/manifest.yaml b/agents/hermes/manifest.yaml index aa3dc7f7181..85cc0bc1fc6 100644 --- a/agents/hermes/manifest.yaml +++ b/agents/hermes/manifest.yaml @@ -40,6 +40,13 @@ dashboard: kind: api # "ui" or "api" label: "OpenAI-compatible API" path: "/v1" # appended to the base URL when displayed +dashboard_ui: + label: "Web dashboard" + port: 9119 + path: "/" + enable_env: NEMOCLAW_HERMES_DASHBOARD + port_env: NEMOCLAW_HERMES_DASHBOARD_PORT + tui_env: NEMOCLAW_HERMES_DASHBOARD_TUI forward_ports: - 8642 diff --git a/agents/hermes/start.sh b/agents/hermes/start.sh index 44d3a9c94f3..67647b438c3 100755 --- a/agents/hermes/start.sh +++ b/agents/hermes/start.sh @@ -9,6 +9,7 @@ # - No device-pairing auto-pair watcher (Hermes has no browser pairing) # - Config is YAML (config.yaml + .env) not JSON (openclaw.json) # - Gateway listens on internal port 18642, socat forwards to 8642 +# - Optional dashboard listens on internal port 19119, socat forwards to 9119 # # SECURITY: The gateway runs as a separate user so the sandboxed agent cannot # kill it or restart it with a tampered config. Config hash is verified at @@ -113,6 +114,10 @@ PUBLIC_PORT=8642 # Hermes binds to 127.0.0.1 regardless of config (upstream bug). # Run it on an internal port and use socat to expose on PUBLIC_PORT. INTERNAL_PORT=18642 +HERMES_DASHBOARD_ENABLED="${NEMOCLAW_HERMES_DASHBOARD:-0}" +HERMES_DASHBOARD_PUBLIC_PORT="${NEMOCLAW_HERMES_DASHBOARD_PORT:-9119}" +HERMES_DASHBOARD_INTERNAL_PORT="${NEMOCLAW_HERMES_DASHBOARD_INTERNAL_PORT:-19119}" +HERMES_DASHBOARD_TUI="${NEMOCLAW_HERMES_DASHBOARD_TUI:-0}" HERMES="$(command -v hermes)" # Resolve once, use absolute path everywhere # Hermes resolves config and runtime state relative to HERMES_HOME. The config @@ -123,6 +128,61 @@ HERMES="$(command -v hermes)" # Resolve once, use absolute path everywhere HERMES_DIR="/sandbox/.hermes" HERMES_HASH_FILE="/etc/nemoclaw/hermes.config-hash" +truthy_env() { + case "$(printf '%s' "${1:-}" | tr '[:upper:]' '[:lower:]')" in + 1 | true | yes | on) return 0 ;; + *) return 1 ;; + esac +} + +validate_tcp_port() { + local name="$1" + local value="$2" + case "$value" in + '' | *[!0-9]*) + echo "[gateway] ERROR: ${name} must be an integer TCP port, got '${value}'" >&2 + exit 1 + ;; + esac + if [ "$value" -lt 1024 ] || [ "$value" -gt 65535 ]; then + echo "[gateway] ERROR: ${name} must be between 1024 and 65535, got '${value}'" >&2 + exit 1 + fi +} + +validate_port_configuration() { + validate_tcp_port PUBLIC_PORT "$PUBLIC_PORT" + validate_tcp_port INTERNAL_PORT "$INTERNAL_PORT" + validate_tcp_port HERMES_DASHBOARD_PUBLIC_PORT "$HERMES_DASHBOARD_PUBLIC_PORT" + validate_tcp_port HERMES_DASHBOARD_INTERNAL_PORT "$HERMES_DASHBOARD_INTERNAL_PORT" + if [ "$HERMES_DASHBOARD_PUBLIC_PORT" -eq "$PUBLIC_PORT" ]; then + echo "[gateway] ERROR: HERMES_DASHBOARD_PUBLIC_PORT must not equal PUBLIC_PORT (${PUBLIC_PORT})" >&2 + exit 1 + fi + if [ "$HERMES_DASHBOARD_INTERNAL_PORT" -eq "$INTERNAL_PORT" ]; then + echo "[gateway] ERROR: HERMES_DASHBOARD_INTERNAL_PORT must not equal INTERNAL_PORT (${INTERNAL_PORT})" >&2 + exit 1 + fi + if [ "$HERMES_DASHBOARD_PUBLIC_PORT" -eq "$INTERNAL_PORT" ]; then + echo "[gateway] ERROR: HERMES_DASHBOARD_PUBLIC_PORT must not equal INTERNAL_PORT (${INTERNAL_PORT})" >&2 + exit 1 + fi + if [ "$HERMES_DASHBOARD_INTERNAL_PORT" -eq "$PUBLIC_PORT" ]; then + echo "[gateway] ERROR: HERMES_DASHBOARD_INTERNAL_PORT must not equal PUBLIC_PORT (${PUBLIC_PORT})" >&2 + exit 1 + fi +} + +validate_port_configuration + +hermes_dashboard_enabled() { + truthy_env "$HERMES_DASHBOARD_ENABLED" +} + +hermes_dashboard_tui_enabled() { + truthy_env "$HERMES_DASHBOARD_TUI" +} + # verify_config_integrity is provided by sandbox-init.sh (parameterized). # configure_messaging_channels is provided by sandbox-init.sh (shared). @@ -132,6 +192,9 @@ print_dashboard_urls() { local_url="http://127.0.0.1:${PUBLIC_PORT}/v1" echo "[gateway] Hermes API: ${local_url}" >&2 echo "[gateway] Health: ${local_url%/v1}/health" >&2 + if hermes_dashboard_enabled; then + echo "[gateway] Dashboard: http://127.0.0.1:${HERMES_DASHBOARD_PUBLIC_PORT}/" >&2 + fi echo "[gateway] Connect any OpenAI-compatible frontend to this endpoint." >&2 } @@ -140,6 +203,11 @@ start_gateway_log_stream() { GATEWAY_LOG_TAIL_PID=$! } +start_dashboard_log_stream() { + { tail -n +1 -F /tmp/hermes-dashboard.log 2>/dev/null | sed -u 's/^/[dashboard-log:] /' >&2; } & + DASHBOARD_LOG_TAIL_PID=$! +} + retry_tirith_marker_if_needed() { local marker="${HERMES_DIR}/.tirith-install-failed" local reason @@ -187,6 +255,8 @@ has_live_hermes_gateway() { cleanup_orphan_socat_forwarders() { local proc_root="${NEMOCLAW_PROC_ROOT:-/proc}" + local dashboard_internal="${HERMES_DASHBOARD_INTERNAL_PORT:-19119}" + local dashboard_public="${HERMES_DASHBOARD_PUBLIC_PORT:-9119}" local cmdline_file pid cmdline for cmdline_file in "${proc_root}"/[0-9]*/cmdline; do @@ -198,6 +268,10 @@ cleanup_orphan_socat_forwarders() { echo "[gateway] Removing orphaned socat forwarder for ${PUBLIC_PORT}->${INTERNAL_PORT} (pid ${pid})" >&2 kill "$pid" 2>/dev/null || true ;; + *socat*"TCP-LISTEN:${dashboard_public}"*"TCP:127.0.0.1:${dashboard_internal}"*) + echo "[gateway] Removing orphaned dashboard socat forwarder for ${dashboard_public}->${dashboard_internal} (pid ${pid})" >&2 + kill "$pid" 2>/dev/null || true + ;; esac done } @@ -328,23 +402,76 @@ cleanup_stale_hermes_gateway_runtime() { # OpenShell needs the port accessible on 0.0.0.0 for port forwarding. # socat bridges 0.0.0.0:PUBLIC_PORT → 127.0.0.1:INTERNAL_PORT. SOCAT_PID="" +DASHBOARD_SOCAT_PID="" start_socat_forwarder() { + local label="${1:-gateway}" + local public_port="${2:-$PUBLIC_PORT}" + local internal_port="${3:-$INTERNAL_PORT}" + local pid_var="${4:-SOCAT_PID}" + if ! command -v socat >/dev/null 2>&1; then - echo "[gateway] socat not available — port forwarding from host may not work" >&2 + echo "[gateway] socat not available — ${label} port forwarding from host may not work" >&2 return fi local attempts=0 while [ "$attempts" -lt 30 ]; do - if ss -tln 2>/dev/null | grep -q "127.0.0.1:${INTERNAL_PORT}"; then + if ss -tln 2>/dev/null | grep -q "127.0.0.1:${internal_port}"; then break fi sleep 1 attempts=$((attempts + 1)) done - nohup socat TCP-LISTEN:"${PUBLIC_PORT}",bind=0.0.0.0,fork,reuseaddr \ - TCP:127.0.0.1:"${INTERNAL_PORT}" >/dev/null 2>&1 & - SOCAT_PID=$! - echo "[gateway] socat forwarder 0.0.0.0:${PUBLIC_PORT} → 127.0.0.1:${INTERNAL_PORT} (pid $SOCAT_PID)" >&2 + nohup socat TCP-LISTEN:"${public_port}",bind=0.0.0.0,fork,reuseaddr \ + TCP:127.0.0.1:"${internal_port}" >/dev/null 2>&1 & + printf -v "$pid_var" '%s' "$!" + echo "[gateway] ${label} socat forwarder 0.0.0.0:${public_port} → 127.0.0.1:${internal_port} (pid ${!pid_var})" >&2 +} + +build_hermes_dashboard_args() { + HERMES_DASHBOARD_ARGS=( + dashboard + --host + 127.0.0.1 + --port + "$HERMES_DASHBOARD_INTERNAL_PORT" + --skip-build + --no-open + ) + if hermes_dashboard_tui_enabled; then + HERMES_DASHBOARD_ARGS+=(--tui) + fi +} + +start_hermes_dashboard_current_user() { + hermes_dashboard_enabled || return 0 + + build_hermes_dashboard_args + prepare_restricted_log /tmp/hermes-dashboard.log "" 600 + HERMES_HOME="${HERMES_DIR}" \ + nohup "$HERMES" "${HERMES_DASHBOARD_ARGS[@]}" >/tmp/hermes-dashboard.log 2>&1 & + HERMES_DASHBOARD_PID=$! + echo "[gateway] hermes dashboard launched (pid $HERMES_DASHBOARD_PID)" >&2 + start_dashboard_log_stream + start_socat_forwarder "dashboard" \ + "$HERMES_DASHBOARD_PUBLIC_PORT" \ + "$HERMES_DASHBOARD_INTERNAL_PORT" \ + DASHBOARD_SOCAT_PID +} + +start_hermes_dashboard_sandbox_user() { + hermes_dashboard_enabled || return 0 + + build_hermes_dashboard_args + prepare_restricted_log /tmp/hermes-dashboard.log sandbox:sandbox 600 + HERMES_HOME="${HERMES_DIR}" \ + nohup "${STEP_DOWN_PREFIX_SANDBOX[@]}" sh -c 'umask 0077; exec "$@" >/tmp/hermes-dashboard.log 2>&1' sh "$HERMES" "${HERMES_DASHBOARD_ARGS[@]}" & + HERMES_DASHBOARD_PID=$! + echo "[gateway] hermes dashboard launched as 'sandbox' user (pid $HERMES_DASHBOARD_PID)" >&2 + start_dashboard_log_stream + start_socat_forwarder "dashboard" \ + "$HERMES_DASHBOARD_PUBLIC_PORT" \ + "$HERMES_DASHBOARD_INTERNAL_PORT" \ + DASHBOARD_SOCAT_PID } # ── Messaging egress ───────────────────────────────────────────── @@ -736,15 +863,19 @@ if [ "$(id -u)" -ne 0 ]; then GATEWAY_PID=$! echo "[gateway] hermes gateway launched (pid $GATEWAY_PID)" >&2 start_gateway_log_stream + start_hermes_dashboard_current_user # NOTE: PIDs are collected after launch; a signal arriving between trap # registration and the final append is a small race window (same as before # the shared-library refactor). Acceptable for entrypoint-level cleanup. SANDBOX_CHILD_PIDS=("$GATEWAY_PID") [ -n "${GATEWAY_LOG_TAIL_PID:-}" ] && SANDBOX_CHILD_PIDS+=("$GATEWAY_LOG_TAIL_PID") + [ -n "${HERMES_DASHBOARD_PID:-}" ] && SANDBOX_CHILD_PIDS+=("$HERMES_DASHBOARD_PID") + [ -n "${DASHBOARD_LOG_TAIL_PID:-}" ] && SANDBOX_CHILD_PIDS+=("$DASHBOARD_LOG_TAIL_PID") + [ -n "${DASHBOARD_SOCAT_PID:-}" ] && SANDBOX_CHILD_PIDS+=("$DASHBOARD_SOCAT_PID") # shellcheck disable=SC2034 # read by cleanup_on_signal from sandbox-init.sh SANDBOX_WAIT_PID="$GATEWAY_PID" trap cleanup_on_signal SIGTERM SIGINT - start_socat_forwarder + start_socat_forwarder "api" "$PUBLIC_PORT" "$INTERNAL_PORT" SOCAT_PID [ -n "${SOCAT_PID:-}" ] && SANDBOX_CHILD_PIDS+=("$SOCAT_PID") print_dashboard_urls @@ -780,15 +911,19 @@ HERMES_HOME="${HERMES_DIR}" \ GATEWAY_PID=$! echo "[gateway] hermes gateway launched as 'gateway' user (pid $GATEWAY_PID)" >&2 start_gateway_log_stream +start_hermes_dashboard_sandbox_user # NOTE: PIDs are collected after launch; a signal arriving between trap # registration and the final append is a small race window (same as before # the shared-library refactor). Acceptable for entrypoint-level cleanup. SANDBOX_CHILD_PIDS=("$GATEWAY_PID") [ -n "${GATEWAY_LOG_TAIL_PID:-}" ] && SANDBOX_CHILD_PIDS+=("$GATEWAY_LOG_TAIL_PID") +[ -n "${HERMES_DASHBOARD_PID:-}" ] && SANDBOX_CHILD_PIDS+=("$HERMES_DASHBOARD_PID") +[ -n "${DASHBOARD_LOG_TAIL_PID:-}" ] && SANDBOX_CHILD_PIDS+=("$DASHBOARD_LOG_TAIL_PID") +[ -n "${DASHBOARD_SOCAT_PID:-}" ] && SANDBOX_CHILD_PIDS+=("$DASHBOARD_SOCAT_PID") # shellcheck disable=SC2034 # read by cleanup_on_signal from sandbox-init.sh SANDBOX_WAIT_PID="$GATEWAY_PID" trap cleanup_on_signal SIGTERM SIGINT -start_socat_forwarder +start_socat_forwarder "api" "$PUBLIC_PORT" "$INTERNAL_PORT" SOCAT_PID [ -n "${SOCAT_PID:-}" ] && SANDBOX_CHILD_PIDS+=("$SOCAT_PID") print_dashboard_urls diff --git a/docs/get-started/quickstart-hermes.mdx b/docs/get-started/quickstart-hermes.mdx index 1a7d9764790..4605301cca5 100644 --- a/docs/get-started/quickstart-hermes.mdx +++ b/docs/get-started/quickstart-hermes.mdx @@ -85,6 +85,18 @@ Use the provider variables from [Inference Options](/inference/inference-options When onboarding completes, NemoClaw prints the sandbox name, model, lifecycle commands, and Hermes API endpoint. Hermes exposes an OpenAI-compatible API on port `8642`, not a browser dashboard. +To also launch the native Hermes web dashboard, opt in before onboarding: + +```bash +export NEMOCLAW_HERMES_DASHBOARD=1 +nemohermes onboard +``` + +The dashboard uses port `9119` by default. +Set `NEMOCLAW_HERMES_DASHBOARD_PORT` before onboarding to choose a different port. +Set `NEMOCLAW_HERMES_DASHBOARD_TUI=1` to enable Hermes' optional in-browser TUI tab. +For upstream dashboard features, see the +[Hermes web dashboard documentation](https://hermes-agent.nousresearch.com/docs/user-guide/features/web-dashboard). ```text ────────────────────────────────────────────────── @@ -99,6 +111,10 @@ Access Port 8642 must be forwarded before connecting. http://127.0.0.1:8642/v1 + Hermes Agent Web dashboard + Port 9119 must be forwarded before opening this URL. + http://127.0.0.1:9119/ + Terminal: nemohermes my-hermes connect @@ -145,6 +161,20 @@ Configure an OpenAI-compatible client with the base URL `http://127.0.0.1:8642/v Hermes uses API header authentication for client requests. Do not append an OpenClaw `#token=` URL fragment to the Hermes endpoint. +## Open the Optional Dashboard + +When `NEMOCLAW_HERMES_DASHBOARD=1` is set during onboarding, NemoClaw starts `hermes dashboard --no-open` inside the sandbox and forwards `http://127.0.0.1:9119/` on the host. +The API endpoint remains separate on `8642`. + +If the dashboard forward is missing after a reboot or terminal restart, start it again: + +```bash +openshell forward start --background 9119 my-hermes +``` + +Treat the dashboard as a local management UI. +Avoid exposing it on shared or public networks unless you put it behind your own access controls. + ## Manage the Sandbox Use the same lifecycle commands as a standard NemoClaw sandbox. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 85451dca94b..c642ebf3e50 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -1221,6 +1221,9 @@ All ports must be non-privileged integers between 1024 and 65535. | `NEMOCLAW_GATEWAY_PORT` | 8080 | OpenShell gateway port | | `NEMOCLAW_GATEWAY_BIND_ADDRESS` | 127.0.0.1 | OpenShell gateway bind address (`127.0.0.1` or `0.0.0.0`) | | `NEMOCLAW_DASHBOARD_PORT` | 18789 (auto-derived from `CHAT_UI_URL` port if set) | Dashboard UI | +| `NEMOCLAW_HERMES_DASHBOARD` | 0 | Optional Hermes native web dashboard (`1`, `true`, `yes`, or `on` enables it) | +| `NEMOCLAW_HERMES_DASHBOARD_PORT` | 9119 | Optional Hermes native web dashboard forward port | +| `NEMOCLAW_HERMES_DASHBOARD_TUI` | 0 | Optional Hermes in-browser TUI tab when the dashboard is enabled | | `NEMOCLAW_VLLM_PORT` | 8000 | vLLM / NIM inference | | `NEMOCLAW_OLLAMA_PORT` | 11434 | Ollama inference | | `NEMOCLAW_OLLAMA_PROXY_PORT` | 11435 | Ollama auth proxy | @@ -1253,6 +1256,8 @@ These overrides apply to onboarding, status checks, health probes, and the unins Defaults are unchanged when no variable is set. If `NEMOCLAW_DASHBOARD_PORT` or the port from `CHAT_UI_URL` is already occupied by another sandbox, onboarding scans `18789` through `18799` and uses the next free dashboard port. Pass `--control-ui-port ` to require a specific port. +For Hermes sandboxes, `NEMOCLAW_HERMES_DASHBOARD=1` starts the native Hermes dashboard separately from the OpenAI-compatible API. +The Hermes API remains on port `8642`; the optional browser dashboard uses `NEMOCLAW_HERMES_DASHBOARD_PORT`. ### Onboarding Configuration diff --git a/src/lib/actions/sandbox/forward-health.ts b/src/lib/actions/sandbox/forward-health.ts new file mode 100644 index 00000000000..e5bb78431e3 --- /dev/null +++ b/src/lib/actions/sandbox/forward-health.ts @@ -0,0 +1,61 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import type { + SandboxForwardHealth, + SandboxForwardListEntry, +} from "./process-recovery"; + +export function classifySandboxForwardHealth( + entries: SandboxForwardListEntry[], + sandboxName: string, + port: string, +): Exclude { + const match = entries.find((entry) => entry.port === port); + if (!match) return false; + if (match.sandboxName !== sandboxName) return "occupied"; + return match.status === "running"; +} + +/** + * Like {@link classifySandboxForwardHealth} but accepts a reachability + * callback that probes whether the local forwarded port actually answers. + * When the entry-based classification would return `false`, the + * reachability check overrides it: a port that answers is healthy + * regardless of what `forward list` reports. The "occupied" verdict is + * preserved — we never silently take over a forward owned by another + * sandbox, even if that forward happens to be reachable. + */ +export function classifyForwardHealthWithReachability( + entries: SandboxForwardListEntry[], + sandboxName: string, + port: string, + isReachable: () => boolean, +): Exclude { + const verdict = classifySandboxForwardHealth(entries, sandboxName, port); + if (verdict !== false) return verdict; + return isReachable() ? true : false; +} + +/** + * Synchronous reachability check for a local port. Used to override a + * negative `openshell forward list` verdict when the forward is actually + * still serving traffic. + */ +export function isLocalForwardReachable(port: number): boolean { + const script = + "const net=require('node:net');" + + `const s=net.createConnection({host:'127.0.0.1',port:${port}});` + + "s.setTimeout(1000);" + + "s.on('connect',()=>{s.destroy();process.exit(0)});" + + "s.on('error',()=>process.exit(1));" + + "s.on('timeout',()=>{s.destroy();process.exit(1)});"; + const result = spawnSync(process.execPath, ["-e", script], { + encoding: "utf-8", + stdio: ["ignore", "ignore", "ignore"], + timeout: 2000, + }); + if (result.error) return false; + return result.status === 0; +} diff --git a/src/lib/actions/sandbox/hermes-dashboard-recovery.test.ts b/src/lib/actions/sandbox/hermes-dashboard-recovery.test.ts new file mode 100644 index 00000000000..8b0be4ae53a --- /dev/null +++ b/src/lib/actions/sandbox/hermes-dashboard-recovery.test.ts @@ -0,0 +1,149 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; +import { + ensureHermesDashboardPortForwardIfEnabled, + getHermesDashboardRecoveryConfig, + recoverHermesDashboardProcessIfEnabled, +} from "../../../../dist/lib/actions/sandbox/hermes-dashboard-recovery"; + +describe("Hermes dashboard recovery helpers", () => { + it("reads recovery config only for enabled Hermes dashboard sandboxes", () => { + expect( + getHermesDashboardRecoveryConfig("alpha", () => ({ + name: "alpha", + agent: "hermes", + hermesDashboardEnabled: true, + hermesDashboardPort: 9119, + hermesDashboardInternalPort: 19119, + hermesDashboardTui: true, + })), + ).toEqual({ publicPort: 9119, internalPort: 19119, tuiEnabled: true }); + + expect( + getHermesDashboardRecoveryConfig("alpha", () => ({ + name: "alpha", + agent: "openclaw", + hermesDashboardEnabled: true, + hermesDashboardPort: 9119, + hermesDashboardInternalPort: 19119, + })), + ).toBeNull(); + + expect( + getHermesDashboardRecoveryConfig("alpha", () => ({ + name: "alpha", + agent: "hermes", + hermesDashboardEnabled: true, + hermesDashboardPort: "9119" as unknown as number, + hermesDashboardInternalPort: 19119, + })), + ).toBeNull(); + + expect( + getHermesDashboardRecoveryConfig("alpha", () => ({ + name: "alpha", + agent: "hermes", + hermesDashboardEnabled: true, + hermesDashboardPort: 1023, + hermesDashboardInternalPort: 19119, + })), + ).toBeNull(); + + expect( + getHermesDashboardRecoveryConfig("alpha", () => ({ + name: "alpha", + agent: "hermes", + hermesDashboardEnabled: true, + hermesDashboardPort: 9119, + hermesDashboardInternalPort: 1023, + })), + ).toBeNull(); + }); + + it("restarts the dashboard forward only when the recorded forward is unhealthy", () => { + const ensurePortForward = vi.fn(() => true); + const getRecoveryConfig = () => ({ + publicPort: 9119, + internalPort: 19119, + tuiEnabled: false, + }); + + expect( + ensureHermesDashboardPortForwardIfEnabled("alpha", { + getRecoveryConfig, + isPortForwardHealthy: () => false, + ensurePortForward, + }), + ).toBe(true); + expect(ensurePortForward).toHaveBeenCalledWith("alpha", 9119); + + expect( + ensureHermesDashboardPortForwardIfEnabled("alpha", { + getRecoveryConfig, + isPortForwardHealthy: () => true, + ensurePortForward, + }), + ).toBe(false); + + const ensurePortForwardWhenOccupied = vi.fn(() => true); + expect( + ensureHermesDashboardPortForwardIfEnabled("alpha", { + getRecoveryConfig, + isPortForwardHealthy: () => "occupied", + ensurePortForward: ensurePortForwardWhenOccupied, + }), + ).toBe(false); + expect(ensurePortForwardWhenOccupied).not.toHaveBeenCalled(); + + expect( + ensureHermesDashboardPortForwardIfEnabled("alpha", { + getRecoveryConfig: () => null, + isPortForwardHealthy: () => false, + ensurePortForward, + }), + ).toBeNull(); + }); + + it("reports dashboard process recovery only for successful recovery markers", () => { + const buildRecoveryScript = vi.fn(() => "dashboard recovery"); + const getRecoveryConfig = () => ({ + publicPort: 9119, + internalPort: 19119, + tuiEnabled: true, + }); + + expect( + recoverHermesDashboardProcessIfEnabled("alpha", { + getRecoveryConfig, + buildRecoveryScript, + executeCommand: (_sandboxName, command) => ({ + status: 0, + stdout: command === "dashboard recovery" ? "DASHBOARD_PID=42" : "", + stderr: "", + }), + }), + ).toBe(true); + expect(buildRecoveryScript).toHaveBeenCalledWith({ + publicPort: 9119, + internalPort: 19119, + tuiEnabled: true, + }); + + expect( + recoverHermesDashboardProcessIfEnabled("alpha", { + getRecoveryConfig, + buildRecoveryScript, + executeCommand: () => ({ status: 1, stdout: "DASHBOARD_FAILED", stderr: "" }), + }), + ).toBe(false); + + expect( + recoverHermesDashboardProcessIfEnabled("alpha", { + getRecoveryConfig: () => null, + executeCommand: () => ({ status: 0, stdout: "DASHBOARD_PID=42", stderr: "" }), + }), + ).toBeNull(); + }); +}); diff --git a/src/lib/actions/sandbox/hermes-dashboard-recovery.ts b/src/lib/actions/sandbox/hermes-dashboard-recovery.ts new file mode 100644 index 00000000000..686723493a7 --- /dev/null +++ b/src/lib/actions/sandbox/hermes-dashboard-recovery.ts @@ -0,0 +1,73 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import * as agentRuntime from "../../agent/runtime"; +import * as registry from "../../state/registry"; +import type { + SandboxCommandResult, + SandboxForwardHealth, +} from "./process-recovery"; + +type RecoveryConfigReader = ( + sandboxName: string, +) => agentRuntime.HermesDashboardRecoveryConfig | null; + +function isValidPort(value: unknown): value is number { + return ( + typeof value === "number" && + Number.isInteger(value) && + value >= 1024 && + value <= 65535 + ); +} + +export function getHermesDashboardRecoveryConfig( + sandboxName: string, + getSandbox: typeof registry.getSandbox = registry.getSandbox, +): agentRuntime.HermesDashboardRecoveryConfig | null { + const sandbox = getSandbox(sandboxName); + if (sandbox?.agent !== "hermes" || sandbox.hermesDashboardEnabled !== true) return null; + if (!isValidPort(sandbox.hermesDashboardPort)) return null; + if (!isValidPort(sandbox.hermesDashboardInternalPort)) return null; + return { + publicPort: sandbox.hermesDashboardPort, + internalPort: sandbox.hermesDashboardInternalPort, + tuiEnabled: sandbox.hermesDashboardTui === true, + }; +} + +export function ensureHermesDashboardPortForwardIfEnabled( + sandboxName: string, + deps: { + getRecoveryConfig?: RecoveryConfigReader; + isPortForwardHealthy(sandboxName: string, port: number): SandboxForwardHealth; + ensurePortForward(sandboxName: string, port: number): boolean; + }, +): boolean | null { + const dashboard = (deps.getRecoveryConfig ?? getHermesDashboardRecoveryConfig)(sandboxName); + if (dashboard === null) return null; + const forwardHealth = deps.isPortForwardHealthy(sandboxName, dashboard.publicPort); + if (forwardHealth === true || forwardHealth === "occupied") return false; + return deps.ensurePortForward(sandboxName, dashboard.publicPort); +} + +export function recoverHermesDashboardProcessIfEnabled( + sandboxName: string, + deps: { + getRecoveryConfig?: RecoveryConfigReader; + executeCommand(sandboxName: string, command: string): SandboxCommandResult | null; + buildRecoveryScript?: typeof agentRuntime.buildHermesDashboardProcessRecoveryScript; + }, +): boolean | null { + const dashboard = (deps.getRecoveryConfig ?? getHermesDashboardRecoveryConfig)(sandboxName); + if (dashboard === null) return null; + const buildRecoveryScript = + deps.buildRecoveryScript ?? agentRuntime.buildHermesDashboardProcessRecoveryScript; + const result = deps.executeCommand(sandboxName, buildRecoveryScript(dashboard)); + return !!( + result && + result.status === 0 && + (result.stdout.includes("DASHBOARD_PID=") || + result.stdout.includes("DASHBOARD_ALREADY_RUNNING")) + ); +} diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index 6086b799d07..6b9e060a99a 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -22,6 +22,20 @@ import { sleepSeconds } from "../../core/wait"; import { ROOT, shellQuote } from "../../runner"; import * as registry from "../../state/registry"; import { parseForwardList } from "../../state/sandbox-session"; +import { + classifyForwardHealthWithReachability, + isLocalForwardReachable, +} from "./forward-health"; +import { + ensureHermesDashboardPortForwardIfEnabled as ensureHermesDashboardPortForward, + getHermesDashboardRecoveryConfig, + recoverHermesDashboardProcessIfEnabled as recoverHermesDashboardProcess, +} from "./hermes-dashboard-recovery"; + +export { + classifyForwardHealthWithReachability, + classifySandboxForwardHealth, +} from "./forward-health"; export type SandboxCommandResult = { status: number; @@ -274,7 +288,9 @@ export async function probeSandboxInferenceGatewayHealth( function recoverSandboxProcesses(sandboxName: string): boolean { const agent = agentRuntime.getSessionAgent(sandboxName); const dashboardPort = resolveSandboxDashboardPort(sandboxName); - const agentScript = agentRuntime.buildRecoveryScript(agent, dashboardPort); + const agentScript = agentRuntime.buildRecoveryScript(agent, dashboardPort, { + hermesDashboard: getHermesDashboardRecoveryConfig(sandboxName), + }); const hasRecoveryMarker = (result: SandboxCommandResult | null) => !!( result && @@ -334,17 +350,7 @@ function waitForRecoveredSandboxGateway(sandboxName: string): boolean { * confirms the new entry is running, false otherwise. */ function ensureSandboxPortForward(sandboxName: string): boolean { - const forwardHealth = isSandboxForwardHealthy(sandboxName); - if (forwardHealth === true) return true; - if (forwardHealth === "occupied") return false; - - const port = String(resolveSandboxDashboardPort(sandboxName)); - runOpenshell(["forward", "stop", port], { ignoreError: true, stdio: "ignore" }); - const startResult = runOpenshell(["forward", "start", "--background", port, sandboxName], { - ignoreError: true, - }); - if (startResult.status !== 0) return false; - return isSandboxForwardHealthy(sandboxName) === true; + return ensureSandboxPortForwardForPort(sandboxName, resolveSandboxDashboardPort(sandboxName)); } /** @@ -367,7 +373,13 @@ function ensureSandboxPortForward(sandboxName: string): boolean { * re-establish" line even though the forward worked. */ function isSandboxForwardHealthy(sandboxName: string): SandboxForwardHealth { - const port = resolveSandboxDashboardPort(sandboxName); + return isSandboxPortForwardHealthy(sandboxName, resolveSandboxDashboardPort(sandboxName)); +} + +function isSandboxPortForwardHealthy( + sandboxName: string, + port: number, +): SandboxForwardHealth { const result = captureOpenshell(["forward", "list"], { ignoreError: true, timeout: OPENSHELL_PROBE_TIMEOUT_MS, @@ -379,59 +391,31 @@ function isSandboxForwardHealthy(sandboxName: string): SandboxForwardHealth { ); } -export function classifySandboxForwardHealth( - entries: SandboxForwardListEntry[], - sandboxName: string, - port: string, -): Exclude { - const match = entries.find((entry) => entry.port === port); - if (!match) return false; - if (match.sandboxName !== sandboxName) return "occupied"; - return match.status === "running"; -} +function ensureSandboxPortForwardForPort(sandboxName: string, port: number): boolean { + const forwardHealth = isSandboxPortForwardHealthy(sandboxName, port); + if (forwardHealth === true) return true; + if (forwardHealth === "occupied") return false; -/** - * Like {@link classifySandboxForwardHealth} but accepts a reachability - * callback that probes whether the local forwarded port actually answers. - * When the entry-based classification would return `false`, the - * reachability check overrides it: a port that answers is healthy - * regardless of what `forward list` reports. The "occupied" verdict is - * preserved — we never silently take over a forward owned by another - * sandbox, even if that forward happens to be reachable. - */ -export function classifyForwardHealthWithReachability( - entries: SandboxForwardListEntry[], - sandboxName: string, - port: string, - isReachable: () => boolean, -): Exclude { - const verdict = classifySandboxForwardHealth(entries, sandboxName, port); - if (verdict !== false) return verdict; - return isReachable() ? true : false; + runOpenshell(["forward", "stop", String(port), sandboxName], { + ignoreError: true, + stdio: "ignore", + }); + const startResult = runOpenshell(["forward", "start", "--background", String(port), sandboxName], { + ignoreError: true, + }); + if (startResult.status !== 0) return false; + return isSandboxPortForwardHealthy(sandboxName, port) === true; } -/** - * Synchronous reachability check for a local port. Used to override a - * negative `openshell forward list` verdict when the forward is actually - * still serving traffic — see {@link classifyForwardHealthWithReachability}. - * Returns false on any error so the existing recovery path stays intact - * when Node can't probe (e.g., restrictive sandbox). - */ -function isLocalForwardReachable(port: number): boolean { - const script = - "const net=require('node:net');" + - `const s=net.createConnection({host:'127.0.0.1',port:${port}});` + - "s.setTimeout(1000);" + - "s.on('connect',()=>{s.destroy();process.exit(0)});" + - "s.on('error',()=>process.exit(1));" + - "s.on('timeout',()=>{s.destroy();process.exit(1)});"; - const result = spawnSync(process.execPath, ["-e", script], { - encoding: "utf-8", - stdio: ["ignore", "ignore", "ignore"], - timeout: 2000, +function ensureHermesDashboardPortForwardIfEnabled(sandboxName: string): boolean | null { + return ensureHermesDashboardPortForward(sandboxName, { + isPortForwardHealthy: isSandboxPortForwardHealthy, + ensurePortForward: ensureSandboxPortForwardForPort, }); - if (result.error) return false; - return result.status === 0; +} + +function recoverHermesDashboardProcessIfEnabled(sandboxName: string): boolean | null { + return recoverHermesDashboardProcess(sandboxName, { executeCommand: executeSandboxCommand }); } /** @@ -455,6 +439,7 @@ export function checkAndRecoverSandboxProcesses( // Gateway is alive but the host-side forward can still be dead or // owned by another sandbox. Probe and re-establish only when // necessary so the live-and-healthy path stays a no-op. + const dashboardProcessRecovered = recoverHermesDashboardProcessIfEnabled(sandboxName); const forwardHealthy = isSandboxForwardHealthy(sandboxName); if (forwardHealthy === false) { if (!quiet) { @@ -463,6 +448,7 @@ export function checkAndRecoverSandboxProcesses( console.log(" Re-establishing..."); } const forwardRecovered = ensureSandboxPortForward(sandboxName); + const dashboardForwardRecovered = ensureHermesDashboardPortForwardIfEnabled(sandboxName); if (!quiet) { if (forwardRecovered) { console.log(` ${G}✓${R} Dashboard port forward re-established.`); @@ -473,7 +459,15 @@ export function checkAndRecoverSandboxProcesses( ); } } - return { checked: true, wasRunning: true, recovered: false, forwardRecovered }; + return { + checked: true, + wasRunning: true, + recovered: false, + forwardRecovered: + forwardRecovered || + dashboardForwardRecovered === true || + dashboardProcessRecovered === true, + }; } if (forwardHealthy === "occupied") { if (!quiet) { @@ -483,7 +477,14 @@ export function checkAndRecoverSandboxProcesses( } return { checked: true, wasRunning: true, recovered: false, forwardRecovered: false }; } - return { checked: true, wasRunning: true, recovered: false, forwardRecovered: false }; + const dashboardForwardRecovered = ensureHermesDashboardPortForwardIfEnabled(sandboxName); + return { + checked: true, + wasRunning: true, + recovered: false, + forwardRecovered: + dashboardForwardRecovered === true || dashboardProcessRecovered === true, + }; } // Gateway not running — attempt recovery @@ -511,6 +512,7 @@ export function checkAndRecoverSandboxProcesses( return { checked: true, wasRunning: false, recovered: false, forwardRecovered: false }; } const forwardRecovered = ensureSandboxPortForward(sandboxName); + const dashboardForwardRecovered = ensureHermesDashboardPortForwardIfEnabled(sandboxName); if (!quiet) { console.log( ` ${G}✓${R} ${agentRuntime.getAgentDisplayName(recoveryAgent)} gateway restarted inside sandbox.`, @@ -524,7 +526,12 @@ export function checkAndRecoverSandboxProcesses( ); } } - return { checked: true, wasRunning: false, recovered, forwardRecovered }; + return { + checked: true, + wasRunning: false, + recovered, + forwardRecovered: forwardRecovered || dashboardForwardRecovered === true, + }; } if (!quiet) { console.error( diff --git a/src/lib/agent/dashboard-ui.ts b/src/lib/agent/dashboard-ui.ts new file mode 100644 index 00000000000..eff8d5be391 --- /dev/null +++ b/src/lib/agent/dashboard-ui.ts @@ -0,0 +1,125 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { isTruthyEnv } from "../hermes-dashboard"; +import type { AgentDefinition } from "./defs"; + +type ManifestRecordLike = Record; + +export interface AgentDashboardUi { + label: string; + port: number; + path: string; + enableEnv: string; + portEnv: string; + tuiEnv: string | null; +} + +function readString(record: ManifestRecordLike, key: string): string | undefined { + const value = record[key]; + return typeof value === "string" ? value : undefined; +} + +function readObject(record: ManifestRecordLike, key: string): ManifestRecordLike | undefined { + const value = record[key]; + if (typeof value !== "object" || value === null || Array.isArray(value)) return undefined; + const prototype = Object.getPrototypeOf(value); + if (prototype !== Object.prototype && prototype !== null) return undefined; + return value as ManifestRecordLike; +} + +function isValidPort(value: unknown): value is number { + return ( + typeof value === "number" && + Number.isInteger(value) && + value >= 1024 && + value <= 65535 + ); +} + +export function readDashboardUi(record: ManifestRecordLike): AgentDashboardUi | null { + const dashboardUi = readObject(record, "dashboard_ui"); + if (!dashboardUi) return null; + + const port = dashboardUi.port; + if (!isValidPort(port)) { + throw new Error( + "Agent manifest field 'dashboard_ui.port' must be an integer TCP port between 1024 and 65535", + ); + } + + const label = readString(dashboardUi, "label")?.trim() || "Web dashboard"; + const rawPath = readString(dashboardUi, "path")?.trim() || "/"; + const enableEnv = readString(dashboardUi, "enable_env")?.trim(); + const portEnv = readString(dashboardUi, "port_env")?.trim(); + const tuiEnv = readString(dashboardUi, "tui_env")?.trim() || null; + if (!enableEnv) { + throw new Error("Agent manifest field 'dashboard_ui.enable_env' is required"); + } + if (!portEnv) { + throw new Error("Agent manifest field 'dashboard_ui.port_env' is required"); + } + + return { + label, + port, + path: rawPath.startsWith("/") ? rawPath : `/${rawPath}`, + enableEnv, + portEnv, + tuiEnv, + }; +} + +function dashboardUiEnabled(agent: AgentDefinition, env: NodeJS.ProcessEnv): boolean { + const dashboardUi = agent.dashboardUi; + return !!dashboardUi && isTruthyEnv(env[dashboardUi.enableEnv]); +} + +function dashboardUiPort(agent: AgentDefinition, env: NodeJS.ProcessEnv): number { + const dashboardUi = agent.dashboardUi; + if (!dashboardUi) return agent.forwardPort; + const raw = env[dashboardUi.portEnv]; + if (raw && /^\d+$/.test(raw.trim())) { + const port = Number(raw.trim()); + if (port >= 1024 && port <= 65535) return port; + } + return dashboardUi.port; +} + +export function printOptionalDashboardUi( + agent: AgentDefinition, + deps: { + buildControlUiUrls: (token: string | null, port: number) => string[]; + redactUrl: (url: string) => string; + env?: NodeJS.ProcessEnv; + writeLine?: (message?: string) => void; + }, +): void { + const dashboardUi = agent.dashboardUi; + const env = deps.env ?? process.env; + if (!dashboardUi || !dashboardUiEnabled(agent, env)) return; + + const writeLine = deps.writeLine ?? console.log; + const port = dashboardUiPort(agent, env); + writeLine(""); + writeLine(` ${agent.displayName} ${dashboardUi.label}`); + writeLine(` Port ${port} must be forwarded before opening this URL.`); + const seen = new Set(); + for (const baseUrl of deps.buildControlUiUrls(null, port)) { + const withoutHash = baseUrl.split("#")[0].replace(/\/$/, ""); + let urlPort = ""; + try { + urlPort = new URL(withoutHash).port; + } catch { + urlPort = ""; + } + if (urlPort !== String(port)) continue; + const url = + dashboardUi.path && dashboardUi.path !== "/" + ? `${withoutHash}${dashboardUi.path}` + : `${withoutHash}/`; + if (seen.has(url)) continue; + seen.add(url); + writeLine(` ${deps.redactUrl(url)}`); + } +} diff --git a/src/lib/agent/defs.test.ts b/src/lib/agent/defs.test.ts index 38ccd263cc1..b188f45dfa8 100644 --- a/src/lib/agent/defs.test.ts +++ b/src/lib/agent/defs.test.ts @@ -72,6 +72,19 @@ describe("agent definitions", () => { }); expect(hermes.inferenceProviderOptions).toEqual(["hermesProvider"]); expect(hermes.healthProbe.url).toBe("http://localhost:8642/health"); + expect(hermes.dashboard).toEqual({ + kind: "api", + label: "OpenAI-compatible API", + path: "/v1", + }); + expect(hermes.dashboardUi).toEqual({ + label: "Web dashboard", + port: 9119, + path: "/", + enableEnv: "NEMOCLAW_HERMES_DASHBOARD", + portEnv: "NEMOCLAW_HERMES_DASHBOARD_PORT", + tuiEnv: "NEMOCLAW_HERMES_DASHBOARD_TUI", + }); expect(hermes.messagingPlatforms).toEqual([ "telegram", "discord", @@ -138,6 +151,24 @@ describe("agent definitions", () => { expect(() => loadAgent(agentName)).toThrow(/health_probe\.port/); }); + it("rejects invalid dashboard_ui.port values in manifests", () => { + const agentName = `invalid-dashboard-ui-port-${String(Date.now())}`; + writeTempAgentManifest( + agentName, + [ + `name: ${agentName}`, + "display_name: Broken Dashboard UI", + "dashboard_ui:", + " label: Web dashboard", + " port: 1023", + " enable_env: NEMOCLAW_TEST_DASHBOARD", + " port_env: NEMOCLAW_TEST_DASHBOARD_PORT", + ].join("\n"), + ); + + expect(() => loadAgent(agentName)).toThrow(/dashboard_ui\.port/); + }); + it("rejects invalid inference provider options in manifests", () => { const agentName = `invalid-inference-options-${String(Date.now())}`; writeTempAgentManifest( diff --git a/src/lib/agent/defs.ts b/src/lib/agent/defs.ts index 20f176221d8..4826aa5d86c 100644 --- a/src/lib/agent/defs.ts +++ b/src/lib/agent/defs.ts @@ -6,9 +6,9 @@ import fs from "node:fs"; import path from "node:path"; - -import { ROOT } from "../runner"; import { DASHBOARD_PORT } from "../core/ports"; +import { ROOT } from "../runner"; +import { type AgentDashboardUi, readDashboardUi } from "./dashboard-ui"; export const AGENTS_DIR = path.join(ROOT, "agents"); @@ -84,6 +84,7 @@ export interface AgentDefinition { readonly healthProbe: AgentHealthProbe; readonly forwardPort: number; readonly dashboard: AgentDashboard; + readonly dashboardUi?: AgentDashboardUi | null; readonly configPaths: AgentConfigPaths; readonly inferenceProviderOptions: string[]; readonly stateDirs: string[]; @@ -340,6 +341,7 @@ export function loadAgent(name: string): AgentDefinition { const phoneHomeHosts = readStringArray(raw, "phone_home_hosts"); const messagingPlatforms = readMessagingPlatforms(raw); const legacyPathConfig = readStringMap(raw, "_legacy_paths"); + const dashboardUi = readDashboardUi(raw); const agent: AgentDefinition = { ...raw, @@ -395,6 +397,10 @@ export function loadAgent(name: string): AgentDefinition { }; }, + get dashboardUi(): AgentDashboardUi | null { + return dashboardUi; + }, + get configPaths(): AgentConfigPaths { return { dir: readString(config ?? {}, "dir") ?? "/sandbox/.openclaw", diff --git a/src/lib/agent/onboard.test.ts b/src/lib/agent/onboard.test.ts index 072c27a7b56..77024c314f6 100644 --- a/src/lib/agent/onboard.test.ts +++ b/src/lib/agent/onboard.test.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, it, expect, beforeEach, afterEach, afterAll, vi } from "vitest"; +import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from "vitest"; // Import from compiled dist/ so coverage is attributed correctly. import { collectHermesStartupDiagnostics, @@ -50,6 +50,14 @@ const apiAgent = makeAgent({ displayName: "Hermes Agent", forwardPort: 8642, dashboard: { kind: "api", label: "OpenAI-compatible API", path: "/v1" }, + dashboardUi: { + label: "Web dashboard", + port: 9119, + path: "/", + enableEnv: "NEMOCLAW_HERMES_DASHBOARD", + portEnv: "NEMOCLAW_HERMES_DASHBOARD_PORT", + tuiEnv: "NEMOCLAW_HERMES_DASHBOARD_TUI", + }, }); const uiAgent = makeAgent({ @@ -78,6 +86,8 @@ describe("printDashboardUi — regression for #2078 (port 8642 is not a chat UI) afterEach(() => { logSpy.mockClear(); + delete process.env.NEMOCLAW_HERMES_DASHBOARD; + delete process.env.NEMOCLAW_HERMES_DASHBOARD_PORT; }); afterAll(() => { @@ -113,6 +123,38 @@ describe("printDashboardUi — regression for #2078 (port 8642 is not a chat UI) expect(noteSpy).not.toHaveBeenCalled(); }); + it("prints the optional Hermes web dashboard URL when dashboard mode is enabled", () => { + process.env.NEMOCLAW_HERMES_DASHBOARD = "1"; + process.env.NEMOCLAW_HERMES_DASHBOARD_PORT = "9120"; + + printDashboardUi("sandbox-x", null, apiAgent, { + note: noteSpy, + buildControlUiUrls: buildUrlsLoopback, + }); + + const output = logSpy.mock.calls.map((args) => String(args[0])).join("\n"); + expect(output).toContain("Hermes Agent OpenAI-compatible API"); + expect(output).toContain("http://127.0.0.1:8642/v1"); + expect(output).toContain("Hermes Agent Web dashboard"); + expect(output).toContain("Port 9120 must be forwarded before opening this URL."); + expect(output).toContain("http://127.0.0.1:9120/"); + }); + + it("falls back to the manifest dashboard port for privileged env override ports", () => { + process.env.NEMOCLAW_HERMES_DASHBOARD = "1"; + process.env.NEMOCLAW_HERMES_DASHBOARD_PORT = "1023"; + + printDashboardUi("sandbox-x", null, apiAgent, { + note: noteSpy, + buildControlUiUrls: buildUrlsLoopback, + }); + + const output = logSpy.mock.calls.map((args) => String(args[0])).join("\n"); + expect(output).toContain("Port 9119 must be forwarded before opening this URL."); + expect(output).toContain("http://127.0.0.1:9119/"); + expect(output).not.toContain("http://127.0.0.1:1023/"); + }); + it("redacts tokenized URLs for UI-kind agents and shows the token retrieval command", () => { const token = "a".repeat(64); printDashboardUi("sandbox-y", token, uiAgent, { diff --git a/src/lib/agent/onboard.ts b/src/lib/agent/onboard.ts index db4c3f5794a..04508067ff4 100644 --- a/src/lib/agent/onboard.ts +++ b/src/lib/agent/onboard.ts @@ -11,8 +11,9 @@ import path from "path"; import { dockerBuild, dockerImageInspect } from "../adapters/docker"; import { getAgentBranding } from "../cli/branding"; -import { getProviderSelectionConfig } from "../inference/config"; import type { JsonObject as LooseObject } from "../core/json-types"; +import { sleepSeconds } from "../core/wait"; +import { getProviderSelectionConfig } from "../inference/config"; import { runSandboxConfigSync } from "../onboard/config-sync"; import { ROOT, redact, run, shellQuote } from "../runner"; import { @@ -20,7 +21,7 @@ import { resolveSandboxBaseImage, SANDBOX_BASE_TAG, } from "../sandbox-base-image"; -import { sleepSeconds } from "../core/wait"; +import { printOptionalDashboardUi } from "./dashboard-ui"; import { type AgentDefinition, loadAgent, resolveAgentName } from "./defs"; export interface OnboardContext { @@ -550,6 +551,7 @@ export function printDashboardUi( seen.add(url); console.log(` ${dashboardUrlForDisplay(url)}`); } + printOptionalDashboardUi(agent, { ...deps, redactUrl: dashboardUrlForDisplay }); return; } @@ -571,4 +573,5 @@ export function printDashboardUi( console.log(` ${dashboardUrlForDisplay(url)}`); } } + printOptionalDashboardUi(agent, { ...deps, redactUrl: dashboardUrlForDisplay }); } diff --git a/src/lib/agent/runtime.test.ts b/src/lib/agent/runtime.test.ts index 81a98e58167..a2cd02a3527 100644 --- a/src/lib/agent/runtime.test.ts +++ b/src/lib/agent/runtime.test.ts @@ -4,6 +4,7 @@ import { describe, it, expect } from "vitest"; // Import from compiled dist/ so coverage is attributed correctly. import { + buildHermesDashboardProcessRecoveryScript, buildManualRecoveryCommand, buildOpenClawRecoveryScript, buildRecoveryScript, @@ -108,6 +109,30 @@ describe("buildRecoveryScript", () => { expect(script).not.toContain("hermes gateway run --port 8642"); }); + it("relaunches the optional Hermes dashboard during recovery", () => { + const script = buildRecoveryScript(hermesAgent, 8642, { + hermesDashboard: { publicPort: 9119, internalPort: 19119, tuiEnabled: true }, + }); + expect(script).toContain("/tmp/hermes-dashboard.log"); + expect(script).toContain( + '"$AGENT_BIN" dashboard --host 127.0.0.1 --port 19119 --skip-build --no-open --tui', + ); + expect(script).toContain("DASHBOARD_PID=$DPID"); + expect(script).toContain("DASHBOARD_FAILED"); + }); + + it("can recover only the optional Hermes dashboard process", () => { + const script = buildHermesDashboardProcessRecoveryScript({ + publicPort: 9119, + internalPort: 19119, + tuiEnabled: false, + }); + expect(script).toContain(". /tmp/nemoclaw-proxy-env.sh"); + expect(script).toContain("/usr/local/bin/hermes"); + expect(script).toContain('"$AGENT_BIN" dashboard --host 127.0.0.1 --port 19119 --skip-build --no-open'); + expect(script).not.toContain("--tui"); + }); + it("does not launch a Hermes decode proxy during recovery", () => { const script = buildRecoveryScript(hermesAgent, 8642); expect(script).not.toContain("/usr/local/bin/nemoclaw-decode-proxy"); diff --git a/src/lib/agent/runtime.ts b/src/lib/agent/runtime.ts index 45906d15f2e..86337499cb6 100644 --- a/src/lib/agent/runtime.ts +++ b/src/lib/agent/runtime.ts @@ -146,6 +146,41 @@ function hermesGatewayEnvPrefix(): string { return "HERMES_HOME=/sandbox/.hermes"; } +export interface HermesDashboardRecoveryConfig { + publicPort: number; + internalPort: number; + tuiEnabled?: boolean; +} + +function buildHermesDashboardRecoveryLines(config: HermesDashboardRecoveryConfig): string[] { + const tuiFlag = config.tuiEnabled ? " --tui" : ""; + const dashboardLogSelection = + '_DASHBOARD_LOG=/tmp/hermes-dashboard.log; if ! : >> "$_DASHBOARD_LOG" 2>/dev/null; then _DASHBOARD_LOG=/tmp/hermes-dashboard-recovery.log; : >> "$_DASHBOARD_LOG" 2>/dev/null || true; fi;'; + return [ + `_DASH_CODE=$(curl -so /dev/null -w '%{http_code}' --max-time 3 http://127.0.0.1:${config.internalPort}/ 2>/dev/null || echo 000); case "$_DASH_CODE" in 200|301|302|307|308) echo DASHBOARD_ALREADY_RUNNING; ;; *)`, + `${buildNoFollowLogSetupCommand("/tmp/hermes-dashboard.log")} || exit 1;`, + dashboardLogSelection, + "_DASHBOARD_PROC_PATTERN='[h]ermes[[:space:]]+dashboard([[:space:]]|$)';", + 'pkill -TERM -f "$_DASHBOARD_PROC_PATTERN" 2>/dev/null || true; sleep 1; pkill -KILL -f "$_DASHBOARD_PROC_PATTERN" 2>/dev/null || true;', + `${hermesGatewayEnvPrefix()} nohup "$AGENT_BIN" dashboard --host 127.0.0.1 --port ${config.internalPort} --skip-build --no-open${tuiFlag} >> "$_DASHBOARD_LOG" 2>&1 &`, + "DPID=$!; sleep 2;", + 'if kill -0 "$DPID" 2>/dev/null; then echo "DASHBOARD_PID=$DPID"; else echo DASHBOARD_FAILED; tail -5 "$_DASHBOARD_LOG" 2>/dev/null; exit 1; fi ;; esac;', + ]; +} + +export function buildHermesDashboardProcessRecoveryScript( + config: HermesDashboardRecoveryConfig, +): string { + return [ + "[ -f ~/.bashrc ] && . ~/.bashrc;", + "export HERMES_HOME=/sandbox/.hermes;", + 'if [ -r /tmp/nemoclaw-proxy-env.sh ]; then . /tmp/nemoclaw-proxy-env.sh; fi;', + 'AGENT_BIN=/usr/local/bin/hermes; if [ ! -x "$AGENT_BIN" ]; then AGENT_BIN="$(command -v hermes)"; fi;', + 'if [ -z "$AGENT_BIN" ]; then echo AGENT_MISSING; exit 1; fi;', + ...buildHermesDashboardRecoveryLines(config), + ].join(" "); +} + /** * Build the OpenClaw recovery shell script used by the default sandbox. */ @@ -176,7 +211,11 @@ export function buildOpenClawRecoveryScript(port: number): string { * Returns the script string, or null if agent is null (use existing inline * OpenClaw script instead). */ -export function buildRecoveryScript(agent: AgentDefinition | null, port: number): string | null { +export function buildRecoveryScript( + agent: AgentDefinition | null, + port: number, + options: { hermesDashboard?: HermesDashboardRecoveryConfig | null } = {}, +): string | null { if (!agent) return null; const probeUrl = getHealthProbeUrl(agent); @@ -232,7 +271,10 @@ export function buildRecoveryScript(agent: AgentDefinition | null, port: number) '[ "$_PE_MISSING" = "0" ] && [ "$_GUARDS_MISSING" = "1" ] && { _E="[gateway-recovery] ERROR: /tmp/nemoclaw-proxy-env.sh present but NODE_OPTIONS missing safety-net preload or ciao preload - refusing unguarded gateway relaunch (#2478)"; echo "$_E" >&2; echo "$_E" >> "$_GATEWAY_LOG"; exit 1; };', launchCommand, "GPID=$!; sleep 2;", - 'if kill -0 "$GPID" 2>/dev/null; then echo "GATEWAY_PID=$GPID"; else echo GATEWAY_FAILED; tail -5 "$_GATEWAY_LOG" 2>/dev/null; fi', + 'if kill -0 "$GPID" 2>/dev/null; then echo "GATEWAY_PID=$GPID"; else echo GATEWAY_FAILED; tail -5 "$_GATEWAY_LOG" 2>/dev/null; exit 1; fi', + ...(isHermes && options.hermesDashboard + ? buildHermesDashboardRecoveryLines(options.hermesDashboard) + : []), ].join(" "); } diff --git a/src/lib/hermes-dashboard.test.ts b/src/lib/hermes-dashboard.test.ts new file mode 100644 index 00000000000..4b2122bfe66 --- /dev/null +++ b/src/lib/hermes-dashboard.test.ts @@ -0,0 +1,55 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + HERMES_DASHBOARD_DEFAULT_INTERNAL_PORT, + HERMES_DASHBOARD_DEFAULT_PORT, + readHermesDashboardConfig, +} from "./hermes-dashboard"; + +describe("Hermes dashboard config", () => { + it("defaults to disabled dashboard settings", () => { + expect(readHermesDashboardConfig({})).toEqual({ + enabled: false, + port: HERMES_DASHBOARD_DEFAULT_PORT, + internalPort: HERMES_DASHBOARD_DEFAULT_INTERNAL_PORT, + tuiEnabled: false, + }); + }); + + it("reads opt-in dashboard settings from env", () => { + expect( + readHermesDashboardConfig({ + NEMOCLAW_HERMES_DASHBOARD: "true", + NEMOCLAW_HERMES_DASHBOARD_PORT: "9120", + NEMOCLAW_HERMES_DASHBOARD_INTERNAL_PORT: "19120", + NEMOCLAW_HERMES_DASHBOARD_TUI: "yes", + }), + ).toEqual({ + enabled: true, + port: 9120, + internalPort: 19120, + tuiEnabled: true, + }); + }); + + it("rejects invalid port values", () => { + expect(() => + readHermesDashboardConfig({ + NEMOCLAW_HERMES_DASHBOARD_PORT: "abc", + }), + ).toThrow(/NEMOCLAW_HERMES_DASHBOARD_PORT/); + expect(() => + readHermesDashboardConfig({ + NEMOCLAW_HERMES_DASHBOARD_PORT: "1023", + }), + ).toThrow(/NEMOCLAW_HERMES_DASHBOARD_PORT/); + expect(() => + readHermesDashboardConfig({ + NEMOCLAW_HERMES_DASHBOARD_PORT: "65536", + }), + ).toThrow(/NEMOCLAW_HERMES_DASHBOARD_PORT/); + }); +}); diff --git a/src/lib/hermes-dashboard.ts b/src/lib/hermes-dashboard.ts new file mode 100644 index 00000000000..7808973a304 --- /dev/null +++ b/src/lib/hermes-dashboard.ts @@ -0,0 +1,66 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export const HERMES_DASHBOARD_ENABLE_ENV = "NEMOCLAW_HERMES_DASHBOARD"; +export const HERMES_DASHBOARD_PORT_ENV = "NEMOCLAW_HERMES_DASHBOARD_PORT"; +export const HERMES_DASHBOARD_INTERNAL_PORT_ENV = "NEMOCLAW_HERMES_DASHBOARD_INTERNAL_PORT"; +export const HERMES_DASHBOARD_TUI_ENV = "NEMOCLAW_HERMES_DASHBOARD_TUI"; + +export const HERMES_DASHBOARD_DEFAULT_PORT = 9119; +export const HERMES_DASHBOARD_DEFAULT_INTERNAL_PORT = 19119; + +export interface HermesDashboardConfig { + enabled: boolean; + port: number; + internalPort: number; + tuiEnabled: boolean; +} + +export function isTruthyEnv(value: string | undefined): boolean { + if (value === undefined) return false; + switch (value.trim().toLowerCase()) { + case "1": + case "true": + case "yes": + case "on": + return true; + default: + return false; + } +} + +function parsePortEnv( + env: NodeJS.ProcessEnv, + name: string, + fallback: number, +): number { + const raw = env[name]; + if (raw === undefined || raw.trim() === "") return fallback; + if (!/^\d+$/.test(raw.trim())) { + throw new Error( + `Invalid port: ${name}="${raw}" must be an integer between 1024 and 65535`, + ); + } + const parsed = Number(raw.trim()); + if (parsed < 1024 || parsed > 65535) { + throw new Error( + `Invalid port: ${name}="${raw}" must be an integer between 1024 and 65535`, + ); + } + return parsed; +} + +export function readHermesDashboardConfig( + env: NodeJS.ProcessEnv = process.env, +): HermesDashboardConfig { + return { + enabled: isTruthyEnv(env[HERMES_DASHBOARD_ENABLE_ENV]), + port: parsePortEnv(env, HERMES_DASHBOARD_PORT_ENV, HERMES_DASHBOARD_DEFAULT_PORT), + internalPort: parsePortEnv( + env, + HERMES_DASHBOARD_INTERNAL_PORT_ENV, + HERMES_DASHBOARD_DEFAULT_INTERNAL_PORT, + ), + tuiEnabled: isTruthyEnv(env[HERMES_DASHBOARD_TUI_ENV]), + }; +} diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 761c3c24540..3e468b4dec6 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -218,7 +218,9 @@ const { const onboardProviders = require("./onboard/providers"); const { ensureResumeProviderReady } = require("./onboard/resume-provider-shim"); const hermesProviderAuth = require("./hermes-provider-auth"); +const onboardHermesDashboard: typeof import("./onboard/hermes-dashboard") = require("./onboard/hermes-dashboard"); const hermesAuth: typeof import("./onboard/hermes-auth") = require("./onboard/hermes-auth"); +const { warnIfLandlockUnsupported } = require("./onboard/landlock-warning"); const { HERMES_AUTH_METHOD_API_KEY, HERMES_AUTH_METHOD_OAUTH, @@ -230,7 +232,6 @@ const { } = hermesAuth; type HermesAuthMethod = import("./onboard/hermes-auth").HermesAuthMethod; - function getHermesToolGatewayBroker(): any { return require("./hermes-tool-gateway-broker"); } @@ -500,7 +501,6 @@ import { readMessagingChannelConfigFromEnv, } from "./messaging-channel-config"; import { streamGatewayStart } from "./onboard/gateway"; -import { runOllamaStartupOrGate } from "./onboard/ollama-startup"; import { mergeRequiredHermesToolGatewayPolicyPresets, normalizeHermesToolGatewaySelections, @@ -515,16 +515,13 @@ import { resolveQrSelectedChannels, } from "./onboard/messaging-state"; import { getValidatedMessagingToken, getValidatedMessagingTokenByEnvKey } from "./onboard/messaging-token"; +import { runOllamaStartupOrGate } from "./onboard/ollama-startup"; import type { DockerDriverBinaryOverrides, OpenShellInstallDeps, OpenShellInstallResult, } from "./onboard/openshell-install"; import { decidePolicyCarryForward } from "./onboard/policy-carryforward"; -import { - backupSandboxBeforeRecreate, - shouldSkipPreRecreateBackup, -} from "./onboard/sandbox-backup-on-recreate"; import { getSuggestedPolicyPresets } from "./onboard/policy-presets"; import { computeSetupPresetSuggestions as computeSetupPresetSuggestionsImpl, @@ -533,6 +530,10 @@ import { type SetupPresetSuggestionOptions, setupPoliciesWithSelection as setupPoliciesWithSelectionImpl, } from "./onboard/policy-selection"; +import { + backupSandboxBeforeRecreate, + shouldSkipPreRecreateBackup, +} from "./onboard/sandbox-backup-on-recreate"; import { getResumeSandboxGpuOverrides, resolveSandboxGpuConfig, @@ -2816,6 +2817,15 @@ async function createSandbox( } else { chatUiUrl = `http://127.0.0.1:${effectivePort}`; } + const hermesDashboardForwarding = onboardHermesDashboard.createHermesDashboardOnboardForwarding({ + agentName: agent?.name, + env: process.env, + ensureForward: ensureAgentFixedForward, + note, + runOpenshell, + getApiForwardPort: () => getDashboardForwardPort(chatUiUrl), + }); + const hermesDashboardState = hermesDashboardForwarding.resolveStateForPort(effectivePort); // Check whether messaging providers will be needed — this must happen before // the sandbox reuse decision so we can detect stale sandboxes that were created @@ -3032,10 +3042,16 @@ async function createSandbox( const selectionDrift = getSelectionDrift(sandboxName, provider, model, { runOpenshell }); const confirmedSelectionDrift = selectionDrift.changed && !selectionDrift.unknown; const sandboxGpuDrift = hasSandboxGpuDrift(sandboxName, effectiveSandboxGpuConfig); + const existingSandboxEntry = registry.getSandbox(sandboxName); const recordedHermesToolGateways = normalizeHermesToolGatewaySelections( - registry.getSandbox(sandboxName)?.hermesToolGateways, + existingSandboxEntry?.hermesToolGateways, ); const hermesToolGatewayDrift = !stringSetsEqual(recordedHermesToolGateways, hermesToolGateways); + const hermesDashboardDrift = onboardHermesDashboard.hasHermesDashboardDrift({ + agentName: agent?.name, + existing: existingSandboxEntry, + state: hermesDashboardState, + }); // Detect whether any messaging credential has been rotated since the // sandbox was created. Provider credentials are resolved once at sandbox @@ -3050,7 +3066,8 @@ async function createSandbox( !needsProviderMigration && !sandboxGpuDrift && !credentialRotation.changed && - !hermesToolGatewayDrift + !hermesToolGatewayDrift && + !hermesDashboardDrift ) { // Guard against reusing a CPU-only sandbox when GPU passthrough is enabled. // Placed before the non-interactive / interactive split so all reuse @@ -3094,7 +3111,11 @@ async function createSandbox( ); } const reusedPort = ensureDashboardForward(sandboxName, chatUiUrl); - process.env.CHAT_UI_URL = `http://127.0.0.1:${reusedPort}`; + chatUiUrl = `http://127.0.0.1:${reusedPort}`; + process.env.CHAT_UI_URL = chatUiUrl; + const reusedHermesDashboardState = + hermesDashboardForwarding.resolveStateForPort(reusedPort); + hermesDashboardForwarding.ensureForState(reusedHermesDashboardState, sandboxName); updateReusedSandboxMetadata( sandboxName, agent, @@ -3104,6 +3125,10 @@ async function createSandbox( !selectionDrift.unknown, effectiveSandboxGpuConfig, ); + registry.updateSandbox( + sandboxName, + onboardHermesDashboard.getHermesDashboardRegistryFields(reusedHermesDashboardState), + ); return sandboxName; } } else { @@ -3131,7 +3156,11 @@ async function createSandbox( if (await promptYesNoOrDefault(" Reuse existing sandbox?", null, true)) { upsertMessagingProviders(messagingTokenDefs); const reusedPort2 = ensureDashboardForward(sandboxName, chatUiUrl); - process.env.CHAT_UI_URL = `http://127.0.0.1:${reusedPort2}`; + chatUiUrl = `http://127.0.0.1:${reusedPort2}`; + process.env.CHAT_UI_URL = chatUiUrl; + const reusedHermesDashboardState2 = + hermesDashboardForwarding.resolveStateForPort(reusedPort2); + hermesDashboardForwarding.ensureForState(reusedHermesDashboardState2, sandboxName); updateReusedSandboxMetadata( sandboxName, agent, @@ -3141,6 +3170,10 @@ async function createSandbox( !selectionDrift.unknown, effectiveSandboxGpuConfig, ); + registry.updateSandbox( + sandboxName, + onboardHermesDashboard.getHermesDashboardRegistryFields(reusedHermesDashboardState2), + ); return sandboxName; } } @@ -3183,6 +3216,8 @@ async function createSandbox( note(` Sandbox '${sandboxName}' exists — recreating to apply sandbox GPU settings.`); } else if (hermesToolGatewayDrift) { note(` Sandbox '${sandboxName}' exists — recreating to apply Hermes managed-tool changes.`); + } else if (hermesDashboardDrift) { + note(` Sandbox '${sandboxName}' exists — recreating to apply Hermes dashboard settings.`); } else if (credentialRotation.changed) { // Message already printed above during backup. } else if (existingSandboxState === "ready") { @@ -3526,6 +3561,11 @@ async function createSandbox( // 18789 and the gateway listens on the wrong port. (#2267, #1925) const effectiveDashboardPort = getDashboardForwardPort(chatUiUrl); envArgs.push(formatEnvAssignment("NEMOCLAW_DASHBOARD_PORT", effectiveDashboardPort)); + onboardHermesDashboard.appendHermesDashboardEnvArgs( + envArgs, + hermesDashboardState, + formatEnvAssignment, + ); // Propagate NEMOCLAW_PROXY_HOST / NEMOCLAW_PROXY_PORT to the runtime // sandbox container. patchStagedDockerfile() already substitutes them // into the build-time Dockerfile ARG/ENV, but `openshell sandbox create @@ -3728,6 +3768,9 @@ async function createSandbox( chatUiUrl = `http://127.0.0.1:${actualDashboardPort}`; } process.env.CHAT_UI_URL = chatUiUrl; + const finalHermesDashboardState = + hermesDashboardForwarding.resolveStateForPort(actualDashboardPort); + hermesDashboardForwarding.ensureForState(finalHermesDashboardState, sandboxName, true); // Register only after confirmed ready — prevents phantom entries const providerCredentialHashes: Record = {}; @@ -3767,6 +3810,7 @@ async function createSandbox( messagingChannelConfig: messagingChannelConfig || undefined, disabledChannels: disabledChannels.length > 0 ? [...disabledChannels] : undefined, hermesToolGateways: hermesToolGateways.length > 0 ? [...hermesToolGateways] : undefined, + ...onboardHermesDashboard.getHermesDashboardRegistryFields(finalHermesDashboardState), dashboardPort: actualDashboardPort, }); registry.setDefault(sandboxName); @@ -3812,39 +3856,7 @@ async function createSandbox( console.log(` ✓ Sandbox '${sandboxName}' created`); - try { - if (process.platform === "darwin") { - const vmKernel = dockerInfoFormat("{{.KernelVersion}}", { - ignoreError: true, - }).trim(); - if (vmKernel) { - const parts = vmKernel.split("."); - const major = parseInt(parts[0], 10); - const minor = parseInt(parts[1], 10); - if (!isNaN(major) && !isNaN(minor) && (major < 5 || (major === 5 && minor < 13))) { - console.warn( - ` ⚠ Landlock: Docker VM kernel ${vmKernel} does not support Landlock (requires ≥5.13).`, - ); - console.warn( - " Sandbox filesystem restrictions will silently degrade (best_effort mode).", - ); - } - } - } else if (process.platform === "linux") { - const uname = runCapture(["uname", "-r"], { ignoreError: true }).trim(); - if (uname) { - const parts = uname.split("."); - const major = parseInt(parts[0], 10); - const minor = parseInt(parts[1], 10); - if (!isNaN(major) && !isNaN(minor) && (major < 5 || (major === 5 && minor < 13))) { - console.warn(` ⚠ Landlock: Kernel ${uname} does not support Landlock (requires ≥5.13).`); - console.warn( - " Sandbox filesystem restrictions will silently degrade (best_effort mode).", - ); - } - } - } - } catch {} + warnIfLandlockUnsupported({ dockerInfoFormat, runCapture }); return sandboxName; } @@ -3853,13 +3865,6 @@ async function createSandbox( type ProviderChoice = { key: string; label: string }; -function providerNameToOptionKey( - name: string | null | undefined, - opts: { hasNimContainer?: boolean } = {}, -): string | null { - return providerRecovery.providerNameToOptionKey(REMOTE_PROVIDER_CONFIG, name, opts); -} - const { readLiveInference, readRecordedProvider, readRecordedNimContainer, readRecordedModel } = providerRecovery.createProviderRecoveryHelpers({ parseGatewayInference, @@ -4154,7 +4159,7 @@ async function setupNim( if (!providerKey) { const recordedProvider = readRecordedProvider(sandboxName); const hasNimContainer = !!readRecordedNimContainer(sandboxName); - const recoveredKey = providerNameToOptionKey(recordedProvider, { hasNimContainer }); + const recoveredKey = providerRecovery.providerNameToOptionKey(REMOTE_PROVIDER_CONFIG, recordedProvider, { hasNimContainer }); if (recoveredKey) { // Refuse to silently switch providers behind the user's back; if // the previously-recorded one is gone, surface the recorded value @@ -6310,16 +6315,10 @@ async function presetsCheckboxSelector( }); } -function computeSetupPresetSuggestions( +const computeSetupPresetSuggestions = ( tierName: string, options: SetupPresetSuggestionOptions = {}, -): string[] { - return computeSetupPresetSuggestionsImpl( - { policies, tiers, localInferenceProviders: LOCAL_INFERENCE_PROVIDERS }, - tierName, - options, - ); -} +): string[] => computeSetupPresetSuggestionsImpl({ policies, tiers, localInferenceProviders: LOCAL_INFERENCE_PROVIDERS }, tierName, options); async function setupPoliciesWithSelection( sandboxName: string, @@ -6355,6 +6354,7 @@ const { buildOrphanedSandboxRollbackMessage, ensureDashboardForward, ensureAgentDashboardForward, + ensureAgentFixedForward, fetchGatewayAuthTokenFromSandbox, getDashboardForwardPort, getWslHostAddress, @@ -7255,7 +7255,7 @@ module.exports = { MESSAGING_CHANNELS, selectOnboardAgent, setupNim, - providerNameToOptionKey, + providerNameToOptionKey: (name: string | null | undefined, opts: { hasNimContainer?: boolean } = {}) => providerRecovery.providerNameToOptionKey(REMOTE_PROVIDER_CONFIG, name, opts), readRecordedProvider, readRecordedModel, readRecordedNimContainer, diff --git a/src/lib/onboard/agent-fixed-forward.ts b/src/lib/onboard/agent-fixed-forward.ts new file mode 100644 index 00000000000..5d47b8c43d7 --- /dev/null +++ b/src/lib/onboard/agent-fixed-forward.ts @@ -0,0 +1,59 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { OPENSHELL_PROBE_TIMEOUT_MS } from "../adapters/openshell/timeouts"; +import { bestEffortForwardStopForSandbox } from "./forward-cleanup"; +import { + buildDetachedForwardStartSpawn, + buildForwardStartProgressLogger, + runDetachedForwardStartWithPortReleaseRetries, +} from "./forward-start"; + +type CommandResult = { status: number | null }; + +export interface AgentFixedForwardDeps { + runOpenshell(args: string[], opts?: Record): CommandResult; + runCaptureOpenshell(args: string[], opts?: Record): string | null; + openshellArgv(args: string[]): string[]; + cliName(): string; + sleep(seconds: number): void; +} + +export function ensureAgentFixedForward( + deps: AgentFixedForwardDeps, + sandboxName: string, + port: number, + label: string, +): boolean { + const forwardTarget = String(port); + const stopForwardForSandbox = (portToStop: string | number) => + bestEffortForwardStopForSandbox( + deps.runOpenshell, + (args, opts) => (deps.runCaptureOpenshell(args, opts) ?? "") as string, + portToStop, + sandboxName, + ); + + stopForwardForSandbox(port); + const { ok, diagnostic } = runDetachedForwardStartWithPortReleaseRetries( + buildDetachedForwardStartSpawn( + deps.openshellArgv(["forward", "start", "--background", forwardTarget, sandboxName]), + ), + () => + (deps.runCaptureOpenshell(["forward", "list"], { timeout: OPENSHELL_PROBE_TIMEOUT_MS }) ?? "") as string, + { port, sandboxName }, + () => { + deps.sleep(1); + stopForwardForSandbox(port); + }, + { onProgress: buildForwardStartProgressLogger(port) }, + ); + if (!ok) { + console.warn( + `! ${label} forward on port ${port} did not start: ${diagnostic.slice(0, 240)}`, + ); + console.warn(` Reconnect after resolving the issue: ${deps.cliName()} ${sandboxName} connect`); + return false; + } + return true; +} diff --git a/src/lib/onboard/dashboard.ts b/src/lib/onboard/dashboard.ts index 63ab72c9e90..b2d7e32ef6f 100644 --- a/src/lib/onboard/dashboard.ts +++ b/src/lib/onboard/dashboard.ts @@ -4,26 +4,26 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; - +import { OPENSHELL_PROBE_TIMEOUT_MS } from "../adapters/openshell/timeouts"; import type { AgentDefinition } from "../agent/defs"; import { DASHBOARD_PORT } from "../core/ports"; import { buildChain, buildControlUiUrls } from "../dashboard/contract"; import * as nim from "../inference/nim"; import { runCapture as defaultRunCapture } from "../runner"; +import { ensureAgentFixedForward as ensureFixedAgentForward } from "./agent-fixed-forward"; import * as dashboardAccess from "./dashboard-access"; import { findAvailableDashboardPort, getOccupiedPorts, isLiveForwardStatus, } from "./dashboard-port"; -import { OPENSHELL_PROBE_TIMEOUT_MS } from "../adapters/openshell/timeouts"; +import { bestEffortForwardStop, bestEffortForwardStopForSandbox } from "./forward-cleanup"; import { buildDetachedForwardStartSpawn, buildForwardStartProgressLogger, looksLikeForwardPortConflict, runDetachedForwardStartWithPortReleaseRetries, } from "./forward-start"; -import { bestEffortForwardStop, bestEffortForwardStopForSandbox } from "./forward-cleanup"; const ANSI_RE = /\x1B(?:\[[0-?]*[ -/]*[@-~]|\][^\x07]*(?:\x07|\x1B\\)|[@-_])/g; export const CONTROL_UI_PORT = DASHBOARD_PORT; @@ -73,6 +73,7 @@ export interface OnboardDashboardHelpers { sandboxName: string, agent: { forwardPort?: number | null }, ): number; + ensureAgentFixedForward(sandboxName: string, port: number, label: string): boolean; fetchGatewayAuthTokenFromSandbox(sandboxName: string): string | null; getDashboardForwardPort( chatUiUrl?: string, @@ -322,6 +323,10 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa return actualAgentDashboardPort; } + function ensureAgentFixedForward(sandboxName: string, port: number, label: string): boolean { + return ensureFixedAgentForward(deps, sandboxName, port, label); + } + function fetchGatewayAuthTokenFromSandbox(sandboxName: string): string | null { const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-token-")); try { @@ -439,6 +444,7 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa buildOrphanedSandboxRollbackMessage, ensureDashboardForward, ensureAgentDashboardForward, + ensureAgentFixedForward, fetchGatewayAuthTokenFromSandbox, getDashboardForwardPort, getDashboardForwardTarget, diff --git a/src/lib/onboard/hermes-dashboard.test.ts b/src/lib/onboard/hermes-dashboard.test.ts new file mode 100644 index 00000000000..d827f796bba --- /dev/null +++ b/src/lib/onboard/hermes-dashboard.test.ts @@ -0,0 +1,69 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { + createHermesDashboardForwardEnsurer, + getHermesDashboardRegistryFields, + hasHermesDashboardDrift, + resolveHermesDashboardOnboardState, +} from "./hermes-dashboard"; + +describe("onboard Hermes dashboard helpers", () => { + it("rejects dashboard/API port overlap before sandbox create", () => { + expect(() => + resolveHermesDashboardOnboardState({ + agentName: "hermes", + effectivePort: 9119, + env: { NEMOCLAW_HERMES_DASHBOARD: "1" }, + }), + ).toThrow(/must not equal the Hermes API port/); + }); + + it("tracks registry drift for enabled dashboard settings", () => { + const state = resolveHermesDashboardOnboardState({ + agentName: "hermes", + effectivePort: 8642, + env: { + NEMOCLAW_HERMES_DASHBOARD: "1", + NEMOCLAW_HERMES_DASHBOARD_PORT: "9120", + }, + }); + + expect(getHermesDashboardRegistryFields(state)).toMatchObject({ + hermesDashboardEnabled: true, + hermesDashboardPort: 9120, + hermesDashboardInternalPort: 19119, + }); + expect( + hasHermesDashboardDrift({ + agentName: "hermes", + state, + existing: { name: "h", agent: "hermes", hermesDashboardEnabled: false }, + }), + ).toBe(true); + }); + + it("rolls back and fails when an opted-in dashboard forward cannot start", () => { + const rollback = vi.fn(); + const fail = vi.fn((message: string): never => { + throw new Error(message); + }); + const ensure = createHermesDashboardForwardEnsurer({ + state: resolveHermesDashboardOnboardState({ + agentName: "hermes", + effectivePort: 8642, + env: { NEMOCLAW_HERMES_DASHBOARD: "1" }, + }), + ensureForward: vi.fn(() => false), + note: vi.fn(), + rollbackSandbox: rollback, + fail, + }); + + expect(() => ensure("my-hermes", true)).toThrow(/Failed to start Hermes dashboard forward/); + expect(rollback).toHaveBeenCalledWith("my-hermes"); + expect(fail).toHaveBeenCalled(); + }); +}); diff --git a/src/lib/onboard/hermes-dashboard.ts b/src/lib/onboard/hermes-dashboard.ts new file mode 100644 index 00000000000..842742b7f3e --- /dev/null +++ b/src/lib/onboard/hermes-dashboard.ts @@ -0,0 +1,209 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + HERMES_DASHBOARD_ENABLE_ENV, + HERMES_DASHBOARD_INTERNAL_PORT_ENV, + HERMES_DASHBOARD_PORT_ENV, + HERMES_DASHBOARD_TUI_ENV, + type HermesDashboardConfig, + readHermesDashboardConfig, +} from "../hermes-dashboard"; +import type { SandboxEntry } from "../state/registry"; + +export interface HermesDashboardOnboardState { + config: HermesDashboardConfig | null; + enabled: boolean; +} + +type RunOpenshell = (args: string[], options: { ignoreError: true }) => unknown; + +export function resolveHermesDashboardOnboardState({ + agentName, + effectivePort, + env, + fail, +}: { + agentName: string | null | undefined; + effectivePort: number; + env: NodeJS.ProcessEnv; + fail?: (message: string) => never; +}): HermesDashboardOnboardState { + if (agentName !== "hermes") return { config: null, enabled: false }; + + let config: HermesDashboardConfig; + try { + config = readHermesDashboardConfig(env); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if (fail) return fail(message); + throw error; + } + + if (config.enabled) { + if (config.port === effectivePort) { + const message = `${HERMES_DASHBOARD_PORT_ENV} must not equal the Hermes API port (${effectivePort}).`; + if (fail) return fail(message); + throw new Error(message); + } + if (config.port === config.internalPort) { + const message = `${HERMES_DASHBOARD_PORT_ENV} must not equal ${HERMES_DASHBOARD_INTERNAL_PORT_ENV}.`; + if (fail) return fail(message); + throw new Error(message); + } + } + + return { config, enabled: config.enabled === true }; +} + +export function getHermesDashboardRegistryFields( + state: HermesDashboardOnboardState, +): Partial { + if (!state.enabled || !state.config) { + return { + hermesDashboardEnabled: undefined, + hermesDashboardPort: undefined, + hermesDashboardInternalPort: undefined, + hermesDashboardTui: undefined, + }; + } + return { + hermesDashboardEnabled: true, + hermesDashboardPort: state.config.port, + hermesDashboardInternalPort: state.config.internalPort, + hermesDashboardTui: state.config.tuiEnabled ? true : undefined, + }; +} + +export function hasHermesDashboardDrift({ + agentName, + existing, + state, +}: { + agentName: string | null | undefined; + existing: SandboxEntry | null | undefined; + state: HermesDashboardOnboardState; +}): boolean { + if (agentName !== "hermes") return false; + const recordedEnabled = existing?.hermesDashboardEnabled === true; + if (recordedEnabled !== state.enabled) return true; + if (!state.enabled || !state.config) return false; + return ( + existing?.hermesDashboardPort !== state.config.port || + existing?.hermesDashboardInternalPort !== state.config.internalPort || + (existing?.hermesDashboardTui === true) !== state.config.tuiEnabled + ); +} + +export function appendHermesDashboardEnvArgs( + envArgs: string[], + state: HermesDashboardOnboardState, + formatEnvAssignment: (name: string, value: string) => string, +): void { + if (!state.enabled || !state.config) return; + envArgs.push(formatEnvAssignment(HERMES_DASHBOARD_ENABLE_ENV, "1")); + envArgs.push(formatEnvAssignment(HERMES_DASHBOARD_PORT_ENV, String(state.config.port))); + envArgs.push( + formatEnvAssignment(HERMES_DASHBOARD_INTERNAL_PORT_ENV, String(state.config.internalPort)), + ); + if (state.config.tuiEnabled) { + envArgs.push(formatEnvAssignment(HERMES_DASHBOARD_TUI_ENV, "1")); + } +} + +export function ensureHermesDashboardForwardIfEnabled({ + state, + sandboxName, + ensureForward, + note, +}: { + state: HermesDashboardOnboardState; + sandboxName: string; + ensureForward: (sandboxName: string, port: number, label: string) => boolean; + note: (message: string) => void; +}): boolean { + if (!state.enabled || !state.config) return true; + if (!ensureForward(sandboxName, state.config.port, "Hermes dashboard")) return false; + note(` ✓ Hermes dashboard forwarded at http://127.0.0.1:${state.config.port}/`); + return true; +} + +export function formatHermesDashboardForwardFailure( + state: HermesDashboardOnboardState, +): string { + const port = state.config?.port ?? "unknown"; + return `Failed to start Hermes dashboard forward on port ${port}. Free the port and re-run onboarding, or set ${HERMES_DASHBOARD_PORT_ENV} to another port.`; +} + +export function createHermesDashboardForwardEnsurer({ + state, + ensureForward, + note, + rollbackSandbox, + fail, +}: { + state: HermesDashboardOnboardState; + ensureForward: (sandboxName: string, port: number, label: string) => boolean; + note: (message: string) => void; + rollbackSandbox: (sandboxName: string) => void; + fail: (message: string) => never; +}): (sandboxName: string, rollback?: boolean) => void { + return (sandboxName: string, rollback = false): void => { + const ok = ensureHermesDashboardForwardIfEnabled({ state, sandboxName, ensureForward, note }); + if (ok) return; + if (rollback) rollbackSandbox(sandboxName); + fail(formatHermesDashboardForwardFailure(state)); + }; +} + +export function createHermesDashboardOnboardForwarding({ + agentName, + env, + ensureForward, + note, + runOpenshell, + getApiForwardPort, + fail, +}: { + agentName: string | null | undefined; + env: NodeJS.ProcessEnv; + ensureForward: (sandboxName: string, port: number, label: string) => boolean; + note: (message: string) => void; + runOpenshell: RunOpenshell; + getApiForwardPort: () => string; + fail?: (message: string) => never; +}) { + const failWithMessage = + fail ?? + ((message: string): never => { + console.error(` ${message}`); + process.exit(1); + }); + const resolveStateForPort = (effectivePort: number) => + resolveHermesDashboardOnboardState({ agentName, effectivePort, env, fail: failWithMessage }); + + const ensureForState = ( + state: HermesDashboardOnboardState, + sandboxName: string, + rollback = false, + ) => + createHermesDashboardForwardEnsurer({ + state, + ensureForward, + note, + rollbackSandbox: (targetSandbox) => { + runOpenshell(["forward", "stop", getApiForwardPort(), targetSandbox], { + ignoreError: true, + }); + if (state.config) { + runOpenshell(["forward", "stop", String(state.config.port), targetSandbox], { + ignoreError: true, + }); + } + runOpenshell(["sandbox", "delete", targetSandbox], { ignoreError: true }); + }, + fail: failWithMessage, + })(sandboxName, rollback); + + return { resolveStateForPort, ensureForState }; +} diff --git a/src/lib/onboard/landlock-warning.ts b/src/lib/onboard/landlock-warning.ts new file mode 100644 index 00000000000..bc1458c2b64 --- /dev/null +++ b/src/lib/onboard/landlock-warning.ts @@ -0,0 +1,41 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +function parseKernelMajorMinor(value: string): { major: number; minor: number } | null { + const parts = value.split("."); + const major = parseInt(parts[0] ?? "", 10); + const minor = parseInt(parts[1] ?? "", 10); + if (Number.isNaN(major) || Number.isNaN(minor)) return null; + return { major, minor }; +} + +function warnIfUnsupported(kernel: string, label: string, warn: (message: string) => void): void { + const parsed = parseKernelMajorMinor(kernel); + if (!parsed || parsed.major > 5 || (parsed.major === 5 && parsed.minor >= 13)) return; + warn(` ⚠ Landlock: ${label} ${kernel} does not support Landlock (requires ≥5.13).`); + warn(" Sandbox filesystem restrictions will silently degrade (best_effort mode)."); +} + +export function warnIfLandlockUnsupported({ + platform = process.platform, + dockerInfoFormat, + runCapture, + warn = console.warn, +}: { + platform?: NodeJS.Platform; + dockerInfoFormat: (format: string, options?: { ignoreError?: boolean }) => string; + runCapture: (args: string[], options?: { ignoreError?: boolean }) => string; + warn?: (message: string) => void; +}): void { + try { + if (platform === "darwin") { + const vmKernel = dockerInfoFormat("{{.KernelVersion}}", { ignoreError: true }).trim(); + if (vmKernel) warnIfUnsupported(vmKernel, "Docker VM kernel", warn); + } else if (platform === "linux") { + const uname = runCapture(["uname", "-r"], { ignoreError: true }).trim(); + if (uname) warnIfUnsupported(uname, "Kernel", warn); + } + } catch { + /* best effort warning */ + } +} diff --git a/src/lib/state/registry.ts b/src/lib/state/registry.ts index 548859d436e..037c99f79f8 100644 --- a/src/lib/state/registry.ts +++ b/src/lib/state/registry.ts @@ -38,6 +38,10 @@ export interface SandboxEntry { messagingChannels?: string[]; messagingChannelConfig?: MessagingChannelConfig; hermesToolGateways?: string[]; + hermesDashboardEnabled?: boolean; + hermesDashboardPort?: number | null; + hermesDashboardInternalPort?: number | null; + hermesDashboardTui?: boolean; disabledChannels?: string[]; dashboardPort?: number | null; } @@ -219,6 +223,10 @@ export function registerSandbox(entry: SandboxEntry): void { Array.isArray(entry.hermesToolGateways) && entry.hermesToolGateways.length > 0 ? [...entry.hermesToolGateways] : undefined, + hermesDashboardEnabled: entry.hermesDashboardEnabled === true ? true : undefined, + hermesDashboardPort: entry.hermesDashboardPort ?? undefined, + hermesDashboardInternalPort: entry.hermesDashboardInternalPort ?? undefined, + hermesDashboardTui: entry.hermesDashboardTui === true ? true : undefined, disabledChannels: Array.isArray(entry.disabledChannels) && entry.disabledChannels.length > 0 ? [...entry.disabledChannels] diff --git a/test/e2e/test-hermes-e2e.sh b/test/e2e/test-hermes-e2e.sh index 94029f182d0..c655c9a0e10 100755 --- a/test/e2e/test-hermes-e2e.sh +++ b/test/e2e/test-hermes-e2e.sh @@ -20,6 +20,8 @@ # NEMOCLAW_AGENT=hermes — auto-set if not already set # NEMOCLAW_SANDBOX_NAME — sandbox name (default: e2e-hermes) # NEMOCLAW_RECREATE_SANDBOX=1 — recreate sandbox if it exists from a previous run +# NEMOCLAW_E2E_HERMES_DASHBOARD=1 — validate optional Hermes web dashboard end-to-end +# NEMOCLAW_HERMES_DASHBOARD=1 — enable optional Hermes web dashboard during onboard # NVIDIA_API_KEY — required for NVIDIA Endpoints inference # # Usage: @@ -105,6 +107,46 @@ except Exception as e: " } +is_truthy_env_value() { + case "${1:-}" in + 1 | true | TRUE | yes | YES | on | ON) return 0 ;; + *) return 1 ;; + esac +} + +hermes_dashboard_e2e_enabled() { + is_truthy_env_value "${NEMOCLAW_E2E_HERMES_DASHBOARD:-}" \ + || is_truthy_env_value "${NEMOCLAW_HERMES_DASHBOARD:-}" +} + +http_status_ok() { + case "$1" in + 2?? | 3??) return 0 ;; + *) return 1 ;; + esac +} + +forward_list_has_running_port() { + local sandbox="$1" + local port="$2" + local forward_list="$3" + FORWARD_LIST_TEXT="$forward_list" python3 - "$sandbox" "$port" <<'PY' +import os +import re +import sys + +ANSI_RE = re.compile(r"\x1B(?:\[[0-?]*[ -/]*[@-~]|\][^\x07]*(?:\x07|\x1B\\)|[@-_])") +sandbox = sys.argv[1] +port = sys.argv[2] +for raw_line in os.environ.get("FORWARD_LIST_TEXT", "").splitlines(): + line = ANSI_RE.sub("", raw_line) + parts = line.split() + if len(parts) >= 5 and parts[0] == sandbox and parts[2] == port and parts[-1].lower() in {"running", "active"}: + sys.exit(0) +sys.exit(1) +PY +} + # Determine repo root if [ -d /workspace ] && [ -f /workspace/install.sh ]; then REPO="/workspace" @@ -124,6 +166,9 @@ register_sandbox_for_teardown "$SANDBOX_NAME" # Hermes health probe endpoint (from agents/hermes/manifest.yaml) HERMES_HEALTH_URL="http://localhost:8642/health" +HERMES_DASHBOARD_PORT="${NEMOCLAW_HERMES_DASHBOARD_PORT:-9119}" +HERMES_DASHBOARD_INTERNAL_PORT="${NEMOCLAW_HERMES_DASHBOARD_INTERNAL_PORT:-19119}" +TIMEOUT_CMD="" # ══════════════════════════════════════════════════════════════════ # Phase 0: Pre-cleanup @@ -258,6 +303,22 @@ else fail "nemoclaw --help failed" fi +if hermes_dashboard_e2e_enabled; then + if grep -Fq "Hermes Agent OpenAI-compatible API" "$INSTALL_LOG" \ + && grep -Fq "http://127.0.0.1:8642/v1" "$INSTALL_LOG"; then + pass "Install output advertises Hermes API on 8642/v1" + else + fail "Install output did not advertise Hermes API on 8642/v1" + fi + + if grep -Fq "Hermes Agent Web dashboard" "$INSTALL_LOG" \ + && grep -Fq "http://127.0.0.1:${HERMES_DASHBOARD_PORT}/" "$INSTALL_LOG"; then + pass "Install output advertises Hermes web dashboard on ${HERMES_DASHBOARD_PORT}" + else + fail "Install output did not advertise Hermes web dashboard on ${HERMES_DASHBOARD_PORT}" + fi +fi + # ══════════════════════════════════════════════════════════════════ # Phase 3: Sandbox verification (Hermes-specific) # ══════════════════════════════════════════════════════════════════ @@ -429,6 +490,112 @@ else fail "Hermes config/state directory not found at /sandbox/.hermes" fi +if hermes_dashboard_e2e_enabled; then + section "Phase 4f: Hermes web dashboard" + + registry_check=$( + python3 - "$SANDBOX_NAME" "$HERMES_DASHBOARD_PORT" "$HERMES_DASHBOARD_INTERNAL_PORT" <<'PY' 2>&1 +import json +import os +import sys + +sandbox_name = sys.argv[1] +public_port = int(sys.argv[2]) +internal_port = int(sys.argv[3]) +registry_path = os.path.join(os.path.expanduser("~"), ".nemoclaw", "sandboxes.json") +with open(registry_path, encoding="utf-8") as fh: + registry = json.load(fh) +sandbox = (registry.get("sandboxes") or {}).get(sandbox_name) +errors = [] +if sandbox is None: + errors.append(f"{sandbox_name} missing from registry") +elif sandbox.get("agent") != "hermes": + errors.append(f"agent={sandbox.get('agent')!r}") +else: + checks = { + "hermesDashboardEnabled": True, + "hermesDashboardPort": public_port, + "hermesDashboardInternalPort": internal_port, + "dashboardPort": 8642, + } + for key, expected in checks.items(): + actual = sandbox.get(key) + if actual != expected: + errors.append(f"{key}={actual!r} expected {expected!r}") +if errors: + print("; ".join(errors)) + sys.exit(1) +print("ok") +PY + ) + if [ "$registry_check" = "ok" ]; then + pass "Registry records Hermes API and optional dashboard ports separately" + else + fail "Registry did not record Hermes dashboard metadata: ${registry_check:0:240}" + fi + + forward_list=$(openshell forward list 2>&1 || true) + if forward_list_has_running_port "$SANDBOX_NAME" "8642" "$forward_list"; then + pass "OpenShell forward list shows Hermes API port 8642 running" + else + fail "OpenShell forward list does not show Hermes API port 8642 running" + info "forward list: ${forward_list:0:300}" + fi + if forward_list_has_running_port "$SANDBOX_NAME" "$HERMES_DASHBOARD_PORT" "$forward_list"; then + pass "OpenShell forward list shows Hermes dashboard port ${HERMES_DASHBOARD_PORT} running" + else + fail "OpenShell forward list does not show Hermes dashboard port ${HERMES_DASHBOARD_PORT} running" + info "forward list: ${forward_list:0:300}" + fi + + dashboard_body="$(mktemp)" + dashboard_url="http://127.0.0.1:${HERMES_DASHBOARD_PORT}/" + dashboard_code="000" + for attempt in $(seq 1 20); do + dashboard_code=$(curl -sS -L --max-time 10 -o "$dashboard_body" -w "%{http_code}" "$dashboard_url" 2>/dev/null || echo "000") + if http_status_ok "$dashboard_code" && [ -s "$dashboard_body" ]; then + break + fi + info "Dashboard host probe attempt ${attempt}/20 returned HTTP ${dashboard_code:-000}; waiting 4s..." + sleep 4 + done + if http_status_ok "$dashboard_code" && [ -s "$dashboard_body" ]; then + pass "Hermes web dashboard responds from host on ${dashboard_url} (HTTP ${dashboard_code})" + else + fail "Hermes web dashboard did not respond from host on ${dashboard_url} (HTTP ${dashboard_code:-000})" + fi + + api_health=$(curl -sf --max-time 10 "http://127.0.0.1:8642/health" 2>&1 || true) + if echo "$api_health" | grep -qi '"ok"'; then + pass "Hermes API health remains on port 8642" + else + fail "Hermes API health did not respond on port 8642: ${api_health:0:160}" + fi + + if [ ! -s "$ssh_config" ]; then + openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config" 2>/dev/null || true + fi + if [ -s "$ssh_config" ]; then + dashboard_internal_code=$($TIMEOUT_CMD ssh -F "$ssh_config" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" \ + "code=\$(curl -sS -L --max-time 10 -o /tmp/hermes-dashboard-e2e-body -w '%{http_code}' http://127.0.0.1:${HERMES_DASHBOARD_INTERNAL_PORT}/ 2>/dev/null || echo 000); if test -s /tmp/hermes-dashboard-e2e-body; then echo \"\$code\"; else echo \"EMPTY:\$code\"; fi" \ + 2>&1) || true + + if http_status_ok "$dashboard_internal_code"; then + pass "Hermes dashboard process responds inside sandbox on ${HERMES_DASHBOARD_INTERNAL_PORT}" + else + fail "Hermes dashboard process did not respond inside sandbox on ${HERMES_DASHBOARD_INTERNAL_PORT}: ${dashboard_internal_code:0:160}" + fi + else + fail "Could not get SSH config for in-sandbox Hermes dashboard probe" + fi + rm -f "$dashboard_body" +fi + rm -f "$ssh_config" # ══════════════════════════════════════════════════════════════════ diff --git a/test/hermes-start.test.ts b/test/hermes-start.test.ts index 7177b5e18b0..3b53e47aa0d 100644 --- a/test/hermes-start.test.ts +++ b/test/hermes-start.test.ts @@ -47,6 +47,42 @@ function extractRuntimeShellEnvBlock(src: string): string { return src.slice(start, end).trimEnd(); } +function runHermesPortValidation(opts: { + publicPort?: number; + internalPort?: number; + dashboardPublicPort?: number; + dashboardInternalPort?: number; +}) { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-port-validation-")); + const scriptPath = path.join(tmpDir, "run.sh"); + const src = fs.readFileSync(START_SCRIPT, "utf-8"); + fs.writeFileSync( + scriptPath, + [ + "#!/usr/bin/env bash", + "set -euo pipefail", + extractShellFunctionFromSource(src, "validate_tcp_port"), + extractShellFunctionFromSource(src, "validate_port_configuration"), + `PUBLIC_PORT=${opts.publicPort ?? 8642}`, + `INTERNAL_PORT=${opts.internalPort ?? 18642}`, + `HERMES_DASHBOARD_PUBLIC_PORT=${opts.dashboardPublicPort ?? 9119}`, + `HERMES_DASHBOARD_INTERNAL_PORT=${opts.dashboardInternalPort ?? 19119}`, + "validate_port_configuration", + ].join("\n"), + { mode: 0o700 }, + ); + + try { + return spawnSync("bash", [scriptPath], { + encoding: "utf-8", + timeout: 5000, + env: process.env, + }); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } +} + function runTirithMarkerBootstrap(opts: { markerReason?: string; symlinkMarker?: boolean; @@ -114,6 +150,7 @@ function lstatIfPresent(entry: string): fs.Stats | null { function runHermesGatewayRuntimeCleanup(opts: { liveGateway?: boolean; orphanSocat?: boolean; + orphanDashboardSocat?: boolean; staleLock?: boolean; stalePid?: boolean; lockedConfigRoot?: boolean; @@ -151,6 +188,13 @@ function runHermesGatewayRuntimeCleanup(opts: { "TCP:127.0.0.1:18642", ]); } + if (opts.orphanDashboardSocat) { + writeFakeProcCmdline(procRoot, 789, [ + "socat", + "TCP-LISTEN:9119,bind=0.0.0.0,fork,reuseaddr", + "TCP:127.0.0.1:19119", + ]); + } const src = fs.readFileSync(START_SCRIPT, "utf-8"); fs.writeFileSync( @@ -193,6 +237,8 @@ function runHermesGatewayRuntimeCleanup(opts: { : "", "PUBLIC_PORT=8642", "INTERNAL_PORT=18642", + "HERMES_DASHBOARD_PUBLIC_PORT=9119", + "HERMES_DASHBOARD_INTERNAL_PORT=19119", "cleanup_stale_hermes_gateway_runtime", ].join("\n"), { mode: 0o700 }, @@ -315,6 +361,26 @@ describe("agents/hermes/start.sh runtime shell env", () => { }); +describe("agents/hermes/start.sh port validation", () => { + it("rejects cross-collisions between API and dashboard ports", () => { + const dashboardPublicOnApiInternal = runHermesPortValidation({ + dashboardPublicPort: 18642, + }); + expect(dashboardPublicOnApiInternal.status).toBe(1); + expect(dashboardPublicOnApiInternal.stderr).toContain( + "HERMES_DASHBOARD_PUBLIC_PORT must not equal INTERNAL_PORT", + ); + + const dashboardInternalOnApiPublic = runHermesPortValidation({ + dashboardInternalPort: 8642, + }); + expect(dashboardInternalOnApiPublic.status).toBe(1); + expect(dashboardInternalOnApiPublic.stderr).toContain( + "HERMES_DASHBOARD_INTERNAL_PORT must not equal PUBLIC_PORT", + ); + }); +}); + describe("agents/hermes/start.sh gateway runtime cleanup", () => { it("removes stale Hermes pid and lock files plus the legacy compatibility pid symlink", () => { const run = runHermesGatewayRuntimeCleanup({}); @@ -375,6 +441,18 @@ describe("agents/hermes/start.sh gateway runtime cleanup", () => { expect(run.result.stderr).toContain("Removing orphaned socat forwarder"); }); + it("kills orphaned dashboard socat forwarders when no Hermes gateway is alive", () => { + const run = runHermesGatewayRuntimeCleanup({ + orphanDashboardSocat: true, + staleLock: false, + stalePid: false, + }); + + expect(run.result.status).toBe(0); + expect(run.killLog.trim()).toBe("789"); + expect(run.result.stderr).toContain("Removing orphaned dashboard socat forwarder"); + }); + it("preserves Hermes runtime state when a gateway process is alive", () => { const run = runHermesGatewayRuntimeCleanup({ liveGateway: true, orphanSocat: true }); diff --git a/test/onboard-dashboard.test.ts b/test/onboard-dashboard.test.ts index 034d5e3a6e7..008f2be2035 100644 --- a/test/onboard-dashboard.test.ts +++ b/test/onboard-dashboard.test.ts @@ -66,6 +66,60 @@ describe("onboard dashboard helpers", () => { ).toBe(false); }); + it("starts a fixed extra agent forward without stopping other sandboxes", () => { + const runOpenshell = vi.fn((_args: string[], _opts?: Record) => ({ + status: 0, + })); + const runCaptureOpenshell = vi.fn(() => "hermes-sandbox 127.0.0.1 9119 123 running"); + const openshellArgv = vi.fn((args: string[]) => [process.execPath, "-e", "", ...args]); + const helpers = createOnboardDashboardHelpers({ + runOpenshell, + runCaptureOpenshell, + openshellArgv, + cliName: () => "nemohermes", + agentProductName: () => "NemoHermes", + getProviderLabel: (provider: string) => provider, + note: vi.fn(), + isWsl: () => false, + redact: (value: unknown) => String(value), + sleep: vi.fn(), + printAgentDashboardUi: vi.fn(), + }); + + vi.stubEnv("NEMOCLAW_DASHBOARD_BIND", "0.0.0.0"); + try { + expect(helpers.ensureAgentFixedForward("hermes-sandbox", 9119, "Hermes dashboard")).toBe( + true, + ); + } finally { + vi.unstubAllEnvs(); + } + + const stopArgs = runOpenshell.mock.calls.map(([args]) => args); + expect(stopArgs).toContainEqual(["forward", "stop", "9119", "hermes-sandbox"]); + expect(openshellArgv).toHaveBeenCalledWith([ + "forward", + "start", + "--background", + "9119", + "hermes-sandbox", + ]); + for (const args of stopArgs) { + if (args[0] === "forward" && args[1] === "stop") { + expect(args.at(-1)).toBe("hermes-sandbox"); + } + } + expect( + stopArgs.some( + (args) => + Array.isArray(args) && + args[0] === "forward" && + args[1] === "stop" && + args.length === 3, + ), + ).toBe(false); + }); + it("prints the dashboard-url command instead of raw gateway-token guidance", () => { const logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined); const nimStatus = vi.fn(() => ({ running: false, container: "nemoclaw-nim-test" })); diff --git a/test/process-recovery.test.ts b/test/process-recovery.test.ts index d88f6a46b87..ec4d3a17c12 100644 --- a/test/process-recovery.test.ts +++ b/test/process-recovery.test.ts @@ -1,15 +1,44 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it } from "vitest"; +import { createRequire } from "node:module"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { + checkAndRecoverSandboxProcesses, classifyForwardHealthWithReachability, classifySandboxForwardHealth, resolveSandboxDashboardPort, type SandboxForwardListEntry, } from "../dist/lib/actions/sandbox/process-recovery.js"; +const requireDist = createRequire(import.meta.url); + +afterEach(() => { + vi.restoreAllMocks(); +}); + +function withFakeOpenshellBinary(fn: () => T): T { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-fake-openshell-")); + const bin = path.join(dir, "openshell"); + const previous = process.env.NEMOCLAW_OPENSHELL_BIN; + fs.writeFileSync(bin, "#!/bin/sh\nexit 0\n", { mode: 0o755 }); + process.env.NEMOCLAW_OPENSHELL_BIN = bin; + try { + return fn(); + } finally { + if (previous === undefined) { + delete process.env.NEMOCLAW_OPENSHELL_BIN; + } else { + process.env.NEMOCLAW_OPENSHELL_BIN = previous; + } + fs.rmSync(dir, { recursive: true, force: true }); + } +} + describe("resolveSandboxDashboardPort", () => { it("uses the recorded OpenClaw dashboard port for multi-sandbox recovery", () => { expect( @@ -143,3 +172,65 @@ describe("classifyForwardHealthWithReachability", () => { ).toBe("occupied"); }); }); + +describe("checkAndRecoverSandboxProcesses", () => { + it("scopes forward stop to the target sandbox when restarting a dead forward", () => { + const openshellRuntime = requireDist("../dist/lib/adapters/openshell/runtime.js"); + const agentRuntime = requireDist("../dist/lib/agent/runtime.js"); + const registry = requireDist("../dist/lib/state/registry.js"); + const forwardHealth = requireDist("../dist/lib/actions/sandbox/forward-health.js"); + const childProcess = requireDist("node:child_process"); + const deadForward = `SANDBOX BIND PORT PID STATUS +beta 127.0.0.1 18789 12345 dead`; + const runningForward = `SANDBOX BIND PORT PID STATUS +beta 127.0.0.1 18789 12345 running`; + let forwardListCalls = 0; + + vi.spyOn(childProcess, "spawnSync").mockReturnValue({ + status: 0, + stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nRUNNING\n", + stderr: "", + } as never); + vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue(null); + vi.spyOn(registry, "getSandbox").mockReturnValue({ + name: "beta", + agent: "openclaw", + dashboardPort: 18789, + }); + vi.spyOn(forwardHealth, "isLocalForwardReachable").mockReturnValue(false); + vi.spyOn(openshellRuntime, "captureOpenshell").mockImplementation((rawArgs: unknown) => { + const args = Array.isArray(rawArgs) ? rawArgs : []; + expect(args).toEqual(["forward", "list"]); + forwardListCalls += 1; + return { + status: 0, + output: forwardListCalls >= 3 ? runningForward : deadForward, + }; + }); + const runOpenshell = vi + .spyOn(openshellRuntime, "runOpenshell") + .mockReturnValue({ status: 0 } as never); + + expect( + withFakeOpenshellBinary(() => checkAndRecoverSandboxProcesses("beta", { quiet: true })), + ).toEqual({ + checked: true, + wasRunning: true, + recovered: false, + forwardRecovered: true, + }); + expect(runOpenshell).toHaveBeenCalledWith( + ["forward", "stop", "18789", "beta"], + { ignoreError: true, stdio: "ignore" }, + ); + expect( + runOpenshell.mock.calls.some( + ([args]) => + Array.isArray(args) && + args[0] === "forward" && + args[1] === "stop" && + args.length === 3, + ), + ).toBe(false); + }); +});