feat: multi-agent architecture with Hermes support

.github/workflows/nightly-e2e.yaml

@@ -11,6 +11,8 @@
 #                            for Telegram + Discord. Uses fake tokens. See PR #1081.
 #   sandbox-survival-e2e     Sandbox survival across gateway restarts (onboard, inference,
 #                            gateway stop/start, verify sandbox + workspace + inference).
+#   hermes-e2e               Hermes Agent E2E — install → onboard --agent hermes → health
+#                            probe → live inference. Validates the multi-agent architecture.
 #   gpu-e2e                  Local Ollama inference on a GPU self-hosted runner.
 #                            Controlled by the GPU_E2E_ENABLED repository variable.
 #                            Set vars.GPU_E2E_ENABLED to "true" in repo settings to enable.
@@ -225,6 +227,37 @@ jobs:
           path: /tmp/nemoclaw-e2e-install.log
           if-no-files-found: ignore
 
+  # ── Hermes Agent E2E ─────────────────────────────────────────
+  # Validates the multi-agent architecture by onboarding with --agent hermes,
+  # verifying the Hermes health probe, and running live inference through the
+  # Hermes sandbox. See: PR #1618
+  hermes-e2e:
+    if: github.repository == 'NVIDIA/NemoClaw'
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Run Hermes Agent E2E test
+        env:
+          NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
+          NEMOCLAW_NON_INTERACTIVE: "1"
+          NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
+          NEMOCLAW_SANDBOX_NAME: "e2e-hermes"
+          NEMOCLAW_RECREATE_SANDBOX: "1"
+          NEMOCLAW_AGENT: "hermes"
+          GITHUB_TOKEN: ${{ github.token }}
+        run: bash test/e2e/test-hermes-e2e.sh
+
+      - name: Upload install log on failure
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: hermes-e2e-install-log
+          path: /tmp/nemoclaw-e2e-hermes-install.log
+          if-no-files-found: ignore
+
   # ── GPU E2E (Ollama local inference) ──────────────────────────
   # Enable by setting repository variable GPU_E2E_ENABLED=true
   # (Settings → Secrets and variables → Actions → Variables)
@@ -277,8 +310,8 @@ jobs:
 
   notify-on-failure:
     runs-on: ubuntu-latest
-    needs: [cloud-e2e, cloud-experimental-e2e, messaging-providers-e2e, sandbox-survival-e2e, gpu-e2e]
-    if: ${{ always() && (needs.cloud-e2e.result == 'failure' || needs.cloud-experimental-e2e.result == 'failure' || needs.messaging-providers-e2e.result == 'failure' || needs.sandbox-survival-e2e.result == 'failure' || needs.gpu-e2e.result == 'failure') }}
+    needs: [cloud-e2e, cloud-experimental-e2e, messaging-providers-e2e, sandbox-survival-e2e, hermes-e2e, gpu-e2e]
+    if: ${{ always() && (needs.cloud-e2e.result == 'failure' || needs.cloud-experimental-e2e.result == 'failure' || needs.messaging-providers-e2e.result == 'failure' || needs.sandbox-survival-e2e.result == 'failure' || needs.hermes-e2e.result == 'failure' || needs.gpu-e2e.result == 'failure') }}
     permissions:
       issues: write
     steps:

agents/hermes/Dockerfile

@@ -0,0 +1,112 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Hermes sandbox image — Hermes Agent + NemoClaw plugin inside OpenShell
+#
+# Layers PR-specific code (plugin, config, startup script) on top of the
+# pre-built Hermes base image. Mirrors the OpenClaw Dockerfile structure.
+
+ARG BASE_IMAGE=ghcr.io/nvidia/nemoclaw/hermes-sandbox-base:latest
+
+FROM ${BASE_IMAGE}
+
+# Harden: remove unnecessary build tools and network probes
+RUN (apt-get remove --purge -y gcc gcc-12 g++ g++-12 cpp cpp-12 make \
+        netcat-openbsd netcat-traditional ncat 2>/dev/null || true) \
+    && apt-get autoremove --purge -y \
+    && rm -rf /var/lib/apt/lists/*
+
+# Patch: disable Hermes's TelegramFallbackTransport inside the sandbox.
+# The fallback transport rewrites api.telegram.org to raw IPs, which
+# OpenShell's L7 proxy rejects (hostname-based policy). Without it,
+# python-telegram-bot uses default httpx transport which respects
+# HTTPS_PROXY and routes through the proxy correctly.
+# Must run as root before USER sandbox.
+# Replace _fallback_ips method to always return empty list, which
+# skips the entire fallback transport code path.
+RUN python3 -c "\
+import pathlib, re; \
+p = pathlib.Path('/usr/local/lib/python3.11/dist-packages/gateway/platforms/telegram.py'); \
+src = p.read_text(); \
+patched = src.replace( \
+    'fallback_ips = self._fallback_ips()', \
+    'fallback_ips = []  # NemoClaw: disabled for sandbox proxy compatibility'); \
+patched = patched.replace( \
+    'if not fallback_ips:', \
+    'if False:  # NemoClaw: skip fallback discovery'); \
+assert 'NemoClaw: disabled' in patched, 'Patch failed: _fallback_ips line not found'; \
+p.write_text(patched); \
+print('[patch] Disabled TelegramFallbackTransport for sandbox proxy compatibility')"
+
+# Copy NemoClaw plugin for Hermes (Python-based)
+COPY agents/hermes/plugin/ /opt/nemoclaw-hermes-plugin/
+
+# Copy config generation script and URL-decode proxy
+COPY agents/hermes/generate-config.ts /opt/nemoclaw-generate-config.ts
+COPY agents/hermes/decode-proxy.py /usr/local/bin/nemoclaw-decode-proxy
+RUN chmod 755 /usr/local/bin/nemoclaw-decode-proxy
+
+# Copy blueprint (shared infrastructure)
+COPY nemoclaw-blueprint/ /opt/nemoclaw-blueprint/
+
+# Copy startup script
+COPY agents/hermes/start.sh /usr/local/bin/nemoclaw-start
+RUN chmod 755 /usr/local/bin/nemoclaw-start
+
+# Build args for config that varies per deployment.
+ARG NEMOCLAW_MODEL=nvidia/nemotron-3-super-120b-a12b
+ARG NEMOCLAW_PROVIDER_KEY=custom
+ARG NEMOCLAW_INFERENCE_BASE_URL=https://inference.local/v1
+ARG CHAT_UI_URL=http://127.0.0.1:8642
+ARG NEMOCLAW_MESSAGING_CHANNELS_B64=W10=
+ARG NEMOCLAW_MESSAGING_ALLOWED_IDS_B64=e30=
+ARG NEMOCLAW_BUILD_ID=default
+
+# Promote build-args to env vars for the config generation script.
+ENV NEMOCLAW_MODEL=${NEMOCLAW_MODEL} \
+    NEMOCLAW_PROVIDER_KEY=${NEMOCLAW_PROVIDER_KEY} \
+    NEMOCLAW_INFERENCE_BASE_URL=${NEMOCLAW_INFERENCE_BASE_URL} \
+    CHAT_UI_URL=${CHAT_UI_URL} \
+    NEMOCLAW_MESSAGING_CHANNELS_B64=${NEMOCLAW_MESSAGING_CHANNELS_B64} \
+    NEMOCLAW_MESSAGING_ALLOWED_IDS_B64=${NEMOCLAW_MESSAGING_ALLOWED_IDS_B64}
+
+WORKDIR /sandbox
+USER sandbox
+
+# Set up blueprint for local resolution
+RUN mkdir -p /sandbox/.nemoclaw/blueprints/0.1.0 \
+    && cp -r /opt/nemoclaw-blueprint/* /sandbox/.nemoclaw/blueprints/0.1.0/
+
+# Generate Hermes config.yaml and .env from build args.
+# config.yaml is immutable at runtime (Landlock read-only on /sandbox/.hermes).
+# .env holds API key placeholders for OpenShell provider pipeline.
+# SECURITY: Uses a separate script file instead of inline code to avoid
+# code injection via build-arg interpolation (same concern as OpenClaw C-2).
+RUN node --experimental-strip-types /opt/nemoclaw-generate-config.ts
+
+# Install NemoClaw plugin into Hermes
+RUN mkdir -p /sandbox/.hermes-data/plugins/nemoclaw \
+    && cp -r /opt/nemoclaw-hermes-plugin/* /sandbox/.hermes-data/plugins/nemoclaw/
+
+# Lock .hermes via DAC: chown to root so sandbox user cannot modify config.
+# Same pattern as OpenClaw — Landlock provides defense-in-depth.
+# hadolint ignore=DL3002
+USER root
+RUN chown root:root /sandbox/.hermes \
+    && rm -rf /root/.cache/pip /sandbox/.cache \
+    && find /sandbox/.hermes -mindepth 1 -maxdepth 1 -exec chown -h root:root {} + \
+    && chmod 755 /sandbox/.hermes \
+    && chmod 444 /sandbox/.hermes/config.yaml \
+    && chmod 444 /sandbox/.hermes/.env
+
+# Pin config hash at build time for integrity verification at startup.
+RUN sha256sum /sandbox/.hermes/config.yaml /sandbox/.hermes/.env \
+    > /sandbox/.hermes/.config-hash \
+    && chmod 444 /sandbox/.hermes/.config-hash \
+    && chown root:root /sandbox/.hermes/.config-hash
+
+# start.sh handles privilege separation: runs as root initially to perform
+# symlink validation and hardening, then drops to 'gateway' user via gosu
+# for the agent process. See the root-path branch in start.sh.
+ENTRYPOINT ["/usr/local/bin/nemoclaw-start"]
+CMD ["/bin/bash"]

agents/hermes/Dockerfile.base

@@ -0,0 +1,100 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Hermes sandbox base image — expensive, rarely-changing layers.
+#
+# Contains: node:22-slim (OpenShell needs Node), apt packages, gosu,
+# user/group setup, .hermes directory structure, Hermes CLI, and PyYAML.
+#
+# Mirrors the OpenClaw Dockerfile.base structure but installs Hermes
+# (Python-based) instead of OpenClaw (Node-based).
+#
+# ── When to rebuild ─────────────────────────────────────────────
+#   1. Hermes version bump      — change the HERMES_VERSION below
+#   2. New apt package needed   — add it to the apt-get install list
+#   3. gosu upgrade             — update URL, checksum, and version
+#   4. node:22-slim digest rot  — update-docker-pin.sh updates both
+#   5. New .hermes subdirectory — add mkdir + symlink below
+# ────────────────────────────────────────────────────────────────
+
+FROM node:22-slim@sha256:4f77a690f2f8946ab16fe1e791a3ac0667ae1c3575c3e4d0d4589e9ed5bfaf3d
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Hermes version pinned for reproducibility.
+# Calver tag v2026.4.8 = semver 0.8.0.
+ARG HERMES_VERSION=v2026.4.8
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        python3=3.11.2-1+b1 \
+        python3-pip=23.0.1+dfsg-1 \
+        python3-venv=3.11.2-1+b1 \
+        curl=7.88.1-10+deb12u14 \
+        git=1:2.39.5-0+deb12u3 \
+        ca-certificates=20230311+deb12u1 \
+        iproute2=6.1.0-3 \
+        iptables=1.8.9-2 \
+        libcap2-bin=1:2.66-4+deb12u2+b2 \
+        socat=1.7.4.4-2 \
+    && rm -rf /var/lib/apt/lists/*
+
+# gosu for privilege separation (gateway vs sandbox user).
+# Identical to OpenClaw base — pinned to 1.19 with checksum.
+# hadolint ignore=DL4006
+RUN arch="$(dpkg --print-architecture)" \
+    && case "$arch" in \
+        amd64) gosu_asset="gosu-amd64"; gosu_sha256="52c8749d0142edd234e9d6bd5237dff2d81e71f43537e2f4f66f75dd4b243dd0" ;; \
+        arm64) gosu_asset="gosu-arm64"; gosu_sha256="3a8ef022d82c0bc4a98bcb144e77da714c25fcfa64dccc57f6aba7ae47ff1a44" ;; \
+        *) echo "Unsupported architecture for gosu: $arch" >&2; exit 1 ;; \
+    esac \
+    && curl -fsSL -o /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/1.19/${gosu_asset}" \
+    && echo "${gosu_sha256}  /usr/local/bin/gosu" | sha256sum -c - \
+    && chmod +x /usr/local/bin/gosu \
+    && gosu --version
+
+# Create sandbox user (matches OpenShell convention) and gateway user.
+# Same UID/GID layout as OpenClaw — OpenShell expects these names.
+RUN groupadd -r gateway && useradd -r -g gateway -d /sandbox -s /usr/sbin/nologin gateway \
+    && groupadd -r sandbox && useradd -r -g sandbox -d /sandbox -s /bin/bash sandbox \
+    && mkdir -p /sandbox/.nemoclaw \
+    && chown -R sandbox:sandbox /sandbox
+
+# Split .hermes into immutable config dir + writable state dir.
+# Same pattern as OpenClaw: Landlock makes /sandbox/.hermes read-only,
+# writable state lives in .hermes-data, reached via symlinks.
+RUN mkdir -p /sandbox/.hermes-data/memories \
+        /sandbox/.hermes-data/sessions \
+        /sandbox/.hermes-data/skills \
+        /sandbox/.hermes-data/plugins \
+        /sandbox/.hermes-data/cron \
+        /sandbox/.hermes-data/logs \
+        /sandbox/.hermes-data/skins \
+        /sandbox/.hermes-data/plans \
+        /sandbox/.hermes-data/workspace \
+        /sandbox/.hermes-data/profiles \
+        /sandbox/.hermes-data/cache \
+        /sandbox/.hermes-data/pairing \
+    && mkdir -p /sandbox/.hermes \
+    && ln -s /sandbox/.hermes-data/memories /sandbox/.hermes/memories \
+    && ln -s /sandbox/.hermes-data/sessions /sandbox/.hermes/sessions \
+    && ln -s /sandbox/.hermes-data/skills /sandbox/.hermes/skills \
+    && ln -s /sandbox/.hermes-data/plugins /sandbox/.hermes/plugins \
+    && ln -s /sandbox/.hermes-data/cron /sandbox/.hermes/cron \
+    && ln -s /sandbox/.hermes-data/logs /sandbox/.hermes/logs \
+    && ln -s /sandbox/.hermes-data/skins /sandbox/.hermes/skins \
+    && ln -s /sandbox/.hermes-data/plans /sandbox/.hermes/plans \
+    && ln -s /sandbox/.hermes-data/workspace /sandbox/.hermes/workspace \
+    && ln -s /sandbox/.hermes-data/profiles /sandbox/.hermes/profiles \
+    && ln -s /sandbox/.hermes-data/cache /sandbox/.hermes/cache \
+    && ln -s /sandbox/.hermes-data/pairing /sandbox/.hermes/pairing \
+    && chown -R sandbox:sandbox /sandbox/.hermes /sandbox/.hermes-data
+
+# Install Hermes Agent from GitHub release.
+# Hermes is not on PyPI — official install is via their install script or
+# direct pip install from the release tarball.
+# hadolint ignore=DL3013
+RUN pip3 install --no-cache-dir --break-system-packages \
+        "hermes-agent @ https://github.com/NousResearch/hermes-agent/archive/refs/tags/${HERMES_VERSION}.tar.gz" \
+        "pyyaml==6.0.3" \
+        "python-telegram-bot>=21.0" \
+    && hermes --version

agents/hermes/decode-proxy.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+"""
+URL-decoding HTTP proxy for OpenShell placeholder rewriting.
+
+Python HTTP clients (httpx) URL-encode colons in URL paths, turning
+openshell:resolve:env:TOKEN into openshell%3Aresolve%3Aenv%3ATOKEN.
+OpenShell's L7 proxy doesn't recognize the encoded form.
+
+This proxy sits between the Python process and the OpenShell proxy,
+URL-decodes the CONNECT target and request paths so the placeholders
+are restored before reaching the L7 proxy.
+
+Usage: Launched by start.sh, listens on 127.0.0.1:3129.
+       HTTPS_PROXY=http://127.0.0.1:3129 hermes gateway run
+"""
+
+import asyncio
+import sys
+from urllib.parse import unquote
+
+
+UPSTREAM_HOST = "10.200.0.1"
+UPSTREAM_PORT = 3128
+LISTEN_HOST = "127.0.0.1"
+LISTEN_PORT = 3129
+
+
+async def handle_client(reader, writer):
+    """Proxy a single connection, URL-decoding the initial request line."""
+    up_writer = None
+    try:
+        first_line = await asyncio.wait_for(reader.readline(), timeout=10)
+        if not first_line:
+            writer.close()
+            return
+
+        # Decode only the request target (second token) so valid percent-encoding
+        # like %2F or %3F in the method/version is preserved. Only the path
+        # contains openshell%3Aresolve placeholders that need decoding.
+        parts = first_line.decode("utf-8", errors="replace").split(" ", 2)
+        if len(parts) == 3:
+            parts[1] = unquote(parts[1])
+        decoded_line = " ".join(parts).encode("utf-8")
+
+        # Read remaining headers
+        headers = bytearray(decoded_line)
+        while True:
+            line = await asyncio.wait_for(reader.readline(), timeout=10)
+            headers.extend(line)
+            if line == b"\r\n" or line == b"\n" or not line:
+                break
+
+        # Connect to upstream proxy
+        up_reader, up_writer = await asyncio.open_connection(
+            UPSTREAM_HOST, UPSTREAM_PORT
+        )
+        up_writer.write(bytes(headers))
+        await up_writer.drain()
+
+        # Bidirectional relay
+        await asyncio.gather(
+            _relay(reader, up_writer),
+            _relay(up_reader, writer),
+        )
+    except (asyncio.TimeoutError, ConnectionError, OSError):
+        pass
+    finally:
+        for w in (up_writer, writer):
+            if w is not None:
+                try:
+                    w.close()
+                    await w.wait_closed()
+                except (ConnectionError, OSError):
+                    pass
+
+
+async def _relay(src, dst):
+    """Copy data from src to dst until EOF."""
+    try:
+        while True:
+            data = await src.read(65536)
+            if not data:
+                break
+            dst.write(data)
+            await dst.drain()
+    except (ConnectionError, OSError):
+        pass
+
+
+async def main():
+    server = await asyncio.start_server(handle_client, LISTEN_HOST, LISTEN_PORT)
+    print(
+        f"[decode-proxy] Listening on {LISTEN_HOST}:{LISTEN_PORT} -> {UPSTREAM_HOST}:{UPSTREAM_PORT}",
+        file=sys.stderr,
+    )
+    async with server:
+        await server.serve_forever()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())