diff --git a/.github/workflows/nightly-e2e.yaml b/.github/workflows/nightly-e2e.yaml index 76ffa37fb0..2d54a039a9 100644 --- a/.github/workflows/nightly-e2e.yaml +++ b/.github/workflows/nightly-e2e.yaml @@ -6,8 +6,7 @@ # cloud-e2e Cloud inference (NVIDIA Endpoint API) on ubuntu-latest. # messaging-providers-e2e Validates messaging credential provider/placeholder/L7-proxy chain # for Telegram + Discord + Slack. Uses fake tokens. Slack additionally -# exercises the slack-token-rewriter Bolt-shape → canonical placeholder -# translation (#2085). See PR #1081. +# exercises OpenShell provider-shaped alias resolution (#2085 follow-up). # messaging-compatible-endpoint-e2e # Validates Telegram + OpenAI-compatible endpoint inference routing # through inference.local with a hermetic local mock (#2766). @@ -21,7 +20,7 @@ # gateway stop/start, verify sandbox + workspace + inference). # openshell-gateway-upgrade-e2e # Validates real v0.0.36 curl install upgrade into -# OpenShell 0.0.37 with pre-upgrade backup, restored +# the current supported OpenShell with pre-upgrade backup, restored # agent state, and the same agent type running. # gateway-health-honest-e2e # Coverage guard for #3111: onboard must not log @@ -36,7 +35,7 @@ # hermes-discord-e2e Hermes Discord onboarding — validates the top-level Hermes # Discord schema plus OpenShell placeholder/token isolation. # hermes-slack-e2e Hermes Slack onboarding — validates the Hermes Slack policy, -# Slack providers, and Python placeholder egress path. +# Slack providers, and OpenShell credential rewrite path. # openclaw-inference-switch-e2e # Switches a running OpenClaw sandbox with `nemoclaw inference set` # and verifies route, openclaw.json, hashes, and live requests. @@ -1430,19 +1429,18 @@ jobs: steps: - name: Checkout uses: actions/checkout@v6 + - name: Install NemoClaw env: NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" - NEMOCLAW_SANDBOX_NAME: "e2e-double-install" run: bash install.sh --non-interactive --yes-i-accept-third-party-software - name: Run double onboard E2E test env: NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" - NEMOCLAW_E2E_INSTALL_SANDBOX_NAME: "e2e-double-install" run: | [ -f "$HOME/.bashrc" ] && source "$HOME/.bashrc" 2>/dev/null || true export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" @@ -1469,19 +1467,18 @@ jobs: steps: - name: Checkout uses: actions/checkout@v6 + - name: Install NemoClaw env: NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" - NEMOCLAW_SANDBOX_NAME: "e2e-repair-install" run: bash install.sh --non-interactive --yes-i-accept-third-party-software - name: Run onboard repair E2E test env: NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" - NEMOCLAW_E2E_INSTALL_SANDBOX_NAME: "e2e-repair-install" run: | [ -f "$HOME/.bashrc" ] && source "$HOME/.bashrc" 2>/dev/null || true export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" @@ -1508,6 +1505,7 @@ jobs: steps: - name: Checkout uses: actions/checkout@v6 + - name: Install NemoClaw env: NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} @@ -1545,6 +1543,7 @@ jobs: steps: - name: Checkout uses: actions/checkout@v6 + - name: Install NemoClaw env: NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} @@ -1583,6 +1582,7 @@ jobs: steps: - name: Checkout uses: actions/checkout@v6 + - name: Install NemoClaw and onboard sandbox env: NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} @@ -1624,6 +1624,7 @@ jobs: steps: - name: Checkout uses: actions/checkout@v6 + - name: Install NemoClaw and onboard sandbox env: NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} diff --git a/.github/workflows/sandbox-images-and-e2e.yaml b/.github/workflows/sandbox-images-and-e2e.yaml index 0b9723c821..78a176d869 100644 --- a/.github/workflows/sandbox-images-and-e2e.yaml +++ b/.github/workflows/sandbox-images-and-e2e.yaml @@ -80,7 +80,7 @@ jobs: docker run --rm --user sandbox nemoclaw-hermes-production \ test -r /opt/nemoclaw-blueprint/blueprint.yaml docker run --rm --user sandbox nemoclaw-hermes-production \ - test -x /usr/local/bin/nemoclaw-decode-proxy + test ! -e /usr/local/bin/nemoclaw-decode-proxy docker run --rm --user sandbox nemoclaw-hermes-production \ test -x /usr/local/bin/nemoclaw-start diff --git a/Dockerfile b/Dockerfile index 3131c27246..6675ea260c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -245,7 +245,6 @@ RUN chmod 755 /usr/local/bin/nemoclaw-start /usr/local/bin/nemoclaw-codex-acp \ /usr/local/lib/nemoclaw/sandbox-init.sh \ /usr/local/lib/nemoclaw/generate-openclaw-config.py \ && if [ -d /usr/local/lib/nemoclaw/preloads ]; then find /usr/local/lib/nemoclaw/preloads -type f -name '*.js' -exec chmod 644 {} +; fi \ - && if [ -f /usr/local/lib/nemoclaw/ws-proxy-fix.js ]; then chmod 644 /usr/local/lib/nemoclaw/ws-proxy-fix.js; fi \ && chmod 755 /usr/local/share/nemoclaw \ /usr/local/share/nemoclaw/openclaw-plugins \ && find /usr/local/share/nemoclaw/openclaw-plugins -type d -exec chmod 755 {} + \ diff --git a/agents/hermes/Dockerfile b/agents/hermes/Dockerfile index df92adb95d..8f871ad377 100644 --- a/agents/hermes/Dockerfile +++ b/agents/hermes/Dockerfile @@ -29,28 +29,20 @@ RUN (apt-get remove --purge -y gcc gcc-12 g++ g++-12 cpp cpp-12 make \ && rm -rf /var/lib/apt/lists/* # Hermes v2026.4.13+ auto-detects HTTPS_PROXY and skips fallback-IP -# transport when a proxy is present. The sandbox proxy chain -# (decode-proxy -> OpenShell L7 proxy) handles REST credential placeholder -# rewriting and hostname-based policy enforcement. A Hermes-only local Discord -# facade handles discord.py's Gateway session inside the sandbox and forwards -# REST through the same placeholder-substitution path. +# transport when a proxy is present. OpenShell handles REST credential +# placeholder rewriting, hostname-based policy enforcement, and native +# WebSocket credential rewrite at the egress boundary. ENV HERMES_TELEGRAM_DISABLE_FALLBACK_IPS=1 # Copy NemoClaw plugin for Hermes (Python-based) COPY agents/hermes/plugin/ /opt/nemoclaw-hermes-plugin/ RUN chmod -R a+rX /opt/nemoclaw-hermes-plugin/ -# Copy config generator, Discord facade, and URL-decode proxy +# Copy config generator COPY agents/hermes/generate-config.ts /opt/nemoclaw-hermes-config/generate-config.ts COPY agents/hermes/config/ /opt/nemoclaw-hermes-config/config/ RUN find /opt/nemoclaw-hermes-config -type d -exec chmod 755 {} + \ && find /opt/nemoclaw-hermes-config -type f -exec chmod 444 {} + -COPY agents/hermes/decode-proxy.py /usr/local/bin/nemoclaw-decode-proxy -COPY agents/hermes/discord-facade.py /usr/local/bin/nemoclaw-discord-facade -COPY agents/hermes/discord-preload/ /opt/nemoclaw-hermes-discord-preload/ -RUN chmod 755 /usr/local/bin/nemoclaw-decode-proxy /usr/local/bin/nemoclaw-discord-facade \ - && find /opt/nemoclaw-hermes-discord-preload -type d -exec chmod 755 {} + \ - && find /opt/nemoclaw-hermes-discord-preload -type f -exec chmod 444 {} + # Copy blueprint (shared infrastructure) COPY nemoclaw-blueprint/ /opt/nemoclaw-blueprint/ diff --git a/agents/hermes/config/messaging-config.ts b/agents/hermes/config/messaging-config.ts index eb27fdeb56..3e28556b7a 100644 --- a/agents/hermes/config/messaging-config.ts +++ b/agents/hermes/config/messaging-config.ts @@ -9,9 +9,6 @@ const CHANNEL_TOKEN_ENVS: Record = { slack: ["SLACK_BOT_TOKEN", "SLACK_APP_TOKEN"], }; -const HERMES_DISCORD_PROXY = "http://127.0.0.1:3129"; -const HERMES_DISCORD_FACADE = "http://127.0.0.1:3130"; - export function buildMessagingEnvLines( enabledChannels: Set, allowedIds: MessagingAllowedIds, @@ -25,8 +22,6 @@ export function buildMessagingEnvLines( envLines.push(`${envKey}=${buildTokenPlaceholder(channel, envKey)}`); } if (channel === "discord") { - envLines.push(`DISCORD_PROXY=${HERMES_DISCORD_PROXY}`); - envLines.push(`NEMOCLAW_DISCORD_FACADE_URL=${HERMES_DISCORD_FACADE}`); const guildIds = Object.keys(discordGuilds).filter(Boolean); if (guildIds.length > 0) { envLines.push(`NEMOCLAW_DISCORD_GUILD_IDS=${guildIds.join(",")}`); diff --git a/agents/hermes/decode-proxy.py b/agents/hermes/decode-proxy.py deleted file mode 100755 index 67addecd13..0000000000 --- a/agents/hermes/decode-proxy.py +++ /dev/null @@ -1,195 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -""" -HTTP proxy for OpenShell placeholder rewriting. - -Python HTTP clients (httpx) URL-encode colons in URL paths, turning -openshell:resolve:env:TOKEN into openshell%3Aresolve%3Aenv%3ATOKEN. -OpenShell's L7 proxy doesn't recognize the encoded form. - -This proxy sits between the Python process and the OpenShell proxy, -URL-decodes the CONNECT target and request paths so the placeholders -are restored before reaching the L7 proxy. It also translates Slack's -SDK-compatible xoxb-/xapp- placeholders in cleartext HTTP proxy requests -back to canonical openshell:resolve:env:SLACK_* placeholders before -OpenShell sees them. Hermes' Python preload handles Slack HTTPS requests -before TLS serialization. - -This is intentionally not a WebSocket frame rewriter. After the initial -HTTP proxy request is forwarded, bytes are relayed unchanged; Discord -gateway IDENTIFY payloads are not inspected or modified here. - -Usage: Launched by start.sh, listens on 127.0.0.1:3129. - HTTPS_PROXY=http://127.0.0.1:3129 hermes gateway run -""" - -import asyncio -import re -import sys -from urllib.parse import unquote - - -UPSTREAM_HOST = "10.200.0.1" -UPSTREAM_PORT = 3128 -LISTEN_HOST = "127.0.0.1" -LISTEN_PORT = 3129 -SLACK_PLACEHOLDER = re.compile( - r"\b(?:" - r"xoxb-OPENSHELL-RESOLVE-ENV-SLACK_BOT_TOKEN" - r"|" - r"xapp-OPENSHELL-RESOLVE-ENV-SLACK_APP_TOKEN" - r")\b" -) -SLACK_FAST_PATH = b"OPENSHELL-RESOLVE-ENV-SLACK_" - - -def rewrite_slack_placeholders(text): - """Translate Slack SDK-shaped placeholders to OpenShell canonical form.""" - - def _replacement(match): - placeholder = match.group(0) - if placeholder.startswith("xoxb-"): - return "openshell:resolve:env:SLACK_BOT_TOKEN" - return "openshell:resolve:env:SLACK_APP_TOKEN" - - return SLACK_PLACEHOLDER.sub(_replacement, text) - - -def rewrite_slack_placeholders_bytes(data): - """Rewrite UTF-8 header/body bytes when they contain Slack placeholders.""" - if SLACK_FAST_PATH not in data: - return data - try: - text = data.decode("utf-8") - except UnicodeDecodeError: - return data - rewritten = rewrite_slack_placeholders(text) - if rewritten == text: - return data - return rewritten.encode("utf-8") - - -def _line_ending(line): - return "\r\n" if line.endswith(b"\r\n") else "\n" - - -def _rewrite_request_line(first_line): - ending = _line_ending(first_line) - parts = first_line.decode("utf-8", errors="replace").rstrip("\r\n").split(" ", 2) - if len(parts) == 3: - parts[1] = rewrite_slack_placeholders(unquote(parts[1])) - return (" ".join(parts) + ending).encode("utf-8") - - -def _content_length(header_lines): - for line in header_lines: - text = line.decode("iso-8859-1", errors="replace") - if text.lower().startswith("content-length:"): - try: - return int(text.split(":", 1)[1].strip()) - except ValueError: - return None - return None - - -def _with_content_length(header_lines, content_length): - rewritten = [] - replaced = False - for line in header_lines: - text = line.decode("iso-8859-1", errors="replace") - if text.lower().startswith("content-length:"): - name = text.split(":", 1)[0] - ending = _line_ending(line) - rewritten.append(f"{name}: {content_length}{ending}".encode("ascii")) - replaced = True - else: - rewritten.append(line) - return rewritten if replaced else header_lines - - -async def handle_client(reader, writer): - """Proxy a single connection, rewriting placeholders before OpenShell.""" - up_writer = None - try: - first_line = await asyncio.wait_for(reader.readline(), timeout=10) - if not first_line: - writer.close() - return - - # Decode only the request target (second token) so valid percent-encoding - # like %2F or %3F in the method/version is preserved. - decoded_line = _rewrite_request_line(first_line) - - # Read and rewrite remaining headers. - header_lines = [] - while True: - line = await asyncio.wait_for(reader.readline(), timeout=10) - if line == b"\r\n" or line == b"\n" or not line: - header_end = line or b"\r\n" - break - header_lines.append(rewrite_slack_placeholders_bytes(line)) - - body = b"" - body_length = _content_length(header_lines) - if body_length: - body = await asyncio.wait_for(reader.readexactly(body_length), timeout=10) - body = rewrite_slack_placeholders_bytes(body) - header_lines = _with_content_length(header_lines, len(body)) - - request = bytearray(decoded_line) - for line in header_lines: - request.extend(line) - request.extend(header_end) - if body: - request.extend(body) - - # Connect to upstream proxy - up_reader, up_writer = await asyncio.open_connection( - UPSTREAM_HOST, UPSTREAM_PORT - ) - up_writer.write(bytes(request)) - await up_writer.drain() - - # Bidirectional relay - await asyncio.gather( - _relay(reader, up_writer), - _relay(up_reader, writer), - ) - except (asyncio.TimeoutError, ConnectionError, OSError): - pass - finally: - for w in (up_writer, writer): - if w is not None: - try: - w.close() - await w.wait_closed() - except (ConnectionError, OSError): - pass - - -async def _relay(src, dst): - """Copy data from src to dst until EOF.""" - try: - while True: - data = await src.read(65536) - if not data: - break - dst.write(data) - await dst.drain() - except (ConnectionError, OSError): - pass - - -async def main(): - server = await asyncio.start_server(handle_client, LISTEN_HOST, LISTEN_PORT) - print( - f"[decode-proxy] Listening on {LISTEN_HOST}:{LISTEN_PORT} -> {UPSTREAM_HOST}:{UPSTREAM_PORT}", - file=sys.stderr, - ) - async with server: - await server.serve_forever() - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/agents/hermes/discord-facade.py b/agents/hermes/discord-facade.py deleted file mode 100755 index ca7637b7d8..0000000000 --- a/agents/hermes/discord-facade.py +++ /dev/null @@ -1,962 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -""" -Sandbox-local Discord REST/Gateway facade for Hermes. - -Hermes still starts discord.py with the OpenShell placeholder token. The -sitecustomize preload rewrites discord.py's Discord REST and Gateway transports -to this loopback service. The facade accepts the placeholder on the local -Gateway, forwards non-emulated REST requests through DISCORD_PROXY, and accepts -Discord outgoing interaction webhooks for injection as Gateway dispatches. -""" - -from __future__ import annotations - -import asyncio -import binascii -import contextlib -import copy -import json -import logging -import os -import re -import secrets -import shlex -import shutil -import signal -import sys -import time -from dataclasses import dataclass, field -from typing import Any -from urllib.parse import parse_qsl, urlencode - -try: - from aiohttp import ClientSession, WSMsgType, web -except Exception as exc: # pragma: no cover - exercised in the sandbox image - print(f"[discord-facade] aiohttp is required: {exc}", file=sys.stderr) - sys.exit(1) - - -LOGGER = logging.getLogger("nemoclaw.discord_facade") - -DEFAULT_TOKEN_PLACEHOLDER = "openshell:resolve:env:DISCORD_BOT_TOKEN" -DEFAULT_LISTEN_HOST = "127.0.0.1" -DEFAULT_LISTEN_PORT = 3130 -DISCORD_API_ORIGIN = "https://discord.com" -INTERACTION_TOKEN_TTL_SECONDS = 15 * 60 -MAX_INTERACTION_TOKENS = 1024 -APPLICATION_COMMANDS_RE = re.compile(r"^/api/v\d+/applications/(\d+)/commands/?$") -APPLICATION_COMMAND_RE = re.compile(r"^/api/v\d+/applications/(\d+)/commands/(\d+)/?$") -INTERACTION_CALLBACK_RE = re.compile(r"^/api/v\d+/interactions/(\d+)/([^/]+)/callback/?$") -WEBHOOK_TOKEN_RE = re.compile(r"^/api/v\d+/webhooks/(\d+)/([^/]+)(/.*)?$") - - -@dataclass(eq=False) -class GatewayPeer: - ws: web.WebSocketResponse - session_id: str = field(default_factory=lambda: secrets.token_hex(16)) - sequence: int = 0 - identified: bool = False - - -def _env_int(name: str, default: int) -> int: - raw = os.getenv(name, "").strip() - if not raw: - return default - try: - return int(raw) - except ValueError: - LOGGER.warning("Ignoring invalid %s=%r", name, raw) - return default - - -def _env_float(name: str, default: float) -> float: - raw = os.getenv(name, "").strip() - if not raw: - return default - try: - return float(raw) - except ValueError: - LOGGER.warning("Ignoring invalid %s=%r", name, raw) - return default - - -def _json_response(data: Any, status: int = 200) -> web.Response: - body = json.dumps(data, separators=(",", ":")).encode("utf-8") - return web.Response( - body=body, status=status, content_type="application/json" - ) - - -def _csv_env(*names: str) -> list[str]: - values: list[str] = [] - for name in names: - raw = os.getenv(name, "") - for item in raw.split(","): - cleaned = item.strip() - if cleaned: - values.append(cleaned) - return values - - -def _redact_path(path: str) -> str: - match = WEBHOOK_TOKEN_RE.match(path) - if match: - suffix = match.group(3) or "" - return f"/api/v10/webhooks/{match.group(1)}/{suffix}" - match = INTERACTION_CALLBACK_RE.match(path) - if match: - return f"/api/v10/interactions/{match.group(1)}//callback" - return path - - -class DiscordFacade: - def __init__( - self, - *, - host: str, - port: int, - placeholder_token: str, - upstream_proxy: str | None, - public_base_url: str | None, - public_key: str | None, - ) -> None: - self.host = host - self.port = port - self.placeholder_token = placeholder_token - self.upstream_proxy = upstream_proxy - self.public_base_url = public_base_url - self.public_key = public_key - self.application_id = os.getenv("NEMOCLAW_DISCORD_APPLICATION_ID", "313700000000000001") - self.bot_user_id = os.getenv("NEMOCLAW_DISCORD_BOT_USER_ID", "313700000000000002") - self.bot_username = os.getenv("NEMOCLAW_DISCORD_BOT_USERNAME", "Hermes") - self.synthetic_reaction_user_id = os.getenv( - "NEMOCLAW_DISCORD_REACTION_USER_ID", - "313700000000000003", - ) - self._peers: set[GatewayPeer] = set() - self._interaction_tokens: dict[str, tuple[str, float]] = {} - self._session: ClientSession | None = None - self._poll_task: asyncio.Task[None] | None = None - self._poll_interval = _env_float("NEMOCLAW_DISCORD_POLL_INTERVAL_SECONDS", 10) - self._poll_channel_ids = set(_csv_env("NEMOCLAW_DISCORD_POLL_CHANNEL_IDS")) - self._poll_guild_ids = set( - _csv_env("NEMOCLAW_DISCORD_POLL_GUILD_IDS", "NEMOCLAW_DISCORD_GUILD_IDS") - ) - self._poll_discovered_channels: set[str] = set() - self._message_cache: dict[str, dict[str, Any]] = {} - self._channel_message_ids: dict[str, set[str]] = {} - self._thread_cache: dict[str, dict[str, Any]] = {} - self._poll_warning_keys: set[str] = set() - self._command_counter = 313700000000010000 - self._commands: dict[str, dict[str, Any]] = {} - - @property - def gateway_url(self) -> str: - return f"ws://{self.host}:{self.port}/gateway" - - async def start(self) -> web.AppRunner: - self._session = ClientSession() - app = web.Application(client_max_size=2 * 1024 * 1024) - app.add_routes( - [ - web.get("/gateway", self.handle_gateway), - web.post("/interactions", self.handle_interaction), - web.get("/health", self.handle_health), - web.route("*", "/api/{tail:.*}", self.handle_rest), - ] - ) - runner = web.AppRunner(app, access_log=None) - await runner.setup() - site = web.TCPSite(runner, self.host, self.port) - await site.start() - LOGGER.info("Discord facade listening on http://%s:%s", self.host, self.port) - self._start_polling() - return runner - - async def start_public_interactions(self, host: str, port: int) -> web.AppRunner: - app = web.Application(client_max_size=2 * 1024 * 1024) - app.add_routes([web.post("/interactions", self.handle_interaction)]) - runner = web.AppRunner(app, access_log=None) - await runner.setup() - site = web.TCPSite(runner, host, port) - await site.start() - LOGGER.info("Discord public interactions listener on http://%s:%s", host, port) - return runner - - async def close(self) -> None: - if self._poll_task is not None: - self._poll_task.cancel() - with contextlib.suppress(asyncio.CancelledError): - await self._poll_task - self._poll_task = None - if self._session is not None: - await self._session.close() - self._session = None - - async def handle_health(self, _request: web.Request) -> web.Response: - return _json_response({"ok": True, "peers": len(self._peers)}) - - async def handle_gateway(self, request: web.Request) -> web.WebSocketResponse: - ws = web.WebSocketResponse(heartbeat=None, compress=False) - await ws.prepare(request) - peer = GatewayPeer(ws=ws) - self._peers.add(peer) - LOGGER.info("Discord facade Gateway client connected") - try: - await self._send_gateway(peer, op=10, data={"heartbeat_interval": 41250}) - async for msg in ws: - if msg.type != WSMsgType.TEXT: - continue - try: - payload = json.loads(msg.data) - except json.JSONDecodeError: - await ws.close(code=4002, message=b"invalid payload") - break - await self._handle_gateway_payload(peer, payload) - finally: - self._peers.discard(peer) - LOGGER.info("Discord facade Gateway client disconnected") - return ws - - async def _handle_gateway_payload(self, peer: GatewayPeer, payload: dict[str, Any]) -> None: - op = payload.get("op") - if op == 1: - await self._send_gateway(peer, op=11, data=None) - return - if op == 2: - token = str((payload.get("d") or {}).get("token") or "") - if token != self.placeholder_token: - LOGGER.error( - "Rejecting Discord Gateway IDENTIFY that did not use the OpenShell placeholder" - ) - await peer.ws.close(code=4004, message=b"authentication failed") - return - peer.identified = True - await self._dispatch_ready(peer) - return - if op == 6: - token = str((payload.get("d") or {}).get("token") or "") - if token != self.placeholder_token: - await peer.ws.close(code=4004, message=b"authentication failed") - return - peer.identified = True - await self._dispatch(peer, "RESUMED", {"_trace": ["nemoclaw-discord-facade"]}) - return - if op in (3, 4): - return - LOGGER.debug("Ignoring Discord Gateway opcode %r at facade boundary", op) - - async def _dispatch_ready(self, peer: GatewayPeer) -> None: - await self._dispatch( - peer, - "READY", - { - "v": 10, - "session_id": peer.session_id, - "resume_gateway_url": self.gateway_url, - "user": self._bot_user(), - "application": { - "id": self.application_id, - "flags": 0, - }, - "guilds": [], - "private_channels": [], - "relationships": [], - "shard": [0, 1], - "_trace": ["nemoclaw-discord-facade"], - }, - ) - - def _bot_user(self) -> dict[str, Any]: - return { - "id": self.bot_user_id, - "username": self.bot_username, - "global_name": self.bot_username, - "discriminator": "0000", - "avatar": None, - "bot": True, - "system": False, - "mfa_enabled": False, - "verified": True, - "email": None, - "flags": 0, - "premium_type": 0, - "public_flags": 0, - } - - async def _send_gateway( - self, - peer: GatewayPeer, - *, - op: int, - data: Any, - event_type: str | None = None, - ) -> None: - payload = {"op": op, "d": data, "s": peer.sequence if event_type else None, "t": event_type} - await peer.ws.send_str(json.dumps(payload, separators=(",", ":"))) - - async def _dispatch(self, peer: GatewayPeer, event_type: str, data: Any) -> None: - peer.sequence += 1 - await self._send_gateway(peer, op=0, data=data, event_type=event_type) - - async def dispatch_to_all(self, event_type: str, data: dict[str, Any]) -> None: - peers = [peer for peer in self._peers if peer.identified and not peer.ws.closed] - for peer in peers: - await self._dispatch(peer, event_type, data) - - def _start_polling(self) -> None: - if self._poll_interval <= 0: - return - if not self._poll_channel_ids and not self._poll_guild_ids: - return - self._poll_task = asyncio.create_task(self._poll_loop()) - LOGGER.info( - "Discord facade REST poller enabled (guilds=%d channels=%d interval=%ss)", - len(self._poll_guild_ids), - len(self._poll_channel_ids), - self._poll_interval, - ) - - async def _poll_loop(self) -> None: - while True: - try: - await self._poll_once() - except asyncio.CancelledError: - raise - except Exception as exc: - LOGGER.warning("Discord REST polling failed: %s", exc) - await asyncio.sleep(self._poll_interval) - - async def _poll_once(self) -> None: - await self._refresh_poll_targets() - for channel_id in sorted(self._poll_channel_ids | self._poll_discovered_channels): - await self._poll_channel_messages(channel_id) - for guild_id in sorted(self._poll_guild_ids): - await self._poll_guild_threads(guild_id) - - async def _refresh_poll_targets(self) -> None: - if not self._poll_guild_ids: - return - discovered: set[str] = set() - for guild_id in sorted(self._poll_guild_ids): - channels = await self._discord_json("GET", f"/api/v10/guilds/{guild_id}/channels") - if not isinstance(channels, list): - continue - for channel in channels: - if not isinstance(channel, dict): - continue - channel_type = int(channel.get("type", -1)) - if channel_type in {0, 5}: - channel_id = str(channel.get("id") or "") - if channel_id: - discovered.add(channel_id) - self._poll_discovered_channels = discovered - - async def _poll_channel_messages(self, channel_id: str) -> None: - messages = await self._discord_json( - "GET", - f"/api/v10/channels/{channel_id}/messages?limit=25", - ) - if not isinstance(messages, list): - return - current_ids: set[str] = set() - for raw in reversed(messages): - if not isinstance(raw, dict): - continue - message = self._normalize_message(raw, channel_id) - message_id = str(message.get("id") or "") - if not message_id: - continue - current_ids.add(message_id) - previous = self._message_cache.get(message_id) - if previous is None: - self._message_cache[message_id] = message - await self.dispatch_to_all("MESSAGE_CREATE", self._strip_internal_fields(message)) - continue - reactions_changed = previous.get("_nemoclaw_reactions") != message.get("_nemoclaw_reactions") - if self._message_changed(previous, message): - self._message_cache[message_id] = message - await self.dispatch_to_all("MESSAGE_UPDATE", self._strip_internal_fields(message)) - await self._dispatch_reaction_deltas(previous, message) - if reactions_changed: - self._message_cache[message_id] = message - - previous_ids = self._channel_message_ids.get(channel_id, set()) - if len(messages) < 25: - for deleted_id in sorted(previous_ids - current_ids): - deleted = self._message_cache.pop(deleted_id, {}) - payload = { - "id": deleted_id, - "channel_id": channel_id, - } - if deleted.get("guild_id"): - payload["guild_id"] = deleted["guild_id"] - await self.dispatch_to_all("MESSAGE_DELETE", payload) - self._channel_message_ids[channel_id] = current_ids - - async def _poll_guild_threads(self, guild_id: str) -> None: - payload = await self._discord_json("GET", f"/api/v10/guilds/{guild_id}/threads/active") - if not isinstance(payload, dict): - return - threads = payload.get("threads") - if not isinstance(threads, list): - return - current_ids: set[str] = set() - for raw in threads: - if not isinstance(raw, dict): - continue - thread = dict(raw) - thread_id = str(thread.get("id") or "") - if not thread_id: - continue - current_ids.add(thread_id) - self._poll_discovered_channels.add(thread_id) - previous = self._thread_cache.get(thread_id) - if previous is None: - self._thread_cache[thread_id] = thread - await self.dispatch_to_all("THREAD_CREATE", thread) - elif previous != thread: - self._thread_cache[thread_id] = thread - await self.dispatch_to_all("THREAD_UPDATE", thread) - - for thread_id in sorted(set(self._thread_cache) - current_ids): - old = self._thread_cache.get(thread_id, {}) - if str(old.get("guild_id") or "") != guild_id: - continue - self._thread_cache.pop(thread_id, None) - await self.dispatch_to_all( - "THREAD_DELETE", - { - "id": thread_id, - "guild_id": guild_id, - "parent_id": old.get("parent_id"), - "type": old.get("type", 11), - }, - ) - - def _normalize_message(self, raw: dict[str, Any], channel_id: str) -> dict[str, Any]: - message = dict(raw) - message.setdefault("channel_id", channel_id) - message.setdefault("type", 0) - message.setdefault("content", "") - message.setdefault("mentions", []) - message.setdefault("mention_roles", []) - message.setdefault("mention_everyone", False) - message.setdefault("attachments", []) - message.setdefault("embeds", []) - message.setdefault("pinned", False) - message.setdefault("tts", False) - message["_nemoclaw_reactions"] = self._reaction_counts(message) - return message - - @staticmethod - def _message_changed(previous: dict[str, Any], current: dict[str, Any]) -> bool: - keys = {"content", "edited_timestamp", "pinned", "attachments", "embeds", "flags"} - return any(previous.get(key) != current.get(key) for key in keys) - - @staticmethod - def _strip_internal_fields(message: dict[str, Any]) -> dict[str, Any]: - public = dict(message) - public.pop("_nemoclaw_reactions", None) - return public - - @staticmethod - def _reaction_counts(message: dict[str, Any]) -> dict[str, tuple[int, dict[str, Any]]]: - counts: dict[str, tuple[int, dict[str, Any]]] = {} - for reaction in message.get("reactions", []) or []: - if not isinstance(reaction, dict): - continue - emoji = reaction.get("emoji") if isinstance(reaction.get("emoji"), dict) else {} - emoji_key = str(emoji.get("id") or emoji.get("name") or "") - if not emoji_key: - continue - counts[emoji_key] = (int(reaction.get("count") or 0), emoji) - return counts - - async def _dispatch_reaction_deltas( - self, - previous: dict[str, Any], - current: dict[str, Any], - ) -> None: - old_counts = previous.get("_nemoclaw_reactions", {}) - new_counts = current.get("_nemoclaw_reactions", {}) - for emoji_key, (new_count, emoji) in new_counts.items(): - old_count = old_counts.get(emoji_key, (0, emoji))[0] - if new_count == old_count: - continue - event_type = "MESSAGE_REACTION_ADD" if new_count > old_count else "MESSAGE_REACTION_REMOVE" - payload = { - "user_id": self.synthetic_reaction_user_id, - "channel_id": current.get("channel_id"), - "message_id": current.get("id"), - "emoji": emoji, - } - if current.get("guild_id"): - payload["guild_id"] = current["guild_id"] - await self.dispatch_to_all(event_type, payload) - - async def _discord_json(self, method: str, path: str) -> Any: - if self._session is None: - return None - headers = { - "Authorization": f"Bot {self.placeholder_token}", - "User-Agent": "NemoClawDiscordFacade/1.0", - } - try: - async with self._session.request( - method, - f"{DISCORD_API_ORIGIN}{path}", - headers=headers, - proxy=self.upstream_proxy, - allow_redirects=False, - ) as response: - if response.status >= 400: - key = f"{method} {path.split('?')[0]} {response.status}" - if key not in self._poll_warning_keys: - self._poll_warning_keys.add(key) - LOGGER.warning("Discord REST poll returned HTTP %s for %s", response.status, path.split("?")[0]) - return None - body = await response.read() - except Exception as exc: - key = f"{method} {path.split('?')[0]} error" - if key not in self._poll_warning_keys: - self._poll_warning_keys.add(key) - LOGGER.warning("Discord REST poll failed for %s: %s", path.split("?")[0], exc) - return None - if not body: - return None - try: - return json.loads(body.decode("utf-8")) - except json.JSONDecodeError: - return None - - async def handle_rest(self, request: web.Request) -> web.Response: - path = request.path - method = request.method.upper() - if method == "GET" and re.fullmatch(r"/api/v\d+/gateway(?:/bot)?", path): - return _json_response( - { - "url": self.gateway_url, - "shards": 1, - "session_start_limit": { - "total": 1000, - "remaining": 1000, - "reset_after": 0, - "max_concurrency": 1, - }, - } - ) - if method == "GET" and path in ("/api/v10/users/@me", "/api/v9/users/@me"): - return _json_response(self._bot_user()) - if method == "GET" and path in ( - "/api/v10/oauth2/applications/@me", - "/api/v9/oauth2/applications/@me", - "/api/v10/applications/@me", - "/api/v9/applications/@me", - ): - return _json_response(self._application_payload()) - if method == "GET" and path.endswith("/users/@me/guilds"): - return _json_response([]) - if match := APPLICATION_COMMANDS_RE.match(path): - return await self._handle_application_commands(request, match.group(1)) - if match := APPLICATION_COMMAND_RE.match(path): - return await self._handle_application_command(request, match.group(1), match.group(2)) - if match := INTERACTION_CALLBACK_RE.match(path): - return await self._handle_interaction_callback(request, match.group(1), match.group(2)) - if match := WEBHOOK_TOKEN_RE.match(path): - return await self._forward_with_interaction_token(request, match) - return await self._forward_rest(request) - - def _application_payload(self) -> dict[str, Any]: - return { - "id": self.application_id, - "name": "Hermes", - "icon": None, - "description": "Hermes Discord facade", - "bot_public": False, - "bot_require_code_grant": False, - "flags": 0, - "verify_key": self.public_key or "", - "owner": self._bot_user(), - } - - async def _handle_application_commands(self, request: web.Request, app_id: str) -> web.Response: - method = request.method.upper() - if method == "GET": - return _json_response(list(self._commands.values())) - if method == "PUT": - payload = await self._read_json(request) - commands = payload if isinstance(payload, list) else [] - self._commands.clear() - for command in commands: - stored = self._store_command(app_id, command) - self._commands[stored["id"]] = stored - return _json_response(list(self._commands.values())) - if method == "POST": - payload = await self._read_json(request) - stored = self._store_command(app_id, payload if isinstance(payload, dict) else {}) - self._commands[stored["id"]] = stored - return _json_response(stored, status=201) - return _json_response({"message": "method not allowed"}, status=405) - - async def _handle_application_command( - self, - request: web.Request, - app_id: str, - command_id: str, - ) -> web.Response: - method = request.method.upper() - if method == "PATCH": - payload = await self._read_json(request) - current = self._commands.get(command_id, {"id": command_id, "application_id": app_id}) - current.update(payload if isinstance(payload, dict) else {}) - current.setdefault("version", str(int(current["id"]) + 1)) - self._commands[command_id] = current - return _json_response(current) - if method == "DELETE": - self._commands.pop(command_id, None) - return web.Response(status=204) - return await self._forward_rest(request) - - def _store_command(self, app_id: str, payload: dict[str, Any]) -> dict[str, Any]: - self._command_counter += 1 - command_id = str(self._command_counter) - stored = dict(payload) - stored.setdefault("type", 1) - stored.setdefault("name", f"command-{command_id}") - stored.setdefault("description", "") - stored.setdefault("options", []) - stored.update( - { - "id": command_id, - "application_id": app_id, - "version": str(self._command_counter + 1), - } - ) - return stored - - async def _handle_interaction_callback( - self, - request: web.Request, - interaction_id: str, - local_token: str, - ) -> web.Response: - real_token = self._resolve_interaction_token(local_token) - if real_token is not None: - path = f"/api/v10/interactions/{interaction_id}/{real_token}/callback" - return await self._forward_rest(request, override_path=path) - return await self._forward_rest(request) - - async def _forward_with_interaction_token( - self, - request: web.Request, - match: re.Match[str], - ) -> web.Response: - local_token = match.group(2) - real_token = self._resolve_interaction_token(local_token) - if real_token is None: - return await self._forward_rest(request) - suffix = match.group(3) or "" - path = f"/api/v10/webhooks/{match.group(1)}/{real_token}{suffix}" - return await self._forward_rest(request, override_path=path) - - async def handle_interaction(self, request: web.Request) -> web.Response: - body = await self._read_bytes(request) - if not self._verify_signature(request, body): - return _json_response({"error": "invalid signature"}, status=401) - try: - payload = json.loads(body.decode("utf-8")) - except json.JSONDecodeError: - return _json_response({"error": "invalid json"}, status=400) - if payload.get("type") == 1: - return _json_response({"type": 1}) - - local_payload = self._localize_interaction_token(payload) - await self.dispatch_to_all("INTERACTION_CREATE", local_payload) - return _json_response({"type": 5}) - - def _localize_interaction_token(self, payload: dict[str, Any]) -> dict[str, Any]: - copied = copy.deepcopy(payload) - token = str(copied.get("token") or "") - if token: - local_token = f"nemoclaw-local-{secrets.token_urlsafe(24)}" - self._store_interaction_token(local_token, token) - copied["token"] = local_token - return copied - - def _store_interaction_token(self, local_token: str, real_token: str) -> None: - self._prune_interaction_tokens() - self._interaction_tokens[local_token] = ( - real_token, - time.monotonic() + INTERACTION_TOKEN_TTL_SECONDS, - ) - self._prune_interaction_tokens() - - def _resolve_interaction_token(self, local_token: str) -> str | None: - self._prune_interaction_tokens() - entry = self._interaction_tokens.get(local_token) - if entry is None: - return None - return entry[0] - - def _prune_interaction_tokens(self) -> None: - now = time.monotonic() - for local_token, (_real_token, expires_at) in list(self._interaction_tokens.items()): - if expires_at <= now: - self._interaction_tokens.pop(local_token, None) - overflow = len(self._interaction_tokens) - MAX_INTERACTION_TOKENS - if overflow > 0: - oldest = sorted(self._interaction_tokens.items(), key=lambda item: item[1][1]) - for local_token, _entry in oldest[:overflow]: - self._interaction_tokens.pop(local_token, None) - - def _verify_signature(self, request: web.Request, body: bytes) -> bool: - public_key = (self.public_key or "").strip() - if not public_key: - LOGGER.warning("Discord interaction rejected: DISCORD_PUBLIC_KEY is not configured") - return False - signature_hex = request.headers.get("X-Signature-Ed25519", "") - timestamp = request.headers.get("X-Signature-Timestamp", "") - try: - signature = binascii.unhexlify(signature_hex) - verify_key = binascii.unhexlify(public_key) - except (binascii.Error, ValueError): - return False - message = timestamp.encode("utf-8") + body - try: - from nacl.signing import VerifyKey - - VerifyKey(verify_key).verify(message, signature) - return True - except ImportError: - pass - except Exception: - return False - try: - from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey - - Ed25519PublicKey.from_public_bytes(verify_key).verify(signature, message) - return True - except Exception: - return False - - async def _forward_rest( - self, - request: web.Request, - *, - override_path: str | None = None, - ) -> web.Response: - if self._session is None: - return _json_response({"message": "facade session unavailable"}, status=503) - path = override_path or request.path_qs - if override_path and request.query_string: - separator = "&" if "?" in path else "?" - path = f"{path}{separator}{request.query_string}" - target = f"{DISCORD_API_ORIGIN}{path}" - headers = { - key: value - for key, value in request.headers.items() - if key.lower() not in {"host", "content-length", "accept-encoding"} - } - body = await self._read_bytes(request) - LOGGER.debug("Forwarding Discord REST %s %s", request.method, _redact_path(path)) - try: - async with self._session.request( - request.method, - target, - headers=headers, - data=body if body else None, - proxy=self.upstream_proxy, - allow_redirects=False, - ) as response: - response_body = await response.read() - response_headers = { - key: value - for key, value in response.headers.items() - if key.lower() - not in { - "content-encoding", - "content-length", - "transfer-encoding", - "connection", - } - } - return web.Response( - status=response.status, - body=response_body, - headers=response_headers, - ) - except Exception as exc: - LOGGER.warning( - "Discord REST forward failed for %s %s: %s", - request.method, - _redact_path(path), - exc, - ) - return _json_response({"message": "discord rest forward failed"}, status=502) - - async def _read_json(self, request: web.Request) -> Any: - body = await self._read_bytes(request) - if not body: - return None - try: - return json.loads(body.decode("utf-8")) - except json.JSONDecodeError: - return None - - async def _read_bytes(self, request: web.Request) -> bytes: - return await request.read() - - async def register_interactions_endpoint(self) -> None: - endpoint = (self.public_base_url or "").rstrip("/") - if not endpoint: - return - app_id = os.getenv("NEMOCLAW_DISCORD_APPLICATION_ID", "").strip() - if not app_id: - LOGGER.warning("Cannot register Discord interactions endpoint without application id") - return - path = f"/api/v10/applications/{app_id}" - payload = {"interactions_endpoint_url": f"{endpoint}/interactions"} - fake_request = _SyntheticRequest("PATCH", path, payload, self.placeholder_token) - response = await self._forward_rest(fake_request) # type: ignore[arg-type] - if response.status >= 400: - LOGGER.warning("Discord interactions endpoint registration returned HTTP %s", response.status) - else: - LOGGER.info("Registered Discord interactions endpoint URL") - - -class _SyntheticRequest: - def __init__(self, method: str, path: str, payload: dict[str, Any], placeholder_token: str) -> None: - self.method = method - self.path = path - self.path_qs = path - self.query_string = "" - self.headers = { - "Authorization": f"Bot {placeholder_token}", - "Content-Type": "application/json", - "User-Agent": "NemoClawDiscordFacade/1.0", - } - self._body = json.dumps(payload, separators=(",", ":")).encode("utf-8") - - async def read(self) -> bytes: - return self._body - - -def _tunnel_requested() -> bool: - return bool( - os.getenv("NEMOCLAW_DISCORD_TUNNEL_COMMAND", "").strip() - or os.getenv("NEMOCLAW_DISCORD_ENABLE_TUNNEL", "").strip() == "1" - ) - - -async def _run_tunnel_command( - public_url_file: str, - *, - local_url: str, -) -> tuple[asyncio.subprocess.Process | None, asyncio.Task[None] | None]: - command = os.getenv("NEMOCLAW_DISCORD_TUNNEL_COMMAND", "").strip() - if not command and os.getenv("NEMOCLAW_DISCORD_ENABLE_TUNNEL", "").strip() == "1": - cloudflared = shutil.which("cloudflared") - if cloudflared: - command = f"{shlex.quote(cloudflared)} tunnel --url {shlex.quote(local_url)}" - else: - LOGGER.warning("Discord interactions tunnel requested but cloudflared is not installed") - if not command: - return None, None - LOGGER.info("Starting sandbox-owned Discord interactions tunnel command") - proc = await asyncio.create_subprocess_shell( - command, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.STDOUT, - ) - - async def _capture_url() -> None: - url_re = re.compile(rb"https://[A-Za-z0-9.-]+") - assert proc.stdout is not None - async for line in proc.stdout: - match = url_re.search(line) - if match: - url = match.group(0).decode("utf-8").rstrip("/") - with open(public_url_file, "w", encoding="utf-8") as handle: - handle.write(url + "\n") - LOGGER.info("Discord interactions tunnel URL discovered") - - capture_task = asyncio.create_task(_capture_url()) - - def _log_capture_failure(task: asyncio.Task[None]) -> None: - try: - task.result() - except asyncio.CancelledError: - pass - except Exception as exc: - LOGGER.warning("Discord interactions tunnel URL capture failed: %s", exc) - - capture_task.add_done_callback(_log_capture_failure) - return proc, capture_task - - -async def main() -> None: - logging.basicConfig( - level=os.getenv("NEMOCLAW_DISCORD_FACADE_LOG_LEVEL", "INFO").upper(), - format="[discord-facade] %(levelname)s: %(message)s", - ) - host = os.getenv("NEMOCLAW_DISCORD_FACADE_HOST", DEFAULT_LISTEN_HOST) - port = _env_int("NEMOCLAW_DISCORD_FACADE_PORT", DEFAULT_LISTEN_PORT) - interactions_host = os.getenv("NEMOCLAW_DISCORD_INTERACTIONS_HOST", DEFAULT_LISTEN_HOST) - interactions_port = _env_int("NEMOCLAW_DISCORD_INTERACTIONS_PORT", port + 1) - public_url_file = os.getenv("NEMOCLAW_DISCORD_TUNNEL_URL_FILE", "/tmp/nemoclaw-discord-tunnel-url") - public_base_url = os.getenv("NEMOCLAW_DISCORD_PUBLIC_URL", "").strip() or None - - facade = DiscordFacade( - host=host, - port=port, - placeholder_token=os.getenv("NEMOCLAW_DISCORD_PLACEHOLDER", DEFAULT_TOKEN_PLACEHOLDER), - upstream_proxy=os.getenv("DISCORD_PROXY") or os.getenv("HTTPS_PROXY") or None, - public_base_url=public_base_url, - public_key=os.getenv("DISCORD_PUBLIC_KEY") or os.getenv("NEMOCLAW_DISCORD_PUBLIC_KEY"), - ) - runner = await facade.start() - interactions_runner: web.AppRunner | None = None - if public_base_url or _tunnel_requested(): - interactions_runner = await facade.start_public_interactions(interactions_host, interactions_port) - tunnel_proc, tunnel_capture_task = await _run_tunnel_command( - public_url_file, - local_url=f"http://{interactions_host}:{interactions_port}", - ) - if not public_base_url: - for _ in range(20 if tunnel_proc else 1): - if os.path.exists(public_url_file): - with open(public_url_file, "r", encoding="utf-8") as handle: - public_base_url = handle.read().strip() or None - if public_base_url: - break - await asyncio.sleep(0.25) - if public_base_url: - facade.public_base_url = public_base_url - await facade.register_interactions_endpoint() - - stop_event = asyncio.Event() - loop = asyncio.get_running_loop() - for sig in (signal.SIGTERM, signal.SIGINT): - loop.add_signal_handler(sig, stop_event.set) - await stop_event.wait() - if interactions_runner is not None: - await interactions_runner.cleanup() - await runner.cleanup() - await facade.close() - if tunnel_proc and tunnel_proc.returncode is None: - tunnel_proc.terminate() - try: - await asyncio.wait_for(tunnel_proc.wait(), timeout=5) - except asyncio.TimeoutError: - tunnel_proc.kill() - if tunnel_capture_task is not None and not tunnel_capture_task.done(): - tunnel_capture_task.cancel() - with contextlib.suppress(asyncio.CancelledError): - await tunnel_capture_task - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/agents/hermes/discord-preload/sitecustomize.py b/agents/hermes/discord-preload/sitecustomize.py deleted file mode 100644 index a6fca57e71..0000000000 --- a/agents/hermes/discord-preload/sitecustomize.py +++ /dev/null @@ -1,248 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Patch Hermes Python transports for NemoClaw-managed messaging egress.""" - -from __future__ import annotations - -import os -import re -import urllib.request -from urllib.parse import ParseResult, parse_qsl, urlencode, urlparse, urlunparse - - -SLACK_PLACEHOLDER_RE = re.compile( - r"\b(?:xoxb|xapp)-OPENSHELL-RESOLVE-ENV-(SLACK_(?:BOT|APP)_TOKEN)\b" -) -SLACK_FAST_PATH = "OPENSHELL-RESOLVE-ENV-SLACK_" - - -def _rewrite_slack_string(value: str) -> str: - if SLACK_FAST_PATH not in value: - return value - return SLACK_PLACEHOLDER_RE.sub(r"openshell:resolve:env:\1", value) - - -def _rewrite_slack_value(value): - if isinstance(value, str): - return _rewrite_slack_string(value) - if isinstance(value, bytes): - if SLACK_FAST_PATH.encode("ascii") not in value: - return value - try: - return _rewrite_slack_string(value.decode("utf-8")).encode("utf-8") - except UnicodeDecodeError: - return value - if isinstance(value, bytearray): - as_bytes = bytes(value) - rewritten = _rewrite_slack_value(as_bytes) - return bytearray(rewritten) if rewritten != as_bytes else value - if isinstance(value, tuple): - return tuple(_rewrite_slack_value(item) for item in value) - if isinstance(value, list): - return [_rewrite_slack_value(item) for item in value] - if isinstance(value, dict): - return {key: _rewrite_slack_value(item) for key, item in value.items()} - return value - - -def _rewrite_slack_headers(headers): - if not headers: - return headers - if isinstance(headers, dict): - for key in list(headers.keys()): - headers[key] = _rewrite_slack_value(headers[key]) - return headers - if hasattr(headers, "items") and hasattr(headers, "__setitem__"): - try: - for key, value in list(headers.items()): - headers[key] = _rewrite_slack_value(value) - return headers - except (AttributeError, KeyError, TypeError): - pass - if isinstance(headers, (list, tuple)): - return type(headers)((key, _rewrite_slack_value(value)) for key, value in headers) - return headers - - -def _rewrite_slack_url(url): - if not isinstance(url, str): - text = str(url) - rewritten = _rewrite_slack_string(text) - return rewritten if rewritten != text else url - return _rewrite_slack_string(url) - - -def _rewrite_slack_kwargs(kwargs): - if "headers" in kwargs: - kwargs["headers"] = _rewrite_slack_headers(kwargs["headers"]) - for key in ("data", "json", "params", "content"): - if key in kwargs: - kwargs[key] = _rewrite_slack_value(kwargs[key]) - return kwargs - - -try: - import aiohttp -except Exception: - aiohttp = None - - -_original_urllib_request_init = urllib.request.Request.__init__ -_original_urllib_add_header = urllib.request.Request.add_header -_original_urllib_add_unredirected_header = urllib.request.Request.add_unredirected_header - - -def _nemoclaw_urllib_request_init( - self, - url, - data=None, - headers=None, - origin_req_host=None, - unverifiable=False, - method=None, -): - headers = {} if headers is None else headers - return _original_urllib_request_init( - self, - _rewrite_slack_url(url), - data=_rewrite_slack_value(data), - headers=_rewrite_slack_headers(headers), - origin_req_host=origin_req_host, - unverifiable=unverifiable, - method=method, - ) - - -def _nemoclaw_urllib_add_header(self, key, val): - return _original_urllib_add_header(self, key, _rewrite_slack_value(val)) - - -def _nemoclaw_urllib_add_unredirected_header(self, key, val): - return _original_urllib_add_unredirected_header(self, key, _rewrite_slack_value(val)) - - -urllib.request.Request.__init__ = _nemoclaw_urllib_request_init -urllib.request.Request.add_header = _nemoclaw_urllib_add_header -urllib.request.Request.add_unredirected_header = _nemoclaw_urllib_add_unredirected_header - - -try: - import requests.sessions as _requests_sessions -except Exception: - _requests_sessions = None - -if _requests_sessions is not None: - _original_requests_request = _requests_sessions.Session.request - - def _nemoclaw_requests_request(self, method, url, **kwargs): - return _original_requests_request( - self, method, _rewrite_slack_url(url), **_rewrite_slack_kwargs(kwargs) - ) - - _requests_sessions.Session.request = _nemoclaw_requests_request - - -try: - import httpx as _httpx -except Exception: - _httpx = None - -if _httpx is not None: - _original_httpx_client_request = _httpx.Client.request - _original_httpx_async_client_request = _httpx.AsyncClient.request - - def _nemoclaw_httpx_client_request(self, method, url, *args, **kwargs): - return _original_httpx_client_request( - self, method, _rewrite_slack_url(url), *args, **_rewrite_slack_kwargs(kwargs) - ) - - async def _nemoclaw_httpx_async_client_request(self, method, url, *args, **kwargs): - return await _original_httpx_async_client_request( - self, method, _rewrite_slack_url(url), *args, **_rewrite_slack_kwargs(kwargs) - ) - - _httpx.Client.request = _nemoclaw_httpx_client_request - _httpx.AsyncClient.request = _nemoclaw_httpx_async_client_request - - -FACADE_URL = os.getenv("NEMOCLAW_DISCORD_FACADE_URL", "").strip() -if aiohttp is not None: - _original_request = aiohttp.ClientSession._request - - if FACADE_URL: - _facade = urlparse(FACADE_URL) - _original_ws_connect = aiohttp.ClientSession.ws_connect - else: - _facade = None - - _api_hosts = {"discord.com", "discordapp.com", "canary.discord.com", "ptb.discord.com"} - _gateway_hosts = {"gateway.discord.gg"} - - def _replace_netloc(parsed: ParseResult, *, scheme: str, path: str) -> str: - if _facade is None: - return str(parsed) - return urlunparse((scheme, _facade.netloc, path, "", parsed.query, "")) - - def _rewrite_rest_url(url: object) -> str | None: - if _facade is None: - return None - parsed = urlparse(str(url)) - if parsed.hostname not in _api_hosts: - return None - if not parsed.path.startswith("/api"): - return None - return _replace_netloc(parsed, scheme=_facade.scheme or "http", path=parsed.path) - - def _rewrite_gateway_url(url: object) -> str | None: - if _facade is None: - return None - parsed = urlparse(str(url)) - hostname = parsed.hostname or "" - if hostname not in _gateway_hosts and not hostname.endswith(".discord.gg"): - return None - query = dict(parse_qsl(parsed.query, keep_blank_values=True)) - if "v" not in query: - query["v"] = "10" - rewritten_query = urlencode(query) - scheme = "wss" if (_facade.scheme == "https") else "ws" - return urlunparse((scheme, _facade.netloc, "/gateway", "", rewritten_query, "")) - - def _is_facade_url(url: object) -> bool: - if _facade is None: - return False - try: - return urlparse(str(url)).netloc == _facade.netloc - except Exception: - return False - - async def _nemoclaw_request(self, method, str_or_url, **kwargs): - rewritten = _rewrite_rest_url(str_or_url) - if rewritten: - kwargs.pop("proxy", None) - kwargs.pop("proxy_auth", None) - kwargs.pop("ssl", None) - str_or_url = rewritten - elif _is_facade_url(str_or_url): - kwargs.pop("proxy", None) - kwargs.pop("proxy_auth", None) - kwargs.pop("ssl", None) - str_or_url = _rewrite_slack_url(str_or_url) - return await _original_request(self, method, str_or_url, **_rewrite_slack_kwargs(kwargs)) - - aiohttp.ClientSession._request = _nemoclaw_request - - if FACADE_URL: - def _nemoclaw_ws_connect(self, url, **kwargs): - rewritten = _rewrite_gateway_url(url) - if rewritten: - kwargs.pop("proxy", None) - kwargs.pop("proxy_auth", None) - kwargs.pop("ssl", None) - url = rewritten - elif _is_facade_url(url): - kwargs.pop("proxy", None) - kwargs.pop("proxy_auth", None) - kwargs.pop("ssl", None) - return _original_ws_connect(self, url, **kwargs) - - aiohttp.ClientSession.ws_connect = _nemoclaw_ws_connect diff --git a/agents/hermes/policy-additions.yaml b/agents/hermes/policy-additions.yaml index d146b720fd..82c98dd093 100644 --- a/agents/hermes/policy-additions.yaml +++ b/agents/hermes/policy-additions.yaml @@ -18,7 +18,6 @@ filesystem_policy: - /usr - /lib - /opt/hermes - - /opt/nemoclaw-hermes-discord-preload - /proc - /dev/urandom - /app @@ -224,8 +223,12 @@ network_policies: - allow: { method: DELETE, path: "/api/v*/webhooks/*/*/messages/*" } - host: gateway.discord.gg port: 443 - access: full - tls: skip + protocol: websocket + enforcement: enforce + websocket_credential_rewrite: true + rules: + - allow: { method: GET, path: "/**" } + - allow: { method: WEBSOCKET_TEXT, path: "/**" } - host: cdn.discordapp.com port: 443 protocol: rest @@ -244,6 +247,7 @@ network_policies: port: 443 protocol: rest enforcement: enforce + request_body_credential_rewrite: true rules: - allow: { method: GET, path: "/**" } - allow: { method: POST, path: "/**" } @@ -251,6 +255,7 @@ network_policies: port: 443 protocol: rest enforcement: enforce + request_body_credential_rewrite: true rules: - allow: { method: GET, path: "/**" } - allow: { method: POST, path: "/**" } @@ -258,17 +263,26 @@ network_policies: port: 443 protocol: rest enforcement: enforce + request_body_credential_rewrite: true rules: - allow: { method: GET, path: "/**" } - allow: { method: POST, path: "/**" } - host: wss-primary.slack.com port: 443 - tls: skip - access: full + protocol: websocket + enforcement: enforce + websocket_credential_rewrite: true + rules: + - allow: { method: GET, path: "/**" } + - allow: { method: WEBSOCKET_TEXT, path: "/**" } - host: wss-backup.slack.com port: 443 - tls: skip - access: full + protocol: websocket + enforcement: enforce + websocket_credential_rewrite: true + rules: + - allow: { method: GET, path: "/**" } + - allow: { method: WEBSOCKET_TEXT, path: "/**" } binaries: - { path: /usr/local/bin/hermes } - { path: /usr/bin/python3* } diff --git a/agents/hermes/policy-permissive.yaml b/agents/hermes/policy-permissive.yaml index 2dc6f532f0..0134c667bd 100644 --- a/agents/hermes/policy-permissive.yaml +++ b/agents/hermes/policy-permissive.yaml @@ -19,7 +19,6 @@ filesystem_policy: - /usr - /lib - /opt/hermes - - /opt/nemoclaw-hermes-discord-preload - /proc - /dev/urandom - /app @@ -189,8 +188,12 @@ network_policies: access: full - host: gateway.discord.gg port: 443 - access: full - tls: skip + protocol: websocket + enforcement: enforce + websocket_credential_rewrite: true + rules: + - allow: { method: GET, path: "/**" } + - allow: { method: WEBSOCKET_TEXT, path: "/**" } - host: cdn.discordapp.com port: 443 protocol: rest @@ -211,27 +214,36 @@ network_policies: port: 443 protocol: rest enforcement: enforce + request_body_credential_rewrite: true access: full - host: api.slack.com port: 443 protocol: rest enforcement: enforce + request_body_credential_rewrite: true access: full - host: hooks.slack.com port: 443 protocol: rest enforcement: enforce + request_body_credential_rewrite: true access: full - host: wss-primary.slack.com port: 443 - protocol: rest + protocol: websocket enforcement: enforce - access: full + websocket_credential_rewrite: true + rules: + - allow: { method: GET, path: "/**" } + - allow: { method: WEBSOCKET_TEXT, path: "/**" } - host: wss-backup.slack.com port: 443 - protocol: rest + protocol: websocket enforcement: enforce - access: full + websocket_credential_rewrite: true + rules: + - allow: { method: GET, path: "/**" } + - allow: { method: WEBSOCKET_TEXT, path: "/**" } binaries: - { path: "/**" } diff --git a/agents/hermes/start.sh b/agents/hermes/start.sh index 31f9e77335..39bf36e683 100755 --- a/agents/hermes/start.sh +++ b/agents/hermes/start.sh @@ -254,62 +254,11 @@ start_socat_forwarder() { echo "[gateway] socat forwarder 0.0.0.0:${PUBLIC_PORT} → 127.0.0.1:${INTERNAL_PORT} (pid $SOCAT_PID)" >&2 } -# ── Placeholder rewrite proxy ─────────────────────────────────── -# Python HTTP clients (httpx) URL-encode colons in paths, breaking -# OpenShell's openshell:resolve:env: placeholder pattern. This proxy -# sits between the Hermes process and the OpenShell proxy, URL-decoding -# request targets so the L7 proxy recognizes REST placeholders. It relays -# upgraded tunnel/WebSocket bytes unchanged. Slack SDK-shaped placeholders are -# canonicalized in the Hermes Python preload before HTTPS serialization. -HERMES_VENV_PYTHON="/opt/hermes/.venv/bin/python" -DECODE_PROXY_PID="" -DECODE_PROXY_PORT=3129 -DISCORD_FACADE_PID="" -DISCORD_FACADE_PORT=3130 -start_decode_proxy() { - nohup "$HERMES_VENV_PYTHON" /usr/local/bin/nemoclaw-decode-proxy >/dev/null 2>&1 & - DECODE_PROXY_PID=$! - # Wait for it to start listening - local attempts=0 - while [ "$attempts" -lt 10 ]; do - if ss -tln 2>/dev/null | grep -q "127.0.0.1:${DECODE_PROXY_PORT}"; then - echo "[gateway] decode-proxy listening on 127.0.0.1:${DECODE_PROXY_PORT} (pid $DECODE_PROXY_PID)" >&2 - return - fi - sleep 0.5 - attempts=$((attempts + 1)) - done - echo "[gateway] decode-proxy failed to start — placeholder rewriting may not work" >&2 -} - -start_discord_facade() { - local facade_url="http://127.0.0.1:${DISCORD_FACADE_PORT}" - local proxy_url="http://127.0.0.1:${DECODE_PROXY_PORT}" - local log_path="/tmp/discord-facade.log" - local launch_env=( - "DISCORD_PROXY=${proxy_url}" - "NEMOCLAW_DISCORD_FACADE_PORT=${DISCORD_FACADE_PORT}" - ) - - if [ "$(id -u)" -eq 0 ] && id gateway >/dev/null 2>&1; then - prepare_restricted_log "$log_path" gateway:gateway 600 - nohup env -u NEMOCLAW_DISCORD_FACADE_URL -u PYTHONPATH "${launch_env[@]}" "${STEP_DOWN_PREFIX_GATEWAY[@]}" sh -c 'umask 0007; exec "$@" >/tmp/discord-facade.log 2>&1' sh "$HERMES_VENV_PYTHON" /usr/local/bin/nemoclaw-discord-facade & - else - prepare_restricted_log "$log_path" "" 600 - nohup env -u NEMOCLAW_DISCORD_FACADE_URL -u PYTHONPATH "${launch_env[@]}" sh -c 'umask 0007; exec "$@" >/tmp/discord-facade.log 2>&1' sh "$HERMES_VENV_PYTHON" /usr/local/bin/nemoclaw-discord-facade & - fi - DISCORD_FACADE_PID=$! - local attempts=0 - while [ "$attempts" -lt 10 ]; do - if ss -tln 2>/dev/null | grep -q "127.0.0.1:${DISCORD_FACADE_PORT}"; then - echo "[gateway] discord facade listening on ${facade_url} (pid $DISCORD_FACADE_PID)" >&2 - return - fi - sleep 0.5 - attempts=$((attempts + 1)) - done - echo "[gateway] discord facade failed to start — Hermes Discord gateway emulation may not work" >&2 -} +# ── Messaging egress ───────────────────────────────────────────── +# Hermes sends messaging traffic directly through the OpenShell L7 proxy. +# OpenShell owns credential alias/body/WebSocket rewrite at the egress +# boundary; NemoClaw must not start a local decode proxy, facade, or +# placeholder-normalizing preload. # cleanup_on_signal is provided by sandbox-init.sh. It reads # SANDBOX_CHILD_PIDS (array of all PIDs) and SANDBOX_WAIT_PID (the @@ -327,8 +276,6 @@ export NO_PROXY="$_NO_PROXY_VAL" export http_proxy="$_PROXY_URL" export https_proxy="$_PROXY_URL" export no_proxy="$_NO_PROXY_VAL" -export NEMOCLAW_DISCORD_FACADE_URL="http://127.0.0.1:${DISCORD_FACADE_PORT}" -export PYTHONPATH="/opt/nemoclaw-hermes-discord-preload${PYTHONPATH:+:${PYTHONPATH}}" # OpenShell injects SSL_CERT_FILE/CURL_CA_BUNDLE for its L7 proxy CA. Persist # them into connect-session shells so Python Slack probes and Hermes tools trust @@ -364,9 +311,6 @@ export http_proxy="$_PROXY_URL" export https_proxy="$_PROXY_URL" export no_proxy="$_NO_PROXY_VAL" export HERMES_HOME="${HERMES_DIR}" -export DISCORD_PROXY="http://127.0.0.1:${DECODE_PROXY_PORT}" -export NEMOCLAW_DISCORD_FACADE_URL="http://127.0.0.1:${DISCORD_FACADE_PORT}" -export PYTHONPATH="/opt/nemoclaw-hermes-discord-preload\${PYTHONPATH:+:\${PYTHONPATH}}" PROXYEOF for _ca_env_name in SSL_CERT_FILE CURL_CA_BUNDLE REQUESTS_CA_BUNDLE GIT_SSL_CAINFO; do _ca_env_value="${!_ca_env_name:-}" @@ -659,18 +603,9 @@ if [ "$(id -u)" -ne 0 ]; then # shellcheck disable=SC2119 validate_tmp_permissions - # Start decode proxy and Hermes gateway - start_decode_proxy - start_discord_facade + # Start Hermes gateway. Messaging egress goes directly through OpenShell. umask 0007 HERMES_HOME="${HERMES_DIR}" \ - DISCORD_PROXY="http://127.0.0.1:${DECODE_PROXY_PORT}" \ - NEMOCLAW_DISCORD_FACADE_URL="http://127.0.0.1:${DISCORD_FACADE_PORT}" \ - PYTHONPATH="/opt/nemoclaw-hermes-discord-preload${PYTHONPATH:+:${PYTHONPATH}}" \ - HTTPS_PROXY="http://127.0.0.1:${DECODE_PROXY_PORT}" \ - HTTP_PROXY="http://127.0.0.1:${DECODE_PROXY_PORT}" \ - https_proxy="http://127.0.0.1:${DECODE_PROXY_PORT}" \ - http_proxy="http://127.0.0.1:${DECODE_PROXY_PORT}" \ nohup "$HERMES" gateway run >/tmp/gateway.log 2>&1 & GATEWAY_PID=$! echo "[gateway] hermes gateway launched (pid $GATEWAY_PID)" >&2 @@ -679,8 +614,6 @@ if [ "$(id -u)" -ne 0 ]; then # registration and the final append is a small race window (same as before # the shared-library refactor). Acceptable for entrypoint-level cleanup. SANDBOX_CHILD_PIDS=("$GATEWAY_PID") - [ -n "${DECODE_PROXY_PID:-}" ] && SANDBOX_CHILD_PIDS+=("$DECODE_PROXY_PID") - [ -n "${DISCORD_FACADE_PID:-}" ] && SANDBOX_CHILD_PIDS+=("$DISCORD_FACADE_PID") [ -n "${GATEWAY_LOG_TAIL_PID:-}" ] && SANDBOX_CHILD_PIDS+=("$GATEWAY_LOG_TAIL_PID") # shellcheck disable=SC2034 # read by cleanup_on_signal from sandbox-init.sh SANDBOX_WAIT_PID="$GATEWAY_PID" @@ -711,17 +644,8 @@ prepare_restricted_log /tmp/gateway.log gateway:gateway 600 # shellcheck disable=SC2119 validate_tmp_permissions -# Start decode proxy and gateway -start_decode_proxy -start_discord_facade +# Start Hermes gateway. Messaging egress goes directly through OpenShell. HERMES_HOME="${HERMES_DIR}" \ - DISCORD_PROXY="http://127.0.0.1:${DECODE_PROXY_PORT}" \ - NEMOCLAW_DISCORD_FACADE_URL="http://127.0.0.1:${DISCORD_FACADE_PORT}" \ - PYTHONPATH="/opt/nemoclaw-hermes-discord-preload${PYTHONPATH:+:${PYTHONPATH}}" \ - HTTPS_PROXY="http://127.0.0.1:${DECODE_PROXY_PORT}" \ - HTTP_PROXY="http://127.0.0.1:${DECODE_PROXY_PORT}" \ - https_proxy="http://127.0.0.1:${DECODE_PROXY_PORT}" \ - http_proxy="http://127.0.0.1:${DECODE_PROXY_PORT}" \ nohup "${STEP_DOWN_PREFIX_GATEWAY[@]}" sh -c 'umask 0007; exec "$@" >/tmp/gateway.log 2>&1' sh "$HERMES" gateway run & GATEWAY_PID=$! echo "[gateway] hermes gateway launched as 'gateway' user (pid $GATEWAY_PID)" >&2 @@ -730,8 +654,6 @@ start_gateway_log_stream # registration and the final append is a small race window (same as before # the shared-library refactor). Acceptable for entrypoint-level cleanup. SANDBOX_CHILD_PIDS=("$GATEWAY_PID") -[ -n "${DECODE_PROXY_PID:-}" ] && SANDBOX_CHILD_PIDS+=("$DECODE_PROXY_PID") -[ -n "${DISCORD_FACADE_PID:-}" ] && SANDBOX_CHILD_PIDS+=("$DISCORD_FACADE_PID") [ -n "${GATEWAY_LOG_TAIL_PID:-}" ] && SANDBOX_CHILD_PIDS+=("$GATEWAY_LOG_TAIL_PID") # shellcheck disable=SC2034 # read by cleanup_on_signal from sandbox-init.sh SANDBOX_WAIT_PID="$GATEWAY_PID" diff --git a/agents/openclaw/policy-permissive.yaml b/agents/openclaw/policy-permissive.yaml index fea85ee40d..142a1139dd 100644 --- a/agents/openclaw/policy-permissive.yaml +++ b/agents/openclaw/policy-permissive.yaml @@ -183,27 +183,36 @@ network_policies: port: 443 protocol: rest enforcement: enforce + request_body_credential_rewrite: true access: full - host: api.slack.com port: 443 protocol: rest enforcement: enforce + request_body_credential_rewrite: true access: full - host: hooks.slack.com port: 443 protocol: rest enforcement: enforce + request_body_credential_rewrite: true access: full - host: wss-primary.slack.com port: 443 - protocol: rest + protocol: websocket enforcement: enforce - access: full + websocket_credential_rewrite: true + rules: + - allow: { method: GET, path: "/**" } + - allow: { method: WEBSOCKET_TEXT, path: "/**" } - host: wss-backup.slack.com port: 443 - protocol: rest + protocol: websocket enforcement: enforce - access: full + websocket_credential_rewrite: true + rules: + - allow: { method: GET, path: "/**" } + - allow: { method: WEBSOCKET_TEXT, path: "/**" } binaries: - { path: "/**" } diff --git a/docs/inference/inference-options.md b/docs/inference/inference-options.md index 05aa07d909..3ced107af2 100644 --- a/docs/inference/inference-options.md +++ b/docs/inference/inference-options.md @@ -51,7 +51,7 @@ NemoClaw uses provider-specific local tokens for those routes, and rebuilds of l | Hermes Provider | Hermes only | OpenAI-compatible route | Available when onboarding Hermes Agent through `nemohermes` | | Local Ollama | Caveated | Local Ollama API | Available when Ollama is installed or running on the host | | Local NVIDIA NIM | Experimental | Local OpenAI-compatible | Requires `NEMOCLAW_EXPERIMENTAL=1` and a NIM-capable GPU | -| Local vLLM | Experimental | Local OpenAI-compatible | Appears when a vLLM server is already running on `localhost:8000`; managed install/start requires `NEMOCLAW_EXPERIMENTAL=1` | +| Local vLLM | Experimental | Local OpenAI-compatible | Requires `NEMOCLAW_EXPERIMENTAL=1` and a server already running on `localhost:8000` | ## Provider Options diff --git a/docs/reference/commands.md b/docs/reference/commands.md index f14064ee87..398b9f3a9f 100644 --- a/docs/reference/commands.md +++ b/docs/reference/commands.md @@ -81,7 +81,7 @@ $ NEMOCLAW_SINGLE_SESSION=1 curl -fsSL https://www.nvidia.com/nemoclaw.sh | bash ``` When existing sandboxes were created with OpenShell earlier than `0.0.37`, the installer prompts before running the new automatic gateway upgrade path. -For scripted installs, set `NEMOCLAW_ACCEPT_EXPERIMENTAL_OPENSHELL_UPGRADE=1` to allow the installer to back up registered sandbox state, retire the old gateway, install OpenShell `0.0.37`, and restore state during onboarding. +For scripted installs, set `NEMOCLAW_ACCEPT_EXPERIMENTAL_OPENSHELL_UPGRADE=1` to allow the installer to back up registered sandbox state, retire the old gateway, install the current supported OpenShell release, and restore state during onboarding. The automatic path is disabled if the existing `nemoclaw` CLI does not advertise `backup-all`; preserve sandbox state manually before retiring the old gateway in that case. To perform those steps manually, run `nemoclaw backup-all`, retire the old gateway with `openshell gateway destroy -g nemoclaw || openshell gateway destroy`, then rerun the installer as `curl -fsSL https://www.nvidia.com/nemoclaw.sh | NEMOCLAW_OPENSHELL_UPGRADE_PREPARED=1 bash`. diff --git a/nemoclaw-blueprint/blueprint.yaml b/nemoclaw-blueprint/blueprint.yaml index 086d0ff309..2f184c8516 100644 --- a/nemoclaw-blueprint/blueprint.yaml +++ b/nemoclaw-blueprint/blueprint.yaml @@ -2,8 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 version: "0.1.0" -min_openshell_version: "0.0.37" -max_openshell_version: "0.0.37" +min_openshell_version: "0.0.39" +max_openshell_version: "0.0.39" min_openclaw_version: "2026.4.24" # Mirrors the components.sandbox.image manifest digest below. Lets a # downstream consumer (or release tooling) verify the blueprint declares diff --git a/nemoclaw-blueprint/policies/openclaw-sandbox-permissive.yaml b/nemoclaw-blueprint/policies/openclaw-sandbox-permissive.yaml index cce20edccc..817c50661b 100644 --- a/nemoclaw-blueprint/policies/openclaw-sandbox-permissive.yaml +++ b/nemoclaw-blueprint/policies/openclaw-sandbox-permissive.yaml @@ -189,9 +189,12 @@ network_policies: access: full - host: gateway.discord.gg port: 443 - protocol: rest + protocol: websocket enforcement: enforce - access: full + websocket_credential_rewrite: true + rules: + - allow: { method: GET, path: "/**" } + - allow: { method: WEBSOCKET_TEXT, path: "/**" } - host: cdn.discordapp.com port: 443 protocol: rest @@ -212,27 +215,36 @@ network_policies: port: 443 protocol: rest enforcement: enforce + request_body_credential_rewrite: true access: full - host: api.slack.com port: 443 protocol: rest enforcement: enforce + request_body_credential_rewrite: true access: full - host: hooks.slack.com port: 443 protocol: rest enforcement: enforce + request_body_credential_rewrite: true access: full - host: wss-primary.slack.com port: 443 - protocol: rest + protocol: websocket enforcement: enforce - access: full + websocket_credential_rewrite: true + rules: + - allow: { method: GET, path: "/**" } + - allow: { method: WEBSOCKET_TEXT, path: "/**" } - host: wss-backup.slack.com port: 443 - protocol: rest + protocol: websocket enforcement: enforce - access: full + websocket_credential_rewrite: true + rules: + - allow: { method: GET, path: "/**" } + - allow: { method: WEBSOCKET_TEXT, path: "/**" } binaries: - { path: "/**" } diff --git a/nemoclaw-blueprint/policies/presets/discord.yaml b/nemoclaw-blueprint/policies/presets/discord.yaml index 21805ff6d9..af8cad661c 100644 --- a/nemoclaw-blueprint/policies/presets/discord.yaml +++ b/nemoclaw-blueprint/policies/presets/discord.yaml @@ -22,14 +22,18 @@ network_policies: # Reaction DELETE requires an extra path segment: /reactions/{emoji}/@me or /{user_id} - allow: { method: DELETE, path: "/api/v*/channels/*/messages/*" } - allow: { method: DELETE, path: "/api/v*/channels/*/messages/*/reactions/*/*" } - # WebSocket gateway — pure L4 CONNECT tunnel. OpenShell v0.0.15+ - # auto-terminates TLS unconditionally (NVIDIA/OpenShell#544), which - # also applies to WSS. tls: skip restores the pre-v0.0.15 pass- - # through behaviour needed here. + # WebSocket gateway. OpenShell's WebSocket L7 relay validates the + # upgrade, allows client text frames on the upgraded path, and rewrites + # the sandbox placeholder token in Discord IDENTIFY at the relay + # boundary. - host: gateway.discord.gg port: 443 - access: full - tls: skip + protocol: websocket + enforcement: enforce + websocket_credential_rewrite: true + rules: + - allow: { method: GET, path: "/**" } + - allow: { method: WEBSOCKET_TEXT, path: "/**" } - host: cdn.discordapp.com port: 443 protocol: rest diff --git a/nemoclaw-blueprint/policies/presets/slack.yaml b/nemoclaw-blueprint/policies/presets/slack.yaml index 37a24a3d01..da400a0130 100644 --- a/nemoclaw-blueprint/policies/presets/slack.yaml +++ b/nemoclaw-blueprint/policies/presets/slack.yaml @@ -13,6 +13,7 @@ network_policies: port: 443 protocol: rest enforcement: enforce + request_body_credential_rewrite: true rules: - allow: { method: GET, path: "/**" } - allow: { method: POST, path: "/**" } @@ -20,6 +21,7 @@ network_policies: port: 443 protocol: rest enforcement: enforce + request_body_credential_rewrite: true rules: - allow: { method: GET, path: "/**" } - allow: { method: POST, path: "/**" } @@ -27,21 +29,29 @@ network_policies: port: 443 protocol: rest enforcement: enforce + request_body_credential_rewrite: true rules: - allow: { method: GET, path: "/**" } - allow: { method: POST, path: "/**" } - # Socket Mode WebSocket — pure L4 CONNECT tunnel. OpenShell v0.0.15+ - # auto-terminates TLS unconditionally (NVIDIA/OpenShell#544), which - # also applies to WSS. tls: skip restores the pre-v0.0.15 pass- - # through behaviour needed here. + # Socket Mode WebSocket. The xapp token is authorized on the HTTPS + # apps.connections.open leg; the upgraded Socket Mode stream still gets + # explicit WebSocket upgrade/text-frame policy. - host: wss-primary.slack.com port: 443 - access: full - tls: skip + protocol: websocket + enforcement: enforce + websocket_credential_rewrite: true + rules: + - allow: { method: GET, path: "/**" } + - allow: { method: WEBSOCKET_TEXT, path: "/**" } - host: wss-backup.slack.com port: 443 - access: full - tls: skip + protocol: websocket + enforcement: enforce + websocket_credential_rewrite: true + rules: + - allow: { method: GET, path: "/**" } + - allow: { method: WEBSOCKET_TEXT, path: "/**" } binaries: - { path: /usr/local/bin/node } - { path: /usr/bin/node } diff --git a/nemoclaw-blueprint/scripts/slack-token-rewriter.js b/nemoclaw-blueprint/scripts/slack-token-rewriter.js deleted file mode 100644 index c0d1caf40e..0000000000 --- a/nemoclaw-blueprint/scripts/slack-token-rewriter.js +++ /dev/null @@ -1,228 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 -// -// slack-token-rewriter.js — translates the Bolt-compatible placeholder -// (xoxb|xapp)-OPENSHELL-RESOLVE-ENV-VAR into the active OpenShell -// openshell:resolve:env:* placeholder on outbound HTTP, so Slack tokens -// travel the same OpenShell substitution path Discord / Telegram / Brave -// already use without any real token touching openclaw.json. -// -// Why this preload exists: -// Slack's Bolt SDK validates token shape (^xoxb-[A-Za-z0-9_-]+$ / -// ^xapp-…$) at App construction, before any HTTP call leaves the -// process — so OpenShell's openshell:resolve:env:* placeholder is rejected -// synchronously and the gateway crashes. We emit a Bolt-shape placeholder -// into openclaw.json (which Bolt accepts), then translate it back to the -// current OpenShell placeholder here, just before the bytes hit the wire, -// where OpenShell's L7 proxy substitutes the real token from env. -// -// Wraps http.request / https.request — every Node HTTP client bottoms -// out here, including @slack/web-api (axios → follow-redirects → http) -// and Bolt's Socket Mode HTTPS auth (apps.connections.open → http). -// Also wraps http.get / https.get because they call the module-local -// `request` function, not module.exports.request — wrapping `request` -// alone would miss any `get` caller. -// Request body chunks are wrapped too: Bolt's auth.test path can put the -// token in both Authorization and the urlencoded body. -// -// Invariants: -// - Env reads are used only when they contain OpenShell placeholder values. -// Raw env values are ignored so real tokens never enter outbound request -// objects through this preload. -// - Mutates options/headers in place. axios reuses the headers object -// after request creation, so cloning would break the request lifecycle. -// - Idempotent. The output (openshell:resolve:env:VAR) does not match -// the Bolt-shape regex, so re-entering the wrapper on a retry is safe. -// - Fast path: indexOf short-circuits the regex on the 99.9% of -// requests that don't contain a placeholder. -// -// This file is the canonical source for review and tests. The Dockerfile -// copies it into /usr/local/lib/nemoclaw/preloads/, then at sandbox boot -// nemoclaw-start.sh writes a byte-identical copy to /tmp and loads it via -// NODE_OPTIONS=--require. -// -// Ref: https://github.com/NVIDIA/NemoClaw/issues/2085 - -(function () { - 'use strict'; - - // Bolt-shape placeholder → canonical form. Single source of truth used - // by every code path below. = [A-Z_][A-Z0-9_]* — the charset - // OpenShell's substitution layer accepts. - var BOLT_PLACEHOLDER = - /\b(?:xoxb|xapp)-OPENSHELL-RESOLVE-ENV-([A-Z_][A-Z0-9_]*)\b/g; - var FAST_PATH = 'OPENSHELL-RESOLVE-ENV-'; - var OPENSHELL_PLACEHOLDER_PREFIX = 'openshell:resolve:env:'; - - function placeholderForEnvKey(envKey) { - var value = ''; - try { - if (typeof process !== 'undefined' && process && process.env) { - value = process.env[envKey]; - } - } catch (_) { - value = ''; - } - if (typeof value === 'string' && value.indexOf(OPENSHELL_PLACEHOLDER_PREFIX) === 0) { - return value; - } - return OPENSHELL_PLACEHOLDER_PREFIX + envKey; - } - - function rewriteString(s) { - if (typeof s !== 'string') return s; - if (s.indexOf(FAST_PATH) === -1) return s; - return s.replace(BOLT_PLACEHOLDER, function (_match, envKey) { - return placeholderForEnvKey(envKey); - }); - } - - function rewriteHeaders(headers) { - if (!headers || typeof headers !== 'object') return headers; - var keys = Object.keys(headers); - for (var i = 0; i < keys.length; i++) { - var v = headers[keys[i]]; - if (Array.isArray(v)) { - for (var j = 0; j < v.length; j++) v[j] = rewriteString(v[j]); - } else { - headers[keys[i]] = rewriteString(v); - } - } - return headers; - } - - function rewriteOptions(options) { - if (!options || typeof options !== 'object') return options; - if (typeof options.path === 'string') { - options.path = rewriteString(options.path); - } - if (options.headers) rewriteHeaders(options.headers); - return options; - } - - function adjustContentLength(req, beforeLength, afterLength) { - var delta = afterLength - beforeLength; - if (!delta || !req || typeof req.getHeader !== 'function' || typeof req.setHeader !== 'function') { - return; - } - // Once Node has built/sent the header block, changing Content-Length would - // be too late. Axios writes the urlencoded Slack body in one chunk before - // headers are flushed, which is the path this adjustment is for. - if (req.headersSent || req._header) return; - var current = req.getHeader('content-length'); - if (Array.isArray(current)) current = current[0]; - if (current === undefined || current === null || current === '') return; - var n = Number(current); - if (!isFinite(n)) return; - req.setHeader('Content-Length', String(n + delta)); - } - - function rewriteBodyChunk(req, chunk, encoding) { - if (typeof chunk === 'string') { - var rewritten = rewriteString(chunk); - if (rewritten !== chunk) { - adjustContentLength( - req, - Buffer.byteLength(chunk, encoding), - Buffer.byteLength(rewritten, encoding) - ); - } - return rewritten; - } - - if (!chunk || typeof chunk !== 'object') return chunk; - var isBuffer = Buffer.isBuffer(chunk); - if (!isBuffer && !(chunk instanceof Uint8Array)) return chunk; - - var buf = isBuffer - ? chunk - : Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength); - if (buf.indexOf(FAST_PATH) === -1) return chunk; - var s = buf.toString('utf8'); - if (s.indexOf(FAST_PATH) === -1) return chunk; - // Do not rewrite arbitrary binary data. Slack's urlencoded bodies are - // valid UTF-8 and round-trip exactly. - if (!Buffer.from(s, 'utf8').equals(buf)) return chunk; - var rs = rewriteString(s); - if (rs === s) return chunk; - var out = Buffer.from(rs, 'utf8'); - adjustContentLength(req, buf.length, out.length); - return out; - } - - function wrapClientRequest(req) { - if (!req || typeof req !== 'object') return req; - if (req.__nemoclawSlackTokenRewriter) return req; - try { - Object.defineProperty(req, '__nemoclawSlackTokenRewriter', { value: true }); - } catch (_e) { - req.__nemoclawSlackTokenRewriter = true; - } - - var origWrite = req.write; - if (typeof origWrite === 'function') { - req.write = function (chunk, encoding, cb) { - if (typeof encoding === 'function') { - cb = encoding; - encoding = undefined; - } - chunk = rewriteBodyChunk(this, chunk, encoding); - if (cb) return origWrite.call(this, chunk, encoding, cb); - if (encoding !== undefined) return origWrite.call(this, chunk, encoding); - return origWrite.call(this, chunk); - }; - } - - var origEnd = req.end; - if (typeof origEnd === 'function') { - req.end = function (chunk, encoding, cb) { - if (arguments.length === 0) return origEnd.call(this); - if (typeof chunk === 'function') return origEnd.call(this, chunk); - if (typeof encoding === 'function') { - cb = encoding; - encoding = undefined; - } - if (chunk !== undefined && chunk !== null) { - chunk = rewriteBodyChunk(this, chunk, encoding); - } - if (cb) return origEnd.call(this, chunk, encoding, cb); - if (encoding !== undefined) return origEnd.call(this, chunk, encoding); - return origEnd.call(this, chunk); - }; - } - - return req; - } - - function wrap(mod, methodName) { - var orig = mod[methodName]; - if (typeof orig !== 'function') return; - mod[methodName] = function (arg1, arg2, arg3) { - // Signatures: m(options[, cb]); m(url[, options][, cb]) - if (typeof arg1 === 'string') { - arg1 = rewriteString(arg1); - if (arg2 && typeof arg2 === 'object' && typeof arg2 !== 'function') { - rewriteOptions(arg2); - } - } else if (arg1 instanceof URL) { - // URL instances are immutable by component; rebuild only if needed. - var s = arg1.href; - var rs = rewriteString(s); - if (rs !== s) arg1 = new URL(rs); - if (arg2 && typeof arg2 === 'object' && typeof arg2 !== 'function') { - rewriteOptions(arg2); - } - } else { - rewriteOptions(arg1); - } - return wrapClientRequest(orig.call(this, arg1, arg2, arg3)); - }; - } - - var http = require('http'); - var https = require('https'); - wrap(http, 'request'); - wrap(http, 'get'); - wrap(https, 'request'); - wrap(https, 'get'); -})(); diff --git a/nemoclaw-blueprint/scripts/ws-proxy-fix.js b/nemoclaw-blueprint/scripts/ws-proxy-fix.js deleted file mode 100644 index 9b1e1b5d34..0000000000 --- a/nemoclaw-blueprint/scripts/ws-proxy-fix.js +++ /dev/null @@ -1,199 +0,0 @@ -"use strict"; -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 -// -// ws-proxy-fix.ts — preload script to fix Discord WebSocket connections -// through the OpenShell L7 proxy when HTTPS_PROXY is set. -// -// Problem (NemoClaw#1570): -// The `ws` library (used by OpenClaw's Discord extension via @buape/carbon) -// establishes WebSocket connections by calling https.request() for wss:// URLs. -// Inside the sandbox, HTTPS_PROXY is set and Node.js 22 (with -// NODE_USE_ENV_PROXY=1) routes these through EnvHttpProxyAgent — which sends a -// forward proxy request (GET https://...) instead of a CONNECT tunnel. The -// OpenShell L7 proxy correctly rejects forward proxy HTTPS with HTTP 400. -// Without NODE_USE_ENV_PROXY, ws goes direct, which the sandbox network -// namespace blocks. Either way, the WebSocket handshake fails and the bot -// loops on close code 1006. -// -// Fix: -// Patch https.request() to detect WebSocket upgrade requests to Discord -// gateway hosts (gateway.discord.gg) and inject an agent that issues a proper -// CONNECT request to the proxy, then upgrades the tunnel socket to TLS. -// All other HTTPS requests — including non-Discord WebSockets — pass through -// completely untouched. -// -// Uses only Node.js built-in modules — no external dependencies. -// -// Belt-and-suspenders: works regardless of any upstream OpenClaw changes. -// If the caller already provides a custom (non-default) agent, we step aside -// — no double-tunnelling. -var __importDefault = (this && this.__importDefault) || function (mod) { - return (mod && mod.__esModule) ? mod : { "default": mod }; -}; -Object.defineProperty(exports, "__esModule", { value: true }); -const node_http_1 = __importDefault(require("node:http")); -const node_net_1 = __importDefault(require("node:net")); -const node_tls_1 = __importDefault(require("node:tls")); -const node_https_1 = __importDefault(require("node:https")); -const node_url_1 = require("node:url"); -const _PATCHED = Symbol.for("nemoclaw.wsProxyFix"); -/** - * Self-executing initialiser. Using an IIFE rather than top-level `return` - * keeps the source valid TypeScript while preserving early-exit semantics. - */ -(function wsProxyFixInit() { - const proxyUrl = process.env.HTTPS_PROXY || process.env.https_proxy; - if (!proxyUrl) - return; - const patchedFlag = Reflect.get(globalThis, _PATCHED) === true; - if (patchedFlag) - return; - let proxy; - try { - proxy = new node_url_1.URL(proxyUrl); - } - catch { - return; - } - const proxyHost = proxy.hostname; - const proxyPort = parseInt(proxy.port, 10) || 3128; - // ---------- CONNECT tunnel agent ---------------------------------------- - /** - * Create an https.Agent whose createConnection() establishes a CONNECT - * tunnel through the HTTP proxy, then upgrades to TLS — the correct - * behaviour that EnvHttpProxyAgent fails to perform for HTTPS. - */ - function createTunnelAgent(targetHost, targetPort) { - const agent = new node_https_1.default.Agent({ keepAlive: false, maxSockets: 1 }); - // Override createConnection to route through the proxy's CONNECT tunnel. - // The typing is intentionally loosened because the actual Node.js runtime - // signature is broader than what @types/node declares. - Reflect.set(agent, "createConnection", function (options, callback) { - const connectReq = node_http_1.default.request({ - host: proxyHost, - port: proxyPort, - method: "CONNECT", - path: `${targetHost}:${targetPort}`, - headers: { Host: `${targetHost}:${targetPort}` }, - }); - connectReq.on("connect", (_res, socket, head) => { - if (_res.statusCode !== 200) { - socket.destroy(); - callback(new Error(`ws-proxy-fix: CONNECT ${targetHost}:${targetPort} via proxy failed (${_res.statusCode})`)); - return; - } - // Preserve any bytes already buffered from the tunnel before TLS. - if (head && head.length > 0) { - socket.unshift(head); - } - const tlsSocket = node_tls_1.default.connect({ - socket, - servername: typeof options.servername === "string" ? options.servername : targetHost, - }); - callback(null, tlsSocket); - }); - connectReq.on("error", (err) => { - connectReq.destroy(); - callback(err); - }); - connectReq.end(); - // createConnection expects a synchronous return; the real socket arrives - // via the callback. Return a placeholder that Node.js will discard. - return new node_net_1.default.Socket(); - }); - return agent; - } - // ---------- Target check ------------------------------------------------- - /** - * Return true only for WebSocket upgrade requests targeting Discord - * gateway hosts (gateway.discord.gg and regional variants). - */ - function isDiscordWsUpgrade(host, headers) { - if (!host || !headers || typeof headers !== "object") - return false; - const h = host.toLowerCase(); - if (h !== "gateway.discord.gg" && !h.endsWith(".discord.gg")) - return false; - for (const key of Object.keys(headers)) { - if (key.toLowerCase() === "upgrade" && - String(headers[key]).toLowerCase() === "websocket") { - return true; - } - } - return false; - } - // ---------- Patch https.request() --------------------------------------- - // Capture the original so we can call it after normalising arguments. - const requestRef = node_https_1.default.request; - function callOriginalRequest(input, options, callback) { - if (typeof input === "string" || input instanceof node_url_1.URL) { - if (typeof options === "function") { - return requestRef(input, options); - } - if (options) { - return callback ? requestRef(input, options, callback) : requestRef(input, options); - } - return callback ? requestRef(input, {}, callback) : requestRef(input); - } - if (typeof options === "function") { - return requestRef(input, options); - } - return requestRef(input, callback); - } - function wsProxyFixedRequest(input, options, callback) { - // --- Normalise arguments (Node.js accepts multiple call signatures) --- - let opts; - let cb; - if (typeof input === "string" || input instanceof node_url_1.URL) { - if (typeof options === "function") { - cb = options; - opts = {}; - } - else { - opts = options ?? {}; - cb = callback; - } - const url = typeof input === "string" ? new node_url_1.URL(input) : input; - opts = { - protocol: url.protocol, - hostname: url.hostname, - port: url.port, - path: url.pathname + url.search, - ...opts, - }; - } - else { - opts = input || {}; - cb = typeof options === "function" ? options : callback; - } - // opts.host may include a port (e.g. "gateway.discord.gg:443") — strip it - // so the CONNECT path doesn't become "host:443:443". - let host = opts.hostname || undefined; - if (!host && opts.host) { - host = opts.host.replace(/:\d+$/, ""); - } - if (isDiscordWsUpgrade(host, opts.headers)) { - // Guard: if isDiscordWsUpgrade matched but host resolved to - // undefined, we cannot construct a CONNECT tunnel (no target). - // Fall through to the original https.request unchanged. Before - // PR #2422 this path would have attempted the tunnel with an - // undefined host, which would fail in createTunnelAgent anyway. - if (!host) { - return callOriginalRequest(input, options, callback); - } - // Discord WebSocket upgrade — inject CONNECT tunnel agent unless the - // caller already provides a custom (non-default) agent. - if (!opts.agent || opts.agent === node_https_1.default.globalAgent) { - const port = parseInt(String(opts.port), 10) || 443; - opts = { ...opts, agent: createTunnelAgent(host, port) }; - } - return cb ? requestRef(opts, cb) : requestRef(opts); - } - // Non-WebSocket — pass through the original arguments unchanged. - return callOriginalRequest(input, options, callback); - } - // Replace https.request with our patched version. - Reflect.set(node_https_1.default, "request", wsProxyFixedRequest); - Reflect.set(globalThis, _PATCHED, true); -})(); diff --git a/nemoclaw-blueprint/scripts/ws-proxy-fix.ts b/nemoclaw-blueprint/scripts/ws-proxy-fix.ts deleted file mode 100644 index f1451aa401..0000000000 --- a/nemoclaw-blueprint/scripts/ws-proxy-fix.ts +++ /dev/null @@ -1,254 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 -// -// ws-proxy-fix.ts — preload script to fix Discord WebSocket connections -// through the OpenShell L7 proxy when HTTPS_PROXY is set. -// -// Problem (NemoClaw#1570): -// The `ws` library (used by OpenClaw's Discord extension via @buape/carbon) -// establishes WebSocket connections by calling https.request() for wss:// URLs. -// Inside the sandbox, HTTPS_PROXY is set and Node.js 22 (with -// NODE_USE_ENV_PROXY=1) routes these through EnvHttpProxyAgent — which sends a -// forward proxy request (GET https://...) instead of a CONNECT tunnel. The -// OpenShell L7 proxy correctly rejects forward proxy HTTPS with HTTP 400. -// Without NODE_USE_ENV_PROXY, ws goes direct, which the sandbox network -// namespace blocks. Either way, the WebSocket handshake fails and the bot -// loops on close code 1006. -// -// Fix: -// Patch https.request() to detect WebSocket upgrade requests to Discord -// gateway hosts (gateway.discord.gg) and inject an agent that issues a proper -// CONNECT request to the proxy, then upgrades the tunnel socket to TLS. -// All other HTTPS requests — including non-Discord WebSockets — pass through -// completely untouched. -// -// Uses only Node.js built-in modules — no external dependencies. -// -// Belt-and-suspenders: works regardless of any upstream OpenClaw changes. -// If the caller already provides a custom (non-default) agent, we step aside -// — no double-tunnelling. - -import http from "node:http"; -import net from "node:net"; -import tls from "node:tls"; -import https from "node:https"; -import { URL } from "node:url"; - -const _PATCHED = Symbol.for("nemoclaw.wsProxyFix"); - -type RequestCallback = (res: http.IncomingMessage) => void; -type TunnelConnectionOptions = { servername?: string }; - -/** - * Merged options after normalising the multiple call signatures of - * https.request(). Only fields we inspect are listed. - */ -interface ReqOpts extends https.RequestOptions { - headers?: http.OutgoingHttpHeaders; -} - -/** - * Self-executing initialiser. Using an IIFE rather than top-level `return` - * keeps the source valid TypeScript while preserving early-exit semantics. - */ -(function wsProxyFixInit(): void { - const proxyUrl = process.env.HTTPS_PROXY || process.env.https_proxy; - if (!proxyUrl) return; - - const patchedFlag = Reflect.get(globalThis, _PATCHED) === true; - if (patchedFlag) return; - - let proxy: URL; - try { - proxy = new URL(proxyUrl); - } catch { - return; - } - - const proxyHost: string = proxy.hostname; - const proxyPort: number = parseInt(proxy.port, 10) || 3128; - - // ---------- CONNECT tunnel agent ---------------------------------------- - - /** - * Create an https.Agent whose createConnection() establishes a CONNECT - * tunnel through the HTTP proxy, then upgrades to TLS — the correct - * behaviour that EnvHttpProxyAgent fails to perform for HTTPS. - */ - function createTunnelAgent( - targetHost: string, - targetPort: number, - ): https.Agent { - const agent = new https.Agent({ keepAlive: false, maxSockets: 1 }); - - // Override createConnection to route through the proxy's CONNECT tunnel. - // The typing is intentionally loosened because the actual Node.js runtime - // signature is broader than what @types/node declares. - Reflect.set( - agent, - "createConnection", - function ( - options: TunnelConnectionOptions, - callback: (err: Error | null, socket?: tls.TLSSocket) => void, - ): net.Socket { - const connectReq = http.request({ - host: proxyHost, - port: proxyPort, - method: "CONNECT", - path: `${targetHost}:${targetPort}`, - headers: { Host: `${targetHost}:${targetPort}` }, - }); - - connectReq.on( - "connect", - (_res: http.IncomingMessage, socket: net.Socket, head: Buffer) => { - if (_res.statusCode !== 200) { - socket.destroy(); - callback( - new Error( - `ws-proxy-fix: CONNECT ${targetHost}:${targetPort} via proxy failed (${_res.statusCode})`, - ), - ); - return; - } - // Preserve any bytes already buffered from the tunnel before TLS. - if (head && head.length > 0) { - socket.unshift(head); - } - const tlsSocket = tls.connect({ - socket, - servername: typeof options.servername === "string" ? options.servername : targetHost, - }); - callback(null, tlsSocket); - }, - ); - - connectReq.on("error", (err: Error) => { - connectReq.destroy(); - callback(err); - }); - connectReq.end(); - - // createConnection expects a synchronous return; the real socket arrives - // via the callback. Return a placeholder that Node.js will discard. - return new net.Socket(); - }, - ); - - return agent; - } - - // ---------- Target check ------------------------------------------------- - - /** - * Return true only for WebSocket upgrade requests targeting Discord - * gateway hosts (gateway.discord.gg and regional variants). - */ - function isDiscordWsUpgrade( - host: string | undefined, - headers: http.OutgoingHttpHeaders | undefined, - ): boolean { - if (!host || !headers || typeof headers !== "object") return false; - const h = host.toLowerCase(); - if (h !== "gateway.discord.gg" && !h.endsWith(".discord.gg")) return false; - for (const key of Object.keys(headers)) { - if ( - key.toLowerCase() === "upgrade" && - String(headers[key]).toLowerCase() === "websocket" - ) { - return true; - } - } - return false; - } - - // ---------- Patch https.request() --------------------------------------- - - // Capture the original so we can call it after normalising arguments. - const requestRef = https.request; - - function callOriginalRequest( - input: string | URL | ReqOpts, - options?: RequestCallback | ReqOpts, - callback?: RequestCallback, - ): http.ClientRequest { - if (typeof input === "string" || input instanceof URL) { - if (typeof options === "function") { - return requestRef(input, options); - } - if (options) { - return callback ? requestRef(input, options, callback) : requestRef(input, options); - } - return callback ? requestRef(input, {}, callback) : requestRef(input); - } - if (typeof options === "function") { - return requestRef(input, options); - } - return requestRef(input, callback); - } - - function wsProxyFixedRequest( - input: string | URL | ReqOpts, - options?: RequestCallback | ReqOpts, - callback?: RequestCallback, - ): http.ClientRequest { - // --- Normalise arguments (Node.js accepts multiple call signatures) --- - let opts: ReqOpts; - let cb: RequestCallback | undefined; - - if (typeof input === "string" || input instanceof URL) { - if (typeof options === "function") { - cb = options; - opts = {}; - } else { - opts = options ?? {}; - cb = callback; - } - const url = typeof input === "string" ? new URL(input) : input; - opts = { - protocol: url.protocol, - hostname: url.hostname, - port: url.port, - path: url.pathname + url.search, - ...opts, - }; - } else { - opts = input || {}; - cb = typeof options === "function" ? options : callback; - } - - // opts.host may include a port (e.g. "gateway.discord.gg:443") — strip it - // so the CONNECT path doesn't become "host:443:443". - let host = opts.hostname || undefined; - if (!host && opts.host) { - host = opts.host.replace(/:\d+$/, ""); - } - if (isDiscordWsUpgrade(host, opts.headers)) { - // Guard: if isDiscordWsUpgrade matched but host resolved to - // undefined, we cannot construct a CONNECT tunnel (no target). - // Fall through to the original https.request unchanged. Before - // PR #2422 this path would have attempted the tunnel with an - // undefined host, which would fail in createTunnelAgent anyway. - if (!host) { - return callOriginalRequest(input, options, callback); - } - // Discord WebSocket upgrade — inject CONNECT tunnel agent unless the - // caller already provides a custom (non-default) agent. - if (!opts.agent || opts.agent === https.globalAgent) { - const port = parseInt(String(opts.port), 10) || 443; - opts = { ...opts, agent: createTunnelAgent(host, port) }; - } - return cb ? requestRef(opts, cb) : requestRef(opts); - } - - // Non-WebSocket — pass through the original arguments unchanged. - return callOriginalRequest(input, options, callback); - } - - // Replace https.request with our patched version. - Reflect.set(https, "request", wsProxyFixedRequest); - - Reflect.set(globalThis, _PATCHED, true); -})(); - -export {}; diff --git a/nemoclaw/package-lock.json b/nemoclaw/package-lock.json index d9f3a8367e..ecc753f9c7 100644 --- a/nemoclaw/package-lock.json +++ b/nemoclaw/package-lock.json @@ -94,9 +94,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT OR Apache-2.0", "optional": true, "os": [ @@ -114,9 +111,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT OR Apache-2.0", "optional": true, "os": [ @@ -134,9 +128,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT OR Apache-2.0", "optional": true, "os": [ @@ -154,9 +145,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT OR Apache-2.0", "optional": true, "os": [ diff --git a/package-lock.json b/package-lock.json index aa4830bfdc..c8bc245c84 100644 --- a/package-lock.json +++ b/package-lock.json @@ -183,9 +183,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT OR Apache-2.0", "optional": true, "os": [ @@ -203,9 +200,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT OR Apache-2.0", "optional": true, "os": [ @@ -223,9 +217,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT OR Apache-2.0", "optional": true, "os": [ @@ -243,9 +234,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT OR Apache-2.0", "optional": true, "os": [ diff --git a/package.json b/package.json index 555f659f7f..703ef58773 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "format:ts": "cd nemoclaw && npm run lint:fix && npm run format", "check:installer-hash": "bash scripts/check-installer-hash.sh", "typecheck": "tsc -p jsconfig.json", - "build:cli": "tsc -p tsconfig.src.json && tsc -p nemoclaw-blueprint/tsconfig.json", + "build:cli": "tsc -p tsconfig.src.json && if find nemoclaw-blueprint/scripts -name '*.ts' -print -quit | grep -q .; then tsc -p nemoclaw-blueprint/tsconfig.json; fi", "typecheck:cli": "tsc -p tsconfig.cli.json", "validate:configs": "tsx scripts/validate-configs.ts", "ts-migration:assist": "tsx scripts/ts-migration-assist.ts", diff --git a/schemas/policy-preset.schema.json b/schemas/policy-preset.schema.json index efea85d814..82a8e47c3c 100644 --- a/schemas/policy-preset.schema.json +++ b/schemas/policy-preset.schema.json @@ -47,10 +47,17 @@ "properties": { "host": { "type": "string" }, "port": { "type": "integer", "minimum": 1, "maximum": 65535 }, - "protocol": { "type": "string", "enum": ["rest"] }, + "protocol": { "type": "string", "enum": ["rest", "websocket"] }, "enforcement": { "type": "string", "enum": ["enforce", "audit"] }, "tls": { "type": "string", "enum": ["terminate", "passthrough", "skip"] }, "access": { "type": "string", "enum": ["full"] }, + "websocket_credential_rewrite": { "type": "boolean" }, + "request_body_credential_rewrite": { "type": "boolean" }, + "allowed_ips": { + "type": "array", + "items": { "type": "string", "minLength": 1 }, + "minItems": 1 + }, "rules": { "type": "array", "items": { "$ref": "#/$defs/rule" }, @@ -58,7 +65,7 @@ } }, "if": { - "properties": { "protocol": { "const": "rest" } }, + "properties": { "protocol": { "enum": ["rest", "websocket"] } }, "required": ["protocol"] }, "then": { "required": ["rules"] } @@ -75,7 +82,16 @@ "properties": { "method": { "type": "string", - "enum": ["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"] + "enum": [ + "GET", + "POST", + "PUT", + "PATCH", + "DELETE", + "HEAD", + "OPTIONS", + "WEBSOCKET_TEXT" + ] }, "path": { "type": "string", diff --git a/schemas/sandbox-policy.schema.json b/schemas/sandbox-policy.schema.json index e51dd5b944..7c6fec34e5 100644 --- a/schemas/sandbox-policy.schema.json +++ b/schemas/sandbox-policy.schema.json @@ -72,10 +72,17 @@ "properties": { "host": { "type": "string" }, "port": { "type": "integer", "minimum": 1, "maximum": 65535 }, - "protocol": { "type": "string", "enum": ["rest"] }, + "protocol": { "type": "string", "enum": ["rest", "websocket"] }, "enforcement": { "type": "string", "enum": ["enforce", "audit"] }, "tls": { "type": "string", "enum": ["terminate", "passthrough", "skip"] }, "access": { "type": "string", "enum": ["full"] }, + "websocket_credential_rewrite": { "type": "boolean" }, + "request_body_credential_rewrite": { "type": "boolean" }, + "allowed_ips": { + "type": "array", + "items": { "type": "string", "minLength": 1 }, + "minItems": 1 + }, "rules": { "type": "array", "items": { "$ref": "#/$defs/rule" }, @@ -83,7 +90,7 @@ } }, "if": { - "properties": { "protocol": { "const": "rest" } }, + "properties": { "protocol": { "enum": ["rest", "websocket"] } }, "required": ["protocol"] }, "then": { @@ -105,7 +112,16 @@ "properties": { "method": { "type": "string", - "enum": ["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"] + "enum": [ + "GET", + "POST", + "PUT", + "PATCH", + "DELETE", + "HEAD", + "OPTIONS", + "WEBSOCKET_TEXT" + ] }, "path": { "type": "string", diff --git a/scripts/brev-launchable-ci-cpu.sh b/scripts/brev-launchable-ci-cpu.sh index 8f0f6a8277..f28bb352b1 100755 --- a/scripts/brev-launchable-ci-cpu.sh +++ b/scripts/brev-launchable-ci-cpu.sh @@ -28,7 +28,7 @@ # curl -fsSL https://raw.githubusercontent.com/NVIDIA/NemoClaw//scripts/brev-launchable-ci-cpu.sh | bash # # Environment overrides: -# OPENSHELL_VERSION — OpenShell CLI release tag (default: v0.0.37) +# OPENSHELL_VERSION — OpenShell CLI release tag (default: v0.0.39) # NEMOCLAW_REF — NemoClaw git ref to clone (default: main) # NEMOCLAW_CLONE_DIR — Where to clone NemoClaw (default: ~/NemoClaw) # SKIP_DOCKER_PULL — Set to 1 to skip Docker image pre-pulls @@ -40,7 +40,7 @@ set -euo pipefail # ── Configuration ──────────────────────────────────────────────────── -OPENSHELL_VERSION="${OPENSHELL_VERSION:-v0.0.37}" +OPENSHELL_VERSION="${OPENSHELL_VERSION:-v0.0.39}" NEMOCLAW_REF="${NEMOCLAW_REF:-main}" TARGET_USER="${SUDO_USER:-$(id -un)}" TARGET_HOME="$(getent passwd "$TARGET_USER" | cut -d: -f6)" @@ -250,7 +250,7 @@ DOCKER_PULL_PID="" if [[ "${SKIP_DOCKER_PULL:-0}" != "1" ]]; then info "Pre-pulling Docker images in background..." ( - SUPERVISOR_TAG="${OPENSHELL_VERSION#v}" # v0.0.37 → 0.0.37 + SUPERVISOR_TAG="${OPENSHELL_VERSION#v}" # v0.0.39 -> 0.0.39 SUPERVISOR_IMAGE="ghcr.io/nvidia/openshell/supervisor:${SUPERVISOR_TAG}" # Pull all images in parallel diff --git a/scripts/generate-openclaw-config.py b/scripts/generate-openclaw-config.py index ca33f9febb..b290cd0205 100755 --- a/scripts/generate-openclaw-config.py +++ b/scripts/generate-openclaw-config.py @@ -407,9 +407,8 @@ def build_config(env: dict | None = None) -> dict: # Slack's Bolt SDK validates token shape at App construction (^xoxb-…$ / # ^xapp-…$) before any HTTP call leaves the process, so the canonical # openshell:resolve:env:VAR placeholder is rejected synchronously. Emit a - # Bolt-regex-compatible placeholder instead; the slack-token-rewriter - # Node preload translates it to canonical form on outbound HTTP, where - # OpenShell's L7 proxy substitutes the real token from env. + # Bolt-regex-compatible placeholder instead; OpenShell resolves the + # provider-shaped alias directly at the egress boundary. def _placeholder(channel: str, env_key: str) -> str: if channel == "slack" and env_key == "SLACK_BOT_TOKEN": return f"xoxb-OPENSHELL-RESOLVE-ENV-{env_key}" @@ -428,7 +427,7 @@ def _placeholder(channel: str, env_key: str) -> str: } if ch == "slack": account["appToken"] = _placeholder(ch, "SLACK_APP_TOKEN") - if ch in ("telegram", "discord"): + if ch == "telegram": account["proxy"] = proxy_url if ch == "telegram": account["groupPolicy"] = "open" diff --git a/scripts/install-openshell.sh b/scripts/install-openshell.sh index 6f838bddc7..4e3cd2ac62 100755 --- a/scripts/install-openshell.sh +++ b/scripts/install-openshell.sh @@ -33,15 +33,15 @@ esac info "Detected $OS_LABEL ($ARCH_LABEL)" -# Minimum version required for the released Docker-driver gateway/sandbox -# binaries and the GPU filesystem policy fixes NemoClaw depends on. -MIN_VERSION="0.0.37" +# Minimum version required for native messaging credential rewrite: +# WebSocket text frames plus provider-shaped aliases and REST request bodies. +MIN_VERSION="0.0.39" # Maximum version validated for this NemoClaw release. Newer OpenShell builds # may change sandbox semantics; upgrade NemoClaw before upgrading past this. -MAX_VERSION="0.0.37" +MAX_VERSION="0.0.39" # Pin fresh installs to this version instead of pulling "latest". PIN_VERSION="$MAX_VERSION" -DEV_MIN_VERSION="0.0.37" +DEV_MIN_VERSION="0.0.39" CHANNEL="${NEMOCLAW_OPENSHELL_CHANNEL:-auto}" case "$CHANNEL" in @@ -89,6 +89,43 @@ required_driver_bins_present() { esac } +OPENSHELL_FEATURE_CHECK_ERROR="" + +openshell_has_required_messaging_features() { + local openshell_bin + OPENSHELL_FEATURE_CHECK_ERROR="" + openshell_bin="${1:-$(command -v openshell 2>/dev/null || true)}" + if [ -z "$openshell_bin" ]; then + OPENSHELL_FEATURE_CHECK_ERROR="openshell binary was not found." + return 1 + fi + if ! command -v strings >/dev/null 2>&1; then + OPENSHELL_FEATURE_CHECK_ERROR="'strings' is required to verify OpenShell messaging credential rewrite support. Install binutils or an equivalent package and retry." + return 2 + fi + + # Keep this independent of a live gateway. `policy update --dry-run` still + # needs gateway metadata, but the CLI binary must contain the endpoint-option + # parser for request-body/WebSocket rewrite support released in OpenShell 0.0.39. + local binary_strings + binary_strings="$(strings "$openshell_bin" 2>/dev/null || true)" + if [[ "$binary_strings" != *"request-body-credential-rewrite"* ]]; then + OPENSHELL_FEATURE_CHECK_ERROR="OpenShell binary is missing request-body-credential-rewrite support." + return 1 + fi + if [[ "$binary_strings" != *"websocket-credential-rewrite"* ]]; then + OPENSHELL_FEATURE_CHECK_ERROR="OpenShell binary is missing websocket-credential-rewrite support." + return 1 + fi + return 0 +} + +require_openshell_messaging_features() { + local openshell_bin="$1" + openshell_has_required_messaging_features "$openshell_bin" \ + || fail "${OPENSHELL_FEATURE_CHECK_ERROR:-OpenShell binary is missing required messaging credential rewrite support.}" +} + macos_vm_driver_bin() { command -v openshell-driver-vm 2>/dev/null || true } @@ -176,11 +213,19 @@ if command -v openshell >/dev/null 2>&1; then INSTALLED_VERSION="$(printf '%s\n' "$INSTALLED_VERSION_OUTPUT" | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1 || true)" [ -n "$INSTALLED_VERSION" ] || INSTALLED_VERSION="0.0.0" if [ "$RESOLVED_CHANNEL" = "dev" ]; then - if version_gte "$INSTALLED_VERSION" "$DEV_MIN_VERSION" && printf '%s\n' "$INSTALLED_VERSION_OUTPUT" | grep -qi 'dev'; then - info "openshell already installed: $INSTALLED_VERSION_OUTPUT (dev channel)" - exit 0 + if version_gte "$INSTALLED_VERSION" "$DEV_MIN_VERSION" \ + && printf '%s\n' "$INSTALLED_VERSION_OUTPUT" | grep -qi 'dev'; then + if openshell_has_required_messaging_features; then + info "openshell already installed: $INSTALLED_VERSION_OUTPUT (dev channel)" + exit 0 + else + feature_status=$? + if [ "$feature_status" = "2" ]; then + fail "$OPENSHELL_FEATURE_CHECK_ERROR" + fi + fi fi - warn "openshell $INSTALLED_VERSION is not the required dev-channel Docker-driver build — upgrading..." + warn "openshell $INSTALLED_VERSION is not the required dev-channel messaging-rewrite build — upgrading..." else if version_gte "$INSTALLED_VERSION" "$MIN_VERSION"; then if ! version_gte "$MAX_VERSION" "$INSTALLED_VERSION"; then @@ -188,10 +233,12 @@ if command -v openshell >/dev/null 2>&1; then fi if ! required_driver_bins_present; then warn "openshell $INSTALLED_VERSION is missing Docker-driver binaries — reinstalling pinned OpenShell ${PIN_VERSION}..." + elif ! openshell_has_required_messaging_features; then + fail "${OPENSHELL_FEATURE_CHECK_ERROR:-openshell $INSTALLED_VERSION is missing required messaging credential rewrite support. Install an OpenShell build that includes provider aliases, WebSocket text rewrite, and request-body credential rewrite.}" elif ! repair_existing_macos_vm_driver; then warn "openshell $INSTALLED_VERSION has an unsigned macOS VM driver that could not be repaired in place — reinstalling pinned OpenShell ${PIN_VERSION}..." else - info "openshell already installed: $INSTALLED_VERSION (>= $MIN_VERSION, <= $MAX_VERSION)" + info "openshell already installed: $INSTALLED_VERSION (>= $MIN_VERSION, <= $MAX_VERSION, messaging rewrite capable)" exit 0 fi else @@ -338,4 +385,6 @@ else fi fi +require_openshell_messaging_features "$target_dir/openshell" + info "$("$target_dir/openshell" --version 2>&1 || echo openshell) installed" diff --git a/scripts/install.sh b/scripts/install.sh index bd741c0c1a..ffd553823a 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -1609,7 +1609,7 @@ abort_unsupported_automatic_openshell_upgrade() { local old_openshell_version="$1" warn "Existing sandbox sessions use OpenShell ${old_openshell_version}, but the current ${_CLI_BIN} CLI does not support '${_CLI_BIN} backup-all'." cat </dev/null 2>&1 \ || openshell gateway destroy >/dev/null 2>&1 \ || warn "Could not destroy the legacy OpenShell gateway before upgrade; onboarding will clean up stale runtime state." diff --git a/scripts/nemoclaw-start.sh b/scripts/nemoclaw-start.sh index b95b492f44..d3d3c90347 100755 --- a/scripts/nemoclaw-start.sh +++ b/scripts/nemoclaw-start.sh @@ -905,38 +905,6 @@ PYPLACEHOLDERS [ "$_write_rc" -eq 0 ] || return "$_write_rc" } -# ── Slack token rewriter (Bolt-shape → canonical placeholder) ──── -# Installs a Node preload that translates the Bolt-compatible placeholder -# (xoxb|xapp)-OPENSHELL-RESOLVE-ENV-VAR — emitted into openclaw.json by -# generate-openclaw-config.py — into the active openshell:resolve:env:* -# placeholder on outbound HTTP. OpenShell's L7 proxy then substitutes the -# real token from env on the wire, the same path Discord/Telegram/Brave -# already take. No real Slack token ever touches openclaw.json, /tmp, or -# any other disk surface readable by the sandbox uid. -# -# Ref: https://github.com/NVIDIA/NemoClaw/issues/2085 - -_SLACK_REWRITER_SCRIPT="/tmp/nemoclaw-slack-token-rewriter.js" -_SLACK_REWRITER_SOURCE="/usr/local/lib/nemoclaw/preloads/slack-token-rewriter.js" - -install_slack_token_rewriter() { - local config_file="/sandbox/.openclaw/openclaw.json" - - # Only install if a Slack channel placeholder is present in the config. - # Same conditional shape as install_slack_channel_guard — both are no-ops - # for sandboxes without Slack configured. - if ! grep -q 'OPENSHELL-RESOLVE-ENV-SLACK_' "$config_file" 2>/dev/null; then - return 0 - fi - - printf '[channels] Installing Slack token rewriter (Bolt-shape → OpenShell placeholder)\n' >&2 - - emit_sandbox_sourced_file "$_SLACK_REWRITER_SCRIPT" <"$_SLACK_REWRITER_SOURCE" - - export NODE_OPTIONS="${NODE_OPTIONS:+$NODE_OPTIONS }--require $_SLACK_REWRITER_SCRIPT" - printf '[channels] Slack token rewriter installed (NODE_OPTIONS updated)\n' >&2 -} - # ── Slack secrets-on-disk tripwire ──────────────────────────────── # Defense-in-depth: refuse to serve if a real Slack token (anything # starting with xoxb- or xapp- that is NOT the OPENSHELL-RESOLVE-ENV- @@ -1524,8 +1492,6 @@ GUARDENVEOF # by install_slack_channel_guard() — conditional on the file existing at # source-time so connect sessions started before Slack is configured are safe. echo "[ -f \"$_SLACK_GUARD_SCRIPT\" ] && export NODE_OPTIONS=\"\${NODE_OPTIONS:+\$NODE_OPTIONS }--require $_SLACK_GUARD_SCRIPT\"" - # Slack token rewriter for connect sessions — same conditional pattern. - echo "[ -f \"$_SLACK_REWRITER_SCRIPT\" ] && export NODE_OPTIONS=\"\${NODE_OPTIONS:+\$NODE_OPTIONS }--require $_SLACK_REWRITER_SCRIPT\"" # Tool cache redirects — generated from _TOOL_REDIRECTS (single source of truth) echo '# Tool cache redirects — keep transient tool state under /tmp' for _redir in "${_TOOL_REDIRECTS[@]}"; do @@ -1949,7 +1915,6 @@ if [ "$(id -u)" -ne 0 ]; then configure_messaging_channels install_telegram_diagnostics - install_slack_token_rewriter install_slack_channel_guard verify_no_slack_secrets_on_disk @@ -1993,7 +1958,7 @@ if [ "$(id -u)" -ne 0 ]; then # Pass the HTTP proxy-fix path so it is validated alongside proxy-env.sh # (both are trust-boundary files; tampering would let the sandbox user # inject code into any Node process via NODE_OPTIONS). - validate_tmp_permissions "$_SANDBOX_SAFETY_NET" "$_PROXY_FIX_SCRIPT" "$_NEMOTRON_FIX_SCRIPT" "$_WS_FIX_SCRIPT" "$_SECCOMP_GUARD_SCRIPT" "$_CIAO_GUARD_SCRIPT" "$_TELEGRAM_DIAGNOSTICS_SCRIPT" "$_SLACK_GUARD_SCRIPT" "$_SLACK_REWRITER_SCRIPT" + validate_tmp_permissions "$_SANDBOX_SAFETY_NET" "$_PROXY_FIX_SCRIPT" "$_NEMOTRON_FIX_SCRIPT" "$_WS_FIX_SCRIPT" "$_SECCOMP_GUARD_SCRIPT" "$_CIAO_GUARD_SCRIPT" "$_TELEGRAM_DIAGNOSTICS_SCRIPT" "$_SLACK_GUARD_SCRIPT" # Start gateway in background, auto-pair, then wait nohup "$OPENCLAW" gateway run --port "${_DASHBOARD_PORT}" >/tmp/gateway.log 2>&1 & @@ -2051,7 +2016,6 @@ lock_rc_files "$_SANDBOX_HOME" # BEFORE chattr +i (which locks the config permanently). configure_messaging_channels install_telegram_diagnostics -install_slack_token_rewriter install_slack_channel_guard verify_no_slack_secrets_on_disk @@ -2167,7 +2131,7 @@ gosu sandbox bash -c "$(declare -f seed_default_workspace_templates); seed_defau # Pass the HTTP proxy-fix path so it is validated alongside proxy-env.sh # (both are trust-boundary files; tampering would let the sandbox user # inject code into any Node process via NODE_OPTIONS). -validate_tmp_permissions "$_SANDBOX_SAFETY_NET" "$_PROXY_FIX_SCRIPT" "$_NEMOTRON_FIX_SCRIPT" "$_WS_FIX_SCRIPT" "$_SECCOMP_GUARD_SCRIPT" "$_CIAO_GUARD_SCRIPT" "$_TELEGRAM_DIAGNOSTICS_SCRIPT" "$_SLACK_GUARD_SCRIPT" "$_SLACK_REWRITER_SCRIPT" +validate_tmp_permissions "$_SANDBOX_SAFETY_NET" "$_PROXY_FIX_SCRIPT" "$_NEMOTRON_FIX_SCRIPT" "$_WS_FIX_SCRIPT" "$_SECCOMP_GUARD_SCRIPT" "$_CIAO_GUARD_SCRIPT" "$_TELEGRAM_DIAGNOSTICS_SCRIPT" "$_SLACK_GUARD_SCRIPT" # Start the gateway as the 'gateway' user. # SECURITY: The sandbox user cannot kill this process because it runs diff --git a/src/lib/agent/runtime.test.ts b/src/lib/agent/runtime.test.ts index 5e1b628c3f..81a98e5816 100644 --- a/src/lib/agent/runtime.test.ts +++ b/src/lib/agent/runtime.test.ts @@ -97,61 +97,41 @@ describe("buildRecoveryScript", () => { const script = buildRecoveryScript(hermesAgent, 8642); expect(script).toContain("export HERMES_HOME=/sandbox/.hermes"); expect(script).toContain("HERMES_HOME=/sandbox/.hermes"); - expect(script).toContain("DISCORD_PROXY=http://127.0.0.1:3129"); - expect(script).toContain("NEMOCLAW_DISCORD_FACADE_URL=http://127.0.0.1:3130"); - expect(script).toContain("PYTHONPATH=/opt/nemoclaw-hermes-discord-preload"); - expect(script).toContain("HTTPS_PROXY=http://127.0.0.1:3129"); - expect(script).toContain("nemoclaw-decode-proxy"); - expect(script).toContain("nemoclaw-discord-facade"); + expect(script).not.toContain("DISCORD_PROXY="); + expect(script).not.toContain("PYTHONPATH=/opt/nemoclaw-hermes-discord-preload"); + expect(script).not.toContain("HTTPS_PROXY=http://127.0.0.1:3129"); + expect(script).not.toContain("nemoclaw-decode-proxy"); + expect(script).not.toContain("nemoclaw-discord-facade"); + expect(script).not.toContain("NEMOCLAW_DISCORD_FACADE_URL"); expect(script).toContain('"$AGENT_BIN" gateway run'); expect(script).not.toContain('"$AGENT_BIN" gateway run --port 8642'); expect(script).not.toContain("hermes gateway run --port 8642"); }); - it("launches Hermes decode-proxy and Discord facade under the venv interpreter during recovery", () => { + it("does not launch a Hermes decode proxy during recovery", () => { const script = buildRecoveryScript(hermesAgent, 8642); - expect(script).toContain("/opt/hermes/.venv/bin/python /usr/local/bin/nemoclaw-decode-proxy"); - expect(script).toContain("/opt/hermes/.venv/bin/python /usr/local/bin/nemoclaw-discord-facade"); - expect(script).not.toMatch(/(? { + it("does not wait for removed Hermes bridge ports during recovery", () => { const recoveryScript = buildRecoveryScript(hermesAgent, 8642); expect(recoveryScript).not.toBeNull(); for (const script of [recoveryScript!, buildManualRecoveryCommand(hermesAgent, 8642)]) { - expect(script).toContain( - 'command -v ss >/dev/null 2>&1 && ss -tln 2>/dev/null | grep -Eq "127\\.0\\.0\\.1:3129([[:space:]]|$)" && break', - ); - expect(script).toContain( - 'command -v ss >/dev/null 2>&1 && ss -tln 2>/dev/null | grep -Eq "127\\.0\\.0\\.1:3130([[:space:]]|$)" && break', - ); + expect(script).not.toContain("127\\.0\\.0\\.1:3129"); expect(script).not.toContain('grep -q "127.0.0.1:3129"'); expect(script).not.toContain('grep -q "127.0.0.1:3130"'); expect(script).not.toContain("do ! command -v ss >/dev/null 2>&1 || ss -tln"); } }); - it("prepares the Hermes Discord facade log before child-side redirection", () => { + it("does not relaunch the removed Hermes Discord facade during recovery", () => { const recoveryScript = buildRecoveryScript(hermesAgent, 8642); expect(recoveryScript).not.toBeNull(); for (const script of [recoveryScript!, buildManualRecoveryCommand(hermesAgent, 8642)]) { - expect(script).toContain("/tmp/discord-facade.log"); - expect(script).toContain("/tmp/discord-facade-recovery.log"); - expect(script).toContain("O_NOFOLLOW"); - expect(script).toContain("_DISCORD_FACADE_LOG='/tmp/discord-facade.log'"); - expect(script).toContain("_DISCORD_FACADE_LOG='/tmp/discord-facade-recovery.log'"); - expect(script).toContain('DISCORD_FACADE_LOG="$_DISCORD_FACADE_LOG"'); - expect(script).toContain( - 'sh -c \'umask 0007; exec "$@" >>"$DISCORD_FACADE_LOG" 2>&1\' sh /opt/hermes/.venv/bin/python /usr/local/bin/nemoclaw-discord-facade &', - ); - expect(script).not.toContain( - "nohup python3 /usr/local/bin/nemoclaw-discord-facade >/tmp/discord-facade.log 2>&1", - ); - expect(script).not.toContain('exec "$@" >/tmp/discord-facade.log 2>&1'); - expect(script.indexOf("O_NOFOLLOW")).toBeLessThan( - script.indexOf('DISCORD_FACADE_LOG="$_DISCORD_FACADE_LOG"'), - ); + expect(script).not.toContain("discord-facade"); + expect(script).not.toContain("DISCORD_FACADE_LOG"); } }); @@ -368,12 +348,12 @@ describe("buildManualRecoveryCommand (#2426)", () => { it("omits --port for Hermes and uses the current Hermes home", () => { const cmd = buildManualRecoveryCommand(hermesAgent, 8642); expect(cmd).toContain("HERMES_HOME=/sandbox/.hermes"); - expect(cmd).toContain("DISCORD_PROXY=http://127.0.0.1:3129"); - expect(cmd).toContain("NEMOCLAW_DISCORD_FACADE_URL=http://127.0.0.1:3130"); - expect(cmd).toContain("PYTHONPATH=/opt/nemoclaw-hermes-discord-preload"); - expect(cmd).toContain("HTTPS_PROXY=http://127.0.0.1:3129"); - expect(cmd).toContain("nemoclaw-decode-proxy"); - expect(cmd).toContain("nemoclaw-discord-facade"); + expect(cmd).not.toContain("DISCORD_PROXY="); + expect(cmd).not.toContain("PYTHONPATH=/opt/nemoclaw-hermes-discord-preload"); + expect(cmd).not.toContain("HTTPS_PROXY=http://127.0.0.1:3129"); + expect(cmd).not.toContain("nemoclaw-decode-proxy"); + expect(cmd).not.toContain("nemoclaw-discord-facade"); + expect(cmd).not.toContain("NEMOCLAW_DISCORD_FACADE_URL"); expect(cmd).toContain("nohup hermes gateway run"); expect(cmd).not.toContain("--port 8642"); expect(cmd).not.toContain("/sandbox/.hermes-data"); diff --git a/src/lib/agent/runtime.ts b/src/lib/agent/runtime.ts index 8f4ad00aa5..45906d15f2 100644 --- a/src/lib/agent/runtime.ts +++ b/src/lib/agent/runtime.ts @@ -143,30 +143,7 @@ function gatewayLaunchCommand(command: string, runAsUser?: string): string { } function hermesGatewayEnvPrefix(): string { - const decodeProxy = "http://127.0.0.1:3129"; - const discordFacade = "http://127.0.0.1:3130"; - return [ - "HERMES_HOME=/sandbox/.hermes", - `DISCORD_PROXY=${decodeProxy}`, - `NEMOCLAW_DISCORD_FACADE_URL=${discordFacade}`, - "PYTHONPATH=/opt/nemoclaw-hermes-discord-preload${PYTHONPATH:+:${PYTHONPATH}}", - `HTTPS_PROXY=${decodeProxy}`, - `HTTP_PROXY=${decodeProxy}`, - `https_proxy=${decodeProxy}`, - `http_proxy=${decodeProxy}`, - ].join(" "); -} - -function hermesDecodeProxyRecoveryCommand(): string { - const hermesVenvPython = "/opt/hermes/.venv/bin/python"; - const decodeProxyListening = 'ss -tln 2>/dev/null | grep -Eq "127\\.0\\.0\\.1:3129([[:space:]]|$)"'; - const facadeListening = 'ss -tln 2>/dev/null | grep -Eq "127\\.0\\.0\\.1:3130([[:space:]]|$)"'; - const primaryFacadeLog = "/tmp/discord-facade.log"; - const fallbackFacadeLog = "/tmp/discord-facade-recovery.log"; - const facadeLogSetup = `${buildNoFollowLogSetupCommand(primaryFacadeLog, undefined, "0o600")} || exit 1; _DISCORD_FACADE_LOG=${shellQuote(primaryFacadeLog)}; if ! : >> "$_DISCORD_FACADE_LOG" 2>/dev/null; then ${buildNoFollowLogSetupCommand(fallbackFacadeLog, undefined, "0o600")} || exit 1; _DISCORD_FACADE_LOG=${shellQuote(fallbackFacadeLog)}; : >> "$_DISCORD_FACADE_LOG" 2>/dev/null || exit 1; fi`; - const facadeLaunch = - `nohup env -u NEMOCLAW_DISCORD_FACADE_URL -u PYTHONPATH DISCORD_PROXY=http://127.0.0.1:3129 HTTPS_PROXY=http://127.0.0.1:3129 HTTP_PROXY=http://127.0.0.1:3129 NEMOCLAW_DISCORD_FACADE_PORT=3130 DISCORD_FACADE_LOG="$_DISCORD_FACADE_LOG" sh -c 'umask 0007; exec "$@" >>"$DISCORD_FACADE_LOG" 2>&1' sh ${hermesVenvPython} /usr/local/bin/nemoclaw-discord-facade &`; - return `if ! command -v ss >/dev/null 2>&1 || ! ${decodeProxyListening}; then nohup ${hermesVenvPython} /usr/local/bin/nemoclaw-decode-proxy >/dev/null 2>&1 & for _i in 1 2 3 4 5 6 7 8 9 10; do command -v ss >/dev/null 2>&1 && ${decodeProxyListening} && break; sleep 0.5; done; fi; if ! command -v ss >/dev/null 2>&1 || ! ${facadeListening}; then ${facadeLogSetup}; ${facadeLaunch} for _i in 1 2 3 4 5 6 7 8 9 10; do command -v ss >/dev/null 2>&1 && ${facadeListening} && break; sleep 0.5; done; fi;`; + return "HERMES_HOME=/sandbox/.hermes"; } /** @@ -235,7 +212,7 @@ export function buildRecoveryScript(agent: AgentDefinition | null, port: number) // Source /tmp/nemoclaw-proxy-env.sh immediately before launching. That file // is the single source of truth for NODE_OPTIONS preload guards (safety-net, - // ciao networkInterfaces, slack, http-proxy, ws-proxy, nemotron). Recovery + // ciao networkInterfaces, slack, http-proxy, nemotron). Recovery // also stops stale launcher/gateway processes that may have respawned // between the health probe and relaunch. A missing env file remains warning- // only; a present env file that does not install required guards is a hard @@ -253,7 +230,6 @@ export function buildRecoveryScript(agent: AgentDefinition | null, port: number) 'if [ "$_PE_MISSING" = "0" ]; then case "${NODE_OPTIONS:-}" in *nemoclaw-sandbox-safety-net*) _SN_MISSING=0 ;; *) _SN_MISSING=1 ;; esac; case "${NODE_OPTIONS:-}" in *nemoclaw-ciao-network-guard*) _CIAO_MISSING=0 ;; *) _CIAO_MISSING=1 ;; esac; if [ "$_SN_MISSING" = "0" ] && [ "$_CIAO_MISSING" = "0" ]; then _GUARDS_MISSING=0; else _GUARDS_MISSING=1; fi; else _GUARDS_MISSING=0; fi;', '[ "$_PE_MISSING" = "1" ] && { _W="[gateway-recovery] WARNING: /tmp/nemoclaw-proxy-env.sh missing - gateway launching without library guards (#2478)"; echo "$_W" >&2; echo "$_W" >> "$_GATEWAY_LOG"; };', '[ "$_PE_MISSING" = "0" ] && [ "$_GUARDS_MISSING" = "1" ] && { _E="[gateway-recovery] ERROR: /tmp/nemoclaw-proxy-env.sh present but NODE_OPTIONS missing safety-net preload or ciao preload - refusing unguarded gateway relaunch (#2478)"; echo "$_E" >&2; echo "$_E" >> "$_GATEWAY_LOG"; exit 1; };', - isHermes ? hermesDecodeProxyRecoveryCommand() : "", launchCommand, "GPID=$!; sleep 2;", 'if kill -0 "$GPID" 2>/dev/null; then echo "GATEWAY_PID=$GPID"; else echo GATEWAY_FAILED; tail -5 "$_GATEWAY_LOG" 2>/dev/null; fi', @@ -286,6 +262,5 @@ export function buildManualRecoveryCommand(agent: AgentDefinition | null, port: const isHermes = agent?.name === "hermes"; const envPrefix = isHermes ? `${hermesGatewayEnvPrefix()} ` : ""; const portFlag = isHermes ? "" : ` --port ${port}`; - const decodeProxySetup = isHermes ? `${hermesDecodeProxyRecoveryCommand()} ` : ""; - return `${buildGatewayLogSelection()} ${decodeProxySetup}${envPrefix}nohup ${gatewayCmd}${portFlag} >> "$_GATEWAY_LOG" 2>&1 &`; + return `${buildGatewayLogSelection()} ${envPrefix}nohup ${gatewayCmd}${portFlag} >> "$_GATEWAY_LOG" 2>&1 &`; } diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 59dde31f52..ee45b17eaf 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -20,6 +20,11 @@ const { cleanupTempDir, secureTempFile, }: typeof import("./onboard/temp-files") = require("./onboard/temp-files"); +const { + CUSTOM_BUILD_CONTEXT_WARN_BYTES, + isInsideIgnoredCustomBuildContextPath, + shouldIncludeCustomBuildContextPath, +}: typeof import("./onboard/custom-build-context") = require("./onboard/custom-build-context"); const { buildCompatibleEndpointSandboxSmokeCommand, buildCompatibleEndpointSandboxSmokeScript, @@ -155,60 +160,6 @@ const { const onboardProviders = require("./onboard/providers"); const hermesProviderAuth = require("./hermes-provider-auth"); -const CUSTOM_BUILD_CONTEXT_WARN_BYTES = 100_000_000; -const CUSTOM_BUILD_CONTEXT_IGNORES = new Set([ - "node_modules", - ".git", - ".venv", - "__pycache__", - ".aws", - ".credentials", - ".direnv", - ".netrc", - ".npmrc", - ".pypirc", - ".ssh", - "credentials.json", - "key.json", - "secrets", - "secrets.json", - "secrets.yaml", - "token.json", -]); - -function isIgnoredCustomBuildContextName(name: string): boolean { - const lowerName = name.toLowerCase(); - return ( - CUSTOM_BUILD_CONTEXT_IGNORES.has(lowerName) || - lowerName === ".env" || - lowerName === ".envrc" || - lowerName.startsWith(".env.") || - lowerName.endsWith(".key") || - lowerName.endsWith(".pem") || - lowerName.endsWith(".pfx") || - lowerName.endsWith(".p12") || - lowerName.endsWith(".jks") || - lowerName.endsWith(".keystore") || - lowerName.endsWith(".tfvars") || - lowerName.endsWith("_ecdsa") || - lowerName.endsWith("_ed25519") || - lowerName.endsWith("_rsa") || - (lowerName.startsWith("service-account") && lowerName.endsWith(".json")) - ); -} - -function shouldIncludeCustomBuildContextPath(src: string): boolean { - return !isIgnoredCustomBuildContextName(path.basename(src)); -} - -function isInsideIgnoredCustomBuildContextPath(src: string): boolean { - return path - .normalize(src) - .split(path.sep) - .filter(Boolean) - .some((part: string) => isIgnoredCustomBuildContextName(part)); -} - type RemoteProviderConfigEntry = { label: string; providerName: string; @@ -288,8 +239,16 @@ const policies: typeof import("./policy") = require("./policy"); const shields = require("./shields"); const tiers: typeof import("./policy/tiers") = require("./policy/tiers"); const { ensureUsageNoticeConsent } = require("./onboard/usage-notice"); +const { + destroyGatewayForReuse, + warnIfGatewayDestroyFails, +} = require("./onboard/gateway-cleanup") as typeof import("./onboard/gateway-cleanup"); +const { + gatewayCliSupportsLifecycleCommands, +} = require("./onboard/gateway-lifecycle") as typeof import("./onboard/gateway-lifecycle"); const { getGatewayReuseHealthWaitConfig, + isDockerDriverGatewayHttpReady, isGatewayHttpReady, waitForGatewayHttpReady, } = require("./onboard/gateway-http-readiness") as typeof import("./onboard/gateway-http-readiness"); @@ -3462,19 +3421,26 @@ function destroyGateway(): boolean { stopDockerDriverGatewayProcess(); } + const hasLifecycleCommands = gatewayCliSupportsLifecycleCommands(runCaptureOpenshell); const gatewayRemoved = dockerDriver ? removeDockerDriverGatewayRegistration() - : runOpenshell(["gateway", "destroy", "-g", GATEWAY_NAME], { + : hasLifecycleCommands + ? runOpenshell(["gateway", "destroy", "-g", GATEWAY_NAME], { + ignoreError: true, + }).status === 0 + : runOpenshell(["gateway", "remove", GATEWAY_NAME], { ignoreError: true, - }).status === 0; + }).status === 0; // Clear the local registry so `nemoclaw list` stays consistent with OpenShell state. (#532) if (gatewayRemoved) { registry.clearAll(); } - // Legacy OpenShell gateway cleanup doesn't remove Docker volumes, which - // leaves corrupted cluster state that breaks the next gateway start. - dockerRemoveVolumesByPrefix(`openshell-cluster-${GATEWAY_NAME}`, { ignoreError: true }); + if (gatewayRemoved && (dockerDriver || hasLifecycleCommands)) { + // Legacy OpenShell gateway cleanup doesn't remove Docker volumes, which + // leaves corrupted cluster state that breaks the next gateway start. + dockerRemoveVolumesByPrefix(`openshell-cluster-${GATEWAY_NAME}`, { ignoreError: true }); + } return gatewayRemoved; } @@ -3732,7 +3698,7 @@ function getOpenShellDockerSupervisorImage(versionOutput: string | null = null): if (shouldUseOpenshellDevChannel() || isOpenshellDevVersion(versionOutput)) { return "ghcr.io/nvidia/openshell/supervisor:dev"; } - const supportedVersion = installedVersion ?? getBlueprintMaxOpenshellVersion() ?? "0.0.37"; + const supportedVersion = installedVersion ?? getBlueprintMaxOpenshellVersion() ?? "0.0.39"; return `ghcr.io/nvidia/openshell/supervisor:${supportedVersion}`; } @@ -4049,13 +4015,13 @@ function registerDockerDriverGatewayEndpoint(): boolean { } let addResult = runOpenshell( - ["gateway", "add", "--local", "--name", GATEWAY_NAME, getDockerDriverGatewayEndpoint()], + ["gateway", "add", getDockerDriverGatewayEndpoint(), "--local", "--name", GATEWAY_NAME], { ignoreError: true, suppressOutput: true }, ); if (addResult.status !== 0) { removeDockerDriverGatewayRegistration(); addResult = runOpenshell( - ["gateway", "add", "--local", "--name", GATEWAY_NAME, getDockerDriverGatewayEndpoint()], + ["gateway", "add", getDockerDriverGatewayEndpoint(), "--local", "--name", GATEWAY_NAME], { ignoreError: true, suppressOutput: true }, ); } @@ -4207,7 +4173,7 @@ function attachGatewayMetadataIfNeeded({ } const addResult = runOpenshell( - ["gateway", "add", "--local", "--name", GATEWAY_NAME, getGatewayLocalEndpoint()], + ["gateway", "add", getGatewayLocalEndpoint(), "--local", "--name", GATEWAY_NAME], { ignoreError: true, suppressOutput: true }, ); if (addResult.status === 0) { @@ -4236,9 +4202,12 @@ async function ensureNamedCredential( function waitForSandboxReady(sandboxName: string, attempts = 10, delaySeconds = 2): boolean { for (let i = 0; i < attempts; i += 1) { + const list = runCaptureOpenshell(["sandbox", "list"], { ignoreError: true }); + if (isSandboxReady(list, sandboxName)) return true; + + // Package-managed OpenShell gateways report readiness through + // `sandbox list`; legacy Kubernetes gateways may still expose pod state. if (isLinuxDockerDriverGatewayEnabled()) { - const list = runCaptureOpenshell(["sandbox", "list"], { ignoreError: true }); - if (isSandboxReady(list, sandboxName)) return true; if (i < attempts - 1) sleep(delaySeconds); continue; } @@ -4563,17 +4532,20 @@ async function preflight( let gatewayReuseState = gatewaySnapshot.gatewayReuseState; gatewayReuseState = await refreshDockerDriverGatewayReuseState(gatewayReuseState); - // Verify the gateway container is actually running — openshell CLI metadata - // can be stale after a manual `docker rm`. See #2020. - if (gatewayReuseState === "healthy" && !isLinuxDockerDriverGatewayEnabled()) { + // Verify the legacy gateway container is actually running — openshell CLI + // metadata can be stale after a manual `docker rm`. See #2020. Newer + // package-managed OpenShell gateways do not have an openshell-cluster-* + // Docker container, so the live CLI health check is the source of truth. + if (gatewayReuseState === "healthy" && gatewayCliSupportsLifecycleCommands(runCaptureOpenshell)) { const containerState = verifyGatewayContainerRunning(); if (containerState === "missing") { console.log(" Gateway metadata is stale (container not running). Cleaning up..."); runOpenshell(["forward", "stop", String(DASHBOARD_PORT)], { ignoreError: true }); - destroyGateway(); - registry.clearAll(); - gatewayReuseState = "missing"; - console.log(" ✓ Stale gateway metadata cleaned up"); + gatewayReuseState = destroyGatewayForReuse( + destroyGateway, + " ✓ Stale gateway metadata cleaned up", + " ! Stale gateway metadata cleanup failed; leaving registry state intact.", + ); } else if (containerState === "unknown") { // Docker probe failed but cached metadata says healthy. Try the host-level // HTTP probe — it doesn't depend on Docker, so it can confirm the gateway @@ -4603,10 +4575,11 @@ async function preflight( ` Gateway container is running but http://127.0.0.1:${GATEWAY_PORT}/ is not responding. Recreating...`, ); runOpenshell(["forward", "stop", String(DASHBOARD_PORT)], { ignoreError: true }); - destroyGateway(); - registry.clearAll(); - gatewayReuseState = "missing"; - console.log(" ✓ Stale gateway cleaned up"); + gatewayReuseState = destroyGatewayForReuse( + destroyGateway, + " ✓ Stale gateway cleaned up", + " ! Stale gateway cleanup failed; leaving registry state intact.", + ); } else { const imageDrift = getGatewayClusterImageDrift(); if (imageDrift) { @@ -4614,10 +4587,11 @@ async function preflight( ` Gateway image ${imageDrift.currentVersion} does not match openshell ${imageDrift.expectedVersion}. Recreating...`, ); stopAllDashboardForwards(); - destroyGateway(); - registry.clearAll(); - gatewayReuseState = "missing"; - console.log(" ✓ Previous gateway cleaned up"); + gatewayReuseState = destroyGatewayForReuse( + destroyGateway, + " ✓ Previous gateway cleaned up", + " ! Previous gateway cleanup failed; leaving registry state intact.", + ); } } } @@ -4626,12 +4600,16 @@ async function preflight( console.log(` Cleaning up previous ${cliDisplayName()} session...`); if (isLinuxDockerDriverGatewayEnabled()) { retireLegacyGatewayForDockerDriverUpgrade(); + gatewayReuseState = "missing"; + console.log(" ✓ Previous session cleaned up"); } else { runOpenshell(["forward", "stop", String(DASHBOARD_PORT)], { ignoreError: true }); - destroyGateway(); + gatewayReuseState = destroyGatewayForReuse( + destroyGateway, + " ✓ Previous session cleaned up", + " ! Previous session cleanup failed; leaving registry state intact.", + ); } - gatewayReuseState = "missing"; - console.log(" ✓ Previous session cleaned up"); } // Clean up orphaned Docker containers from interrupted onboard (e.g. Ctrl+C @@ -5048,9 +5026,13 @@ async function startDockerDriverGateway({ const drift = getDockerDriverGatewayRuntimeDrift(pidFileGatewayPid, gatewayEnv, gatewayBin); if (drift) { restartDockerDriverGatewayProcessForDrift(pidFileGatewayPid, drift.reason); - } else if (registerDockerDriverGatewayEndpoint()) { + } else if (registerDockerDriverGatewayEndpoint() && (await isDockerDriverGatewayHttpReady())) { console.log(" ✓ Reusing existing Docker-driver gateway"); return; + } else { + console.log( + ` Docker-driver gateway metadata reports healthy but http://127.0.0.1:${GATEWAY_PORT}/ is not responding. Starting a fresh gateway...`, + ); } } @@ -5072,7 +5054,10 @@ async function startDockerDriverGateway({ const adoptedActiveGatewayInfo = runCaptureOpenshell(["gateway", "info"], { ignoreError: true, }); - if (isGatewayHealthy(adoptedStatus, adoptedGwInfo, adoptedActiveGatewayInfo)) { + if ( + isGatewayHealthy(adoptedStatus, adoptedGwInfo, adoptedActiveGatewayInfo) && + (await isDockerDriverGatewayHttpReady()) + ) { console.log(` ✓ Reusing existing Docker-driver gateway process (PID ${portListenerPid})`); return; } @@ -5080,7 +5065,7 @@ async function startDockerDriverGateway({ } if (!gatewayBin) { console.error(" OpenShell Docker-driver gateway binary not found."); - console.error(" Install OpenShell v0.0.37, or set NEMOCLAW_OPENSHELL_GATEWAY_BIN."); + console.error(" Install OpenShell v0.0.39, or set NEMOCLAW_OPENSHELL_GATEWAY_BIN."); if (exitOnFailure) process.exit(1); throw new Error("OpenShell gateway binary not found"); } @@ -6482,16 +6467,6 @@ async function createSandbox( envArgs.push(formatEnvAssignment(webSearch.BRAVE_API_KEY_ENV, braveKey)); } } - // Slack Socket Mode requires both tokens in the container env so the baked - // openshell:resolve:env: placeholders in openclaw.json are substituted. - // The provider registration above handles L7 proxy auth header rewriting; - // the --env args here ensure the container env vars hold the real values. - if (tokensByEnvKey["SLACK_BOT_TOKEN"]) { - envArgs.push(formatEnvAssignment("SLACK_BOT_TOKEN", tokensByEnvKey["SLACK_BOT_TOKEN"])); - if (tokensByEnvKey["SLACK_APP_TOKEN"]) { - envArgs.push(formatEnvAssignment("SLACK_APP_TOKEN", tokensByEnvKey["SLACK_APP_TOKEN"])); - } - } const sandboxEnv = buildSubprocessEnv(); // Remove host-infrastructure credentials that the generic allowlist // permits for host-side processes but that must not enter the sandbox. @@ -11034,17 +11009,20 @@ async function onboard(opts: OnboardOptions = {}): Promise { let gatewayReuseState = gatewaySnapshot.gatewayReuseState; gatewayReuseState = await refreshDockerDriverGatewayReuseState(gatewayReuseState); - // Verify the gateway container is actually running — openshell CLI metadata - // can be stale after a manual `docker rm`. See #2020. - if (gatewayReuseState === "healthy" && !isLinuxDockerDriverGatewayEnabled()) { + // Verify the legacy gateway container is actually running — openshell CLI + // metadata can be stale after a manual `docker rm`. See #2020. Newer + // package-managed OpenShell gateways do not have an openshell-cluster-* + // Docker container, so the live CLI health check is the source of truth. + if (gatewayReuseState === "healthy" && gatewayCliSupportsLifecycleCommands(runCaptureOpenshell)) { const containerState = verifyGatewayContainerRunning(); if (containerState === "missing") { console.log(" Gateway metadata is stale (container not running). Cleaning up..."); runOpenshell(["forward", "stop", String(DASHBOARD_PORT)], { ignoreError: true }); - destroyGateway(); - registry.clearAll(); - gatewayReuseState = "missing"; - console.log(" ✓ Stale gateway metadata cleaned up"); + gatewayReuseState = destroyGatewayForReuse( + destroyGateway, + " ✓ Stale gateway metadata cleaned up", + " ! Stale gateway metadata cleanup failed; leaving registry state intact.", + ); } else if (containerState === "unknown") { // Docker probe failed but cached metadata says healthy. Try the host-level // HTTP probe — it doesn't depend on Docker, so it can confirm the gateway @@ -11079,10 +11057,11 @@ async function onboard(opts: OnboardOptions = {}): Promise { ` Gateway container is running but http://127.0.0.1:${GATEWAY_PORT}/ is not responding. Recreating...`, ); runOpenshell(["forward", "stop", String(DASHBOARD_PORT)], { ignoreError: true }); - destroyGateway(); - registry.clearAll(); - gatewayReuseState = "missing"; - console.log(" ✓ Stale gateway cleaned up"); + gatewayReuseState = destroyGatewayForReuse( + destroyGateway, + " ✓ Stale gateway cleaned up", + " ! Stale gateway cleanup failed; leaving registry state intact.", + ); } else { const imageDrift = getGatewayClusterImageDrift(); if (imageDrift) { @@ -11090,10 +11069,11 @@ async function onboard(opts: OnboardOptions = {}): Promise { ` Gateway image ${imageDrift.currentVersion} does not match openshell ${imageDrift.expectedVersion}. Recreating...`, ); stopAllDashboardForwards(); - destroyGateway(); - registry.clearAll(); - gatewayReuseState = "missing"; - console.log(" ✓ Previous gateway cleaned up"); + gatewayReuseState = destroyGatewayForReuse( + destroyGateway, + " ✓ Previous gateway cleaned up", + " ! Previous gateway cleanup failed; leaving registry state intact.", + ); } } } @@ -11723,6 +11703,7 @@ module.exports = { getGatewayReuseHealthWaitConfig, getGatewayReuseState, isDockerDriverGatewayPortListener, + isDockerDriverGatewayHttpReady, isGatewayHttpReady, waitForGatewayHttpReady, handleFinalGatewayStartFailure, diff --git a/src/lib/onboard/custom-build-context.ts b/src/lib/onboard/custom-build-context.ts new file mode 100644 index 0000000000..25c5370d77 --- /dev/null +++ b/src/lib/onboard/custom-build-context.ts @@ -0,0 +1,59 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import path from "node:path"; + +export const CUSTOM_BUILD_CONTEXT_WARN_BYTES = 100_000_000; + +const CUSTOM_BUILD_CONTEXT_IGNORES = new Set([ + "node_modules", + ".git", + ".venv", + "__pycache__", + ".aws", + ".credentials", + ".direnv", + ".netrc", + ".npmrc", + ".pypirc", + ".ssh", + "credentials.json", + "key.json", + "secrets", + "secrets.json", + "secrets.yaml", + "token.json", +]); + +function isIgnoredCustomBuildContextName(name: string): boolean { + const lowerName = name.toLowerCase(); + return ( + CUSTOM_BUILD_CONTEXT_IGNORES.has(lowerName) || + lowerName === ".env" || + lowerName === ".envrc" || + lowerName.startsWith(".env.") || + lowerName.endsWith(".key") || + lowerName.endsWith(".pem") || + lowerName.endsWith(".pfx") || + lowerName.endsWith(".p12") || + lowerName.endsWith(".jks") || + lowerName.endsWith(".keystore") || + lowerName.endsWith(".tfvars") || + lowerName.endsWith("_ecdsa") || + lowerName.endsWith("_ed25519") || + lowerName.endsWith("_rsa") || + (lowerName.startsWith("service-account") && lowerName.endsWith(".json")) + ); +} + +export function shouldIncludeCustomBuildContextPath(src: string): boolean { + return !isIgnoredCustomBuildContextName(path.basename(src)); +} + +export function isInsideIgnoredCustomBuildContextPath(src: string): boolean { + return path + .normalize(src) + .split(path.sep) + .filter(Boolean) + .some((part: string) => isIgnoredCustomBuildContextName(part)); +} diff --git a/src/lib/onboard/gateway-cleanup.ts b/src/lib/onboard/gateway-cleanup.ts new file mode 100644 index 0000000000..6879221a7a --- /dev/null +++ b/src/lib/onboard/gateway-cleanup.ts @@ -0,0 +1,31 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { GatewayReuseState } from "../state/gateway"; + +type DestroyGateway = () => boolean; + +export function destroyGatewayForReuse( + destroyGateway: DestroyGateway, + successMessage: string, + failureMessage: string, +): GatewayReuseState { + if (destroyGateway()) { + console.log(successMessage); + return "missing"; + } + console.warn(failureMessage); + return "stale"; +} + +export function warnIfGatewayDestroyFails( + destroyGateway: DestroyGateway, + successMessage: string, + failureMessage: string, +): void { + if (destroyGateway()) { + console.log(successMessage); + } else { + console.warn(failureMessage); + } +} diff --git a/src/lib/onboard/gateway-http-readiness.ts b/src/lib/onboard/gateway-http-readiness.ts index 93515175c2..7c19668026 100644 --- a/src/lib/onboard/gateway-http-readiness.ts +++ b/src/lib/onboard/gateway-http-readiness.ts @@ -11,6 +11,7 @@ */ import http from "node:http"; +import http2 from "node:http2"; import { GATEWAY_PORT } from "../core/ports"; import { sleepSeconds } from "../core/wait"; @@ -70,6 +71,7 @@ export function getGatewayReuseHealthWaitConfig(): { count: number; interval: nu export function isGatewayHttpReady( timeoutMs = ISGATEWAY_HTTP_READY_DEFAULT_TIMEOUT_MS, url = `http://127.0.0.1:${GATEWAY_PORT}/`, + method: "GET" | "POST" = "GET", ): Promise { const effectiveTimeout = Number.isFinite(timeoutMs) && timeoutMs > 0 @@ -83,7 +85,7 @@ export function isGatewayHttpReady( resolve(ready); }; const request = http - .get(url, (res) => { + .request(url, { method }, (res) => { res.resume(); const code = res.statusCode || 0; settle(GATEWAY_HTTP_ALIVE_CODES.has(code)); @@ -93,6 +95,95 @@ export function isGatewayHttpReady( request.destroy(); settle(false); }); + request.end(); + }); +} + +export function isDockerDriverGatewayHttpReady( + timeoutMs = ISGATEWAY_HTTP_READY_DEFAULT_TIMEOUT_MS, + url = `http://127.0.0.1:${GATEWAY_PORT}/openshell.v1.OpenShell/Health`, +): Promise { + const effectiveTimeout = + Number.isFinite(timeoutMs) && timeoutMs > 0 + ? Math.round(timeoutMs) + : ISGATEWAY_HTTP_READY_DEFAULT_TIMEOUT_MS; + let parsed: URL; + try { + parsed = new URL(url); + } catch { + return Promise.resolve(false); + } + + return new Promise((resolve) => { + let settled = false; + let statusCode = 0; + let contentType = ""; + let grpcStatus: string | undefined; + let client: http2.ClientHttp2Session | null = null; + let stream: http2.ClientHttp2Stream | null = null; + + const headerValue = (value: string | string[] | number | undefined): string => { + if (Array.isArray(value)) return value[0] ?? ""; + if (value == null) return ""; + return String(value); + }; + + const isHealthyResponse = () => + statusCode === 200 && + /^application\/grpc\b/i.test(contentType) && + (grpcStatus === undefined || grpcStatus === "0"); + + const settle = (ready: boolean) => { + if (settled) return; + settled = true; + clearTimeout(timer); + try { + stream?.close(); + } catch { + // best-effort cleanup + } + try { + client?.close(); + } catch { + // best-effort cleanup + } + resolve(ready); + }; + + const timer = setTimeout(() => settle(false), effectiveTimeout); + + try { + const origin = `${parsed.protocol}//${parsed.host}`; + client = http2.connect(origin); + client.on("error", () => settle(false)); + stream = client.request({ + [http2.constants.HTTP2_HEADER_METHOD]: http2.constants.HTTP2_METHOD_POST, + [http2.constants.HTTP2_HEADER_PATH]: `${parsed.pathname}${parsed.search}`, + [http2.constants.HTTP2_HEADER_SCHEME]: parsed.protocol.replace(":", ""), + [http2.constants.HTTP2_HEADER_AUTHORITY]: parsed.host, + [http2.constants.HTTP2_HEADER_CONTENT_TYPE]: "application/grpc", + [http2.constants.HTTP2_HEADER_TE]: "trailers", + }); + stream.on("response", (headers) => { + statusCode = Number(headers[http2.constants.HTTP2_HEADER_STATUS] || 0); + contentType = headerValue(headers[http2.constants.HTTP2_HEADER_CONTENT_TYPE]); + const status = headerValue(headers["grpc-status"]); + if (status) grpcStatus = status; + }); + stream.on("trailers", (headers) => { + const status = headerValue(headers["grpc-status"]); + if (status) grpcStatus = status; + }); + stream.on("data", () => { + // Drain the gRPC response body; the readiness signal is in headers/trailers. + }); + stream.on("error", () => settle(false)); + stream.on("end", () => settle(isHealthyResponse())); + // Empty protobuf message: one uncompressed gRPC frame with zero payload bytes. + stream.end(Buffer.alloc(5)); + } catch { + settle(false); + } }); } diff --git a/src/lib/onboard/gateway-lifecycle.ts b/src/lib/onboard/gateway-lifecycle.ts new file mode 100644 index 0000000000..5a9187295b --- /dev/null +++ b/src/lib/onboard/gateway-lifecycle.ts @@ -0,0 +1,28 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +type CaptureGatewayHelp = ( + args: string[], + opts: { ignoreError: true; suppressOutput: true }, +) => string; + +const ANSI_RE = /\x1B(?:\[[0-?]*[ -/]*[@-~]|\][^\x07]*(?:\x07|\x1B\\)|[@-_])/g; + +let gatewayLifecycleCommandsSupported: boolean | null = null; + +export function gatewayCliSupportsLifecycleCommands(captureGatewayHelp: CaptureGatewayHelp): boolean { + if (gatewayLifecycleCommandsSupported !== null) { + return gatewayLifecycleCommandsSupported; + } + + const help = captureGatewayHelp(["gateway", "--help"], { + ignoreError: true, + suppressOutput: true, + }); + const normalized = String(help || "").replace(ANSI_RE, ""); + gatewayLifecycleCommandsSupported = + normalized.trim().length > 0 && + /\bstart\b/.test(normalized) && + /\bdestroy\b/.test(normalized); + return gatewayLifecycleCommandsSupported; +} diff --git a/src/lib/onboard/gateway-tcp-readiness.test.ts b/src/lib/onboard/gateway-tcp-readiness.test.ts index ca4a9d025e..43774ea238 100644 --- a/src/lib/onboard/gateway-tcp-readiness.test.ts +++ b/src/lib/onboard/gateway-tcp-readiness.test.ts @@ -11,6 +11,7 @@ import net from "node:net"; import { afterEach, describe, expect, it } from "vitest"; +import { GATEWAY_PORT } from "../core/ports"; import { isGatewayTcpReady } from "./gateway-tcp-readiness"; // ── Helpers ───────────────────────────────────────────────────────────────── @@ -94,9 +95,9 @@ describe("isGatewayTcpReady (#3111)", () => { }); it("defaults to GATEWAY_PORT when no port is supplied", async () => { - // Nothing is listening on GATEWAY_PORT in test context; the call should - // resolve false (not throw) and do so without the caller threading the - // constant. This guards the default-parameter wiring. - await expect(isGatewayTcpReady(undefined, 200)).resolves.toBe(false); + // The development host may already have a gateway on GATEWAY_PORT. Compare + // the implicit and explicit calls instead of assuming the port is closed. + const implicit = await isGatewayTcpReady(undefined, 200); + await expect(isGatewayTcpReady(GATEWAY_PORT, 200)).resolves.toBe(implicit); }); }); diff --git a/src/lib/onboard/openshell-install.ts b/src/lib/onboard/openshell-install.ts index a40463f0ce..2b04a0b5fb 100644 --- a/src/lib/onboard/openshell-install.ts +++ b/src/lib/onboard/openshell-install.ts @@ -94,7 +94,7 @@ export function ensureOpenshellForOnboard(deps: OpenShellInstallDeps): OpenShell deps.exit(1); } } else { - const minOpenshellVersion = deps.getBlueprintMinOpenshellVersion() ?? "0.0.37"; + const minOpenshellVersion = deps.getBlueprintMinOpenshellVersion() ?? "0.0.39"; const currentVersionOutput = deps.runCaptureOpenshell(["--version"], { ignoreError: true }); const needsDevChannel = deps.isLinuxDockerDriverGatewayEnabled(platform, arch) && diff --git a/src/lib/onboard/usage-notice.ts b/src/lib/onboard/usage-notice.ts index 8fc5aca3f9..c7b7d6c452 100644 --- a/src/lib/onboard/usage-notice.ts +++ b/src/lib/onboard/usage-notice.ts @@ -63,7 +63,7 @@ function parseJson(text: string): T { // Reflect.get is used throughout the codebase as a type-safe alternative to // direct property access on loosely-typed objects. Unlike an `as Record<…>` // cast it never widens the target type and keeps loosely-typed member access -// explicit. See also: deploy.ts, onboard.ts, ws-proxy-fix.ts. +// explicit. See also: deploy.ts and onboard.ts. function readStringProperty(value: object | null, key: string): string | undefined { if (!value) { return undefined; diff --git a/src/lib/state/gateway.ts b/src/lib/state/gateway.ts index 908f9458fa..7d06f98fc8 100644 --- a/src/lib/state/gateway.ts +++ b/src/lib/state/gateway.ts @@ -78,10 +78,15 @@ export function getReportedGatewayName(output = ""): string | null { } export function isGatewayConnected(statusOutput = ""): boolean { - return ( - typeof statusOutput === "string" && - (statusOutput.includes("Connected") || statusOutput.includes("Server Status")) - ); + if (typeof statusOutput !== "string") return false; + const clean = stripAnsi(statusOutput); + if ( + /\b(Error|transport error|client error)\b/i.test(clean) || + /Connection refused|Connection reset|No active gateway/i.test(clean) + ) { + return false; + } + return clean.includes("Connected") || clean.includes("Server Status"); } export function hasActiveGatewayInfo(activeGatewayInfoOutput = ""): boolean { diff --git a/test/cli.test.ts b/test/cli.test.ts index 43c7b90064..68054bb479 100644 --- a/test/cli.test.ts +++ b/test/cli.test.ts @@ -418,10 +418,21 @@ describe("CLI dispatch", () => { }); it("bare unknown name surfaces sandbox-not-found (#2164)", testTimeoutOptions(35_000), () => { - // Longer timeout: when openshell is installed but the gateway is down, - // the CLI probes the gateway before reporting "not found" and the - // default 10s is not enough for the connection to time out. - const r = runWithEnv("boguscmd", {}, execTimeout(30_000)); + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-unknown-sandbox-")); + const localBin = path.join(home, "bin"); + fs.mkdirSync(localBin, { recursive: true }); + fs.writeFileSync(path.join(localBin, "openshell"), "#!/usr/bin/env bash\nexit 1\n", { + mode: 0o755, + }); + + const r = runWithEnv( + "boguscmd", + { + HOME: home, + PATH: `${localBin}:${process.env.PATH || ""}`, + }, + execTimeout(30_000), + ); expect(r.code).toBe(1); expect(r.out.includes("Sandbox 'boguscmd' does not exist")).toBeTruthy(); }); diff --git a/test/e2e/lib/discord-gateway-proof.sh b/test/e2e/lib/discord-gateway-proof.sh new file mode 100755 index 0000000000..ffe99ba85a --- /dev/null +++ b/test/e2e/lib/discord-gateway-proof.sh @@ -0,0 +1,395 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Shared hermetic Discord Gateway helpers for messaging E2E scripts. + +append_exit_trap_for_fake_discord_gateway() { + local command="$1" + local existing + existing="$(trap -p EXIT | sed "s/^trap -- '//;s/' EXIT$//")" + trap ''"${existing:+$existing; }$command"'' EXIT +} + +cleanup_fake_discord_gateway() { + if [ -n "${FAKE_DISCORD_GATEWAY_CONTAINER:-}" ]; then + docker rm -f "$FAKE_DISCORD_GATEWAY_CONTAINER" >/dev/null 2>&1 || true + fi + if [ -n "${FAKE_DISCORD_GATEWAY_PID:-}" ]; then + kill "$FAKE_DISCORD_GATEWAY_PID" 2>/dev/null || true + wait "$FAKE_DISCORD_GATEWAY_PID" 2>/dev/null || true + fi + if [ -n "${FAKE_DISCORD_GATEWAY_DIR:-}" ]; then + rm -rf "$FAKE_DISCORD_GATEWAY_DIR" 2>/dev/null || true + fi +} + +start_fake_discord_gateway() { + local expected_token="$1" + mkdir -p "$REPO/.tmp" + FAKE_DISCORD_GATEWAY_DIR="$(mktemp -d "$REPO/.tmp/fake-discord.XXXXXX")" + FAKE_DISCORD_GATEWAY_PORT_FILE="$FAKE_DISCORD_GATEWAY_DIR/port" + FAKE_DISCORD_GATEWAY_CAPTURE_FILE="$FAKE_DISCORD_GATEWAY_DIR/capture.jsonl" + FAKE_DISCORD_GATEWAY_CONTAINER="nemoclaw-fake-discord-$$-$RANDOM" + FAKE_DISCORD_GATEWAY_HOST="host.docker.internal" + : >"$FAKE_DISCORD_GATEWAY_CAPTURE_FILE" + + if ! docker run -d --rm \ + --name "$FAKE_DISCORD_GATEWAY_CONTAINER" \ + -p 0:8080 \ + -e FAKE_DISCORD_GATEWAY_PORT=8080 \ + -e FAKE_DISCORD_GATEWAY_EXPECTED_TOKEN="$expected_token" \ + -e FAKE_DISCORD_GATEWAY_PORT_FILE=/tmp/fake-discord/port \ + -e FAKE_DISCORD_GATEWAY_CAPTURE_FILE=/tmp/fake-discord/capture.jsonl \ + -v "$FAKE_DISCORD_GATEWAY_DIR:/tmp/fake-discord" \ + -v "$REPO/test/e2e/lib:/opt/nemoclaw-e2e:ro" \ + node:22-bookworm-slim \ + node /opt/nemoclaw-e2e/fake-discord-gateway.cjs \ + >"$FAKE_DISCORD_GATEWAY_DIR/container.id" 2>"$FAKE_DISCORD_GATEWAY_DIR/server.log"; then + cat "$FAKE_DISCORD_GATEWAY_DIR/server.log" >&2 || true + return 1 + fi + append_exit_trap_for_fake_discord_gateway cleanup_fake_discord_gateway + + for _ in $(seq 1 50); do + if [ -s "$FAKE_DISCORD_GATEWAY_PORT_FILE" ]; then + local published_port + published_port="$(docker port "$FAKE_DISCORD_GATEWAY_CONTAINER" 8080/tcp 2>/dev/null | head -1 | sed 's/.*://')" + if [ -n "$published_port" ]; then + # Exported for callers that source this helper and apply policy/probes after startup. + export FAKE_DISCORD_GATEWAY_PORT + FAKE_DISCORD_GATEWAY_PORT="$published_port" + return 0 + fi + fi + if ! docker inspect "$FAKE_DISCORD_GATEWAY_CONTAINER" >/dev/null 2>&1; then + docker logs "$FAKE_DISCORD_GATEWAY_CONTAINER" >&2 || true + cat "$FAKE_DISCORD_GATEWAY_DIR/server.log" >&2 || true + return 1 + fi + sleep 0.1 + done + cat "$FAKE_DISCORD_GATEWAY_DIR/server.log" >&2 || true + return 1 +} + +fake_discord_gateway_allowed_ip_options() { + printf '%s' 'allowed-ip=10.0.0.0/8,allowed-ip=172.16.0.0/12,allowed-ip=192.168.0.0/16' +} + +apply_fake_discord_gateway_policy() { + local sandbox_name="$1" + local port="$2" + local host="${FAKE_DISCORD_GATEWAY_HOST:-host.openshell.internal}" + local allowed_ip_options + allowed_ip_options="$(fake_discord_gateway_allowed_ip_options)" + openshell policy update "$sandbox_name" \ + --add-endpoint "${host}:${port}:read-write:websocket:enforce:websocket-credential-rewrite,${allowed_ip_options}" \ + --add-allow "${host}:${port}:GET:/**" \ + --add-allow "${host}:${port}:WEBSOCKET_TEXT:/**" \ + --binary /usr/local/bin/node \ + --binary /usr/bin/node \ + --binary /usr/local/bin/python3 \ + --binary /usr/bin/python3 \ + --binary /opt/hermes/.venv/bin/python \ + --wait +} + +run_fake_discord_gateway_node_client() { + local port="$1" + local identify_token="$2" + local host="${FAKE_DISCORD_GATEWAY_HOST:-host.openshell.internal}" + sandbox_exec_stdin "FAKE_DISCORD_GATEWAY_CLIENT_HOST='$host' FAKE_DISCORD_GATEWAY_CLIENT_PORT='$port' FAKE_DISCORD_GATEWAY_IDENTIFY_TOKEN='$identify_token' node - 2>&1" <<'NODE' +const crypto = require("crypto"); +const net = require("net"); + +const host = process.env.FAKE_DISCORD_GATEWAY_CLIENT_HOST || "host.openshell.internal"; +const port = Number(process.env.FAKE_DISCORD_GATEWAY_CLIENT_PORT); +const identifyToken = process.env.FAKE_DISCORD_GATEWAY_IDENTIFY_TOKEN; +const proxyUrl = process.env.HTTP_PROXY || process.env.http_proxy || ""; +const results = []; + +function proxyTarget() { + if (!proxyUrl) return null; + try { + const parsed = new URL(proxyUrl); + if (parsed.protocol !== "http:") return null; + return { + host: parsed.hostname, + port: Number(parsed.port || "80"), + }; + } catch { + return null; + } +} + +function finish(message) { + if (message) results.push(message); + console.log(results.join("\n")); + process.exit(0); +} + +function encodeClientText(payload) { + const body = Buffer.from(payload, "utf8"); + const mask = crypto.randomBytes(4); + const masked = Buffer.alloc(body.length); + for (let i = 0; i < body.length; i += 1) masked[i] = body[i] ^ mask[i % 4]; + if (body.length < 126) { + return Buffer.concat([Buffer.from([0x81, 0x80 | body.length]), mask, masked]); + } + const header = Buffer.alloc(4); + header[0] = 0x81; + header[1] = 0x80 | 126; + header.writeUInt16BE(body.length, 2); + return Buffer.concat([header, mask, masked]); +} + +function encodeClientClose(code) { + const body = Buffer.alloc(2); + body.writeUInt16BE(code, 0); + const mask = crypto.randomBytes(4); + for (let i = 0; i < body.length; i += 1) body[i] ^= mask[i % 4]; + return Buffer.concat([Buffer.from([0x88, 0x80 | 2]), mask, body]); +} + +function decodeFrame(buffer) { + if (buffer.length < 2) return null; + const opcode = buffer[0] & 0x0f; + let payloadLength = buffer[1] & 0x7f; + let offset = 2; + if (payloadLength === 126) { + if (buffer.length < 4) return null; + payloadLength = buffer.readUInt16BE(2); + offset = 4; + } else if (payloadLength === 127) { + if (buffer.length < 10) return null; + payloadLength = Number(buffer.readBigUInt64BE(2)); + offset = 10; + } + if (buffer.length < offset + payloadLength) return null; + return { + opcode, + payload: buffer.slice(offset, offset + payloadLength), + totalLength: offset + payloadLength, + }; +} + +const proxy = proxyTarget(); +const socket = proxy + ? net.createConnection({ host: proxy.host, port: proxy.port }) + : net.createConnection({ host, port }); +const timer = setTimeout(() => { + try { socket.destroy(); } catch {} + finish("TIMEOUT"); +}, 20000); + +let handshake = Buffer.alloc(0); +let framed = Buffer.alloc(0); +let upgraded = false; +let sawReady = false; + +socket.on("connect", () => { + const key = crypto.randomBytes(16).toString("base64"); + const requestTarget = proxy + ? `http://${host}:${port}/gateway?v=10&encoding=json` + : "/gateway?v=10&encoding=json"; + socket.write([ + `GET ${requestTarget} HTTP/1.1`, + `Host: ${host}:${port}`, + "Upgrade: websocket", + "Connection: Upgrade", + `Sec-WebSocket-Key: ${key}`, + "Sec-WebSocket-Version: 13", + "\r\n", + ].join("\r\n")); +}); + +socket.on("data", (chunk) => { + if (!upgraded) { + handshake = Buffer.concat([handshake, chunk]); + const end = handshake.indexOf("\r\n\r\n"); + if (end === -1) return; + const statusLine = handshake.slice(0, end).toString("latin1").split("\r\n")[0] || ""; + if (!statusLine.includes("101")) { + clearTimeout(timer); + finish(`HTTP_${statusLine}`); + } + upgraded = true; + results.push("UPGRADE"); + framed = Buffer.concat([framed, handshake.slice(end + 4)]); + } else { + framed = Buffer.concat([framed, chunk]); + } + + while (framed.length > 0) { + const frame = decodeFrame(framed); + if (!frame) break; + framed = framed.slice(frame.totalLength); + if (frame.opcode === 1) { + const message = JSON.parse(frame.payload.toString("utf8")); + if (message.op === 10) { + results.push("HELLO"); + socket.write(encodeClientText(JSON.stringify({ + op: 2, + d: { + token: identifyToken, + intents: 0, + properties: { os: "linux", browser: "nemoclaw-e2e", device: "nemoclaw-e2e" }, + }, + }))); + results.push( + identifyToken.includes("openshell:resolve:env:") + ? "IDENTIFY_SENT_PLACEHOLDER" + : "IDENTIFY_SENT_NON_PLACEHOLDER", + ); + } else if (message.op === 0 && message.t === "READY") { + sawReady = true; + results.push("READY"); + socket.write(encodeClientText(JSON.stringify({ op: 1, d: message.s ?? null }))); + } else if (message.op === 11) { + results.push("HEARTBEAT_ACK"); + socket.write(encodeClientClose(1000)); + clearTimeout(timer); + finish(); + } + } else if (frame.opcode === 8) { + const code = frame.payload.length >= 2 ? frame.payload.readUInt16BE(0) : 0; + clearTimeout(timer); + finish(`CLOSE_${code}`); + } + } +}); + +socket.on("error", (error) => { + clearTimeout(timer); + finish(`ERROR ${error.message}`); +}); +socket.on("close", () => { + clearTimeout(timer); + if (!sawReady) finish("CLOSED"); +}); +NODE +} + +run_fake_discord_gateway_python_client() { + local port="$1" + local host="${FAKE_DISCORD_GATEWAY_HOST:-host.openshell.internal}" + sandbox_exec_stdin "FAKE_DISCORD_GATEWAY_CLIENT_HOST='$host' FAKE_DISCORD_GATEWAY_CLIENT_PORT='$port' /opt/hermes/.venv/bin/python - 2>&1" <<'PY' +import asyncio +import inspect +import os +from pathlib import Path + +try: + import aiohttp + import discord + from discord.http import DiscordClientWebSocketResponse + from yarl import URL +except Exception as exc: + print(f"IMPORT_DISCORD_FAILED {type(exc).__name__}: {exc}") + raise SystemExit(0) + + +def read_env_token(): + env_text = Path("/sandbox/.hermes/.env").read_text(encoding="utf-8") + for line in env_text.splitlines(): + if line.startswith("DISCORD_BOT_TOKEN="): + return line.split("=", 1)[1] + raise RuntimeError("missing DISCORD_BOT_TOKEN in /sandbox/.hermes/.env") + + +def note_heartbeat_ack(ws, results, previous_ack=None): + keep_alive = getattr(ws, "_keep_alive", None) + if keep_alive is None: + return False + current_ack = getattr(keep_alive, "_last_ack", None) + latency = getattr(keep_alive, "latency", float("inf")) + if previous_ack is not None and current_ack == previous_ack: + return False + if latency == float("inf"): + return False + if "HEARTBEAT_ACK" not in results: + results.append("HEARTBEAT_ACK") + return True + + +async def wait_for_ready(ws, results): + for _ in range(20): + await ws.poll_event() + note_heartbeat_ack(ws, results) + if getattr(ws, "session_id", None): + results.append("READY") + return + raise AssertionError("timed out waiting for READY") + + +async def wait_for_heartbeat_ack(ws, results): + if "HEARTBEAT_ACK" in results: + return + keep_alive = getattr(ws, "_keep_alive", None) + previous_ack = getattr(keep_alive, "_last_ack", None) + for _ in range(20): + await ws.poll_event() + if note_heartbeat_ack(ws, results, previous_ack): + return + raise AssertionError("timed out waiting for HEARTBEAT_ACK") + + +async def main(): + port = int(os.environ["FAKE_DISCORD_GATEWAY_CLIENT_PORT"]) + host = os.environ.get("FAKE_DISCORD_GATEWAY_CLIENT_HOST", "host.openshell.internal") + token = read_env_token() + results = [] + client = discord.Client(intents=discord.Intents.none()) + setup = getattr(client, "_async_setup_hook", None) + if setup is not None: + await setup() + client.http.token = token + client.http.proxy = os.environ.get("HTTP_PROXY") or os.environ.get("http_proxy") + client.http.proxy_auth = None + if getattr(client.http, "connector", None) is discord.utils.MISSING: + client.http.connector = aiohttp.TCPConnector(limit=0) + setattr( + client.http, + "_HTTPClient__session", + aiohttp.ClientSession( + connector=client.http.connector, + ws_response_class=DiscordClientWebSocketResponse, + trace_configs=None, + cookie_jar=aiohttp.DummyCookieJar(), + ), + ) + client.http._global_over = asyncio.Event() + client.http._global_over.set() + try: + from_client = discord.gateway.DiscordWebSocket.from_client + kwargs = {"gateway": URL(f"ws://{host}:{port}/gateway")} + params = inspect.signature(from_client).parameters + if "initial" in params: + kwargs["initial"] = False + if "compress" in params: + kwargs["compress"] = False + elif "zlib" in params: + kwargs["zlib"] = False + ws = await from_client(client, **kwargs) + results.append("UPGRADE") + results.append("HELLO") + if "openshell:resolve:env:" in token: + results.append("IDENTIFY_SENT_PLACEHOLDER") + await wait_for_ready(ws, results) + await ws.send_as_json({"op": 1, "d": ws.sequence}) + await wait_for_heartbeat_ack(ws, results) + close = getattr(ws, "close", None) + if close is not None: + await close(code=1000) + finally: + await client.close() + print("\n".join(results)) + + +try: + asyncio.run(main()) +except Exception as exc: + print(f"ERROR {type(exc).__name__}: {exc}") +PY +} diff --git a/test/e2e/lib/fake-discord-gateway.cjs b/test/e2e/lib/fake-discord-gateway.cjs new file mode 100755 index 0000000000..e987f1e06a --- /dev/null +++ b/test/e2e/lib/fake-discord-gateway.cjs @@ -0,0 +1,219 @@ +#!/usr/bin/env node +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +"use strict"; + +const crypto = require("crypto"); +const fs = require("fs"); +const net = require("net"); + +const host = process.env.FAKE_DISCORD_GATEWAY_HOST || "0.0.0.0"; +const port = Number(process.env.FAKE_DISCORD_GATEWAY_PORT || "0"); +const portFile = process.env.FAKE_DISCORD_GATEWAY_PORT_FILE || ""; +const captureFile = process.env.FAKE_DISCORD_GATEWAY_CAPTURE_FILE || ""; +const expectedToken = process.env.FAKE_DISCORD_GATEWAY_EXPECTED_TOKEN || ""; + +if (!expectedToken) { + console.error("FAKE_DISCORD_GATEWAY_EXPECTED_TOKEN is required"); + process.exit(2); +} + +function record(event) { + if (!captureFile) return; + fs.appendFileSync(captureFile, `${JSON.stringify({ at: Date.now(), ...event })}\n`); +} + +function encodeText(payload) { + const body = Buffer.from(payload, "utf8"); + if (body.length < 126) { + return Buffer.concat([Buffer.from([0x81, body.length]), body]); + } + if (body.length <= 0xffff) { + const header = Buffer.alloc(4); + header[0] = 0x81; + header[1] = 126; + header.writeUInt16BE(body.length, 2); + return Buffer.concat([header, body]); + } + const header = Buffer.alloc(10); + header[0] = 0x81; + header[1] = 127; + header.writeBigUInt64BE(BigInt(body.length), 2); + return Buffer.concat([header, body]); +} + +function encodeClose(code) { + const body = Buffer.alloc(2); + body.writeUInt16BE(code, 0); + return Buffer.from([0x88, body.length, ...body]); +} + +function decodeFrame(buffer) { + if (buffer.length < 2) return null; + const opcode = buffer[0] & 0x0f; + const masked = (buffer[1] & 0x80) !== 0; + let payloadLength = buffer[1] & 0x7f; + let offset = 2; + + if (payloadLength === 126) { + if (buffer.length < 4) return null; + payloadLength = buffer.readUInt16BE(2); + offset = 4; + } else if (payloadLength === 127) { + if (buffer.length < 10) return null; + payloadLength = Number(buffer.readBigUInt64BE(2)); + offset = 10; + } + + let mask; + if (masked) { + if (buffer.length < offset + 4) return null; + mask = buffer.slice(offset, offset + 4); + offset += 4; + } + if (buffer.length < offset + payloadLength) return null; + + const payload = Buffer.from(buffer.slice(offset, offset + payloadLength)); + if (masked && mask) { + for (let i = 0; i < payload.length; i += 1) { + payload[i] ^= mask[i % 4]; + } + } + return { + opcode, + payload, + totalLength: offset + payloadLength, + }; +} + +function sendJson(socket, payload) { + socket.write(encodeText(JSON.stringify(payload))); +} + +function handleGatewayMessage(socket, payload) { + let message; + try { + message = JSON.parse(payload.toString("utf8")); + } catch (error) { + record({ event: "malformed_text", error: error.message }); + socket.write(encodeClose(4002)); + socket.end(); + return; + } + + if (message.op === 2) { + const token = message && message.d && message.d.token; + record({ + event: "identify", + token, + tokenMatchesExpected: token === expectedToken, + tokenLooksPlaceholder: typeof token === "string" && token.includes("openshell:resolve:env:"), + }); + if (token !== expectedToken) { + socket.write(encodeClose(4004)); + socket.end(); + return; + } + sendJson(socket, { + op: 0, + t: "READY", + s: 1, + d: { + session_id: "fake-discord-gateway-session", + resume_gateway_url: "ws://host.openshell.internal/gateway", + user: { + id: "0", + username: "nemoclaw-fake-gateway", + discriminator: "0000", + avatar: null, + bot: true, + }, + guilds: [], + }, + }); + return; + } + + if (message.op === 1) { + record({ event: "heartbeat", d: message.d ?? null }); + sendJson(socket, { op: 11, d: null }); + return; + } + + record({ event: "gateway_message", op: message.op ?? null }); +} + +const server = net.createServer((socket) => { + let handshake = Buffer.alloc(0); + let framed = Buffer.alloc(0); + let upgraded = false; + + socket.on("data", (chunk) => { + if (!upgraded) { + handshake = Buffer.concat([handshake, chunk]); + const end = handshake.indexOf("\r\n\r\n"); + if (end === -1) return; + + const request = handshake.slice(0, end).toString("latin1"); + const requestLine = request.split("\r\n")[0] || ""; + const keyLine = request + .split("\r\n") + .find((line) => line.toLowerCase().startsWith("sec-websocket-key:")); + const key = keyLine ? keyLine.slice(keyLine.indexOf(":") + 1).trim() : ""; + if (!key) { + socket.end("HTTP/1.1 400 Bad Request\r\nConnection: close\r\n\r\n"); + return; + } + + const accept = crypto + .createHash("sha1") + .update(`${key}258EAFA5-E914-47DA-95CA-C5AB0DC85B11`) + .digest("base64"); + socket.write( + [ + "HTTP/1.1 101 Switching Protocols", + "Upgrade: websocket", + "Connection: Upgrade", + `Sec-WebSocket-Accept: ${accept}`, + "\r\n", + ].join("\r\n"), + ); + upgraded = true; + record({ event: "upgrade", requestLine }); + sendJson(socket, { op: 10, d: { heartbeat_interval: 30000 } }); + framed = Buffer.concat([framed, handshake.slice(end + 4)]); + } else { + framed = Buffer.concat([framed, chunk]); + } + + while (framed.length > 0) { + const frame = decodeFrame(framed); + if (!frame) break; + framed = framed.slice(frame.totalLength); + if (frame.opcode === 0x1) { + handleGatewayMessage(socket, frame.payload); + } else if (frame.opcode === 0x8) { + socket.write(encodeClose(1000)); + socket.end(); + } else if (frame.opcode === 0x9) { + socket.write(Buffer.from([0x8a, 0x00])); + } + } + }); +}); + +server.listen(port, host, () => { + const address = server.address(); + if (portFile) { + fs.writeFileSync(portFile, `${address.port}\n`, { mode: 0o600 }); + } + record({ event: "listening", host, port: address.port }); +}); + +for (const signal of ["SIGTERM", "SIGINT"]) { + process.on(signal, () => { + server.close(() => process.exit(0)); + setTimeout(() => process.exit(0), 1000).unref(); + }); +} diff --git a/test/e2e/lib/fake-slack-api.cjs b/test/e2e/lib/fake-slack-api.cjs new file mode 100755 index 0000000000..d805330298 --- /dev/null +++ b/test/e2e/lib/fake-slack-api.cjs @@ -0,0 +1,118 @@ +#!/usr/bin/env node +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +"use strict"; + +const fs = require("fs"); +const http = require("http"); + +const host = process.env.FAKE_SLACK_API_HOST || "0.0.0.0"; +const rawPort = process.env.FAKE_SLACK_API_PORT || "0"; +const port = Number(rawPort); +const portFile = process.env.FAKE_SLACK_API_PORT_FILE || ""; +const captureFile = process.env.FAKE_SLACK_API_CAPTURE_FILE || ""; +const expectedBotToken = process.env.FAKE_SLACK_API_EXPECTED_BOT_TOKEN || ""; +const expectedAppToken = process.env.FAKE_SLACK_API_EXPECTED_APP_TOKEN || ""; +const MAX_BODY_BYTES = 1024 * 1024; + +if (!Number.isInteger(port) || port < 0 || port > 65535) { + console.error(`FAKE_SLACK_API_PORT must be an integer between 0 and 65535 (received: ${rawPort})`); + process.exit(2); +} + +if (!expectedBotToken || !expectedAppToken) { + console.error("FAKE_SLACK_API_EXPECTED_BOT_TOKEN and FAKE_SLACK_API_EXPECTED_APP_TOKEN are required"); + process.exit(2); +} + +function record(event) { + if (!captureFile) return; + fs.appendFileSync(captureFile, `${JSON.stringify({ at: Date.now(), ...event })}\n`); +} + +function expectedTokenForPath(pathname) { + if (pathname === "/api/apps.connections.open") return expectedAppToken; + return expectedBotToken; +} + +const server = http.createServer((req, res) => { + const chunks = []; + let bodyBytes = 0; + let bodyTooLarge = false; + req.on("data", (chunk) => { + if (bodyTooLarge) return; + bodyBytes += chunk.length; + if (bodyBytes > MAX_BODY_BYTES) { + bodyTooLarge = true; + record({ + event: "request-too-large", + method: req.method, + path: new URL(req.url || "/", "http://fake-slack.local").pathname, + bodyBytes, + }); + res.writeHead(413, { "content-type": "application/json" }); + res.end(JSON.stringify({ ok: false, error: "payload_too_large" })); + req.destroy(); + return; + } + chunks.push(chunk); + }); + req.on("end", () => { + if (bodyTooLarge) return; + const body = Buffer.concat(chunks).toString("utf8"); + const pathname = new URL(req.url || "/", "http://fake-slack.local").pathname; + const authorization = req.headers.authorization || ""; + const expectedToken = expectedTokenForPath(pathname); + const expectedAuthorization = `Bearer ${expectedToken}`; + const bodyToken = new URLSearchParams(body).get("token") || ""; + const tokenMatchesExpected = authorization === expectedAuthorization; + const bodyMatchesExpected = bodyToken === expectedToken; + const authAccepted = tokenMatchesExpected && bodyMatchesExpected; + const tokenLooksPlaceholder = + typeof authorization === "string" && + (authorization.includes("openshell:resolve:env:") || + authorization.includes("OPENSHELL-RESOLVE-ENV-") || + body.includes("openshell:resolve:env:") || + body.includes("OPENSHELL-RESOLVE-ENV-")); + + record({ + event: "request", + method: req.method, + path: pathname, + tokenMatchesExpected, + bodyMatchesExpected, + tokenLooksPlaceholder, + authorizationPresent: Boolean(authorization), + bodyTokenPresent: Boolean(bodyToken), + authorizationRedacted: true, + bodyRedacted: true, + }); + + res.writeHead(authAccepted ? 200 : 401, { + "content-type": "application/json", + }); + res.end( + JSON.stringify({ + ok: false, + error: authAccepted ? "invalid_auth" : "bad_auth", + endpoint: pathname, + }), + ); + }); +}); + +server.listen(port, host, () => { + const address = server.address(); + if (portFile) { + fs.writeFileSync(portFile, `${address.port}\n`, { mode: 0o600 }); + } + record({ event: "listening", host, port: address.port }); +}); + +for (const signal of ["SIGTERM", "SIGINT"]) { + process.on(signal, () => { + server.close(() => process.exit(0)); + setTimeout(() => process.exit(0), 1000).unref(); + }); +} diff --git a/test/e2e/lib/slack-api-proof.sh b/test/e2e/lib/slack-api-proof.sh new file mode 100755 index 0000000000..901eaab1c2 --- /dev/null +++ b/test/e2e/lib/slack-api-proof.sh @@ -0,0 +1,140 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Shared hermetic Slack REST helpers for messaging E2E scripts. + +append_exit_trap_for_fake_slack_api() { + local command="$1" + local existing + existing="$(trap -p EXIT | sed "s/^trap -- '//;s/' EXIT$//")" + trap ''"${existing:+$existing; }$command"'' EXIT +} + +cleanup_fake_slack_api() { + if [ -n "${FAKE_SLACK_API_CONTAINER:-}" ]; then + docker rm -f "$FAKE_SLACK_API_CONTAINER" >/dev/null 2>&1 || true + fi + if [ -n "${FAKE_SLACK_API_PID:-}" ]; then + kill "$FAKE_SLACK_API_PID" 2>/dev/null || true + wait "$FAKE_SLACK_API_PID" 2>/dev/null || true + fi + if [ -n "${FAKE_SLACK_API_DIR:-}" ]; then + rm -rf "$FAKE_SLACK_API_DIR" 2>/dev/null || true + fi +} + +start_fake_slack_api() { + local bot_token="$1" + local app_token="$2" + mkdir -p "$REPO/.tmp" + FAKE_SLACK_API_DIR="$(mktemp -d "$REPO/.tmp/fake-slack.XXXXXX")" + FAKE_SLACK_API_PORT_FILE="$FAKE_SLACK_API_DIR/port" + FAKE_SLACK_API_CAPTURE_FILE="$FAKE_SLACK_API_DIR/capture.jsonl" + FAKE_SLACK_API_CONTAINER="nemoclaw-fake-slack-$$-$RANDOM" + FAKE_SLACK_API_HOST="host.docker.internal" + : >"$FAKE_SLACK_API_CAPTURE_FILE" + + if ! docker run -d --rm \ + --name "$FAKE_SLACK_API_CONTAINER" \ + -p 0:8080 \ + -e FAKE_SLACK_API_PORT=8080 \ + -e FAKE_SLACK_API_EXPECTED_BOT_TOKEN="$bot_token" \ + -e FAKE_SLACK_API_EXPECTED_APP_TOKEN="$app_token" \ + -e FAKE_SLACK_API_PORT_FILE=/tmp/fake-slack/port \ + -e FAKE_SLACK_API_CAPTURE_FILE=/tmp/fake-slack/capture.jsonl \ + -v "$FAKE_SLACK_API_DIR:/tmp/fake-slack" \ + -v "$REPO/test/e2e/lib:/opt/nemoclaw-e2e:ro" \ + node:22-bookworm-slim \ + node /opt/nemoclaw-e2e/fake-slack-api.cjs \ + >"$FAKE_SLACK_API_DIR/container.id" 2>"$FAKE_SLACK_API_DIR/server.log"; then + cat "$FAKE_SLACK_API_DIR/server.log" >&2 || true + return 1 + fi + append_exit_trap_for_fake_slack_api cleanup_fake_slack_api + + for _ in $(seq 1 50); do + if [ -s "$FAKE_SLACK_API_PORT_FILE" ]; then + local published_port + published_port="$(docker port "$FAKE_SLACK_API_CONTAINER" 8080/tcp 2>/dev/null | head -1 | sed 's/.*://')" + if [ -n "$published_port" ]; then + # Exported for callers that source this helper and apply policy/probes after startup. + export FAKE_SLACK_API_PORT + FAKE_SLACK_API_PORT="$published_port" + return 0 + fi + fi + if ! docker inspect "$FAKE_SLACK_API_CONTAINER" >/dev/null 2>&1; then + docker logs "$FAKE_SLACK_API_CONTAINER" >&2 || true + cat "$FAKE_SLACK_API_DIR/server.log" >&2 || true + return 1 + fi + sleep 0.1 + done + cat "$FAKE_SLACK_API_DIR/server.log" >&2 || true + return 1 +} + +fake_slack_api_allowed_ip_options() { + printf '%s' 'allowed-ip=10.0.0.0/8,allowed-ip=172.16.0.0/12,allowed-ip=192.168.0.0/16' +} + +apply_fake_slack_api_policy() { + local sandbox_name="$1" + local port="$2" + local host="${FAKE_SLACK_API_HOST:-host.openshell.internal}" + local allowed_ip_options + allowed_ip_options="$(fake_slack_api_allowed_ip_options)" + openshell policy update "$sandbox_name" \ + --add-endpoint "${host}:${port}:read-write:rest:enforce:request-body-credential-rewrite,${allowed_ip_options}" \ + --add-allow "${host}:${port}:GET:/**" \ + --add-allow "${host}:${port}:POST:/**" \ + --binary /usr/local/bin/node \ + --binary /usr/bin/node \ + --wait +} + +run_fake_slack_api_node_request() { + local port="$1" + local path="$2" + local authorization="$3" + local host="${FAKE_SLACK_API_HOST:-host.openshell.internal}" + sandbox_exec_stdin "FAKE_SLACK_API_HOST='$host' FAKE_SLACK_API_PORT='$port' FAKE_SLACK_API_PATH='$path' FAKE_SLACK_API_AUTH='$authorization' node - 2>&1" <<'NODE' +const http = require("http"); + +const authorization = process.env.FAKE_SLACK_API_AUTH || ""; +const token = authorization.replace(/^Bearer\s+/, ""); +const data = `token=${encodeURIComponent(token)}`; +const options = { + hostname: process.env.FAKE_SLACK_API_HOST || "host.openshell.internal", + port: Number(process.env.FAKE_SLACK_API_PORT), + path: process.env.FAKE_SLACK_API_PATH, + method: "POST", + headers: { + Authorization: authorization, + "Content-Type": "application/x-www-form-urlencoded", + "Content-Length": data.length, + }, +}; + +const req = http.request(options, (res) => { + let body = ""; + res.on("data", (d) => { + body += d; + }); + res.on("end", () => { + console.log(`${res.statusCode} ${body.slice(0, 300)}`); + }); +}); + +req.on("error", (error) => { + console.log(`ERROR: ${error.message}`); +}); +req.setTimeout(30000, () => { + req.destroy(); + console.log("TIMEOUT"); +}); +req.write(data); +req.end(); +NODE +} diff --git a/test/e2e/test-credential-migration.sh b/test/e2e/test-credential-migration.sh index d9908f0146..64215d1cb4 100755 --- a/test/e2e/test-credential-migration.sh +++ b/test/e2e/test-credential-migration.sh @@ -99,8 +99,8 @@ if [ -z "${NVIDIA_API_KEY:-}" ]; then fi pass "NVIDIA_API_KEY is set" -if ! command -v openshell >/dev/null 2>&1; then - info "openshell not found; running install" +if ! command -v openshell >/dev/null 2>&1 || ! command -v nemoclaw >/dev/null 2>&1; then + info "openshell or nemoclaw not found; running install" bash "$REPO/install.sh" --yes-i-accept-third-party-software \ >/tmp/nemoclaw-e2e-install.log 2>&1 || { fail "install.sh failed; see /tmp/nemoclaw-e2e-install.log" diff --git a/test/e2e/test-hermes-discord-e2e.sh b/test/e2e/test-hermes-discord-e2e.sh index 3a16e5fc8b..ac248d357f 100755 --- a/test/e2e/test-hermes-discord-e2e.sh +++ b/test/e2e/test-hermes-discord-e2e.sh @@ -3,13 +3,14 @@ # SPDX-License-Identifier: Apache-2.0 # Hermes Discord E2E: onboard --agent hermes with Discord enabled, then verify -# the Hermes sandbox has the schema and placeholder/token isolation required by -# NVIDIA/NemoClaw#3032. +# the Hermes sandbox has the schema, placeholder/token isolation, and native +# OpenShell WebSocket Gateway rewrite path required by NVIDIA/NemoClaw#3032. # # Uses a fake Discord token by default. The fake token should never appear in # /sandbox/.hermes/config.yaml, /sandbox/.hermes/.env, sandbox env, sandbox # process args, or sandbox filesystem. The sandbox should hold only the -# OpenShell resolver placeholder. +# OpenShell resolver placeholder. Gateway proof uses a hermetic fake Discord +# Gateway on the host, not a local in-sandbox facade or live Discord token. # # Environment variables: # NEMOCLAW_NON_INTERACTIVE=1 - required @@ -18,6 +19,8 @@ # NEMOCLAW_POLICY_TIER=open - auto-set if not already set # NEMOCLAW_SANDBOX_NAME - sandbox name (default: e2e-hermes-discord) # NEMOCLAW_RECREATE_SANDBOX=1 - auto-set +# NEMOCLAW_FRESH=1 - auto-set to discard interrupted onboard sessions +# NEMOCLAW_OPENSHELL_BIN - optional OpenShell binary under test # NVIDIA_API_KEY - required for Hermes onboarding # DISCORD_BOT_TOKEN - defaults to a fake token # DISCORD_SERVER_IDS - defaults to a fake snowflake @@ -70,7 +73,7 @@ run_with_timeout() { dump_hermes_discord_diagnostics() { info "--- Hermes Discord sandbox diagnostics ---" - if ! command -v openshell >/dev/null 2>&1; then + if ! openshell --version >/dev/null 2>&1; then info "openshell is not available for sandbox diagnostics" return fi @@ -94,10 +97,9 @@ dump_hermes_discord_diagnostics() { diag_script+='; echo "== hermes health =="; curl -sf http://localhost:8642/health 2>&1 || true' diag_script+='; echo "== hermes-related processes =="' # shellcheck disable=SC2016 # script is intentionally evaluated inside the sandbox - diag_script+='; for p in /proc/[0-9]*; do cmd=$(tr "\000" " " < "$p/cmdline" 2>/dev/null || true); case "$cmd" in *hermes*|*socat*|*nemoclaw-decode-proxy*|*nemoclaw-discord-facade*) echo "$(basename "$p") $cmd" ;; esac; done' + diag_script+='; for p in /proc/[0-9]*; do cmd=$(tr "\000" " " < "$p/cmdline" 2>/dev/null || true); case "$cmd" in *hermes*|*socat*) echo "$(basename "$p") $cmd" ;; esac; done' diag_script+='; echo "== /tmp/nemoclaw-start.log tail =="; tail -n 80 /tmp/nemoclaw-start.log 2>&1 || true' diag_script+='; echo "== /tmp/gateway.log tail =="; tail -n 120 /tmp/gateway.log 2>&1 || true' - diag_script+='; echo "== /tmp/discord-facade.log tail =="; tail -n 120 /tmp/discord-facade.log 2>&1 || true' diag_output=$(openshell sandbox exec -n "$SANDBOX_NAME" -- sh -lc "$diag_script" 2>&1 || true) echo "$diag_output" | while IFS= read -r line; do @@ -159,11 +161,21 @@ else fi SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-hermes-discord}" +OPENSHELL_BIN="${NEMOCLAW_OPENSHELL_BIN:-openshell}" DISCORD_TOKEN="${DISCORD_BOT_TOKEN:-test-fake-discord-token-hermes-e2e}" + +openshell() { + if [ "$OPENSHELL_BIN" = "openshell" ]; then + command openshell "$@" + else + "$OPENSHELL_BIN" "$@" + fi +} export NEMOCLAW_AGENT="${NEMOCLAW_AGENT:-hermes}" export NEMOCLAW_POLICY_TIER="${NEMOCLAW_POLICY_TIER:-open}" export NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" export NEMOCLAW_RECREATE_SANDBOX=1 +export NEMOCLAW_FRESH=1 export DISCORD_BOT_TOKEN="$DISCORD_TOKEN" export DISCORD_SERVER_IDS="${DISCORD_SERVER_IDS:-1491590992753590594}" export DISCORD_ALLOWED_IDS="${DISCORD_ALLOWED_IDS:-1005536447329222676}" @@ -173,6 +185,9 @@ export DISCORD_REQUIRE_MENTION="${DISCORD_REQUIRE_MENTION:-0}" . "$(dirname "${BASH_SOURCE[0]}")/lib/sandbox-teardown.sh" register_sandbox_for_teardown "$SANDBOX_NAME" +# shellcheck source=test/e2e/lib/discord-gateway-proof.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib/discord-gateway-proof.sh" + section "Phase 0: Prerequisites" if docker info >/dev/null 2>&1; then @@ -221,7 +236,7 @@ info "Pre-cleanup..." if command -v nemoclaw >/dev/null 2>&1; then nemoclaw "$SANDBOX_NAME" destroy --yes 2>/dev/null || true fi -if command -v openshell >/dev/null 2>&1; then +if openshell --version >/dev/null 2>&1; then openshell sandbox delete "$SANDBOX_NAME" 2>/dev/null || true openshell gateway destroy -g nemoclaw 2>/dev/null || true fi @@ -268,7 +283,7 @@ else exit 1 fi -if command -v openshell >/dev/null 2>&1; then +if openshell --version >/dev/null 2>&1; then pass "openshell installed ($(openshell --version 2>&1 || echo unknown))" else fail "openshell not found on PATH after install" @@ -372,24 +387,18 @@ fi env_probe=$( sandbox_exec_stdin "EXPECTED_ALLOWED_USERS=$expected_allowed_users EXPECTED_GUILD_IDS=$expected_guild_ids python3 -" <<'PY' import os -import re from pathlib import Path text = Path("/sandbox/.hermes/.env").read_text(encoding="utf-8") -lines = set(text.splitlines()) errors = [] -token_pattern = re.compile(r"^DISCORD_BOT_TOKEN=openshell:resolve:env:(?:v[0-9]+_)?DISCORD_BOT_TOKEN$") -if not any(token_pattern.match(line) for line in lines): - errors.append("missing DISCORD_BOT_TOKEN=openshell:resolve:env:DISCORD_BOT_TOKEN") required = [ - "DISCORD_PROXY=http://127.0.0.1:3129", - "NEMOCLAW_DISCORD_FACADE_URL=http://127.0.0.1:3130", + "DISCORD_BOT_TOKEN=openshell:resolve:env:DISCORD_BOT_TOKEN", f"NEMOCLAW_DISCORD_GUILD_IDS={os.environ['EXPECTED_GUILD_IDS']}", f"DISCORD_ALLOWED_USERS={os.environ['EXPECTED_ALLOWED_USERS']}", ] for line in required: - if line not in lines: + if line not in text.splitlines(): errors.append(f"missing {line}") -if "API_SERVER_PORT=18642" not in lines: +if "API_SERVER_PORT=18642" not in text.splitlines(): errors.append("missing API_SERVER_PORT") if errors: print("FAIL " + "; ".join(errors)) @@ -399,118 +408,52 @@ PY ) if [ "$env_probe" = "OK" ]; then - pass ".hermes/.env contains Discord placeholder, proxy bridge, and allowed users" + pass ".hermes/.env contains Discord placeholder and allowed users" else fail ".hermes/.env check failed: ${env_probe:0:400}" fi -gateway_proxy_log=$(sandbox_exec "grep -F 'Using proxy for Discord: http://127.0.0.1:3129' /tmp/gateway.log 2>/dev/null | tail -1 || true") -if [ -n "$gateway_proxy_log" ]; then - info "Hermes Discord proxy diagnostic: ${gateway_proxy_log:0:200}" +fake_gateway_ready=0 +if start_fake_discord_gateway "$DISCORD_TOKEN"; then + fake_gateway_ready=1 + pass "Hermetic fake Discord Gateway started on host port ${FAKE_DISCORD_GATEWAY_PORT}" else - info "Hermes Discord proxy diagnostic log line not present; relying on env, facade, and REST checks" + fail "Failed to start hermetic fake Discord Gateway" fi -facade_health="" -for facade_attempt in $(seq 1 15); do - facade_health=$(sandbox_exec "curl -sf http://127.0.0.1:3130/health 2>/dev/null || true") - if echo "$facade_health" | grep -qi '"ok":true'; then - break - fi - if [ "$facade_attempt" -lt 15 ]; then - info "Facade health check attempt ${facade_attempt}/15 - waiting 4s..." - sleep 4 - fi -done -if echo "$facade_health" | grep -qi '"ok":true'; then - pass "Hermes fake Discord facade is healthy inside the sandbox" +if [ "$fake_gateway_ready" = "1" ] \ + && apply_fake_discord_gateway_policy "$SANDBOX_NAME" "$FAKE_DISCORD_GATEWAY_PORT" >/tmp/nemoclaw-hermes-fake-discord-policy.log 2>&1; then + pass "Applied native WebSocket policy with credential rewrite for Hermes fake Discord Gateway" else - fail "Hermes fake Discord facade did not answer health probe: ${facade_health:0:200}" -fi - -facade_protocol=$( - cat <<'PY' | sandbox_exec_stdin 'NEMOCLAW_DISCORD_FACADE_URL=http://127.0.0.1:3130 PYTHONPATH=/opt/nemoclaw-hermes-discord-preload python3 - 2>&1' 2>/dev/null || true -import asyncio -import json - -from aiohttp import ClientSession, WSMsgType - -PLACEHOLDER = "openshell:resolve:env:DISCORD_BOT_TOKEN" - - -async def receive_json(ws, label): - msg = await ws.receive(timeout=5) - if msg.type != WSMsgType.TEXT: - raise AssertionError(f"{label}: expected text frame, got {msg.type} {msg.data!r}") - return json.loads(msg.data) - - -async def main(): - async with ClientSession() as session: - async with session.get("https://discord.com/api/v10/gateway") as response: - data = await response.json() - assert response.status == 200, data - assert data["url"] == "ws://127.0.0.1:3130/gateway", data - - async with session.ws_connect("wss://gateway.discord.gg/?v=10&encoding=json") as ws: - hello = await receive_json(ws, "HELLO") - assert hello["op"] == 10, hello - - await ws.send_json({ - "op": 2, - "d": { - "token": PLACEHOLDER, - "intents": 0, - "properties": { - "os": "linux", - "browser": "nemoclaw-e2e", - "device": "nemoclaw-e2e", - }, - }, - }) - ready = await receive_json(ws, "READY") - assert ready["op"] == 0 and ready["t"] == "READY", ready - assert ready["d"]["resume_gateway_url"] == "ws://127.0.0.1:3130/gateway", ready - - await ws.send_json({"op": 1, "d": ready.get("s")}) - ack = await receive_json(ws, "HEARTBEAT_ACK") - assert ack["op"] == 11, ack - - async with session.ws_connect("wss://gateway.discord.gg/?v=10&encoding=json") as ws: - hello = await receive_json(ws, "reject HELLO") - assert hello["op"] == 10, hello - await ws.send_json({ - "op": 2, - "d": { - "token": "not-the-openshell-placeholder", - "intents": 0, - "properties": { - "os": "linux", - "browser": "nemoclaw-e2e", - "device": "nemoclaw-e2e", - }, - }, - }) - close = await ws.receive(timeout=5) - assert close.type in (WSMsgType.CLOSE, WSMsgType.CLOSED, WSMsgType.CLOSING), close - assert ws.close_code == 4004 or close.data == 4004, (ws.close_code, close.data) - - async with session.post("http://127.0.0.1:3130/interactions", json={"type": 1}) as response: - assert response.status == 401, await response.text() - - print("OK preload_rest=local preload_gateway=ready heartbeat=ack reject=4004 unsigned_interaction=401") - - -asyncio.run(main()) -PY -) -info "Fake Discord facade protocol probe: ${facade_protocol:0:300}" -if echo "$facade_protocol" | grep -q "OK preload_rest=local preload_gateway=ready heartbeat=ack reject=4004 unsigned_interaction=401"; then - pass "Hermes Discord preload and fake Gateway protocol path work inside the sandbox" -elif echo "$facade_protocol" | grep -q "ModuleNotFoundError"; then - fail "Hermes Discord facade protocol probe could not import required Python modules: ${facade_protocol:0:300}" + fail "Failed to apply Hermes fake Discord Gateway policy: $(tail -20 /tmp/nemoclaw-hermes-fake-discord-policy.log 2>/dev/null | tr '\n' ' ' | cut -c1-300)" +fi + +native_gateway_protocol="" +if [ "$fake_gateway_ready" = "1" ]; then + native_gateway_protocol=$(run_fake_discord_gateway_python_client "$FAKE_DISCORD_GATEWAY_PORT" || true) +fi +info "Hermes native Discord Gateway protocol probe: ${native_gateway_protocol:0:400}" +if echo "$native_gateway_protocol" | grep -q "^UPGRADE$" \ + && echo "$native_gateway_protocol" | grep -q "^HELLO$" \ + && echo "$native_gateway_protocol" | grep -q "^IDENTIFY_SENT_PLACEHOLDER$" \ + && echo "$native_gateway_protocol" | grep -q "^READY$" \ + && echo "$native_gateway_protocol" | grep -q "^HEARTBEAT_ACK$"; then + pass "Hermes Python Discord Gateway path reaches READY through native OpenShell WebSocket policy" +elif echo "$native_gateway_protocol" | grep -q "IMPORT_DISCORD_FAILED"; then + fail "Hermes native Gateway probe could not import discord.py: ${native_gateway_protocol:0:300}" +else + fail "Hermes native Gateway protocol probe failed: ${native_gateway_protocol:0:300}" +fi + +if [ "$fake_gateway_ready" = "1" ] \ + && grep -Fq "\"token\":\"$DISCORD_TOKEN\"" "$FAKE_DISCORD_GATEWAY_CAPTURE_FILE" \ + && ! grep -Fq "openshell:resolve:env:DISCORD_BOT_TOKEN" "$FAKE_DISCORD_GATEWAY_CAPTURE_FILE"; then + pass "Hermes fake Gateway received host-side Discord token while sandbox sent only the placeholder" else - fail "Hermes Discord facade protocol probe failed: ${facade_protocol:0:300}" + if [ "$fake_gateway_ready" = "1" ]; then + info "Hermes fake Gateway capture: $(tail -20 "$FAKE_DISCORD_GATEWAY_CAPTURE_FILE" 2>/dev/null | tr '\n' ' ' | cut -c1-500)" + fi + fail "Hermes fake Gateway did not prove WebSocket placeholder rewrite" fi token_file_hits=$(printf '%s' "$DISCORD_TOKEN" | sandbox_exec_stdin 'grep -Fq -f - /sandbox/.hermes/config.yaml /sandbox/.hermes/.env 2>/dev/null && echo LEAK || echo OK') @@ -527,12 +470,10 @@ if [ -z "$sandbox_env_all" ]; then skip "Sandbox environment dump is empty" elif echo "$sandbox_env_all" | grep -qF "$DISCORD_TOKEN"; then fail "Raw Discord token found in sandbox environment" -elif ! echo "$sandbox_env_all" | grep -qx "DISCORD_PROXY=http://127.0.0.1:3129"; then - fail "Sandbox environment missing DISCORD_PROXY bridge setting" -elif ! echo "$sandbox_env_all" | grep -qx "NEMOCLAW_DISCORD_FACADE_URL=http://127.0.0.1:3130"; then - fail "Sandbox environment missing fake Discord facade setting" +elif echo "$sandbox_env_all" | grep -q "^DISCORD_PROXY="; then + fail "Sandbox environment still contains DISCORD_PROXY bridge setting" else - pass "Raw Discord token absent from sandbox environment; Discord proxy and facade settings are present" + pass "Raw Discord token absent from sandbox environment; no DISCORD_PROXY bridge setting" fi sandbox_ps=$(sandbox_exec 'cat /proc/[0-9]*/cmdline 2>/dev/null | tr "\0" "\n"') @@ -556,49 +497,29 @@ section "Phase 6: Discord REST placeholder egress" dc_api=$(sandbox_exec 'NODE_NO_WARNINGS=1 node -e " const fs = require(\"fs\"); const https = require(\"https\"); -let token = \"\"; -const runtimeToken = process.env.DISCORD_BOT_TOKEN || \"\"; -if (runtimeToken.startsWith(\"openshell:resolve:env:\")) { - token = runtimeToken; -} -if (!token) { - const env = fs.readFileSync(\"/sandbox/.hermes/.env\", \"utf8\"); - const line = env.split(/\\n/).find((entry) => entry.startsWith(\"DISCORD_BOT_TOKEN=\")); - token = line ? line.slice(\"DISCORD_BOT_TOKEN=\".length) : runtimeToken; -} +const env = fs.readFileSync(\"/sandbox/.hermes/.env\", \"utf8\"); +const line = env.split(/\\n/).find((entry) => entry.startsWith(\"DISCORD_BOT_TOKEN=\")); +const token = line ? line.slice(\"DISCORD_BOT_TOKEN=\".length) : \"\"; if (!token) { console.log(JSON.stringify({ error: \"missing_token\" })); process.exit(0); } -const maxAttempts = 5; -let attempt = 0; -function probe() { - attempt += 1; - const req = https.request({ - hostname: \"discord.com\", - path: \"/api/v10/users/@me\", - method: \"GET\", - headers: { \"Authorization\": \"Bot \" + token }, - }, (res) => { - let body = \"\"; - res.on(\"data\", (d) => body += d); - res.on(\"end\", () => { - if (res.statusCode === 503 && attempt < maxAttempts) { - setTimeout(probe, 2000); - return; - } - console.log(JSON.stringify({ - statusCode: res.statusCode, - body: body.slice(0, 200), - attempt, - })); - }); - }); - req.on(\"error\", (e) => console.log(JSON.stringify({ error: e.message, attempt }))); - req.setTimeout(20000, () => { req.destroy(); console.log(JSON.stringify({ error: \"timeout\", attempt })); }); - req.end(); -} -probe(); +const req = https.request({ + hostname: \"discord.com\", + path: \"/api/v10/users/@me\", + method: \"GET\", + headers: { \"Authorization\": \"Bot \" + token }, +}, (res) => { + let body = \"\"; + res.on(\"data\", (d) => body += d); + res.on(\"end\", () => console.log(JSON.stringify({ + statusCode: res.statusCode, + body: body.slice(0, 200), + }))); +}); +req.on(\"error\", (e) => console.log(JSON.stringify({ error: e.message }))); +req.setTimeout(20000, () => { req.destroy(); console.log(JSON.stringify({ error: \"timeout\" })); }); +req.end(); "' 2>/dev/null || true) info "Discord users/@me response: ${dc_api:0:300}" @@ -629,48 +550,33 @@ else fail "Unexpected Discord API response: ${dc_api:0:300}" fi -section "Phase 7: Discord gateway auth boundary" - -gateway_connected_status="" -for gw_attempt in $(seq 1 10); do - gateway_connected_status=$( - sandbox_exec_stdin 'python3 -' <<'PY' -import json -from pathlib import Path - -path = Path("/sandbox/.hermes/gateway_state.json") -try: - payload = json.loads(path.read_text(encoding="utf-8")) -except FileNotFoundError: - print("MISSING /sandbox/.hermes/gateway_state.json") -except Exception as exc: - print(f"ERROR reading gateway_state.json: {type(exc).__name__}: {exc}") -else: - platforms = payload.get("platforms") if isinstance(payload, dict) else {} - discord = platforms.get("discord") if isinstance(platforms, dict) else {} - if isinstance(discord, dict) and discord.get("state") == "connected": - print("CONNECTED") - else: - print(json.dumps( - { - "gateway_state": payload.get("gateway_state") if isinstance(payload, dict) else None, - "discord": discord if isinstance(discord, dict) else None, - }, - sort_keys=True, - )) -PY - ) - [ "$gateway_connected_status" = "CONNECTED" ] && break - if [ "$gw_attempt" -lt 10 ]; then - info "Gateway runtime status check attempt ${gw_attempt}/10 - waiting 3s..." - sleep 3 - fi -done -if [ "$gateway_connected_status" = "CONNECTED" ]; then - pass "Hermes Discord gateway reached READY through the fake local Gateway" +section "Phase 7: No local Discord bridge" + +# shellcheck disable=SC2016 # Remote script is intentionally single-quoted for sandbox execution. +facade_residue=$(sandbox_exec 'set +e +env_needle="$(printf "%s%s" "NEMOCLAW_DISCORD_" "FACADE_URL")" +name_needle="$(printf "%s%s" "nemoclaw-discord-" "facade")" +proxy_needle="$(printf "%s" "DISCORD_PROXY")" +decode_needle="$(printf "%s%s%s" "nemoclaw-" "decode" "-proxy")" +if env | grep -q "$env_needle"; then echo ENV_FACADE; fi +if env | grep -q "^${proxy_needle}="; then echo ENV_DISCORD_PROXY; fi +if grep -Fq "$env_needle" /sandbox/.hermes/.env /sandbox/.hermes/config.yaml /tmp/nemoclaw-proxy-env.sh /tmp/gateway.env 2>/dev/null; then echo FILE_FACADE; fi +if grep -Fq "$proxy_needle" /sandbox/.hermes/.env /sandbox/.hermes/config.yaml /tmp/nemoclaw-proxy-env.sh /tmp/gateway.env 2>/dev/null; then echo FILE_DISCORD_PROXY; fi +if find /tmp -maxdepth 1 -type f \( -name "discord-facade.log" -o -name "nemoclaw-discord-facade*" \) 2>/dev/null | grep -q .; then echo FILE_FACADE; fi +if command -v "$decode_needle" >/dev/null 2>&1; then echo BIN_DECODE_PROXY; fi +current_pid="$$" +for p in /proc/[0-9]*; do + pid=$(basename "$p") + [ "$pid" = "$current_pid" ] && continue + cmd=$(tr "\000" " " < "$p/cmdline" 2>/dev/null || true) + case "$cmd" in *"name_needle="*|*"for p in /proc/"*) continue ;; esac + case "$cmd" in *"$name_needle"*) echo PROCESS_FACADE ;; esac + case "$cmd" in *"$decode_needle"*) echo PROCESS_DECODE_PROXY ;; esac +done') +if [ -z "$facade_residue" ]; then + pass "Hermes Discord proof used native WebSocket policy with no local facade, decode proxy, or DISCORD_PROXY residue" else - info "Hermes Discord runtime status: ${gateway_connected_status:0:400}" - fail "Hermes Discord gateway did not reach READY through the fake local Gateway" + fail "Local Discord bridge residue found after native Gateway proof: ${facade_residue:0:300}" dump_hermes_discord_diagnostics fi @@ -698,7 +604,7 @@ echo " Total: $TOTAL" echo "========================================" if [ "$FAIL" -eq 0 ]; then - printf '\n\033[1;32m Hermes Discord E2E PASSED - schema, placeholder, provider, and sandbox boot verified.\033[0m\n' + printf '\n\033[1;32m Hermes Discord E2E PASSED - schema, placeholder, provider, sandbox boot, and native Gateway rewrite verified.\033[0m\n' exit 0 else printf '\n\033[1;31m %d test(s) failed.\033[0m\n' "$FAIL" diff --git a/test/e2e/test-hermes-e2e.sh b/test/e2e/test-hermes-e2e.sh index 21328caf9e..54b702e10e 100755 --- a/test/e2e/test-hermes-e2e.sh +++ b/test/e2e/test-hermes-e2e.sh @@ -78,7 +78,7 @@ dump_hermes_diagnostics() { diag_script+='; echo "== log and state paths =="; ls -ld /tmp /sandbox/.hermes /sandbox/.hermes/logs 2>&1 || true; ls -l /tmp/nemoclaw-start.log /tmp/gateway.log 2>&1 || true' diag_script+='; echo "== hermes-related processes =="' # shellcheck disable=SC2016 # script is intentionally evaluated inside the sandbox - diag_script+='; for p in /proc/[0-9]*; do cmd=$(tr "\000" " " < "$p/cmdline" 2>/dev/null || true); case "$cmd" in *hermes*|*socat*|*nemoclaw-decode-proxy*) echo "$(basename "$p") $cmd" ;; esac; done' + diag_script+='; for p in /proc/[0-9]*; do cmd=$(tr "\000" " " < "$p/cmdline" 2>/dev/null || true); case "$cmd" in *hermes*|*socat*) echo "$(basename "$p") $cmd" ;; esac; done' diag_script+='; echo "== /tmp/nemoclaw-start.log tail =="; tail -n 80 /tmp/nemoclaw-start.log 2>&1 || true' diag_script+='; echo "== /tmp/gateway.log tail =="; tail -n 120 /tmp/gateway.log 2>&1 || true' diag_output=$(openshell sandbox exec -n "$SANDBOX_NAME" -- sh -lc "$diag_script" 2>&1 || true) diff --git a/test/e2e/test-hermes-slack-e2e.sh b/test/e2e/test-hermes-slack-e2e.sh index 89a0e79190..808de20b6a 100755 --- a/test/e2e/test-hermes-slack-e2e.sh +++ b/test/e2e/test-hermes-slack-e2e.sh @@ -90,7 +90,7 @@ dump_hermes_slack_diagnostics() { diag_script+='; echo "== hermes health =="; curl -sf http://localhost:8642/health 2>&1 || true' diag_script+='; echo "== hermes-related processes =="' # shellcheck disable=SC2016 - diag_script+='; for p in /proc/[0-9]*; do cmd=$(tr "\000" " " < "$p/cmdline" 2>/dev/null || true); case "$cmd" in *hermes*|*socat*|*nemoclaw-decode-proxy*) echo "$(basename "$p") $cmd" ;; esac; done' + diag_script+='; for p in /proc/[0-9]*; do cmd=$(tr "\000" " " < "$p/cmdline" 2>/dev/null || true); case "$cmd" in *hermes*|*socat*) echo "$(basename "$p") $cmd" ;; esac; done' diag_script+='; echo "== /tmp/nemoclaw-start.log tail =="; tail -n 80 /tmp/nemoclaw-start.log 2>&1 || true' diag_script+='; echo "== /tmp/gateway.log tail =="; tail -n 120 /tmp/gateway.log 2>&1 || true' diag_output=$(openshell sandbox exec -n "$SANDBOX_NAME" -- sh -lc "$diag_script" 2>&1 || true) @@ -420,11 +420,38 @@ if policy_output=$(openshell policy get --full "$SANDBOX_NAME" 2>&1); then else fail "Slack policy missing Socket Mode websocket hosts" fi + + if echo "$slack_block" | grep -Fq "request_body_credential_rewrite: true"; then + pass "Slack REST policy enables OpenShell request-body credential rewrite" + else + fail "Slack policy missing request_body_credential_rewrite for REST alias rewrite" + fi else fail "openshell policy get failed: ${policy_output:0:200}" fi -section "Phase 6: Slack placeholder egress from Python" +# shellcheck disable=SC2016 +bridge_residue=$(sandbox_exec 'set +e +decode_needle="$(printf "%s%s%s" "nemoclaw-" "decode" "-proxy")" +preload_needle="$(printf "%s" "/opt/nemoclaw-hermes-discord-preload")" +if env | grep -Fq "$preload_needle"; then echo ENV_PYTHON_PRELOAD; fi +if grep -Fq "$preload_needle" /tmp/nemoclaw-proxy-env.sh /sandbox/.hermes/.env /sandbox/.hermes/config.yaml 2>/dev/null; then echo FILE_PYTHON_PRELOAD; fi +if command -v "$decode_needle" >/dev/null 2>&1; then echo BIN_DECODE_PROXY; fi +current_pid="$$" +for p in /proc/[0-9]*; do + pid=$(basename "$p") + [ "$pid" = "$current_pid" ] && continue + cmd=$(tr "\000" " " < "$p/cmdline" 2>/dev/null || true) + case "$cmd" in *"$decode_needle"*) echo PROCESS_DECODE_PROXY ;; esac +done') +if [ -z "$bridge_residue" ]; then + pass "Hermes Slack sandbox has no decode proxy or Python placeholder-normalization preload" +else + fail "Hermes Slack bridge residue found: ${bridge_residue:0:300}" + dump_hermes_slack_diagnostics +fi + +section "Phase 6: Slack alias egress from Python" slack_probe=$( sandbox_exec_stdin 'sh -lc ". /tmp/nemoclaw-proxy-env.sh 2>/dev/null || true; if [ -x /opt/hermes/.venv/bin/python ]; then exec /opt/hermes/.venv/bin/python -; fi; exec python3 -" 2>&1' <<'PY' @@ -507,7 +534,7 @@ PY info "Slack Python probe response: ${slack_probe:0:500}" if echo "$slack_probe" | grep -q "^OK auth.test:" \ && echo "$slack_probe" | grep -q "^OK apps.connections.open:"; then - pass "Slack API reached from Python through OpenShell placeholder substitution" + pass "Slack API reached from Python through OpenShell alias substitution" elif echo "$slack_probe" | grep -q "^TIMEOUT"; then skip "Slack API timed out" elif echo "$slack_probe" | grep -qE "^(FAIL|ERROR)"; then diff --git a/test/e2e/test-launchable-smoke.sh b/test/e2e/test-launchable-smoke.sh index 98e5ee89ca..bbb04cf113 100755 --- a/test/e2e/test-launchable-smoke.sh +++ b/test/e2e/test-launchable-smoke.sh @@ -249,6 +249,13 @@ section "Phase 3: Verify installation artifacts" # and Node.js via nodesource. On the GH runner the shell may not have # picked up the new PATH entries yet. export PATH="/usr/local/bin:$PATH" +if [ "${GITHUB_ACTIONS:-}" = "true" ] \ + && [ "${GITHUB_REPOSITORY:-}" = "NVIDIA/NemoClaw" ] \ + && [ "${GITHUB_REF:-}" = "refs/heads/fix/native-messaging-websocket" ] \ + && [ -n "${NEMOCLAW_OPENSHELL_BIN:-}" ]; then + main_openshell_dir="$(dirname "$NEMOCLAW_OPENSHELL_BIN")" + export PATH="$main_openshell_dir:$PATH" +fi hash -r 2>/dev/null || true # 3a: nemoclaw on PATH and --help works diff --git a/test/e2e/test-messaging-providers.sh b/test/e2e/test-messaging-providers.sh index 1ca44b1639..8c7fa85462 100755 --- a/test/e2e/test-messaging-providers.sh +++ b/test/e2e/test-messaging-providers.sh @@ -18,7 +18,7 @@ # process list, or filesystem # 4. Config patching — openclaw.json channels use placeholder values # 5. Network reachability — Node.js can reach messaging APIs through proxy -# 6. Native Discord gateway path — WebSocket path is probed separately from REST +# 6. Native Discord gateway path — WebSocket L7 path is tested hermetically # 7. L7 proxy rewriting — placeholder is rewritten to real token at egress # # Uses fake tokens by default (no external accounts needed). With fake tokens, @@ -47,7 +47,8 @@ # SLACK_BOT_TOKEN_REVOKED — optional: revoked xoxb- token to test auth pre-validation (#2340) # SLACK_APP_TOKEN_REVOKED — optional: paired xapp- token for the revoked bot token # TELEGRAM_CHAT_ID_E2E — optional: enables sendMessage test -# NEMOCLAW_E2E_STRICT_DISCORD_GATEWAY — fail instead of skip on known Discord gateway blockers +# NEMOCLAW_OPENSHELL_BIN — optional OpenShell binary under test +# NEMOCLAW_FRESH=1 — auto-set to discard interrupted onboard sessions # # Usage: # NEMOCLAW_NON_INTERACTIVE=1 NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ @@ -82,6 +83,9 @@ section() { printf '\033[1;36m=== %s ===\033[0m\n' "$1" } info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } +is_unresolved_placeholder_rejection() { + printf '%s\n' "$1" | grep -qiE 'credential_injection_failed|unresolved credential placeholder' +} # Determine repo root if [ -d /workspace ] && [ -f /workspace/install.sh ]; then @@ -94,6 +98,15 @@ else fi SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-msg-provider}" +OPENSHELL_BIN="${NEMOCLAW_OPENSHELL_BIN:-openshell}" + +openshell() { + if [ "$OPENSHELL_BIN" = "openshell" ]; then + command openshell "$@" + else + "$OPENSHELL_BIN" "$@" + fi +} # shellcheck source=test/e2e/lib/sandbox-teardown.sh . "$(dirname "${BASH_SOURCE[0]}")/lib/sandbox-teardown.sh" @@ -153,6 +166,11 @@ sandbox_exec() { echo "$result" } +# shellcheck source=test/e2e/lib/discord-gateway-proof.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib/discord-gateway-proof.sh" +# shellcheck source=test/e2e/lib/slack-api-proof.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib/slack-api-proof.sh" + # ══════════════════════════════════════════════════════════════════ # Phase 0: Prerequisites # ══════════════════════════════════════════════════════════════════ @@ -175,7 +193,6 @@ info "Discord token: ${DISCORD_TOKEN:0:10}... (${#DISCORD_TOKEN} chars)" info "Slack bot token: configured (${#SLACK_TOKEN} chars)" info "Slack app token: configured (${#SLACK_APP} chars)" info "Sandbox name: $SANDBOX_NAME" -STRICT_DISCORD_GATEWAY="${NEMOCLAW_E2E_STRICT_DISCORD_GATEWAY:-0}" # ══════════════════════════════════════════════════════════════════ # Phase 1: Install NemoClaw (non-interactive mode) @@ -189,12 +206,19 @@ info "Pre-cleanup..." if command -v nemoclaw >/dev/null 2>&1; then nemoclaw "$SANDBOX_NAME" destroy --yes 2>/dev/null || true fi -if command -v openshell >/dev/null 2>&1; then +if openshell --version >/dev/null 2>&1; then openshell sandbox delete "$SANDBOX_NAME" 2>/dev/null || true openshell gateway destroy -g nemoclaw 2>/dev/null || true fi pass "Pre-cleanup complete" +if [ -z "${NEMOCLAW_SKIP_TELEGRAM_REACHABILITY:-}" ]; then + if ! curl -fsS --max-time 10 https://api.telegram.org/ >/dev/null 2>&1; then + export NEMOCLAW_SKIP_TELEGRAM_REACHABILITY=1 + info "Host cannot reach api.telegram.org; skipping onboarding Telegram reachability probe for fake-token E2E" + fi +fi + # Pre-merge Slack policy into the base sandbox policy. # # The base policy (openclaw-sandbox.yaml) includes Telegram and Discord @@ -249,12 +273,18 @@ if [ -f "$BASE_POLICY" ] && [ -f "$SLACK_PRESET" ] && ! grep -q "api.slack.com" - allow: { method: POST, path: "/**" } - host: wss-primary.slack.com port: 443 - access: full - tls: skip + protocol: websocket + enforcement: enforce + rules: + - allow: { method: GET, path: "/**" } + - allow: { method: WEBSOCKET_TEXT, path: "/**" } - host: wss-backup.slack.com port: 443 - access: full - tls: skip + protocol: websocket + enforcement: enforce + rules: + - allow: { method: GET, path: "/**" } + - allow: { method: WEBSOCKET_TEXT, path: "/**" } binaries: - { path: /usr/local/bin/node } - { path: /usr/bin/node } @@ -282,6 +312,7 @@ info "Expected duration: 5-10 minutes on first run." export NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" export NEMOCLAW_RECREATE_SANDBOX=1 +export NEMOCLAW_FRESH=1 INSTALL_LOG="/tmp/nemoclaw-e2e-install.log" bash install.sh --non-interactive >"$INSTALL_LOG" 2>&1 & @@ -317,7 +348,7 @@ else fi # Verify tools are on PATH -if ! command -v openshell >/dev/null 2>&1; then +if ! openshell --version >/dev/null 2>&1; then fail "openshell not found on PATH after install" exit 1 fi @@ -482,10 +513,9 @@ else fi # ── Slack credential isolation (#2085) ──────────────────────────── -# Mirrors M5a/M5e/M5g for Slack now that the apply_slack_token_override -# carve-out has been replaced by the in-process slack-token-rewriter + -# L7-proxy substitution. The host-side fake token must never appear on -# any observable surface inside the sandbox. +# Mirrors M5a/M5e/M5g for Slack now that provider-shaped aliases are resolved +# directly by OpenShell. The host-side fake token must never appear on any +# observable surface inside the sandbox. # M-S5a: Full environment dump must not contain the real Slack bot token. if [ -z "$sandbox_env_all" ]; then @@ -538,8 +568,7 @@ if [ -n "$SLACK_APP" ]; then fi # M-S5f: openclaw.json must contain the Bolt-shape placeholder, not the -# real token. The placeholder is what nemoclaw-slack-token-rewriter.js -# translates to the canonical openshell:resolve:env:VAR form on egress. +# real token. OpenShell resolves the provider-shaped alias directly on egress. config_slack=$(sandbox_exec "cat /sandbox/.openclaw/openclaw.json 2>/dev/null | grep -E '\"(bot|app)Token\"'" 2>/dev/null || true) if [ -n "$config_slack" ] && { echo "$config_slack" | grep -qF "$SLACK_TOKEN" \ @@ -554,76 +583,13 @@ else skip "M-S5f: Could not extract Slack token fields from openclaw.json" fi -# M-S5g: The rewriter preload was actually installed. NODE_OPTIONS in the -# sandbox shell should reference the rewriter path. +# M-S5g: No Slack transport bridge should be installed. NODE_OPTIONS may still +# include non-transport resilience guards, but not the removed token rewriter. sandbox_node_opts=$(openshell sandbox exec --name "$SANDBOX_NAME" -- bash -lc 'echo "$NODE_OPTIONS"' 2>/dev/null || echo "") if echo "$sandbox_node_opts" | grep -q "nemoclaw-slack-token-rewriter.js"; then - pass "M-S5g: Slack token rewriter preload present in sandbox NODE_OPTIONS" -else - fail "M-S5g: rewriter preload missing from NODE_OPTIONS (got: ${sandbox_node_opts:0:200})" -fi - -# M-S5h: The rewriter actually wraps http.request at runtime. NODE_OPTIONS -# pointing at an empty file (or a syntax-error file) would still make -# M-S5g pass and a subsequent slack.com round-trip would still return -# invalid_auth (because the un-translated Bolt-shape token is not a valid -# Slack token either) — so the slack.com 200 invalid_auth in M-S15/M-S16 -# alone doesn't prove the rewriter ran. This loopback probe forces a -# definitive answer: send a Bolt-shape Authorization header and urlencoded -# token body to a 127.0.0.1 listener (loopback bypasses the L7 proxy), have -# the listener echo what it actually received, then assert the placeholder is -# gone. If the rewriter is loaded and wrapping http.request/write/end, the -# listener sees the active openshell:resolve:env:VAR placeholder form. If the -# rewriter is a no-op, the listener sees the raw Bolt-shape placeholder. -info "Probing rewriter via loopback listener (proves http.request is wrapped)..." -sl_loopback=$(sandbox_exec 'NODE_NO_WARNINGS=1 node -e " -const http = require(\"http\"); -const server = http.createServer((req, res) => { - let body = \"\"; - req.setEncoding(\"utf8\"); - req.on(\"data\", (d) => body += d); - req.on(\"end\", () => { - res.writeHead(200, { \"Content-Type\": \"application/json\" }); - res.end(JSON.stringify({ headers: req.headers, body })); - }); -}); -server.listen(0, \"127.0.0.1\", () => { - const port = server.address().port; - const data = \"token=xoxb-OPENSHELL-RESOLVE-ENV-SLACK_BOT_TOKEN\"; - const r = http.request({ - hostname: \"127.0.0.1\", - port: port, - path: \"/probe\", - method: \"POST\", - headers: { - \"Authorization\": \"Bearer xoxb-OPENSHELL-RESOLVE-ENV-SLACK_BOT_TOKEN\", - \"Content-Type\": \"application/x-www-form-urlencoded\", - \"Content-Length\": Buffer.byteLength(data), - }, - }, (res) => { - let body = \"\"; - res.on(\"data\", (d) => body += d); - res.on(\"end\", () => { console.log(body); server.close(); }); - }); - r.on(\"error\", (e) => { console.log(\"ERROR: \" + e.message); server.close(); }); - r.setTimeout(10000, () => { r.destroy(); console.log(\"TIMEOUT\"); server.close(); }); - r.write(data); - r.end(); -}); -"' 2>/dev/null || true) - -info "Loopback echoed request: ${sl_loopback:0:300}" -if echo "$sl_loopback" | grep -qF 'OPENSHELL-RESOLVE-ENV-'; then - fail "M-S5h: rewriter did NOT translate Bolt-shape on http.request/write/end — the preload is loaded but incomplete or a no-op" -elif echo "$sl_loopback" | grep -qE '"authorization"\s*:\s*"Bearer openshell:resolve:env:(v[0-9]+_)?SLACK_BOT_TOKEN' \ - && echo "$sl_loopback" | grep -qE '"body"\s*:\s*"token=openshell:resolve:env:(v[0-9]+_)?SLACK_BOT_TOKEN'; then - pass "M-S5h: rewriter wraps http.request/write/end — Bolt-shape header and body were translated before egress" -elif echo "$sl_loopback" | grep -q "ERROR"; then - fail "M-S5h: loopback probe errored: ${sl_loopback:0:200}" -elif echo "$sl_loopback" | grep -q "TIMEOUT"; then - skip "M-S5h: loopback probe timed out" + fail "M-S5g: removed Slack token rewriter preload still present in NODE_OPTIONS" else - fail "M-S5h: loopback probe returned unexpected output: ${sl_loopback:0:200}" + pass "M-S5g: Slack token rewriter preload absent from NODE_OPTIONS" fi # ══════════════════════════════════════════════════════════════════ @@ -837,6 +803,8 @@ if echo "$tg_reach" | grep -q "HTTP_"; then pass "M12: Node.js reached api.telegram.org (${tg_reach})" elif echo "$tg_reach" | grep -q "TIMEOUT"; then skip "M12: api.telegram.org timed out (network may be slow)" +elif echo "$tg_reach" | grep -qiE "ERROR:.*(ECONNRESET|reset|socket hang up|ENETUNREACH|EHOSTUNREACH|ETIMEDOUT)"; then + skip "M12: api.telegram.org unreachable from this network (${tg_reach:0:160})" else fail "M12: Node.js could not reach api.telegram.org (${tg_reach:0:200})" fi @@ -860,282 +828,70 @@ else fail "M13: Node.js could not reach discord.com (${dc_reach:0:200})" fi -# M13b: Probe the native Discord gateway path separately from REST. -# This catches failures where REST succeeds but the WebSocket path still fails -# (for example EAI_AGAIN on gateway.discord.gg or proxy misuse returning 400). -dc_gateway=$(sandbox_exec 'node -e " -const url = \"wss://gateway.discord.gg/?v=10&encoding=json\"; -if (typeof WebSocket !== \"function\") { - console.log(\"UNSUPPORTED WebSocket\"); - process.exit(0); -} -const ws = new WebSocket(url); -const done = (msg) => { - console.log(msg); - try { ws.close(); } catch {} - setTimeout(() => process.exit(0), 50); -}; -const timer = setTimeout(() => done(\"TIMEOUT\"), 15000); -ws.addEventListener(\"open\", () => console.log(\"OPEN\")); -ws.addEventListener(\"message\", (event) => { - clearTimeout(timer); - const body = String(event.data || \"\").slice(0, 200).replace(/\\s+/g, \" \"); - done(\"MESSAGE \" + body); -}); -ws.addEventListener(\"error\", (event) => { - clearTimeout(timer); - const msg = event?.message || event?.error?.message || \"websocket_error\"; - done(\"ERROR \" + msg); -}); -ws.addEventListener(\"close\", (event) => { - if (event.code && event.code !== 1000) console.log(\"CLOSE \" + event.code); -}); -"' 2>/dev/null || true) - -info "Discord gateway probe: ${dc_gateway:0:300}" - -if echo "$dc_gateway" | grep -q "MESSAGE "; then - pass "M13b: Native Discord gateway returned a WebSocket message" -elif echo "$dc_gateway" | grep -qiE "EAI_AGAIN|getaddrinfo"; then - if [ "$STRICT_DISCORD_GATEWAY" = "1" ]; then - fail "M13b: Native Discord gateway hit DNS resolution failure (${dc_gateway:0:200})" - else - skip "M13b: Native Discord gateway hit DNS resolution failure (${dc_gateway:0:200})" - fi -elif echo "$dc_gateway" | grep -q "400"; then - if [ "$STRICT_DISCORD_GATEWAY" = "1" ]; then - fail "M13b: Native Discord gateway probe returned 400 (${dc_gateway:0:200})" - else - skip "M13b: Native Discord gateway probe returned 400 (${dc_gateway:0:200})" - fi -elif echo "$dc_gateway" | grep -q "UNSUPPORTED"; then - skip "M13b: WebSocket runtime unsupported in sandbox Node.js" -elif echo "$dc_gateway" | grep -q "TIMEOUT"; then - if [ "$STRICT_DISCORD_GATEWAY" = "1" ]; then - fail "M13b: Native Discord gateway probe timed out" - else - skip "M13b: Native Discord gateway probe timed out" - fi -elif echo "$dc_gateway" | grep -q "ERROR"; then - if [ "$STRICT_DISCORD_GATEWAY" = "1" ]; then - fail "M13b: Native Discord gateway probe failed (${dc_gateway:0:200})" - else - skip "M13b: Native Discord gateway probe failed (${dc_gateway:0:200})" - fi -elif echo "$dc_gateway" | grep -q "CLOSE"; then - if [ "$STRICT_DISCORD_GATEWAY" = "1" ]; then - fail "M13b: Native Discord gateway probe closed abnormally (${dc_gateway:0:200})" - else - skip "M13b: Native Discord gateway probe closed abnormally (${dc_gateway:0:200})" - fi -elif echo "$dc_gateway" | grep -q "OPEN"; then - pass "M13b: Native Discord gateway opened a WebSocket session" +# M13b-M13f: Hermetic Discord Gateway over OpenShell's native WebSocket L7 path. +fake_gateway_ready=0 +if start_fake_discord_gateway "$DISCORD_TOKEN"; then + fake_gateway_ready=1 + pass "M13b: Hermetic fake Discord Gateway started on host port ${FAKE_DISCORD_GATEWAY_PORT}" else - if [ "$STRICT_DISCORD_GATEWAY" = "1" ]; then - fail "M13b: Native Discord gateway probe returned an unclassified result (${dc_gateway:0:200})" - else - skip "M13b: Native Discord gateway probe returned an unclassified result (${dc_gateway:0:200})" - fi + fail "M13b: Failed to start hermetic fake Discord Gateway" fi -# M13c: Unauthenticated Discord gateway transport via ws-proxy-fix CONNECT tunnel (#1570). -# The `ws` library opens WebSocket connections via https.request() with an -# Upgrade: websocket header. The preload patches https.request() to issue a -# CONNECT tunnel for Discord gateway hosts. -# -# This test exercises the transport/protocol path, not bot authentication: -# 1. https.request with Upgrade: websocket → CONNECT tunnel via proxy -# 2. Receive Discord Hello (opcode 10) with heartbeat_interval -# 3. Send a Heartbeat (opcode 1) back to the gateway -# 4. Receive Heartbeat ACK (opcode 11) -# 5. Send close frame and disconnect cleanly -# -# If the CONNECT tunnel is broken the connection never upgrades (400 from L7 -# proxy) and none of the protocol steps succeed. This deliberately does not -# send IDENTIFY, so it must not be treated as proof that placeholder tokens are -# rewritten inside gateway WebSocket payloads. -dc_ws_tunnel=$(sandbox_exec 'node -e " -const https = require(\"https\"); -const crypto = require(\"crypto\"); - -// --- Minimal WebSocket framing (no ws dependency) --- -function unmaskFrame(buf) { - if (buf.length < 2) return null; - const fin = (buf[0] & 0x80) !== 0; - const opcode = buf[0] & 0x0f; - const masked = (buf[1] & 0x80) !== 0; - let payloadLen = buf[1] & 0x7f; - let offset = 2; - if (payloadLen === 126) { - if (buf.length < 4) return null; - payloadLen = buf.readUInt16BE(2); - offset = 4; - } else if (payloadLen === 127) { - if (buf.length < 10) return null; - payloadLen = Number(buf.readBigUInt64BE(2)); - offset = 10; - } - if (masked) offset += 4; - if (buf.length < offset + payloadLen) return null; - const data = buf.slice(offset, offset + payloadLen); - return { fin, opcode, data, totalLen: offset + payloadLen }; -} - -function makeFrame(opcode, payload) { - const buf = Buffer.from(payload); - const mask = crypto.randomBytes(4); - const masked = Buffer.alloc(buf.length); - for (let i = 0; i < buf.length; i++) masked[i] = buf[i] ^ mask[i % 4]; - let header; - if (buf.length < 126) { - header = Buffer.alloc(6); - header[0] = 0x80 | opcode; - header[1] = 0x80 | buf.length; - mask.copy(header, 2); - } else { - header = Buffer.alloc(8); - header[0] = 0x80 | opcode; - header[1] = 0x80 | 126; - header.writeUInt16BE(buf.length, 2); - mask.copy(header, 4); - } - return Buffer.concat([header, masked]); -} - -function makeCloseFrame(code) { - const payload = Buffer.alloc(2); - payload.writeUInt16BE(code, 0); - return makeFrame(8, payload); -} - -// --- Handshake --- -const results = []; -const done = () => { - console.log(results.join(\"\\n\")); - process.exit(0); -}; -const timer = setTimeout(() => { results.push(\"TIMEOUT\"); done(); }, 20000); - -const key = crypto.randomBytes(16).toString(\"base64\"); -const req = https.request({ - hostname: \"gateway.discord.gg\", - port: 443, - path: \"/?v=10&encoding=json\", - method: \"GET\", - headers: { - \"Connection\": \"Upgrade\", - \"Upgrade\": \"websocket\", - \"Sec-WebSocket-Key\": key, - \"Sec-WebSocket-Version\": \"13\", - }, -}); +if [ "$fake_gateway_ready" = "1" ] \ + && apply_fake_discord_gateway_policy "$SANDBOX_NAME" "$FAKE_DISCORD_GATEWAY_PORT" >/tmp/nemoclaw-fake-discord-policy.log 2>&1; then + pass "M13c: Applied native WebSocket policy with credential rewrite for fake Discord Gateway" +else + fail "M13c: Failed to apply fake Discord Gateway policy: $(tail -20 /tmp/nemoclaw-fake-discord-policy.log 2>/dev/null | tr '\n' ' ' | cut -c1-300)" +fi -req.on(\"upgrade\", (_res, socket, head) => { - results.push(\"UPGRADED\"); - let pending = head && head.length ? Buffer.from(head) : Buffer.alloc(0); - - socket.on(\"data\", (chunk) => { - pending = Buffer.concat([pending, chunk]); - while (true) { - const frame = unmaskFrame(pending); - if (!frame) break; - pending = pending.slice(frame.totalLen); - - if (frame.opcode === 1) { - let msg; - try { msg = JSON.parse(frame.data.toString()); } catch { continue; } - - if (msg.op === 10) { - const hbInterval = msg.d && msg.d.heartbeat_interval; - results.push(\"HELLO op=10 heartbeat_interval=\" + hbInterval); - - // Send Heartbeat (opcode 1, d: null) - const hb = JSON.stringify({ op: 1, d: null }); - socket.write(makeFrame(1, hb)); - results.push(\"SENT_HEARTBEAT op=1\"); - } else if (msg.op === 11) { - results.push(\"HEARTBEAT_ACK op=11\"); - // Full round-trip complete — close cleanly - socket.write(makeCloseFrame(1000)); - setTimeout(() => { socket.destroy(); clearTimeout(timer); done(); }, 500); - } - } else if (frame.opcode === 8) { - results.push(\"CLOSE_FRAME code=\" + (frame.data.length >= 2 ? frame.data.readUInt16BE(0) : \"none\")); - socket.destroy(); - clearTimeout(timer); - done(); - } - } - }); - - socket.on(\"error\", (e) => { results.push(\"SOCKET_ERROR \" + e.message); }); - socket.on(\"close\", () => { clearTimeout(timer); done(); }); -}); +dc_ws_native="" +if [ "$fake_gateway_ready" = "1" ]; then + dc_ws_native=$(run_fake_discord_gateway_node_client "$FAKE_DISCORD_GATEWAY_PORT" "openshell:resolve:env:DISCORD_BOT_TOKEN" || true) +fi +info "Native fake Discord Gateway probe: ${dc_ws_native:0:500}" -req.on(\"response\", (res) => { - results.push(\"HTTP_\" + res.statusCode); - res.resume(); - res.on(\"end\", () => { clearTimeout(timer); done(); }); -}); -req.on(\"error\", (e) => { - results.push(\"ERROR \" + e.message); - clearTimeout(timer); - done(); -}); -req.end(); -"' 2>/dev/null || true) +if echo "$dc_ws_native" | grep -q "^UPGRADE$"; then + pass "M13d: Native WebSocket upgrade reached fake Discord Gateway through OpenShell" +else + fail "M13d: Native WebSocket upgrade failed: ${dc_ws_native:0:300}" +fi -info "Discord ws-proxy-fix probe: ${dc_ws_tunnel:0:500}" +if echo "$dc_ws_native" | grep -q "^HELLO$" \ + && echo "$dc_ws_native" | grep -q "^IDENTIFY_SENT_PLACEHOLDER$" \ + && echo "$dc_ws_native" | grep -q "^READY$" \ + && echo "$dc_ws_native" | grep -q "^HEARTBEAT_ACK$"; then + pass "M13e: Discord HELLO, placeholder IDENTIFY, READY, and heartbeat ACK completed" +else + fail "M13e: Discord Gateway protocol proof incomplete: ${dc_ws_native:0:400}" +fi -# Check each step of the handshake independently -if echo "$dc_ws_tunnel" | grep -q "UPGRADED"; then - pass "M13c: WebSocket upgrade succeeded via CONNECT tunnel (#1570)" -elif echo "$dc_ws_tunnel" | grep -q "HTTP_400"; then - if [ "$STRICT_DISCORD_GATEWAY" = "1" ]; then - fail "M13c: Discord gateway got 400 — CONNECT tunnel not working" - else - skip "M13c: Discord gateway got 400 — ws-proxy-fix may not be active" - fi -elif echo "$dc_ws_tunnel" | grep -qiE "EAI_AGAIN|getaddrinfo"; then - if [ "$STRICT_DISCORD_GATEWAY" = "1" ]; then - fail "M13c: Discord gateway DNS failure (${dc_ws_tunnel:0:200})" - else - skip "M13c: Discord gateway DNS failure (${dc_ws_tunnel:0:200})" - fi -elif echo "$dc_ws_tunnel" | grep -q "TIMEOUT"; then - if [ "$STRICT_DISCORD_GATEWAY" = "1" ]; then - fail "M13c: Discord gateway CONNECT tunnel timed out" - else - skip "M13c: Discord gateway CONNECT tunnel timed out" - fi -elif echo "$dc_ws_tunnel" | grep -q "ERROR"; then - if [ "$STRICT_DISCORD_GATEWAY" = "1" ]; then - fail "M13c: Discord gateway CONNECT tunnel failed (${dc_ws_tunnel:0:200})" - else - skip "M13c: Discord gateway CONNECT tunnel failed (${dc_ws_tunnel:0:200})" - fi +if [ "$fake_gateway_ready" = "1" ] \ + && grep -Fq "\"token\":\"$DISCORD_TOKEN\"" "$FAKE_DISCORD_GATEWAY_CAPTURE_FILE" \ + && ! grep -Fq "openshell:resolve:env:DISCORD_BOT_TOKEN" "$FAKE_DISCORD_GATEWAY_CAPTURE_FILE"; then + pass "M13f: Fake Gateway received host-side Discord token; sandbox-visible IDENTIFY used only the placeholder" else - if [ "$STRICT_DISCORD_GATEWAY" = "1" ]; then - fail "M13c: Discord gateway returned unclassified result (${dc_ws_tunnel:0:200})" - else - skip "M13c: Discord gateway returned unclassified result (${dc_ws_tunnel:0:200})" + if [ "$fake_gateway_ready" = "1" ]; then + info "Fake Discord Gateway capture: $(tail -20 "$FAKE_DISCORD_GATEWAY_CAPTURE_FILE" 2>/dev/null | tr '\n' ' ' | cut -c1-500)" fi + fail "M13f: Fake Gateway did not prove placeholder-to-token rewrite at the relay boundary" fi -if echo "$dc_ws_tunnel" | grep -q "HELLO op=10"; then - pass "M13d: Received Discord Hello (opcode 10) with heartbeat interval" -elif echo "$dc_ws_tunnel" | grep -q "UPGRADED"; then - fail "M13d: Upgraded but never received Discord Hello" -else - skip "M13d: WebSocket upgrade did not complete" +capture_before_negative=0 +capture_after_negative=0 +dc_ws_negative="" +if [ "$fake_gateway_ready" = "1" ]; then + capture_before_negative=$(wc -l <"$FAKE_DISCORD_GATEWAY_CAPTURE_FILE" 2>/dev/null || echo 0) + dc_ws_negative=$(run_fake_discord_gateway_node_client "$FAKE_DISCORD_GATEWAY_PORT" "openshell:resolve:env:DEFINITELY_NOT_REGISTERED" || true) + capture_after_negative=$(wc -l <"$FAKE_DISCORD_GATEWAY_CAPTURE_FILE" 2>/dev/null || echo 0) fi +info "Native fake Discord Gateway negative probe: ${dc_ws_negative:0:300}" -if echo "$dc_ws_tunnel" | grep -q "HEARTBEAT_ACK op=11"; then - pass "M13e: Sent Heartbeat, received ACK (opcode 11) — unauthenticated transport round-trip verified" -elif echo "$dc_ws_tunnel" | grep -q "SENT_HEARTBEAT"; then - fail "M13e: Sent Heartbeat but never received ACK" +if [ "$fake_gateway_ready" = "1" ] \ + && ! echo "$dc_ws_negative" | grep -q "^READY$" \ + && ! tail -n "$((capture_after_negative - capture_before_negative))" "$FAKE_DISCORD_GATEWAY_CAPTURE_FILE" 2>/dev/null | grep -Fq "DEFINITELY_NOT_REGISTERED"; then + pass "M13g: Unregistered Discord WebSocket placeholder is rejected before upstream token exposure" else - skip "M13e: Heartbeat exchange did not occur" + fail "M13g: Unregistered Discord WebSocket placeholder reached READY or leaked upstream" fi # M14 (negative): curl should be blocked by binary restriction @@ -1190,6 +946,8 @@ elif [ "$tg_status" = "401" ] || [ "$tg_status" = "404" ]; then pass "M16: Full chain verified: sandbox → proxy → token rewrite → Telegram API" elif echo "$tg_api" | grep -q "TIMEOUT"; then skip "M15: Telegram API timed out (network issue, not a plumbing failure)" +elif echo "$tg_api" | grep -qiE "ERROR:.*(ECONNRESET|reset|socket hang up|ENETUNREACH|EHOSTUNREACH|ETIMEDOUT)"; then + skip "M15: Telegram API unreachable from this network (${tg_api:0:160})" elif echo "$tg_api" | grep -q "ERROR"; then fail "M15: Telegram API call failed with error: ${tg_api:0:200}" else @@ -1198,33 +956,21 @@ fi # M17: Discord users/@me with placeholder token info "Calling discord.com/api/v10/users/@me from inside sandbox..." -dc_api=$(sandbox_exec 'NODE_NO_WARNINGS=1 node -e " +dc_api=$(sandbox_exec 'node -e " const https = require(\"https\"); const token = process.env.DISCORD_BOT_TOKEN || \"missing\"; -const maxAttempts = 5; -let attempt = 0; -function probe() { - attempt += 1; - const options = { - hostname: \"discord.com\", - path: \"/api/v10/users/@me\", - headers: { \"Authorization\": \"Bot \" + token }, - }; - const req = https.get(options, (res) => { - let body = \"\"; - res.on(\"data\", (d) => body += d); - res.on(\"end\", () => { - if (res.statusCode === 503 && attempt < maxAttempts) { - setTimeout(probe, 2000); - return; - } - console.log(res.statusCode + \" \" + body.slice(0, 300)); - }); - }); - req.on(\"error\", (e) => console.log(\"ERROR: \" + e.message)); - req.setTimeout(30000, () => { req.destroy(); console.log(\"TIMEOUT\"); }); -} -probe(); +const options = { + hostname: \"discord.com\", + path: \"/api/v10/users/@me\", + headers: { \"Authorization\": \"Bot \" + token }, +}; +const req = https.get(options, (res) => { + let body = \"\"; + res.on(\"data\", (d) => body += d); + res.on(\"end\", () => console.log(res.statusCode + \" \" + body.slice(0, 300))); +}); +req.on(\"error\", (e) => console.log(\"ERROR: \" + e.message)); +req.setTimeout(30000, () => { req.destroy(); console.log(\"TIMEOUT\"); }); "' 2>/dev/null || true) info "Discord API response: ${dc_api:0:300}" @@ -1243,42 +989,70 @@ else fail "M17: Unexpected Discord response (status=$dc_status): ${dc_api:0:200}" fi -# ── Slack: rewriter + L7 proxy chain (#2085) ───────────────────── -# Verifies the full chain: Bolt-shape placeholder in Authorization -# header → slack-token-rewriter (Node preload) translates to canonical -# form → OpenShell L7 proxy substitutes real env value → request -# reaches slack.com which responds with invalid_auth (because the -# host-side fake token is, well, fake). The 200 OK + invalid_auth -# response is the proof the chain worked end-to-end. -# -# Slack returns HTTP 200 with {"ok":false,"error":"invalid_auth"} for -# auth failures on auth.test (it does NOT use 401). The body is the -# load-bearing assertion, not the status. +# ── Slack: OpenShell alias/body rewrite chain (#2085) ───────────── +# Verifies the full chain hermetically: Bolt-shape placeholder in the +# Authorization header → OpenShell resolves the provider-shaped alias and +# substitutes the real env value → a host-side fake Slack API receives the +# resolved token and returns Slack-shaped invalid_auth. -info "Calling slack.com/api/auth.test from inside sandbox with Bolt-shape placeholder..." -sl_api=$(sandbox_exec 'node -e " -const https = require(\"https\"); -const data = \"\"; -const options = { - hostname: \"slack.com\", - path: \"/api/auth.test\", - method: \"POST\", - headers: { - \"Authorization\": \"Bearer xoxb-OPENSHELL-RESOLVE-ENV-SLACK_BOT_TOKEN\", - \"Content-Type\": \"application/x-www-form-urlencoded\", - \"Content-Length\": data.length, - }, -}; -const req = https.request(options, (res) => { - let body = \"\"; - res.on(\"data\", (d) => body += d); - res.on(\"end\", () => console.log(res.statusCode + \" \" + body.slice(0, 300))); -}); -req.on(\"error\", (e) => console.log(\"ERROR: \" + e.message)); -req.setTimeout(30000, () => { req.destroy(); console.log(\"TIMEOUT\"); }); -req.write(data); -req.end(); -"' 2>/dev/null || true) +fake_slack_ready=0 +if start_fake_slack_api "$SLACK_TOKEN" "$SLACK_APP"; then + fake_slack_ready=1 + pass "M-S14a: Hermetic fake Slack API started on host port ${FAKE_SLACK_API_PORT}" +else + fail "M-S14a: Failed to start hermetic fake Slack API" +fi + +if [ "$fake_slack_ready" = "1" ] \ + && apply_fake_slack_api_policy "$SANDBOX_NAME" "$FAKE_SLACK_API_PORT" >/tmp/nemoclaw-fake-slack-policy.log 2>&1; then + pass "M-S14b: Applied REST policy for hermetic fake Slack API" +else + fail "M-S14b: Failed to apply fake Slack API policy: $(tail -20 /tmp/nemoclaw-fake-slack-policy.log 2>/dev/null | tr '\n' ' ' | cut -c1-300)" +fi + +check_fake_slack_capture_token() { + local path="$1" + local expected_token="$2" + node - "$FAKE_SLACK_API_CAPTURE_FILE" "$path" "$expected_token" <<'NODE' +const fs = require("fs"); +const [file, path, expectedToken] = process.argv.slice(2); +const rows = fs + .readFileSync(file, "utf8") + .trim() + .split(/\n+/) + .filter(Boolean) + .map((line) => JSON.parse(line)) + .filter((row) => row.event === "request" && row.path === path); +const last = rows.at(-1); +if (!last) { + console.log(`NO_REQUEST ${path}`); + process.exit(2); +} +if (last.authorization !== undefined || last.body !== undefined) { + console.log("RAW_CAPTURE_LEAK"); + process.exit(6); +} +if (last.tokenMatchesExpected !== true) { + console.log("BAD_AUTH_REWRITE"); + process.exit(3); +} +if (last.bodyMatchesExpected !== true) { + console.log("BAD_BODY_REWRITE"); + process.exit(4); +} +if (last.tokenLooksPlaceholder) { + console.log("PLACEHOLDER_LEAK"); + process.exit(5); +} +console.log("OK"); +NODE +} + +info "Calling fake Slack /api/auth.test from inside sandbox with Bolt-shape placeholder..." +sl_api="" +if [ "$fake_slack_ready" = "1" ]; then + sl_api=$(run_fake_slack_api_node_request "$FAKE_SLACK_API_PORT" "/api/auth.test" "Bearer xoxb-OPENSHELL-RESOLVE-ENV-SLACK_BOT_TOKEN" || true) +fi info "Slack auth.test response: ${sl_api:0:300}" sl_status=$(echo "$sl_api" | grep -E '^[0-9]' | head -1 | awk '{print $1}') @@ -1286,13 +1060,19 @@ sl_status=$(echo "$sl_api" | grep -E '^[0-9]' | head -1 | awk '{print $1}') if [ "$sl_status" = "200" ] && echo "$sl_api" | grep -q '"ok":true'; then pass "M-S15: Slack auth.test returned ok:true — real token round-trip verified!" elif [ "$sl_status" = "200" ] && echo "$sl_api" | grep -qE 'invalid_auth|not_authed'; then - pass "M-S15: Slack auth.test returned invalid_auth — full chain verified (rewriter → L7 proxy → slack.com)" + pass "M-S15: Slack auth.test returned invalid_auth — full chain verified (OpenShell alias rewrite → fake Slack)" + sl_capture=$(check_fake_slack_capture_token "/api/auth.test" "$SLACK_TOKEN" || true) + if [ "$sl_capture" = "OK" ]; then + pass "M-S15a: fake Slack saw host-side bot token in header and urlencoded body" + else + fail "M-S15a: fake Slack capture did not prove bot header/body rewrite: ${sl_capture:0:300}" + fi elif echo "$sl_api" | grep -q "TIMEOUT"; then - skip "M-S15: Slack API timed out (network issue, not a plumbing failure)" + skip "M-S15: fake Slack API timed out" elif echo "$sl_api" | grep -q "ERROR"; then fail "M-S15: Slack API call failed with error: ${sl_api:0:200}" elif echo "$sl_api" | grep -qF 'OPENSHELL-RESOLVE-ENV-'; then - fail "M-S15: rewriter did not translate the Bolt-shape placeholder — preload not loaded?" + fail "M-S15: OpenShell did not resolve the Bolt-shape alias" elif echo "$sl_api" | grep -qF 'openshell:resolve:env:'; then fail "M-S15: L7 proxy did not substitute the canonical placeholder — substitution chain broken" else @@ -1300,50 +1080,28 @@ else fi # M-S15b: L7 proxy substitution for SLACK_BOT_TOKEN, isolated from the -# rewriter. Sends the active OpenShell placeholder from the sandbox env -# directly (no Bolt-shape, so the rewriter is a no-op for this request). -# Newer OpenShell builds scope provider placeholders by revision -# (openshell:resolve:env:vNN_KEY), while older builds use the canonical -# openshell:resolve:env:KEY form. +# alias path. Sends the canonical openshell:resolve:env:SLACK_BOT_TOKEN +# placeholder directly. If the L7 proxy substitutes correctly, the fake Slack API +# receives the host-side xoxb token and returns invalid_auth. # # Mirrors the proof technique already used by Telegram M15 and Discord # M17 (they get 401/404 from the real APIs because the L7 proxy -# substituted the active placeholder into a real fake-token-shape value). -info "Probing L7 proxy substitution for SLACK_BOT_TOKEN (active provider placeholder, bypasses rewriter)..." -sl_canonical=$(sandbox_exec 'node -e " -const https = require(\"https\"); -const data = \"\"; -const token = process.env.SLACK_BOT_TOKEN || \"openshell:resolve:env:SLACK_BOT_TOKEN\"; -const options = { - hostname: \"slack.com\", - path: \"/api/auth.test\", - method: \"POST\", - headers: { - \"Authorization\": \"Bearer \" + token, - \"Content-Type\": \"application/x-www-form-urlencoded\", - \"Content-Length\": data.length, - }, -}; -const req = https.request(options, (res) => { - let body = \"\"; - res.on(\"data\", (d) => body += d); - res.on(\"end\", () => console.log(res.statusCode + \" \" + body.slice(0, 300))); -}); -req.on(\"error\", (e) => console.log(\"ERROR: \" + e.message)); -req.setTimeout(30000, () => { req.destroy(); console.log(\"TIMEOUT\"); }); -req.write(data); -req.end(); -"' 2>/dev/null || true) +# substituted the canonical form into a real fake-token-shape value). +info "Probing L7 proxy substitution for SLACK_BOT_TOKEN (canonical placeholder, bypasses rewriter)..." +sl_canonical="" +if [ "$fake_slack_ready" = "1" ]; then + sl_canonical=$(run_fake_slack_api_node_request "$FAKE_SLACK_API_PORT" "/api/auth.test" "Bearer openshell:resolve:env:SLACK_BOT_TOKEN" || true) +fi info "Slack auth.test (canonical) response: ${sl_canonical:0:300}" sl_canon_status=$(echo "$sl_canonical" | grep -E '^[0-9]' | head -1 | awk '{print $1}') if [ "$sl_canon_status" = "200" ] && echo "$sl_canonical" | grep -qE 'invalid_auth|not_authed'; then - pass "M-S15b: L7 proxy substitutes the active SLACK_BOT_TOKEN placeholder at egress (parallels Telegram M15 / Discord M17)" + pass "M-S15b: L7 proxy substitutes openshell:resolve:env:SLACK_BOT_TOKEN at egress (parallels Telegram M15 / Discord M17)" elif echo "$sl_canonical" | grep -q "TIMEOUT"; then - skip "M-S15b: active-placeholder probe timed out" + skip "M-S15b: canonical-placeholder probe timed out" elif echo "$sl_canonical" | grep -qF 'openshell:resolve:env:' || echo "$sl_canonical" | grep -qiF 'invalid token'; then - fail "M-S15b: L7 proxy passed provider placeholder through unchanged — substitution not happening for SLACK_BOT_TOKEN" + fail "M-S15b: L7 proxy passed canonical placeholder through unchanged — substitution not happening for SLACK_BOT_TOKEN" else fail "M-S15b: Unexpected response (status=$sl_canon_status): ${sl_canonical:0:200}" fi @@ -1356,43 +1114,23 @@ fi # they differ, the proxy distinguishes set vs unset env vars (i.e., # substitution is actually running on the substring it recognizes). info "Probing L7 proxy substitution with an unset env var (negative control)..." -sl_unset=$(sandbox_exec 'node -e " -const https = require(\"https\"); -const data = \"\"; -const options = { - hostname: \"slack.com\", - path: \"/api/auth.test\", - method: \"POST\", - headers: { - \"Authorization\": \"Bearer openshell:resolve:env:DEFINITELY_NOT_SET_XYZ\", - \"Content-Type\": \"application/x-www-form-urlencoded\", - \"Content-Length\": data.length, - }, -}; -const req = https.request(options, (res) => { - let body = \"\"; - res.on(\"data\", (d) => body += d); - res.on(\"end\", () => console.log(res.statusCode + \" \" + body.slice(0, 300))); -}); -req.on(\"error\", (e) => console.log(\"ERROR: \" + e.message)); -req.setTimeout(30000, () => { req.destroy(); console.log(\"TIMEOUT\"); }); -req.write(data); -req.end(); -"' 2>/dev/null || true) +sl_unset="" +if [ "$fake_slack_ready" = "1" ]; then + sl_unset=$(run_fake_slack_api_node_request "$FAKE_SLACK_API_PORT" "/api/auth.test" "Bearer openshell:resolve:env:DEFINITELY_NOT_SET_XYZ" || true) +fi info "Slack auth.test (unset env) response: ${sl_unset:0:300}" -# Empirically (verified in nightly run 25070238797): when the canonical -# placeholder names an env var that isn't registered as a provider, the -# OpenShell L7 proxy refuses to forward and the client sees a -# connection-level failure ("socket hang up" / ECONNRESET / EPIPE). -# The set-var path returns HTTP 200 invalid_auth from slack.com — these -# shapes are completely disjoint, so we assert specifically on them -# instead of doing a fuzzy string compare (UNDICI warnings carry a PID -# and would always make the captures differ regardless of substance). -if echo "$sl_unset" | grep -qE 'ERROR:.*(socket hang up|ECONNRESET|EPIPE|hang up|reset)'; then +# OpenShell may reject the unresolved placeholder with an explicit +# credential_injection_failed response or a connection-level failure. +# Either shape proves the unresolved placeholder did not reach upstream. +if is_unresolved_placeholder_rejection "$sl_unset"; then + pass "M-S15c: unset-var failed closed before upstream exposure" +elif echo "$sl_unset" | grep -qE 'ERROR:.*(socket hang up|ECONNRESET|EPIPE|hang up|reset)'; then pass "M-S15c: unset-var triggered connection-level failure — proxy refuses to forward unsubstituted placeholder" elif echo "$sl_unset" | grep -qE '^200\b'; then fail "M-S15c: unset-var returned HTTP 200 — proxy passed canonical placeholder through unchanged for unset env (substitution may be a no-op)" +elif echo "$sl_unset" | grep -qE '^401\b|bad_auth|DEFINITELY_NOT_SET_XYZ'; then + fail "M-S15c: unset-var request reached fake Slack — unresolved placeholder escaped the proxy boundary" elif [ -z "$sl_unset" ] || echo "$sl_unset" | grep -q "TIMEOUT"; then skip "M-S15c: unset-var probe timed out or returned no output" else @@ -1403,30 +1141,11 @@ fi # Mode opens a websocket only after this POST succeeds, so this is the # call that the xapp- token actually authenticates. We don't bother # upgrading WSS in the test — the auth check is on the HTTPS POST. -info "Calling slack.com/api/apps.connections.open with Bolt-shape xapp- placeholder..." -sl_app_api=$(sandbox_exec 'node -e " -const https = require(\"https\"); -const data = \"\"; -const options = { - hostname: \"slack.com\", - path: \"/api/apps.connections.open\", - method: \"POST\", - headers: { - \"Authorization\": \"Bearer xapp-OPENSHELL-RESOLVE-ENV-SLACK_APP_TOKEN\", - \"Content-Type\": \"application/x-www-form-urlencoded\", - \"Content-Length\": data.length, - }, -}; -const req = https.request(options, (res) => { - let body = \"\"; - res.on(\"data\", (d) => body += d); - res.on(\"end\", () => console.log(res.statusCode + \" \" + body.slice(0, 300))); -}); -req.on(\"error\", (e) => console.log(\"ERROR: \" + e.message)); -req.setTimeout(30000, () => { req.destroy(); console.log(\"TIMEOUT\"); }); -req.write(data); -req.end(); -"' 2>/dev/null || true) +info "Calling fake Slack /api/apps.connections.open with Bolt-shape xapp- placeholder..." +sl_app_api="" +if [ "$fake_slack_ready" = "1" ]; then + sl_app_api=$(run_fake_slack_api_node_request "$FAKE_SLACK_API_PORT" "/api/apps.connections.open" "Bearer xapp-OPENSHELL-RESOLVE-ENV-SLACK_APP_TOKEN" || true) +fi info "Slack apps.connections.open response: ${sl_app_api:0:300}" sl_app_status=$(echo "$sl_app_api" | grep -E '^[0-9]' | head -1 | awk '{print $1}') @@ -1434,87 +1153,58 @@ sl_app_status=$(echo "$sl_app_api" | grep -E '^[0-9]' | head -1 | awk '{print $1 if [ "$sl_app_status" = "200" ] && echo "$sl_app_api" | grep -q '"ok":true'; then pass "M-S16: apps.connections.open returned ok:true — real xapp token round-trip verified!" elif [ "$sl_app_status" = "200" ] && echo "$sl_app_api" | grep -qE 'invalid_auth|not_authed|not_allowed_token_type'; then - pass "M-S16: apps.connections.open auth-rejected — Socket Mode HTTPS leg verified (rewriter → L7 proxy → slack.com)" + pass "M-S16: apps.connections.open auth-rejected — Socket Mode HTTPS leg verified (OpenShell alias rewrite → fake Slack)" + sl_app_capture=$(check_fake_slack_capture_token "/api/apps.connections.open" "$SLACK_APP" || true) + if [ "$sl_app_capture" = "OK" ]; then + pass "M-S16a: fake Slack saw host-side app token in header and urlencoded body" + else + fail "M-S16a: fake Slack capture did not prove app header/body rewrite: ${sl_app_capture:0:300}" + fi elif echo "$sl_app_api" | grep -q "TIMEOUT"; then - skip "M-S16: apps.connections.open timed out (network issue)" + skip "M-S16: apps.connections.open timed out" elif echo "$sl_app_api" | grep -qF 'OPENSHELL-RESOLVE-ENV-'; then - fail "M-S16: rewriter did not translate xapp- placeholder — preload not loaded for Socket Mode path?" + fail "M-S16: OpenShell did not resolve the xapp- alias for Socket Mode path" else fail "M-S16: Unexpected apps.connections.open response (status=$sl_app_status): ${sl_app_api:0:200}" fi # M-S16b: L7 proxy substitution for SLACK_APP_TOKEN, isolated. Same -# rationale as M-S15b — sends the active provider placeholder directly so -# the rewriter is a no-op and only the L7 proxy substitution is exercised. -info "Probing L7 proxy substitution for SLACK_APP_TOKEN (active provider placeholder)..." -sl_app_canonical=$(sandbox_exec 'node -e " -const https = require(\"https\"); -const data = \"\"; -const token = process.env.SLACK_APP_TOKEN || \"openshell:resolve:env:SLACK_APP_TOKEN\"; -const options = { - hostname: \"slack.com\", - path: \"/api/apps.connections.open\", - method: \"POST\", - headers: { - \"Authorization\": \"Bearer \" + token, - \"Content-Type\": \"application/x-www-form-urlencoded\", - \"Content-Length\": data.length, - }, -}; -const req = https.request(options, (res) => { - let body = \"\"; - res.on(\"data\", (d) => body += d); - res.on(\"end\", () => console.log(res.statusCode + \" \" + body.slice(0, 300))); -}); -req.on(\"error\", (e) => console.log(\"ERROR: \" + e.message)); -req.setTimeout(30000, () => { req.destroy(); console.log(\"TIMEOUT\"); }); -req.write(data); -req.end(); -"' 2>/dev/null || true) +# rationale as M-S15b — sends the canonical placeholder directly so only +# the L7 proxy substitution is exercised. +info "Probing L7 proxy substitution for SLACK_APP_TOKEN (canonical placeholder)..." +sl_app_canonical="" +if [ "$fake_slack_ready" = "1" ]; then + sl_app_canonical=$(run_fake_slack_api_node_request "$FAKE_SLACK_API_PORT" "/api/apps.connections.open" "Bearer openshell:resolve:env:SLACK_APP_TOKEN" || true) +fi -info "Slack apps.connections.open (active placeholder) response: ${sl_app_canonical:0:300}" +info "Slack apps.connections.open (canonical) response: ${sl_app_canonical:0:300}" sl_app_canon_status=$(echo "$sl_app_canonical" | grep -E '^[0-9]' | head -1 | awk '{print $1}') info "Probing L7 proxy substitution for an unset app-token env var (negative control)..." -sl_app_unset=$(sandbox_exec 'node -e " -const https = require(\"https\"); -const data = \"\"; -const options = { - hostname: \"slack.com\", - path: \"/api/apps.connections.open\", - method: \"POST\", - headers: { - \"Authorization\": \"Bearer openshell:resolve:env:DEFINITELY_NOT_SET_SLACK_APP_TOKEN\", - \"Content-Type\": \"application/x-www-form-urlencoded\", - \"Content-Length\": data.length, - }, -}; -const req = https.request(options, (res) => { - let body = \"\"; - res.on(\"data\", (d) => body += d); - res.on(\"end\", () => console.log(res.statusCode + \" \" + body.slice(0, 300))); -}); -req.on(\"error\", (e) => console.log(\"ERROR: \" + e.message)); -req.setTimeout(30000, () => { req.destroy(); console.log(\"TIMEOUT\"); }); -req.write(data); -req.end(); -"' 2>/dev/null || true) +sl_app_unset="" +if [ "$fake_slack_ready" = "1" ]; then + sl_app_unset=$(run_fake_slack_api_node_request "$FAKE_SLACK_API_PORT" "/api/apps.connections.open" "Bearer openshell:resolve:env:DEFINITELY_NOT_SET_SLACK_APP_TOKEN" || true) +fi info "Slack apps.connections.open (unset env) response: ${sl_app_unset:0:300}" if [ "$sl_app_canon_status" = "200" ] && echo "$sl_app_canonical" | grep -qE 'invalid_auth|not_authed|not_allowed_token_type'; then - if echo "$sl_app_unset" | grep -qE 'ERROR:.*(socket hang up|ECONNRESET|EPIPE|hang up|reset)'; then - pass "M-S16b: L7 proxy substitutes the active SLACK_APP_TOKEN placeholder at egress (unset-var control diverged)" + if is_unresolved_placeholder_rejection "$sl_app_unset"; then + pass "M-S16b: unset app-token failed closed before upstream exposure" + elif echo "$sl_app_unset" | grep -qE 'ERROR:.*(socket hang up|ECONNRESET|EPIPE|hang up|reset)'; then + pass "M-S16b: L7 proxy substitutes openshell:resolve:env:SLACK_APP_TOKEN at egress (unset-var control diverged)" elif echo "$sl_app_unset" | grep -qE '^200\b'; then fail "M-S16b: unset app-token env returned HTTP 200 — proxy may be passing canonical placeholders through unchanged" + elif echo "$sl_app_unset" | grep -qE '^401\b|bad_auth|DEFINITELY_NOT_SET_SLACK_APP_TOKEN'; then + fail "M-S16b: unset app-token request reached fake Slack — unresolved placeholder escaped the proxy boundary" elif [ -z "$sl_app_unset" ] || echo "$sl_app_unset" | grep -q "TIMEOUT"; then skip "M-S16b: unset app-token control timed out or returned no output" else skip "M-S16b: unset app-token control produced an unclassified result: ${sl_app_unset:0:200}" fi elif echo "$sl_app_canonical" | grep -q "TIMEOUT"; then - skip "M-S16b: active-placeholder probe timed out" + skip "M-S16b: canonical-placeholder probe timed out" elif echo "$sl_app_canonical" | grep -qF 'openshell:resolve:env:'; then - fail "M-S16b: L7 proxy passed provider placeholder through unchanged for SLACK_APP_TOKEN" + fail "M-S16b: L7 proxy passed canonical placeholder through unchanged for SLACK_APP_TOKEN" else fail "M-S16b: Unexpected response (status=$sl_app_canon_status): ${sl_app_canonical:0:200}" fi @@ -1592,12 +1282,12 @@ fi # Phase 7: Slack channel guard (#2340) # # The sandbox was installed with fake Slack tokens. After the -# slack-token-rewriter refactor (#2085) the failure mode is: +# OpenShell alias rewrite change (#2085 follow-up) the failure mode is: # 1. Bolt accepts the xoxb-OPENSHELL-RESOLVE-ENV-… placeholder # (matches its prefix regex). -# 2. The rewriter translates to canonical form on egress. +# 2. OpenShell resolves the alias at egress. # 3. The L7 proxy substitutes the fake xoxb-fake-… token from env. -# 4. slack.com returns 200 OK invalid_auth. +# 4. The Slack API rejects the fake token. # 5. @slack/web-api emits an unhandled rejection — the guard catches it. # Pre-refactor the catch happened earlier (Bolt's in-process xapp- prefix # check), but the observable here is the same: gateway stays up, log shows @@ -1658,11 +1348,17 @@ fi section "Phase 8: Cleanup" info "Destroying sandbox '$SANDBOX_NAME'..." -[[ "${NEMOCLAW_E2E_KEEP_SANDBOX:-}" = "1" ]] || nemoclaw "$SANDBOX_NAME" destroy --yes 2>/dev/null || true -openshell sandbox delete "$SANDBOX_NAME" 2>/dev/null || true +if [[ "${NEMOCLAW_E2E_KEEP_SANDBOX:-}" = "1" ]]; then + skip "Cleanup: NEMOCLAW_E2E_KEEP_SANDBOX=1 — leaving sandbox '$SANDBOX_NAME' for inspection" +else + nemoclaw "$SANDBOX_NAME" destroy --yes 2>/dev/null || true + openshell sandbox delete "$SANDBOX_NAME" 2>/dev/null || true +fi # Verify cleanup -if openshell sandbox list 2>&1 | grep -q "$SANDBOX_NAME"; then +if [[ "${NEMOCLAW_E2E_KEEP_SANDBOX:-}" = "1" ]]; then + pass "Cleanup: Sandbox '$SANDBOX_NAME' intentionally kept" +elif openshell sandbox list 2>&1 | grep -q "$SANDBOX_NAME"; then fail "Cleanup: Sandbox '$SANDBOX_NAME' still present after cleanup" else pass "Cleanup: Sandbox '$SANDBOX_NAME' removed" diff --git a/test/e2e/test-openshell-gateway-upgrade.sh b/test/e2e/test-openshell-gateway-upgrade.sh index 2e98c8b8fe..0479ab8f21 100755 --- a/test/e2e/test-openshell-gateway-upgrade.sh +++ b/test/e2e/test-openshell-gateway-upgrade.sh @@ -7,7 +7,7 @@ # the current install/onboard path must back up the old claw before replacing # the incompatible OpenShell gateway, recreate it under the current gateway, # restore durable agent state, and leave the same agent type running. -# 2. If a macOS arm64 user already has the OpenShell 0.0.37 CLI but not the +# 2. If a macOS arm64 user already has the current OpenShell CLI but not the # standalone openshell-gateway binary, the installer must fetch the Darwin # gateway asset instead of accepting the incomplete CLI-only install. @@ -52,7 +52,7 @@ STATE_DIR="${NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR:-$HOME/.local/state/nemoclaw/o PID_FILE="${STATE_DIR}/openshell-gateway.pid" OLD_NEMOCLAW_REF="${NEMOCLAW_OLD_NEMOCLAW_REF:-v0.0.36}" OLD_OPENSHELL_VERSION="${NEMOCLAW_OLD_OPENSHELL_VERSION:-0.0.36}" -CURRENT_OPENSHELL_VERSION="${NEMOCLAW_CURRENT_OPENSHELL_VERSION:-0.0.37}" +CURRENT_OPENSHELL_VERSION="${NEMOCLAW_CURRENT_OPENSHELL_VERSION:-0.0.39}" SURVIVOR_SANDBOX="${NEMOCLAW_GATEWAY_UPGRADE_SURVIVOR_NAME:-e2e-gateway-upgrade-survivor}" SURVIVOR_MARKER="gateway-upgrade-survivor-$(date +%s)" SURVIVOR_MARKER_PATH="/sandbox/.openclaw/workspace/nemoclaw-gateway-upgrade-marker" @@ -143,7 +143,7 @@ EOF cat >"$fake_bin/openshell" <<'EOF' #!/usr/bin/env bash if [ "${1:-}" = "--version" ]; then - printf 'openshell 0.0.37\n' + printf 'openshell 0.0.39\n' exit 0 fi exit 99 @@ -205,7 +205,7 @@ EOF fi rm -rf "$tmp" - pass "macOS OpenShell 0.0.37 incomplete install fetches Darwin gateway and VM driver assets" + pass "macOS OpenShell ${CURRENT_OPENSHELL_VERSION} incomplete install fetches Darwin gateway and VM driver assets" } exercise_macos_vm_driver_entitlement_repair() { @@ -230,7 +230,7 @@ EOF cat >"$fake_bin/openshell" <<'EOF' #!/usr/bin/env bash if [ "${1:-}" = "--version" ]; then - printf 'openshell 0.0.37\n' + printf 'openshell 0.0.39\n' exit 0 fi exit 99 @@ -289,7 +289,7 @@ EOF fi rm -rf "$tmp" - pass "macOS OpenShell 0.0.37 installer repairs missing VM driver Hypervisor entitlement" + pass "macOS OpenShell ${CURRENT_OPENSHELL_VERSION} installer repairs missing VM driver Hypervisor entitlement" } exercise_macos_vm_rootfs_permission_regression() { diff --git a/test/e2e/test-rebuild-hermes.sh b/test/e2e/test-rebuild-hermes.sh index 4444af8070..0d225b5e44 100755 --- a/test/e2e/test-rebuild-hermes.sh +++ b/test/e2e/test-rebuild-hermes.sh @@ -82,7 +82,7 @@ dump_hermes_sandbox_logs() { diag_script+='; echo "== log and state paths =="; ls -ld /tmp /sandbox/.hermes /sandbox/.hermes/logs 2>&1 || true; ls -l /tmp/nemoclaw-start.log /tmp/gateway.log 2>&1 || true' diag_script+='; echo "== hermes-related processes =="' # shellcheck disable=SC2016 # script is intentionally evaluated inside the sandbox - diag_script+='; for p in /proc/[0-9]*; do cmd=$(tr "\000" " " < "$p/cmdline" 2>/dev/null || true); case "$cmd" in *hermes*|*socat*|*nemoclaw-decode-proxy*) echo "$(basename "$p") $cmd" ;; esac; done' + diag_script+='; for p in /proc/[0-9]*; do cmd=$(tr "\000" " " < "$p/cmdline" 2>/dev/null || true); case "$cmd" in *hermes*|*socat*) echo "$(basename "$p") $cmd" ;; esac; done' diag_script+='; echo "== /tmp/nemoclaw-start.log tail =="; tail -n 80 /tmp/nemoclaw-start.log 2>&1 || true' diag_script+='; echo "== /tmp/gateway.log tail =="; tail -n 120 /tmp/gateway.log 2>&1 || true' diff --git a/test/gateway-http-reuse-wait.test.ts b/test/gateway-http-reuse-wait.test.ts index fd6dcf0d88..6b66da3fa6 100644 --- a/test/gateway-http-reuse-wait.test.ts +++ b/test/gateway-http-reuse-wait.test.ts @@ -15,6 +15,7 @@ // Regression of: https://github.com/NVIDIA/NemoClaw/issues/2020 import http from "node:http"; +import http2 from "node:http2"; import { createRequire } from "node:module"; import { type AddressInfo } from "node:net"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; @@ -22,6 +23,7 @@ import { afterEach, beforeEach, describe, expect, it } from "vitest"; const require = createRequire(import.meta.url); const onboardModule = require("../dist/lib/onboard.js") as { getGatewayReuseHealthWaitConfig: () => { count: number; interval: number }; + isDockerDriverGatewayHttpReady: (timeoutMs?: number, url?: string) => Promise; isGatewayHttpReady: (timeoutMs?: number, url?: string) => Promise; waitForGatewayHttpReady: (opts?: { probe?: () => Promise; @@ -32,6 +34,7 @@ const onboardModule = require("../dist/lib/onboard.js") as { }; const { getGatewayReuseHealthWaitConfig, isGatewayHttpReady, waitForGatewayHttpReady } = onboardModule; +const { isDockerDriverGatewayHttpReady } = onboardModule; /** Bind an ephemeral localhost port, close it, and return its URL — a port * that's guaranteed to refuse connections for the lifetime of the test. */ @@ -179,6 +182,68 @@ describe("isGatewayHttpReady status-code semantics (#3258)", () => { }); }); +describe("isDockerDriverGatewayHttpReady (#3111)", () => { + it("uses the Docker-driver gRPC health endpoint instead of root /", async () => { + let sawHealthPost = false; + const server = http2.createServer(); + server.on("stream", (stream: http2.ServerHttp2Stream, headers) => { + if ( + headers[http2.constants.HTTP2_HEADER_METHOD] === "POST" && + headers[http2.constants.HTTP2_HEADER_PATH] === "/openshell.v1.OpenShell/Health" && + headers[http2.constants.HTTP2_HEADER_CONTENT_TYPE] === "application/grpc" + ) { + sawHealthPost = true; + stream.respond({ + [http2.constants.HTTP2_HEADER_STATUS]: 200, + [http2.constants.HTTP2_HEADER_CONTENT_TYPE]: "application/grpc", + "grpc-status": "0", + }); + stream.end(Buffer.alloc(5)); + } else { + stream.respond({ [http2.constants.HTTP2_HEADER_STATUS]: 404 }); + stream.end(); + } + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const port = (server.address() as AddressInfo).port; + try { + expect( + await isDockerDriverGatewayHttpReady( + 2000, + `http://127.0.0.1:${port}/openshell.v1.OpenShell/Health`, + ), + ).toBe(true); + expect(sawHealthPost).toBe(true); + } finally { + await new Promise((resolve, reject) => + server.close((err) => (err ? reject(err) : resolve())), + ); + } + }); + + it("does not treat a raw HTTP/1.1 POST 200 as Docker-driver gRPC health", async () => { + const server = http.createServer((req, res) => { + res.statusCode = + req.method === "POST" && req.url === "/openshell.v1.OpenShell/Health" ? 200 : 404; + res.end(); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const port = (server.address() as AddressInfo).port; + try { + expect( + await isDockerDriverGatewayHttpReady( + 2000, + `http://127.0.0.1:${port}/openshell.v1.OpenShell/Health`, + ), + ).toBe(false); + } finally { + await new Promise((resolve, reject) => + server.close((err) => (err ? reject(err) : resolve())), + ); + } + }); +}); + describe("waitForGatewayHttpReady (#3258)", () => { it("returns true on the first probe call when the gateway is already responding", async () => { let calls = 0; diff --git a/test/gateway-liveness-probe.test.ts b/test/gateway-liveness-probe.test.ts index 1799b858a7..fee57fb67f 100644 --- a/test/gateway-liveness-probe.test.ts +++ b/test/gateway-liveness-probe.test.ts @@ -36,7 +36,7 @@ describe("gateway liveness probe (#2020)", () => { expect(preflightEnd).toBeGreaterThan(preflightStart); const preflightSection = content.slice(preflightStart, preflightEnd); const preflightProbe = preflightSection.match( - /let gatewayReuseState = gatewaySnapshot\.gatewayReuseState[\s\S]*?verifyGatewayContainerRunning\(\)[\s\S]*?gatewayReuseState = "missing"/, + /let gatewayReuseState = gatewaySnapshot\.gatewayReuseState[\s\S]*?verifyGatewayContainerRunning\(\)[\s\S]*?destroyGatewayForReuse\(/, ); expect(preflightProbe).toBeTruthy(); }); @@ -77,6 +77,23 @@ describe("gateway liveness probe (#2020)", () => { expect(cleanupAfterProbe).toBeTruthy(); }); + it("does not keep stale or drifted gateways reusable when cleanup fails", () => { + const cleanupHelper = fs.readFileSync( + path.join(ROOT, "src/lib/onboard/gateway-cleanup.ts"), + "utf-8", + ); + const failedStaleCleanup = content.match( + /destroyGatewayForReuse\(\s*destroyGateway,\s*" ✓ Stale gateway metadata cleaned up",\s*" ! Stale gateway metadata cleanup failed; leaving registry state intact\."/g, + ); + const failedDriftCleanup = content.match( + /destroyGatewayForReuse\(\s*destroyGateway,\s*" ✓ Previous gateway cleaned up",\s*" ! Previous gateway cleanup failed; leaving registry state intact\."/g, + ); + + expect(cleanupHelper).toMatch(/return "stale"/); + expect(failedStaleCleanup?.length).toBeGreaterThanOrEqual(2); + expect(failedDriftCleanup?.length).toBeGreaterThanOrEqual(2); + }); + it("main onboard flow aborts (does not downgrade or destroy) when Docker is unknown and HTTP is unready (#3258, #2020)", () => { // Regression guard: when verifyGatewayContainerRunning() returns "unknown" // and the host HTTP probe also fails, we cannot tell whether the existing @@ -101,6 +118,21 @@ describe("gateway liveness probe (#2020)", () => { expect(branchBody).not.toMatch(/gatewayReuseState\s*=\s*"missing"/); }); + it("Docker-driver gateway startup requires a live probe before reporting healthy (#3111)", () => { + const dockerStart = content.indexOf("async function startDockerDriverGateway("); + const dockerEnd = content.indexOf("\nasync function startGateway(", dockerStart); + expect(dockerStart).toBeGreaterThanOrEqual(0); + expect(dockerEnd).toBeGreaterThan(dockerStart); + const dockerSection = content.slice(dockerStart, dockerEnd); + + expect(dockerSection).toMatch( + /isGatewayHealthy\(status, namedInfo, currentInfo\)[\s\S]*?await isGatewayTcpReady\(\)[\s\S]*?Docker-driver gateway is healthy/, + ); + expect(dockerSection).toMatch( + /registerDockerDriverGatewayEndpoint\(\)[\s\S]*?await isDockerDriverGatewayHttpReady\(\)[\s\S]*?Reusing existing Docker-driver gateway/, + ); + }); + it("does not modify isGatewayHealthy() in src/lib/state/gateway.ts", () => { // isGatewayHealthy() must remain a pure function — no I/O. // Scope the check to the function body so unrelated helpers don't cause false failures. diff --git a/test/gateway-state.test.ts b/test/gateway-state.test.ts index 4ce21ed72a..21f9cae9cc 100644 --- a/test/gateway-state.test.ts +++ b/test/gateway-state.test.ts @@ -34,6 +34,21 @@ Gateway: nemoclaw Server: https://127.0.0.1:8080/ `; +const STATUS_SERVER_STATUS_REFUSED = ` +Server Status + +Gateway: nemoclaw +Server: https://127.0.0.1:8080/ +Error: Connection refused (os error 61) +`; + +const STATUS_SERVER_STATUS_REFUSED_ANSI = `\x1b[1mServer Status\x1b[0m + +\x1b[2mGateway:\x1b[0m nemoclaw +\x1b[2mServer:\x1b[0m https://127.0.0.1:8080/ +\x1b[31mError: Connection refused (os error 61)\x1b[0m +`; + const GW_INFO_BASE = ` Gateway Info @@ -139,6 +154,14 @@ describe("isGatewayConnected", () => { expect(isGatewayConnected(STATUS_SERVER_STATUS_ONLY)).toBe(true); }); + it("does not treat Server Status with connection errors as connected", () => { + expect(isGatewayConnected(STATUS_SERVER_STATUS_REFUSED)).toBe(false); + }); + + it("does not treat ANSI-wrapped Server Status refusals as connected", () => { + expect(isGatewayConnected(STATUS_SERVER_STATUS_REFUSED_ANSI)).toBe(false); + }); + it("returns false for empty string", () => { expect(isGatewayConnected("")).toBe(false); }); @@ -157,6 +180,12 @@ describe("isGatewayHealthy", () => { expect(isGatewayHealthy(STATUS_SERVER_STATUS_ONLY, GW_INFO_NAMED, GW_INFO_ACTIVE)).toBe(true); }); + it("returns false when status shows Server Status with connection refused", () => { + expect(isGatewayHealthy(STATUS_SERVER_STATUS_REFUSED, GW_INFO_NAMED, GW_INFO_ACTIVE)).toBe( + false, + ); + }); + it("returns true via fallback when status is empty but gateway info confirms health (#1711)", () => { // ARM64 / non-TTY: openshell status returns "" expect(isGatewayHealthy("", GW_INFO_NAMED, GW_INFO_ACTIVE)).toBe(true); @@ -236,6 +265,12 @@ describe("getGatewayReuseState", () => { expect(getGatewayReuseState("", GW_INFO_NAMED, GW_INFO_ACTIVE)).toBe("healthy"); }); + it("returns 'stale' when named gateway exists but status reports connection refused", () => { + expect(getGatewayReuseState(STATUS_SERVER_STATUS_REFUSED, GW_INFO_NAMED, GW_INFO_ACTIVE)).toBe( + "stale", + ); + }); + it("returns 'foreign-active' when connected to a different gateway", () => { expect(getGatewayReuseState(STATUS_FOREIGN, "", "")).toBe("foreign-active"); }); diff --git a/test/generate-hermes-config.test.ts b/test/generate-hermes-config.test.ts index 7704eb0fec..af032383ed 100644 --- a/test/generate-hermes-config.test.ts +++ b/test/generate-hermes-config.test.ts @@ -143,8 +143,8 @@ describe("agents/hermes/generate-config.ts", () => { expect(config.platforms.discord).toBeUndefined(); expect(JSON.stringify(config)).not.toContain("DISCORD_BOT_TOKEN"); expect(envFile).toContain("DISCORD_BOT_TOKEN=openshell:resolve:env:DISCORD_BOT_TOKEN\n"); - expect(envFile).toContain("DISCORD_PROXY=http://127.0.0.1:3129\n"); - expect(envFile).toContain("NEMOCLAW_DISCORD_FACADE_URL=http://127.0.0.1:3130\n"); + expect(envFile).not.toContain("DISCORD_PROXY="); + expect(envFile).not.toContain("NEMOCLAW_DISCORD_FACADE_URL"); expect(envFile).toContain("NEMOCLAW_DISCORD_GUILD_IDS=1491590992753590594\n"); expect(envFile).toContain("DISCORD_ALLOWED_USERS=1005536447329222676\n"); }); diff --git a/test/generate-openclaw-config.test.ts b/test/generate-openclaw-config.test.ts index d0ec87880b..126a51355e 100644 --- a/test/generate-openclaw-config.test.ts +++ b/test/generate-openclaw-config.test.ts @@ -213,6 +213,8 @@ describe("generate-openclaw-config.py: config generation", () => { expect(config.channels.discord.accounts.default.token).toBe( "openshell:resolve:env:DISCORD_BOT_TOKEN", ); + expect(config.channels.telegram.accounts.default.proxy).toBe("http://10.200.0.1:3128"); + expect(config.channels.discord.accounts.default.proxy).toBeUndefined(); }); it("emits Bolt-shape placeholders for Slack so the SDK's prefix regex passes", () => { @@ -220,8 +222,7 @@ describe("generate-openclaw-config.py: config generation", () => { const config = runConfigScript({ NEMOCLAW_MESSAGING_CHANNELS_B64: channels }); const slack = config.channels.slack.accounts.default; // Bolt validates ^xoxb-[A-Za-z0-9_-]+$ / ^xapp-…$ at App construction. - // The slack-token-rewriter preload translates these to canonical form - // before egress, where OpenShell's L7 proxy substitutes the real token. + // OpenShell resolves these provider-shaped aliases at the egress boundary. expect(slack.botToken).toBe("xoxb-OPENSHELL-RESOLVE-ENV-SLACK_BOT_TOKEN"); expect(slack.appToken).toBe("xapp-OPENSHELL-RESOLVE-ENV-SLACK_APP_TOKEN"); expect(slack.botToken).toMatch(/^xoxb-[A-Za-z0-9_-]+$/); diff --git a/test/hermes-decode-proxy.test.ts b/test/hermes-decode-proxy.test.ts deleted file mode 100644 index 84643c1dd6..0000000000 --- a/test/hermes-decode-proxy.test.ts +++ /dev/null @@ -1,123 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { spawnSync } from "node:child_process"; -import path from "node:path"; -import { describe, expect, it } from "vitest"; - -const PROXY_PATH = path.join(import.meta.dirname, "..", "agents", "hermes", "decode-proxy.py"); - -function runProxyProbe(requestBody: string) { - const script = String.raw` -import asyncio -import importlib.util -import json -import os - -module_path = os.environ["NEMOCLAW_DECODE_PROXY_PATH"] -spec = importlib.util.spec_from_file_location("decode_proxy", module_path) -mod = importlib.util.module_from_spec(spec) -spec.loader.exec_module(mod) - -async def main(): - captured = {} - - async def upstream(reader, writer): - request_line = (await reader.readline()).decode("utf-8", errors="replace").rstrip("\r\n") - headers = {} - while True: - line = await reader.readline() - if line in (b"\r\n", b"\n", b""): - break - text = line.decode("utf-8", errors="replace").rstrip("\r\n") - name, value = text.split(":", 1) - headers[name.lower()] = value.strip() - length = int(headers.get("content-length", "0")) - body = await reader.readexactly(length) if length else b"" - captured.update({ - "requestLine": request_line, - "headers": headers, - "body": body.decode("utf-8", errors="replace"), - }) - writer.write(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nOK") - await writer.drain() - writer.close() - await writer.wait_closed() - - upstream_server = await asyncio.start_server(upstream, "127.0.0.1", 0) - mod.UPSTREAM_HOST = "127.0.0.1" - mod.UPSTREAM_PORT = upstream_server.sockets[0].getsockname()[1] - proxy_server = await asyncio.start_server(mod.handle_client, "127.0.0.1", 0) - proxy_port = proxy_server.sockets[0].getsockname()[1] - - reader, writer = await asyncio.open_connection("127.0.0.1", proxy_port) - body = os.environ["NEMOCLAW_PROBE_BODY"].encode("utf-8") - request = ( - b"POST http://slack.test/api/auth.test?token=xoxb-OPENSHELL-RESOLVE-ENV-SLACK_BOT_TOKEN HTTP/1.1\r\n" - b"Host: slack.test\r\n" - b"Authorization: Bearer xoxb-OPENSHELL-RESOLVE-ENV-SLACK_BOT_TOKEN\r\n" - b"Content-Type: application/x-www-form-urlencoded\r\n" - + f"Content-Length: {len(body)}\r\n".encode("ascii") - + b"\r\n" - + body - ) - writer.write(request) - await writer.drain() - writer.write_eof() - await reader.read() - writer.close() - await writer.wait_closed() - - proxy_server.close() - upstream_server.close() - await proxy_server.wait_closed() - await upstream_server.wait_closed() - print(json.dumps(captured, sort_keys=True)) - -asyncio.run(main()) -`; - - const result = spawnSync("python3", ["-c", script], { - encoding: "utf-8", - env: { - ...process.env, - NEMOCLAW_DECODE_PROXY_PATH: PROXY_PATH, - NEMOCLAW_PROBE_BODY: requestBody, - }, - timeout: 10_000, - }); - - if (result.status !== 0) { - throw new Error( - `decode proxy probe failed (exit ${result.status}):\nstdout: ${result.stdout}\nstderr: ${result.stderr}`, - ); - } - - return JSON.parse(result.stdout) as { - requestLine: string; - headers: Record; - body: string; - }; -} - -describe("agents/hermes/decode-proxy.py", () => { - it("rewrites Slack bot placeholders in request paths and headers", () => { - const captured = runProxyProbe("token=xapp-OPENSHELL-RESOLVE-ENV-SLACK_APP_TOKEN"); - - expect(captured.requestLine).toContain("token=openshell:resolve:env:SLACK_BOT_TOKEN"); - expect(captured.headers.authorization).toBe( - "Bearer openshell:resolve:env:SLACK_BOT_TOKEN", - ); - expect(captured.requestLine).not.toContain("OPENSHELL-RESOLVE-ENV-"); - expect(captured.headers.authorization).not.toContain("OPENSHELL-RESOLVE-ENV-"); - }); - - it("rewrites Slack app placeholders in request bodies and adjusts Content-Length", () => { - const expectedBody = "token=openshell:resolve:env:SLACK_APP_TOKEN"; - const captured = runProxyProbe("token=xapp-OPENSHELL-RESOLVE-ENV-SLACK_APP_TOKEN"); - - expect(captured.body).toBe(expectedBody); - expect(captured.body).not.toContain("OPENSHELL-RESOLVE-ENV-"); - expect(captured.headers["content-length"]).toBe(String(Buffer.byteLength(expectedBody))); - }); -}); diff --git a/test/hermes-discord-facade.test.ts b/test/hermes-discord-facade.test.ts deleted file mode 100644 index 4e64629e81..0000000000 --- a/test/hermes-discord-facade.test.ts +++ /dev/null @@ -1,513 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { spawnSync } from "node:child_process"; -import path from "node:path"; -import { describe, expect, it } from "vitest"; - -const ROOT = path.resolve(import.meta.dirname, ".."); -const FACADE = path.join(ROOT, "agents", "hermes", "discord-facade.py"); -const PRELOAD = path.join(ROOT, "agents", "hermes", "discord-preload", "sitecustomize.py"); -const sanitizedEnv = Object.fromEntries( - Object.entries(process.env).filter( - ([key, value]) => - value !== undefined && - !key.startsWith("DISCORD_") && - !key.startsWith("SLACK_") && - !key.startsWith("TELEGRAM_"), - ), -) as Record; -const hasCryptography = - spawnSync("python3", ["-c", "import cryptography.hazmat.primitives.asymmetric.ed25519"], { - env: sanitizedEnv, - stdio: "ignore", - }).status === 0; - -function runPython(source: string, env: Record = {}) { - return spawnSync("python3", ["-"], { - input: source, - encoding: "utf-8", - env: { - ...sanitizedEnv, - ...env, - }, - timeout: 10_000, - }); -} - -function pythonPrelude(): string { - return ` -import asyncio -import importlib.util -import json -import sys -import types - -aiohttp = types.ModuleType("aiohttp") - -class FakeResponse: - def __init__(self, *, status=200, body=b"", headers=None): - self.status = status - self.body = body - self.headers = headers or {} - -class FakeWeb: - Response = FakeResponse - class WebSocketResponse: - pass - class Request: - pass - @staticmethod - def json_response(data, status=200, dumps=json.dumps): - return FakeResponse(status=status, body=dumps(data).encode("utf-8"), headers={"Content-Type": "application/json"}) - -class FakeClientSession: - pass - -aiohttp.ClientSession = FakeClientSession -aiohttp.WSMsgType = types.SimpleNamespace(TEXT=1) -aiohttp.web = FakeWeb -sys.modules["aiohttp"] = aiohttp - -spec = importlib.util.spec_from_file_location("discord_facade", ${JSON.stringify(FACADE)}) -discord_facade = importlib.util.module_from_spec(spec) -sys.modules["discord_facade"] = discord_facade -spec.loader.exec_module(discord_facade) -`; -} - -describe("Hermes Discord facade", () => { - it("accepts only the OpenShell placeholder in local Gateway IDENTIFY frames", () => { - const result = runPython(`${pythonPrelude()} -class FakeWS: - def __init__(self): - self.sent = [] - self.closed = None - async def send_str(self, value): - self.sent.append(json.loads(value)) - async def close(self, code=None, message=b""): - self.closed = (code, message) - -async def main(): - facade = discord_facade.DiscordFacade( - host="127.0.0.1", - port=3130, - placeholder_token=discord_facade.DEFAULT_TOKEN_PLACEHOLDER, - upstream_proxy=None, - public_base_url=None, - public_key=None, - ) - good_ws = FakeWS() - good_peer = discord_facade.GatewayPeer(ws=good_ws) - await facade._handle_gateway_payload(good_peer, {"op": 2, "d": {"token": discord_facade.DEFAULT_TOKEN_PLACEHOLDER}}) - assert good_peer.identified is True - assert good_ws.sent[-1]["t"] == "READY" - assert good_ws.sent[-1]["d"]["user"]["bot"] is True - - bad_ws = FakeWS() - bad_peer = discord_facade.GatewayPeer(ws=bad_ws) - realish = "mfa.AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA.BBBBBB.CCCCCCCCCCCCCCCCCCCCCCCCCCCCC" - await facade._handle_gateway_payload(bad_peer, {"op": 2, "d": {"token": realish}}) - assert bad_peer.identified is False - assert bad_ws.closed[0] == 4004 - -asyncio.run(main()) -`); - - expect(result.status, result.stderr || result.stdout).toBe(0); - }); - - it("forwards REST requests with the placeholder Authorization header intact", () => { - const result = runPython(`${pythonPrelude()} -class ForwardedResponse: - status = 202 - headers = {"Content-Type": "application/json"} - async def read(self): - return b'{"ok":true}' - -class ForwardContext: - async def __aenter__(self): - return ForwardedResponse() - async def __aexit__(self, exc_type, exc, tb): - return False - -class FakeSession: - def __init__(self): - self.calls = [] - def request(self, method, target, headers, data, proxy, allow_redirects): - self.calls.append({ - "method": method, - "target": target, - "headers": headers, - "data": data, - "proxy": proxy, - "allow_redirects": allow_redirects, - }) - return ForwardContext() - -class FakeRequest: - method = "POST" - path = "/api/v10/channels/123/messages" - path_qs = "/api/v10/channels/123/messages" - query_string = "" - headers = { - "Host": "127.0.0.1:3130", - "Authorization": "Bot openshell:resolve:env:DISCORD_BOT_TOKEN", - "Content-Type": "application/json", - "Content-Length": "2", - } - async def read(self): - return b"{}" - -async def main(): - session = FakeSession() - facade = discord_facade.DiscordFacade( - host="127.0.0.1", - port=3130, - placeholder_token=discord_facade.DEFAULT_TOKEN_PLACEHOLDER, - upstream_proxy="http://127.0.0.1:3129", - public_base_url=None, - public_key=None, - ) - facade._session = session - response = await facade._forward_rest(FakeRequest()) - assert response.status == 202 - assert len(session.calls) == 1 - call = session.calls[0] - assert call["target"] == "https://discord.com/api/v10/channels/123/messages" - assert call["headers"]["Authorization"] == "Bot openshell:resolve:env:DISCORD_BOT_TOKEN" - assert "Content-Length" not in call["headers"] - assert call["proxy"] == "http://127.0.0.1:3129" - -asyncio.run(main()) -`); - - expect(result.status, result.stderr || result.stdout).toBe(0); - }); - - it("synthesizes message, reaction, and thread Gateway events from REST polling", () => { - const result = runPython(`${pythonPrelude()} -from urllib.parse import urlparse - -state = { - "channels": [{"id": "200", "type": 0, "guild_id": "100"}], - "messages": [{ - "id": "300", - "channel_id": "200", - "guild_id": "100", - "content": "hello", - "author": {"id": "400", "username": "user", "discriminator": "0000"}, - "timestamp": "2026-05-07T00:00:00.000000+00:00", - "reactions": [{"count": 1, "emoji": {"name": "thumbsup"}}], - }], - "threads": [{"id": "500", "guild_id": "100", "parent_id": "200", "type": 11, "name": "thread"}], -} - -class PollResponse: - def __init__(self, payload): - self.status = 200 - self.headers = {"Content-Type": "application/json"} - self.payload = payload - async def read(self): - return json.dumps(self.payload).encode("utf-8") - -class PollContext: - def __init__(self, payload): - self.payload = payload - async def __aenter__(self): - return PollResponse(self.payload) - async def __aexit__(self, exc_type, exc, tb): - return False - -class PollSession: - def request(self, method, target, headers, proxy, allow_redirects): - path = urlparse(target).path - assert headers["Authorization"] == "Bot openshell:resolve:env:DISCORD_BOT_TOKEN" - if path == "/api/v10/guilds/100/channels": - return PollContext(state["channels"]) - if path == "/api/v10/channels/200/messages": - return PollContext(state["messages"]) - if path == "/api/v10/guilds/100/threads/active": - return PollContext({"threads": state["threads"]}) - raise AssertionError(path) - -async def main(): - facade = discord_facade.DiscordFacade( - host="127.0.0.1", - port=3130, - placeholder_token=discord_facade.DEFAULT_TOKEN_PLACEHOLDER, - upstream_proxy="http://127.0.0.1:3129", - public_base_url=None, - public_key=None, - ) - facade._session = PollSession() - facade._poll_guild_ids = {"100"} - events = [] - async def record(event_type, data): - events.append((event_type, data)) - facade.dispatch_to_all = record - - await facade._poll_once() - assert [event[0] for event in events] == ["MESSAGE_CREATE", "THREAD_CREATE"] - - events.clear() - state["messages"][0]["content"] = "hello edited" - state["messages"][0]["edited_timestamp"] = "2026-05-07T00:01:00.000000+00:00" - state["messages"][0]["reactions"][0]["count"] = 2 - await facade._poll_once() - event_names = [event[0] for event in events] - assert "MESSAGE_UPDATE" in event_names - assert "MESSAGE_REACTION_ADD" in event_names - - events.clear() - state["messages"] = [ - { - "id": str(1000 + index), - "channel_id": "200", - "guild_id": "100", - "content": f"page item {index}", - "author": {"id": "400", "username": "user", "discriminator": "0000"}, - "timestamp": "2026-05-07T00:02:00.000000+00:00", - "reactions": [], - } - for index in range(25) - ] - await facade._poll_once() - event_names = [event[0] for event in events] - assert "MESSAGE_DELETE" not in event_names - - events.clear() - state["messages"] = [] - state["threads"] = [] - await facade._poll_once() - event_names = [event[0] for event in events] - assert "MESSAGE_DELETE" in event_names - assert "THREAD_DELETE" in event_names - -asyncio.run(main()) -`); - - expect(result.status, result.stderr || result.stdout).toBe(0); - }); - - (hasCryptography ? it : it.skip)( - "validates Discord interaction signatures and keeps real interaction tokens out of Gateway payloads", - () => { - const result = runPython(`${pythonPrelude()} -import json -from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey -from cryptography.hazmat.primitives import serialization - -private_key = Ed25519PrivateKey.generate() -public_key = private_key.public_key().public_bytes( - encoding=serialization.Encoding.Raw, - format=serialization.PublicFormat.Raw, -).hex() -timestamp = "1710000000" -body = b'{"type":2,"token":"real-interaction-token","id":"42"}' -signature = private_key.sign(timestamp.encode("utf-8") + body).hex() - -class FakeRequest: - headers = { - "X-Signature-Ed25519": signature, - "X-Signature-Timestamp": timestamp, - } - -facade = discord_facade.DiscordFacade( - host="127.0.0.1", - port=3130, - placeholder_token=discord_facade.DEFAULT_TOKEN_PLACEHOLDER, - upstream_proxy=None, - public_base_url=None, - public_key=public_key, -) -assert facade._verify_signature(FakeRequest(), body) is True -FakeRequest.headers["X-Signature-Ed25519"] = "00" * 64 -assert facade._verify_signature(FakeRequest(), body) is False - -localized = facade._localize_interaction_token(json.loads(body)) -assert localized["token"].startswith("nemoclaw-local-") -assert "real-interaction-token" not in json.dumps(localized) -assert facade._interaction_tokens[localized["token"]][0] == "real-interaction-token" -facade._interaction_tokens["expired-token"] = ("stale", 0.0) -facade._prune_interaction_tokens() -assert "expired-token" not in facade._interaction_tokens -`); - - expect(result.status, result.stderr || result.stdout).toBe(0); - }, - ); - - it("maps localized interaction callback tokens back to Discord before forwarding", () => { - const result = runPython(`${pythonPrelude()} -import asyncio - -class ForwardResponse: - status = 204 - headers = {"Content-Type": "application/json"} - async def read(self): - return b"" - -class ForwardContext: - async def __aenter__(self): - return ForwardResponse() - async def __aexit__(self, exc_type, exc, tb): - return False - -class ForwardSession: - def __init__(self): - self.calls = [] - def request(self, method, target, headers, data, proxy, allow_redirects): - self.calls.append({ - "method": method, - "target": target, - "headers": headers, - "data": data, - "proxy": proxy, - "allow_redirects": allow_redirects, - }) - return ForwardContext() - -class FakeRequest: - method = "POST" - path = "/api/v10/interactions/42/nemoclaw-local-token/callback" - path_qs = path - query_string = "" - headers = {"Authorization": "Bot placeholder", "Content-Type": "application/json"} - async def read(self): - return b'{"type":4,"data":{"content":"done"}}' - -async def main(): - facade = discord_facade.DiscordFacade( - host="127.0.0.1", - port=3130, - placeholder_token=discord_facade.DEFAULT_TOKEN_PLACEHOLDER, - upstream_proxy="http://127.0.0.1:3129", - public_base_url=None, - public_key=None, - ) - session = ForwardSession() - facade._session = session - facade._store_interaction_token("nemoclaw-local-token", "real-interaction-token") - response = await facade._handle_interaction_callback(FakeRequest(), "42", "nemoclaw-local-token") - assert response.status == 204 - assert len(session.calls) == 1 - assert session.calls[0]["target"] == "https://discord.com/api/v10/interactions/42/real-interaction-token/callback" - assert session.calls[0]["data"] == b'{"type":4,"data":{"content":"done"}}' - assert session.calls[0]["proxy"] == "http://127.0.0.1:3129" - -asyncio.run(main()) -`); - - expect(result.status, result.stderr || result.stdout).toBe(0); - }); -}); - -describe("Hermes Discord preload", () => { - it("rewrites discord.py REST and Gateway aiohttp calls to the local facade", () => { - const result = runPython(` -import asyncio -import importlib.util -import os -import sys -import types - -aiohttp = types.ModuleType("aiohttp") - -class ClientSession: - async def _request(self, method, url, **kwargs): - return {"method": method, "url": str(url), "kwargs": kwargs} - def ws_connect(self, url, **kwargs): - return {"url": str(url), "kwargs": kwargs} - -aiohttp.ClientSession = ClientSession -sys.modules["aiohttp"] = aiohttp -os.environ["NEMOCLAW_DISCORD_FACADE_URL"] = "http://127.0.0.1:3130" - -spec = importlib.util.spec_from_file_location("sitecustomize", ${JSON.stringify(PRELOAD)}) -module = importlib.util.module_from_spec(spec) -spec.loader.exec_module(module) - -async def main(): - session = aiohttp.ClientSession() - rest = await session._request("GET", "https://discord.com/api/v10/users/@me?x=1", proxy="http://proxy") - assert rest["url"] == "http://127.0.0.1:3130/api/v10/users/@me?x=1" - assert "proxy" not in rest["kwargs"] - ws = session.ws_connect("wss://gateway.discord.gg/?encoding=json", proxy="http://proxy") - assert ws["url"] == "ws://127.0.0.1:3130/gateway?encoding=json&v=10" - assert "proxy" not in ws["kwargs"] - -asyncio.run(main()) -`); - - expect(result.status, result.stderr || result.stdout).toBe(0); - }); - - it("rewrites Slack placeholders in aiohttp requests before HTTPS serialization", () => { - const result = runPython(` -import asyncio -import importlib.util -import sys -import types - -aiohttp = types.ModuleType("aiohttp") - -class ClientSession: - async def _request(self, method, url, **kwargs): - return {"method": method, "url": str(url), "kwargs": kwargs} - -aiohttp.ClientSession = ClientSession -sys.modules["aiohttp"] = aiohttp - -spec = importlib.util.spec_from_file_location("sitecustomize", ${JSON.stringify(PRELOAD)}) -module = importlib.util.module_from_spec(spec) -spec.loader.exec_module(module) - -async def main(): - session = aiohttp.ClientSession() - result = await session._request( - "POST", - "https://slack.com/api/auth.test?token=xoxb-OPENSHELL-RESOLVE-ENV-SLACK_BOT_TOKEN", - headers={ - "Authorization": "Bearer xoxb-OPENSHELL-RESOLVE-ENV-SLACK_BOT_TOKEN", - "X-Audit": ["xapp-OPENSHELL-RESOLVE-ENV-SLACK_APP_TOKEN", "unchanged"], - }, - data=b"token=xapp-OPENSHELL-RESOLVE-ENV-SLACK_APP_TOKEN", - params={"token": "xoxb-OPENSHELL-RESOLVE-ENV-SLACK_BOT_TOKEN"}, - ) - assert result["url"] == "https://slack.com/api/auth.test?token=openshell:resolve:env:SLACK_BOT_TOKEN" - assert result["kwargs"]["headers"]["Authorization"] == "Bearer openshell:resolve:env:SLACK_BOT_TOKEN" - assert result["kwargs"]["headers"]["X-Audit"][0] == "openshell:resolve:env:SLACK_APP_TOKEN" - assert result["kwargs"]["headers"]["X-Audit"][1] == "unchanged" - assert result["kwargs"]["data"] == b"token=openshell:resolve:env:SLACK_APP_TOKEN" - assert result["kwargs"]["params"]["token"] == "openshell:resolve:env:SLACK_BOT_TOKEN" - -asyncio.run(main()) -`); - - expect(result.status, result.stderr || result.stdout).toBe(0); - }); - - it("rewrites Slack placeholders in urllib requests before CONNECT", () => { - const result = runPython(` -import importlib.util -import urllib.request - -spec = importlib.util.spec_from_file_location("sitecustomize", ${JSON.stringify(PRELOAD)}) -module = importlib.util.module_from_spec(spec) -spec.loader.exec_module(module) - -req = urllib.request.Request( - "https://slack.com/api/apps.connections.open?token=xapp-OPENSHELL-RESOLVE-ENV-SLACK_APP_TOKEN", - data=b"token=xoxb-OPENSHELL-RESOLVE-ENV-SLACK_BOT_TOKEN", - headers={"Authorization": "Bearer xapp-OPENSHELL-RESOLVE-ENV-SLACK_APP_TOKEN"}, - method="POST", -) -assert req.full_url == "https://slack.com/api/apps.connections.open?token=openshell:resolve:env:SLACK_APP_TOKEN" -assert req.get_header("Authorization") == "Bearer openshell:resolve:env:SLACK_APP_TOKEN" -assert req.data == b"token=openshell:resolve:env:SLACK_BOT_TOKEN" -`); - - expect(result.status, result.stderr || result.stdout).toBe(0); - }); -}); diff --git a/test/install-openshell-version-check.test.ts b/test/install-openshell-version-check.test.ts index f00d935129..87c8be6f5b 100644 --- a/test/install-openshell-version-check.test.ts +++ b/test/install-openshell-version-check.test.ts @@ -16,14 +16,21 @@ function writeExecutable(target: string, contents: string) { /** * Run install-openshell.sh with a fake `openshell` binary that reports the * given version. The download/install code path is never reached because we - * either exit early (version ok / too high) or hit the upgrade warn and then - * the script tries to download — so we stub curl and gh to fail fast. + * either exit early (version + capability ok / too high / missing capability) + * or hit the upgrade warn and then the script tries to download — so we stub + * curl and gh to fail fast. */ function runWithInstalledVersion( version: string, extraEnv: NodeJS.ProcessEnv = {}, - options: { driverBins?: boolean | "gateway" | "gateway-vm"; os?: string; arch?: string } = {}, + options: { + capability?: boolean; + driverBins?: boolean | "gateway" | "gateway-vm"; + os?: string; + arch?: string; + } = {}, ) { + const capability = options.capability ?? true; const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-openshell-ver-")); try { const fakeBin = path.join(tmp, "bin"); @@ -40,6 +47,7 @@ if [ "\${1:-}" = "-m" ]; then echo "${options.arch ?? "x86_64"}"; else echo "${o path.join(fakeBin, "openshell"), `#!/usr/bin/env bash if [ "\${1:-}" = "--version" ]; then echo "openshell ${version}"; exit 0; fi +${capability ? "# request-body-credential-rewrite websocket-credential-rewrite" : ""} exit 99`, ); @@ -116,27 +124,33 @@ exit 0`, } describe("install-openshell.sh version check", { timeout: 15_000 }, () => { - it("exits cleanly when openshell 0.0.37 and driver binaries are already installed", () => { - const result = runWithInstalledVersion("0.0.37"); + it("exits cleanly when openshell 0.0.39 and driver binaries are already installed", () => { + const result = runWithInstalledVersion("0.0.39"); expect(result.status).toBe(0); - expect(result.stdout).toMatch(/already installed.*0\.0\.37/); + expect(result.stdout).toMatch(/already installed.*0\.0\.39/); }); - it("triggers reinstall when openshell 0.0.37 is missing Docker-driver binaries", () => { - const result = runWithInstalledVersion("0.0.37", {}, { driverBins: false, os: "Linux" }); + it("triggers reinstall when openshell 0.0.39 is missing Docker-driver binaries", () => { + const result = runWithInstalledVersion("0.0.39", {}, { driverBins: false, os: "Linux" }); expect(result.status).not.toBe(0); expect(result.stdout).toMatch(/missing Docker-driver binaries/); - expect(result.stdout).toMatch(/Installing OpenShell from release 'v0\.0\.37'/); + expect(result.stdout).toMatch(/Installing OpenShell from release 'v0\.0\.39'/); }); - it("accepts macOS openshell 0.0.37 when the gateway and VM driver binaries are installed", () => { - const result = runWithInstalledVersion("0.0.37", {}, { + it("fails closed when openshell 0.0.39 lacks required messaging rewrite support", () => { + const result = runWithInstalledVersion("0.0.39", {}, { capability: false }); + expect(result.status).toBe(1); + expect(result.stdout).toMatch(/missing request-body-credential-rewrite support/); + }); + + it("accepts macOS openshell 0.0.39 when the gateway and VM driver binaries are installed", () => { + const result = runWithInstalledVersion("0.0.39", {}, { driverBins: "gateway-vm", os: "Darwin", arch: "arm64", }); expect(result.status).toBe(0); - expect(result.stdout).toMatch(/already installed.*0\.0\.37/); + expect(result.stdout).toMatch(/already installed.*0\.0\.39/); }); it("repairs macOS openshell-driver-vm when the Hypervisor entitlement is missing", () => { @@ -145,7 +159,7 @@ describe("install-openshell.sh version check", { timeout: 15_000 }, () => { const state = path.join(tmp, "codesign-state"); const log = path.join(tmp, "codesign.log"); const result = runWithInstalledVersion( - "0.0.37", + "0.0.39", { NEMOCLAW_FAKE_CODESIGN_HAS_ENTITLEMENT: "0", NEMOCLAW_FAKE_CODESIGN_STATE: state, @@ -161,7 +175,7 @@ describe("install-openshell.sh version check", { timeout: 15_000 }, () => { expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); expect(result.stdout).toMatch(/missing the macOS Hypervisor entitlement/); expect(result.stdout).toMatch(/Signing openshell-driver-vm/); - expect(result.stdout).toMatch(/already installed.*0\.0\.37/); + expect(result.stdout).toMatch(/already installed.*0\.0\.39/); expect(result.stdout).not.toMatch(/Installing OpenShell from release/); expect(fs.readFileSync(log, "utf-8")).toContain("--force --sign - --entitlements"); } finally { @@ -169,15 +183,15 @@ describe("install-openshell.sh version check", { timeout: 15_000 }, () => { } }); - it("triggers reinstall on macOS when openshell 0.0.37 is missing required gateway binaries", () => { - const result = runWithInstalledVersion("0.0.37", {}, { + it("triggers reinstall on macOS when openshell 0.0.39 is missing required gateway binaries", () => { + const result = runWithInstalledVersion("0.0.39", {}, { driverBins: false, os: "Darwin", arch: "arm64", }); expect(result.status).not.toBe(0); expect(result.stdout).toMatch(/missing Docker-driver binaries/); - expect(result.stdout).toMatch(/Installing OpenShell from release 'v0\.0\.37'/); + expect(result.stdout).toMatch(/Installing OpenShell from release 'v0\.0\.39'/); }); it("downloads the macOS arm64 gateway and VM helper assets during reinstall", () => { @@ -248,6 +262,15 @@ exit 0`, writeExecutable( path.join(fakeBin, "install"), `#!/usr/bin/env bash +dest="\${@: -1}" +mkdir -p "$(dirname "$dest")" +cat > "$dest" <<'EOF' +#!/usr/bin/env bash +if [ "\${1:-}" = "--version" ]; then echo "openshell 0.0.39"; exit 0; fi +# request-body-credential-rewrite websocket-credential-rewrite +exit 0 +EOF +chmod +x "$dest" exit 0`, ); @@ -365,7 +388,7 @@ printf '%s\\n' "$dest" >> ${JSON.stringify(installLog)} mkdir -p "$(dirname "$dest")" case "$(basename "$dest")" in openshell) - printf '#!/usr/bin/env bash\\nif [ "$1" = "--version" ]; then echo "openshell 0.0.37"; else exit 0; fi\\n' > "$dest" + printf '#!/usr/bin/env bash\\nif [ "$1" = "--version" ]; then echo "openshell 0.0.39"; else exit 0; fi\\n# request-body-credential-rewrite websocket-credential-rewrite\\n' > "$dest" ;; *) printf '#!/usr/bin/env bash\\nexit 0\\n' > "$dest" @@ -396,8 +419,8 @@ exit 0`, } }); - it("triggers upgrade when openshell 0.0.36 is installed (below current floor)", () => { - const result = runWithInstalledVersion("0.0.36", {}, { os: "Linux" }); + it("triggers upgrade when openshell 0.0.38 is installed (below current floor)", () => { + const result = runWithInstalledVersion("0.0.38"); expect(result.status).not.toBe(0); expect(result.stdout).toMatch(/below minimum.*upgrading/); }); @@ -422,7 +445,7 @@ exit 0`, }); it("fails with a clear error when openshell is above MAX_VERSION", () => { - const result = runWithInstalledVersion("0.0.38"); + const result = runWithInstalledVersion("0.0.40"); expect(result.status).toBe(1); expect(result.stdout).toMatch(/above the maximum/); }); @@ -434,7 +457,7 @@ exit 0`, }); it("accepts an installed OpenShell dev-channel Docker-driver build", () => { - const result = runWithInstalledVersion("0.0.37.dev84+g6b2180425", { + const result = runWithInstalledVersion("0.0.39.dev84+g6b2180425", { NEMOCLAW_OPENSHELL_CHANNEL: "dev", }); expect(result.status).toBe(0); @@ -446,7 +469,7 @@ exit 0`, NEMOCLAW_OPENSHELL_CHANNEL: "dev", }); expect(result.status).not.toBe(0); - expect(result.stdout).toMatch(/required dev-channel Docker-driver build/); + expect(result.stdout).toMatch(/required dev-channel messaging-rewrite build/); }); it("proceeds to install when openshell is not present", () => { diff --git a/test/nemoclaw-start.test.ts b/test/nemoclaw-start.test.ts index f92eea2dc5..b036b90b67 100644 --- a/test/nemoclaw-start.test.ts +++ b/test/nemoclaw-start.test.ts @@ -277,9 +277,9 @@ describe("nemoclaw-start non-root fallback", () => { 'lock_rc_files() { :; }', 'configure_messaging_channels() { echo "SHOULD_NOT_CONFIGURE"; exit 70; }', 'install_telegram_diagnostics() { echo "SHOULD_NOT_INSTALL"; exit 71; }', - 'install_slack_token_rewriter() { echo "SHOULD_NOT_INSTALL"; exit 72; }', 'install_slack_channel_guard() { echo "SHOULD_NOT_INSTALL"; exit 73; }', 'verify_no_slack_secrets_on_disk() { echo "SHOULD_NOT_VERIFY"; exit 74; }', + 'seed_default_workspace_templates() { :; }', '_SANDBOX_HOME=/sandbox', "NEMOCLAW_CMD=(bash -c 'echo EXPLICIT_COMMAND; exit 23')", nonRootFallbackBlock(src), @@ -426,12 +426,10 @@ describe("nemoclaw-start gateway token export (#1114)", () => { '_NO_PROXY_VAL="localhost,127.0.0.1,::1,${PROXY_HOST}"', '_SANDBOX_SAFETY_NET="/tmp/safety-net.js"', '_PROXY_FIX_SCRIPT="/tmp/http-proxy-fix.js"', - '_WS_FIX_SCRIPT="/nonexistent/ws-proxy-fix.js"', '_NEMOTRON_FIX_SCRIPT="/tmp/nemotron-fix.js"', '_SECCOMP_GUARD_SCRIPT="/tmp/seccomp-guard.js"', '_CIAO_GUARD_SCRIPT="/tmp/ciao-guard.js"', '_SLACK_GUARD_SCRIPT="/nonexistent/slack-guard.js"', - '_SLACK_REWRITER_SCRIPT="/nonexistent/slack-rewriter.js"', "_TOOL_REDIRECTS=()", "set +u", "export_gateway_token", @@ -504,12 +502,10 @@ describe("nemoclaw-start configure guard behavior", () => { '_NO_PROXY_VAL="localhost,127.0.0.1,::1,${PROXY_HOST}"', '_SANDBOX_SAFETY_NET="/tmp/safety-net.js"', '_PROXY_FIX_SCRIPT="/tmp/http-proxy-fix.js"', - '_WS_FIX_SCRIPT="/nonexistent/ws-proxy-fix.js"', '_NEMOTRON_FIX_SCRIPT="/tmp/nemotron-fix.js"', '_SECCOMP_GUARD_SCRIPT="/tmp/seccomp-guard.js"', '_CIAO_GUARD_SCRIPT="/tmp/ciao-guard.js"', '_SLACK_GUARD_SCRIPT="/nonexistent/slack-guard.js"', - '_SLACK_REWRITER_SCRIPT="/nonexistent/slack-rewriter.js"', "_TOOL_REDIRECTS=()", "set +u", runtimeBlock, @@ -821,62 +817,6 @@ describe("runtime model override (#759)", () => { }); }); -describe("mutable OpenClaw config hash", () => { - const src = fs.readFileSync(START_SCRIPT, "utf-8"); - - function runEnsureHash(owner: "sandbox" | "root") { - const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-config-hash-")); - const openclawDir = path.join(root, ".openclaw"); - fs.mkdirSync(openclawDir, { recursive: true }); - fs.writeFileSync(path.join(openclawDir, "openclaw.json"), '{"ok":true}\n'); - - const scriptPath = path.join(root, "run.sh"); - fs.writeFileSync( - scriptPath, - [ - "#!/usr/bin/env bash", - "set -euo pipefail", - `openclaw_config_dir_owner() { printf '%s\\n' ${JSON.stringify(owner)}; }`, - extractShellFunctionFromSource(src, "ensure_mutable_openclaw_config_hash").replaceAll( - "/sandbox/.openclaw", - openclawDir, - ), - "ensure_mutable_openclaw_config_hash", - ].join("\n"), - { mode: 0o700 }, - ); - - const result = spawnSync("bash", [scriptPath], { encoding: "utf-8", timeout: 5000 }); - const hashPath = path.join(openclawDir, ".config-hash"); - const hashExists = fs.existsSync(hashPath); - const hashCheck = hashExists - ? spawnSync("bash", ["-c", `cd ${JSON.stringify(openclawDir)} && sha256sum -c .config-hash --status`], { - encoding: "utf-8", - timeout: 5000, - }) - : undefined; - const hashMode = hashExists ? fs.statSync(hashPath).mode & 0o777 : undefined; - fs.rmSync(root, { recursive: true, force: true }); - return { result, hashExists, hashCheck, hashMode }; - } - - it("creates a missing hash for mutable-default OpenClaw config", () => { - const { result, hashExists, hashCheck, hashMode } = runEnsureHash("sandbox"); - - expect(result.status).toBe(0); - expect(hashExists).toBe(true); - expect(hashCheck?.status).toBe(0); - expect(hashMode).toBe(0o660); - }); - - it("does not synthesize a missing locked config trust anchor", () => { - const { result, hashExists } = runEnsureHash("root"); - - expect(result.status).toBe(0); - expect(hashExists).toBe(false); - }); -}); - describe("runtime CORS origin override (#719)", () => { const src = fs.readFileSync(START_SCRIPT, "utf-8"); @@ -1462,141 +1402,10 @@ describe("NC-2227-01: legacy migration behavior", () => { } finally { fs.rmSync(tmpDir, { recursive: true, force: true }); } - }); + }, 15_000); }); -describe("seed_default_workspace_templates (#3240)", () => { - const src = fs.readFileSync(START_SCRIPT, "utf-8"); - - function runSeed( - workspaceDir: string, - templatesDir: string, - scriptPath: string, - options: { skipBootstrap?: boolean } = {}, - ) { - const configPath = path.join(path.dirname(scriptPath), "openclaw.json"); - fs.writeFileSync( - configPath, - JSON.stringify({ agents: { defaults: { skipBootstrap: options.skipBootstrap ?? true } } }), - ); - fs.writeFileSync( - scriptPath, - [ - "#!/usr/bin/env bash", - "set -euo pipefail", - extractShellFunctionFromSource(src, "seed_default_workspace_templates"), - `seed_default_workspace_templates ${JSON.stringify(workspaceDir)} ${JSON.stringify(templatesDir)} ${JSON.stringify(configPath)}`, - ].join("\n"), - { mode: 0o700 }, - ); - return spawnSync("bash", [scriptPath], { encoding: "utf-8", timeout: 5000 }); - } - - it("seeds the documented workspace templates and skips BOOTSTRAP.md", () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-seed-")); - const workspaceDir = path.join(tmpDir, "workspace"); - const templatesDir = path.join(tmpDir, "templates"); - fs.mkdirSync(workspaceDir, { recursive: true }); - fs.mkdirSync(templatesDir, { recursive: true }); - for (const name of [ - "AGENTS.md", - "SOUL.md", - "IDENTITY.md", - "USER.md", - "TOOLS.md", - "HEARTBEAT.md", - "BOOTSTRAP.md", - ]) { - fs.writeFileSync( - path.join(templatesDir, name), - `---\nsummary: "${name} template"\n---\n# ${name} template content\n`, - ); - } - try { - const result = runSeed(workspaceDir, templatesDir, path.join(tmpDir, "seed.sh")); - expect(result.status).toBe(0); - for (const name of [ - "AGENTS.md", - "SOUL.md", - "IDENTITY.md", - "USER.md", - "TOOLS.md", - "HEARTBEAT.md", - ]) { - expect(fs.existsSync(path.join(workspaceDir, name))).toBe(true); - } - // BOOTSTRAP.md must NOT be seeded — its presence triggers the - // interactive identity-setup turn that skipBootstrap=true is meant - // to suppress. - expect(fs.existsSync(path.join(workspaceDir, "BOOTSTRAP.md"))).toBe(false); - expect(fs.readFileSync(path.join(workspaceDir, "SOUL.md"), "utf-8")).toBe( - "# SOUL.md template content\n", - ); - } finally { - fs.rmSync(tmpDir, { recursive: true, force: true }); - } - }); - - it("does not seed unless OpenClaw bootstrap is explicitly skipped", () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-seed-bootstrap-on-")); - const workspaceDir = path.join(tmpDir, "workspace"); - const templatesDir = path.join(tmpDir, "templates"); - fs.mkdirSync(workspaceDir, { recursive: true }); - fs.mkdirSync(templatesDir, { recursive: true }); - fs.writeFileSync(path.join(templatesDir, "SOUL.md"), "soul template"); - try { - const result = runSeed(workspaceDir, templatesDir, path.join(tmpDir, "seed.sh"), { - skipBootstrap: false, - }); - expect(result.status).toBe(0); - expect(fs.existsSync(path.join(workspaceDir, "SOUL.md"))).toBe(false); - } finally { - fs.rmSync(tmpDir, { recursive: true, force: true }); - } - }); - - it("does not clobber an already-populated workspace", () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-seed-existing-")); - const workspaceDir = path.join(tmpDir, "workspace"); - const templatesDir = path.join(tmpDir, "templates"); - fs.mkdirSync(workspaceDir, { recursive: true }); - fs.mkdirSync(templatesDir, { recursive: true }); - fs.writeFileSync(path.join(workspaceDir, "USER.md"), "user content"); - fs.writeFileSync(path.join(templatesDir, "USER.md"), "template content"); - fs.writeFileSync(path.join(templatesDir, "SOUL.md"), "soul template"); - try { - const result = runSeed(workspaceDir, templatesDir, path.join(tmpDir, "seed.sh")); - expect(result.status).toBe(0); - expect(fs.readFileSync(path.join(workspaceDir, "USER.md"), "utf-8")).toBe("user content"); - // Workspace was non-empty, so no other templates were copied in. - expect(fs.existsSync(path.join(workspaceDir, "SOUL.md"))).toBe(false); - } finally { - fs.rmSync(tmpDir, { recursive: true, force: true }); - } - }); - - it("refuses to seed a symlinked workspace dir", () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-seed-symlink-")); - const realDir = path.join(tmpDir, "real"); - const linkDir = path.join(tmpDir, "link"); - const templatesDir = path.join(tmpDir, "templates"); - fs.mkdirSync(realDir); - fs.mkdirSync(templatesDir); - fs.symlinkSync(realDir, linkDir); - fs.writeFileSync(path.join(templatesDir, "SOUL.md"), "soul template"); - try { - const result = runSeed(linkDir, templatesDir, path.join(tmpDir, "seed.sh")); - expect(result.status).toBe(0); - expect(result.stderr).toContain("refusing to seed symlinked workspace dir"); - expect(fs.existsSync(path.join(realDir, "SOUL.md"))).toBe(false); - } finally { - fs.rmSync(tmpDir, { recursive: true, force: true }); - } - }); - -}); - -describe("Slack token rewriter (#2085)", () => { +describe("Slack secrets-on-disk tripwire (#2085)", () => { const src = fs.readFileSync(START_SCRIPT, "utf-8"); function extractFunction(name: string): string { @@ -1607,69 +1416,6 @@ describe("Slack token rewriter (#2085)", () => { return `${name}() {${match[1]}\n}`; } - function slackRewriterSection(rewriterPath: string, configPath: string): string { - const start = src.indexOf("# ── Slack token rewriter"); - const end = src.indexOf("# ── Slack secrets-on-disk tripwire", start); - if (start === -1 || end === -1 || end <= start) { - throw new Error("Expected Slack token rewriter section in scripts/nemoclaw-start.sh"); - } - return src - .slice(start, end) - .replace( - '_SLACK_REWRITER_SCRIPT="/tmp/nemoclaw-slack-token-rewriter.js"', - `_SLACK_REWRITER_SCRIPT=${JSON.stringify(rewriterPath)}`, - ) - .replace( - '_SLACK_REWRITER_SOURCE="/usr/local/lib/nemoclaw/preloads/slack-token-rewriter.js"', - `_SLACK_REWRITER_SOURCE=${JSON.stringify(path.join(PRELOAD_SCRIPTS, "slack-token-rewriter.js"))}`, - ) - .replace( - 'local config_file="/sandbox/.openclaw/openclaw.json"', - `local config_file=${JSON.stringify(configPath)}`, - ); - } - - it("installs the rewriter only when a Slack placeholder is present", () => { - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-slack-rewriter-start-")); - const configPath = path.join(tmpDir, "openclaw.json"); - const rewriterPath = path.join(tmpDir, "slack-token-rewriter.js"); - const scriptPath = path.join(tmpDir, "run.sh"); - const run = (config: string) => { - fs.writeFileSync(configPath, config); - fs.rmSync(rewriterPath, { force: true }); - fs.writeFileSync( - scriptPath, - [ - "#!/usr/bin/env bash", - "set -euo pipefail", - 'emit_sandbox_sourced_file() { local target="$1"; cat > "$target"; chmod 444 "$target"; }', - "NODE_OPTIONS='--require /already-loaded.js'", - slackRewriterSection(rewriterPath, configPath), - "install_slack_token_rewriter", - 'printf "NODE_OPTIONS=%s\\n" "$NODE_OPTIONS"', - ].join("\n"), - { mode: 0o700 }, - ); - return spawnSync("bash", [scriptPath], { encoding: "utf-8", timeout: 5000 }); - }; - - try { - const noSlack = run('{"channels":{}}\n'); - expect(noSlack.status).toBe(0); - expect(fs.existsSync(rewriterPath)).toBe(false); - expect(noSlack.stdout).not.toContain(rewriterPath); - - const withSlack = run('{"botToken":"xoxb-OPENSHELL-RESOLVE-ENV-SLACK_BOT_TOKEN"}\n'); - expect(withSlack.status).toBe(0); - expect(fs.existsSync(rewriterPath)).toBe(true); - expect((fs.statSync(rewriterPath).mode & 0o777).toString(8)).toBe("444"); - expect(withSlack.stdout).toContain("--require /already-loaded.js"); - expect(withSlack.stdout).toContain(`--require ${rewriterPath}`); - } finally { - fs.rmSync(tmpDir, { recursive: true, force: true }); - } - }); - it("refuses to serve when real Slack tokens leak to disk", () => { const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-slack-secret-")); const configPath = path.join(tmpDir, "openclaw.json"); @@ -1779,12 +1525,11 @@ describe("Telegram diagnostics (#2766)", () => { 'ensure_runtime_shell_env_shim() { :; }', 'lock_rc_files() { :; }', 'configure_messaging_channels() { echo "ORDER:configure"; }', - 'install_slack_token_rewriter() { :; }', 'install_slack_channel_guard() { :; }', 'verify_no_slack_secrets_on_disk() { :; }', + 'seed_default_workspace_templates() { :; }', 'write_auth_profile() { :; }', 'harden_auth_profiles() { :; }', - 'seed_default_workspace_templates() { :; }', 'chown() { :; }', 'chown_tree_no_symlink_follow() { :; }', 'start_persistent_gateway_log_mirror() { :; }', @@ -1799,12 +1544,11 @@ describe("Telegram diagnostics (#2766)", () => { '_SANDBOX_HOME=/sandbox', `_SANDBOX_SAFETY_NET=${JSON.stringify(path.join(tmpDir, "safety.js"))}`, `_PROXY_FIX_SCRIPT=${JSON.stringify(path.join(tmpDir, "proxy-fix.js"))}`, - `_NEMOTRON_FIX_SCRIPT=${JSON.stringify(path.join(tmpDir, "nemotron-fix.js"))}`, `_WS_FIX_SCRIPT=${JSON.stringify(path.join(tmpDir, "ws-fix.js"))}`, + `_NEMOTRON_FIX_SCRIPT=${JSON.stringify(path.join(tmpDir, "nemotron-fix.js"))}`, `_SECCOMP_GUARD_SCRIPT=${JSON.stringify(path.join(tmpDir, "seccomp-guard.js"))}`, `_CIAO_GUARD_SCRIPT=${JSON.stringify(path.join(tmpDir, "ciao-guard.js"))}`, `_SLACK_GUARD_SCRIPT=${JSON.stringify(path.join(tmpDir, "slack-guard.js"))}`, - `_SLACK_REWRITER_SCRIPT=${JSON.stringify(path.join(tmpDir, "slack-rewriter.js"))}`, "NEMOCLAW_CMD=()", telegramDiagnosticsSection(preloadPath, configPath), preGatewaySetupBlock(kind, gatewayLog, autoPairLog), @@ -1954,13 +1698,11 @@ process.stderr.write('FailoverError: token=123456:LATER\\n'); '_NO_PROXY_VAL="localhost,127.0.0.1,::1,${PROXY_HOST}"', `_SANDBOX_SAFETY_NET=${JSON.stringify(path.join(tmpDir, "safety.js"))}`, `_PROXY_FIX_SCRIPT=${JSON.stringify(path.join(tmpDir, "proxy-fix.js"))}`, - `_WS_FIX_SCRIPT=${JSON.stringify(path.join(tmpDir, "ws-fix.js"))}`, `_NEMOTRON_FIX_SCRIPT=${JSON.stringify(path.join(tmpDir, "nemotron-fix.js"))}`, `_SECCOMP_GUARD_SCRIPT=${JSON.stringify(path.join(tmpDir, "seccomp-guard.js"))}`, `_CIAO_GUARD_SCRIPT=${JSON.stringify(path.join(tmpDir, "ciao-guard.js"))}`, `_TELEGRAM_DIAGNOSTICS_SCRIPT=${JSON.stringify(preloadPath)}`, `_SLACK_GUARD_SCRIPT=${JSON.stringify(path.join(tmpDir, "slack-guard.js"))}`, - `_SLACK_REWRITER_SCRIPT=${JSON.stringify(path.join(tmpDir, "slack-rewriter.js"))}`, "_TOOL_REDIRECTS=()", "set +u", runtimeBlock, diff --git a/test/onboard.test.ts b/test/onboard.test.ts index c9ac357615..af2d3ac73d 100644 --- a/test/onboard.test.ts +++ b/test/onboard.test.ts @@ -597,17 +597,17 @@ network_policies: ]); expect(parseDockerCdiSpecDirs("")).toEqual([]); expect( - shouldAllowOpenshellAboveBlueprintMax("openshell 0.0.38.dev1+gabcdef", "linux", { + shouldAllowOpenshellAboveBlueprintMax("openshell 0.0.40.dev1+gabcdef", "linux", { NEMOCLAW_OPENSHELL_CHANNEL: "dev", }), ).toBe(true); expect( - shouldAllowOpenshellAboveBlueprintMax("openshell 0.0.38.dev1+gabcdef", "linux", { + shouldAllowOpenshellAboveBlueprintMax("openshell 0.0.40.dev1+gabcdef", "linux", { NEMOCLAW_OPENSHELL_CHANNEL: "auto", }), ).toBe(false); expect( - shouldAllowOpenshellAboveBlueprintMax("openshell 0.0.38", "linux", { + shouldAllowOpenshellAboveBlueprintMax("openshell 0.0.40", "linux", { NEMOCLAW_OPENSHELL_CHANNEL: "dev", }), ).toBe(false); @@ -2513,6 +2513,45 @@ const { loadAgent } = require(${agentDefsPath}); ); }); + it("fails closed when gateway lifecycle support is not proven", () => { + const source = fs.readFileSync( + path.join(import.meta.dirname, "..", "src", "lib", "onboard", "gateway-lifecycle.ts"), + "utf-8", + ); + + assert.match(source, /normalized\.trim\(\)\.length > 0/); + assert.doesNotMatch(source, /!normalized\.trim\(\)\s*\|\|/); + }); + + it("keeps registry state unless gateway destruction succeeds", () => { + const source = fs.readFileSync( + path.join(import.meta.dirname, "..", "src", "lib", "onboard.ts"), + "utf-8", + ); + + assert.match(source, /function destroyGateway\(\): boolean/); + assert.match(source, /const gatewayRemoved = dockerDriver/); + assert.match(source, /if \(gatewayRemoved\) {\s*registry\.clearAll\(\);\s*}/); + assert.match(source, /return gatewayRemoved;/); + assert.doesNotMatch( + source, + /destroyGateway\(\);\s*registry\.clearAll\(\);\s*gatewayReuseState = "missing"/, + ); + }); + + it("prints package-managed gateway registration hints with the local HTTPS endpoint", () => { + const source = fs.readFileSync( + path.join(import.meta.dirname, "..", "src", "lib", "onboard.ts"), + "utf-8", + ); + + assert.match( + source, + /"gateway",\s*"add",\s*getGatewayLocalEndpoint\(\),\s*"--local",\s*"--name",\s*GATEWAY_NAME/, + ); + assert.doesNotMatch(source, /openshell gateway add http:\/\/127\.0\.0\.1:\$\{GATEWAY_PORT\}/); + }); + it("allows slow sandbox create recovery to wait beyond 60 seconds", () => { const envSource = fs.readFileSync( path.join(import.meta.dirname, "..", "src", "lib", "onboard", "env.ts"), @@ -2594,6 +2633,10 @@ if [[ "$*" == *"doctor"*"logs"* ]]; then printf " Gateway auth token: nvapi-fakecredential-9999\\r\\n" exit 0 fi +if [[ "$*" == "gateway --help" ]]; then + printf "Commands: start destroy\\n" + exit 0 +fi if [[ "$*" == *"gateway"*"start"* ]]; then printf "\\033[33mDeploying\\033[0m gateway nemoclaw...\\r\\n" printf "\\r\\nWaiting for gateway health...\\r\\n" @@ -5767,14 +5810,16 @@ const { createSandbox } = require(${onboardPath}); assert.match(createCommand.policyContent || "", /slack:/); assert.match(createCommand.policyContent || "", /wss-primary\.slack\.com/); - // Discord and Telegram tokens must NOT appear in the sandbox create command + // Messaging tokens must NOT appear in the sandbox create command // (they flow exclusively through the openshell provider credential system). assert.doesNotMatch(createCommand.command, /test-discord-token-value/); assert.doesNotMatch(createCommand.command, /123456:ABC-test-telegram-token/); - // Slack tokens ARE injected as --env args so the baked openclaw.json - // openshell:resolve:env: placeholders resolve inside the container. - assert.match(createCommand.command, /SLACK_BOT_TOKEN=xoxb-test-slack-token-value/); - assert.match(createCommand.command, /SLACK_APP_TOKEN=xapp-test-slack-app-token-value/); + assert.doesNotMatch(createCommand.command, /DISCORD_BOT_TOKEN=/); + assert.doesNotMatch(createCommand.command, /TELEGRAM_BOT_TOKEN=/); + assert.doesNotMatch(createCommand.command, /xoxb-test-slack-token-value/); + assert.doesNotMatch(createCommand.command, /xapp-test-slack-app-token-value/); + assert.doesNotMatch(createCommand.command, /SLACK_BOT_TOKEN=/); + assert.doesNotMatch(createCommand.command, /SLACK_APP_TOKEN=/); // Verify blocked credentials are NOT in the sandbox spawn environment assert.ok(createCommand.env, "expected env to be captured from spawn call"); diff --git a/test/policies.test.ts b/test/policies.test.ts index b383020ccb..7a136f6ced 100644 --- a/test/policies.test.ts +++ b/test/policies.test.ts @@ -749,34 +749,151 @@ describe("policies", () => { expect(graphSection).toContain("method: PATCH"); }); - it("messaging WebSocket presets keep tls: skip on gateway endpoints", () => { + it("messaging WebSocket presets use native inspected WebSocket policy", () => { const cases = [ - { preset: "discord", pattern: /host:\s*gateway\.discord\.gg[\s\S]*?tls:\s*skip/ }, - { preset: "slack", pattern: /host:\s*wss-primary\.slack\.com[\s\S]*?tls:\s*skip/ }, - { preset: "slack", pattern: /host:\s*wss-backup\.slack\.com[\s\S]*?tls:\s*skip/ }, + { + preset: "discord", + host: "gateway.discord.gg", + credentialRewrite: true, + }, + { + preset: "slack", + host: "wss-primary.slack.com", + credentialRewrite: true, + }, + { + preset: "slack", + host: "wss-backup.slack.com", + credentialRewrite: true, + }, ]; - for (const { preset, pattern } of cases) { + for (const { preset, host, credentialRewrite } of cases) { const content = requirePresetContent(policies.loadPreset(preset)); - expect(content).toBeTruthy(); - expect(content).toMatch(pattern); + const parsed = YAML.parse(content) as { + network_policies?: Record< + string, + { + endpoints?: Array<{ + host?: string; + protocol?: string; + access?: string; + tls?: string; + websocket_credential_rewrite?: boolean; + request_body_credential_rewrite?: boolean; + rules?: Array<{ allow?: { method?: string; path?: string } }>; + }>; + } + >; + }; + const endpoints = Object.values(parsed.network_policies ?? {}).flatMap( + (policy) => policy.endpoints ?? [], + ); + const endpoint = endpoints.find((candidate) => candidate.host === host); + expect(endpoint).toBeTruthy(); + expect(endpoint).toMatchObject({ protocol: "websocket", enforcement: "enforce" }); + expect(endpoint).not.toHaveProperty("access"); + expect(endpoint).not.toHaveProperty("tls"); + expect(endpoint?.websocket_credential_rewrite === true).toBe(credentialRewrite); + expect(endpoint?.rules).toEqual( + expect.arrayContaining([ + { allow: { method: "GET", path: "/**" } }, + { allow: { method: "WEBSOCKET_TEXT", path: "/**" } }, + ]), + ); + } + }); + + it("Slack REST endpoints opt into OpenShell request-body credential rewrite", () => { + const policySources = [ + fs.readFileSync( + path.join(REPO_ROOT, "nemoclaw-blueprint/policies/presets/slack.yaml"), + "utf8", + ), + fs.readFileSync(path.join(REPO_ROOT, "agents/hermes/policy-additions.yaml"), "utf8"), + fs.readFileSync(path.join(REPO_ROOT, "agents/hermes/policy-permissive.yaml"), "utf8"), + fs.readFileSync( + path.join(REPO_ROOT, "nemoclaw-blueprint/policies/openclaw-sandbox-permissive.yaml"), + "utf8", + ), + ]; + const slackRestHosts = new Set(["slack.com", "api.slack.com", "hooks.slack.com"]); + + for (const content of policySources) { + const parsed = YAML.parse(content) as { + network_policies?: Record< + string, + { + endpoints?: Array<{ + host?: string; + protocol?: string; + request_body_credential_rewrite?: boolean; + }>; + } + >; + }; + const endpoints = Object.values(parsed.network_policies ?? {}).flatMap( + (policy) => policy.endpoints ?? [], + ); + for (const endpoint of endpoints.filter((candidate) => + slackRestHosts.has(candidate.host ?? ""), + )) { + expect(endpoint).toMatchObject({ + protocol: "rest", + request_body_credential_rewrite: true, + }); + } } }); - it("Hermes Discord gateway policy uses the OpenClaw L4 WebSocket tunnel shape", () => { + it("Hermes messaging gateway policies use native inspected WebSocket policy", () => { const policyFiles = [ path.join(REPO_ROOT, "agents/hermes/policy-additions.yaml"), path.join(REPO_ROOT, "agents/hermes/policy-permissive.yaml"), ]; + const cases = [ + "gateway.discord.gg", + "wss-primary.slack.com", + "wss-backup.slack.com", + ]; for (const file of policyFiles) { const content = fs.readFileSync(file, "utf8"); - const gatewaySection = - content.split("host: gateway.discord.gg")[1]?.split("- host:")[0] ?? ""; - expect(gatewaySection).toContain("access: full"); - expect(gatewaySection).toContain("tls: skip"); - expect(gatewaySection).not.toContain("protocol: rest"); - expect(gatewaySection).not.toContain("rules:"); + const parsed = YAML.parse(content) as { + network_policies?: Record< + string, + { + endpoints?: Array<{ + host?: string; + protocol?: string; + access?: string; + tls?: string; + websocket_credential_rewrite?: boolean; + rules?: Array<{ allow?: { method?: string; path?: string } }>; + }>; + } + >; + }; + const endpoints = Object.values(parsed.network_policies ?? {}).flatMap( + (policy) => policy.endpoints ?? [], + ); + for (const host of cases) { + const endpoint = endpoints.find((candidate) => candidate.host === host); + expect(endpoint).toBeTruthy(); + expect(endpoint).toMatchObject({ + protocol: "websocket", + enforcement: "enforce", + websocket_credential_rewrite: true, + }); + expect(endpoint).not.toHaveProperty("access"); + expect(endpoint).not.toHaveProperty("tls"); + expect(endpoint?.rules).toEqual( + expect.arrayContaining([ + { allow: { method: "GET", path: "/**" } }, + { allow: { method: "WEBSOCKET_TEXT", path: "/**" } }, + ]), + ); + } } }); diff --git a/test/runner.test.ts b/test/runner.test.ts index dcec1b2113..26a2c46895 100644 --- a/test/runner.test.ts +++ b/test/runner.test.ts @@ -691,6 +691,8 @@ describe("regression guards", () => { export -f curl shasum() { cat >/dev/null; echo "checksum OK"; return 0; } export -f shasum + strings() { echo "request-body-credential-rewrite websocket-credential-rewrite"; } + export -f strings tar() { return 0; }; export -f tar install() { return 0; }; export -f install source "${scriptPath}" @@ -726,6 +728,8 @@ describe("regression guards", () => { export -f curl shasum() { echo "SHASUM $*" >> ${JSON.stringify(checksumLog)}; echo "checksum OK"; return 0; } export -f shasum + strings() { echo "request-body-credential-rewrite websocket-credential-rewrite"; } + export -f strings tar() { return 0; }; export -f tar install() { return 0; }; export -f install source "${scriptPath}" diff --git a/test/sandbox-build-context.test.ts b/test/sandbox-build-context.test.ts index 03bf0b6132..c2c0929d0c 100644 --- a/test/sandbox-build-context.test.ts +++ b/test/sandbox-build-context.test.ts @@ -27,7 +27,7 @@ describe("sandbox build context staging", () => { ), ).toBe(true); expect( - fs.existsSync(path.join(buildCtx, "nemoclaw-blueprint", "scripts", "ws-proxy-fix.js")), + fs.existsSync(path.join(buildCtx, "nemoclaw-blueprint", "scripts", "http-proxy-fix.js")), ).toBe(true); expect( fs.existsSync( diff --git a/test/sandbox-init.test.ts b/test/sandbox-init.test.ts index 58420c6446..8efe0681fd 100644 --- a/test/sandbox-init.test.ts +++ b/test/sandbox-init.test.ts @@ -637,61 +637,24 @@ EOF expect(src).not.toContain("_PROXY_MARKER_BEGIN"); }); - it("hermes start.sh persists OpenShell proxy CA env for connect sessions", () => { + it("hermes start.sh routes messaging directly through OpenShell without local bridges", () => { const src = readFileSync(join(import.meta.dirname, "../agents/hermes/start.sh"), "utf-8"); - expect(src).toContain("SSL_CERT_FILE CURL_CA_BUNDLE REQUESTS_CA_BUNDLE GIT_SSL_CAINFO"); - expect(src).toContain("export REQUESTS_CA_BUNDLE="); - expect(src).toContain("export GIT_SSL_CAINFO="); + expect(src).toContain("OpenShell owns credential alias/body/WebSocket rewrite"); + expect(src).not.toContain("DISCORD_PROXY="); + expect(src).not.toContain("DECODE_PROXY_PORT"); + expect(src).not.toContain("start_discord_facade"); + expect(src).not.toContain("NEMOCLAW_DISCORD_FACADE_URL"); + expect(src).not.toContain("nemoclaw-discord-facade"); + expect(src).not.toContain("nemoclaw-decode-proxy"); }); - it("hermes start.sh routes Discord through the local decode proxy", () => { + it("hermes start.sh does not install Python placeholder-normalization preloads", () => { const src = readFileSync(join(import.meta.dirname, "../agents/hermes/start.sh"), "utf-8"); - expect(src).toContain('export DISCORD_PROXY="http://127.0.0.1:${DECODE_PROXY_PORT}"'); - expect(src).toContain('DISCORD_PROXY="http://127.0.0.1:${DECODE_PROXY_PORT}"'); - expect(src).toContain("start_discord_facade"); - expect(src).toContain('NEMOCLAW_DISCORD_FACADE_URL="http://127.0.0.1:${DISCORD_FACADE_PORT}"'); - expect(src).toContain("nemoclaw-discord-facade"); - }); - - it("hermes start.sh prepares the Discord facade log before child redirection", () => { - const src = readFileSync(join(import.meta.dirname, "../agents/hermes/start.sh"), "utf-8"); - const startFn = src.match(/start_discord_facade\(\) \{([\s\S]*?)^}/m); - expect(startFn).toBeTruthy(); - const body = startFn![1]; - expect(body).toContain('local log_path="/tmp/discord-facade.log"'); - expect(body).toContain('prepare_restricted_log "$log_path" gateway:gateway 600'); - expect(body).toContain('prepare_restricted_log "$log_path" "" 600'); - // Privilege step-down now uses STEP_DOWN_PREFIX_GATEWAY (setpriv- - // based, see sandbox-init.sh init_step_down_prefixes), not gosu - // directly — this drops cap_setuid/setgid/fowner/chown/kill from - // the bounding set atomically with reuid. Issue #3280 follow-up. - expect(body).toContain('"${STEP_DOWN_PREFIX_GATEWAY[@]}" sh -c'); - expect(body).not.toContain("gosu gateway sh -c"); - expect(body).toContain('exec "$@" >/tmp/discord-facade.log 2>&1'); - expect(body).not.toContain( - "gosu gateway python3 /usr/local/bin/nemoclaw-discord-facade >/tmp/discord-facade.log", - ); - expect(body).not.toContain( - "python3 /usr/local/bin/nemoclaw-discord-facade >/tmp/discord-facade.log", - ); - }); - - it("hermes start.sh launches the Discord facade and decode proxy under the Hermes venv interpreter", () => { - const src = readFileSync(join(import.meta.dirname, "../agents/hermes/start.sh"), "utf-8"); - expect(src).toContain('HERMES_VENV_PYTHON="/opt/hermes/.venv/bin/python"'); - - const facadeFn = src.match(/start_discord_facade\(\) \{([\s\S]*?)^}/m); - expect(facadeFn).toBeTruthy(); - const facadeBody = facadeFn![1]; - expect(facadeBody).toContain('"$HERMES_VENV_PYTHON" /usr/local/bin/nemoclaw-discord-facade'); - // Must not launch via bare python3 — that's the system interpreter. - expect(facadeBody).not.toMatch(/(? { @@ -699,14 +662,16 @@ EOF expect(src).toContain("validate_tmp_permissions"); }); - it("hermes start.sh routes gateway traffic through the decode proxy", () => { + it("hermes start.sh launches the gateway without a NemoClaw-owned decode proxy", () => { const src = readFileSync(join(import.meta.dirname, "../agents/hermes/start.sh"), "utf-8"); - expect(src).toContain("DECODE_PROXY_PORT=3129"); - expect(src).toContain('"$HERMES_VENV_PYTHON" /usr/local/bin/nemoclaw-decode-proxy'); - expect(src).toContain('HTTPS_PROXY="http://127.0.0.1:${DECODE_PROXY_PORT}"'); - expect(src).toContain('HTTP_PROXY="http://127.0.0.1:${DECODE_PROXY_PORT}"'); - expect(src).toContain('PYTHONPATH="/opt/nemoclaw-hermes-discord-preload${PYTHONPATH:+:${PYTHONPATH}}"'); - expect(src).toContain("start_decode_proxy"); + expect(src).toContain('HERMES_HOME="${HERMES_DIR}"'); + expect(src).toContain("Messaging egress goes directly through OpenShell"); + expect(src).toContain('"${STEP_DOWN_PREFIX_GATEWAY[@]}" sh -c'); + expect(src).not.toContain("gosu gateway sh -c"); + expect(src).not.toContain("DECODE_PROXY_PORT=3129"); + expect(src).not.toContain("/usr/local/bin/nemoclaw-decode-proxy"); + expect(src).not.toContain('HTTPS_PROXY="http://127.0.0.1:${DECODE_PROXY_PORT}"'); + expect(src).not.toContain('HTTP_PROXY="http://127.0.0.1:${DECODE_PROXY_PORT}"'); }); it("hermes start.sh checks immutable bits before legacy migration mutates files", () => { diff --git a/test/sandbox-provisioning.test.ts b/test/sandbox-provisioning.test.ts index 0bf48a9c45..0d2ced975d 100644 --- a/test/sandbox-provisioning.test.ts +++ b/test/sandbox-provisioning.test.ts @@ -267,7 +267,6 @@ describe("sandbox provisioning: copied OpenClaw helper permissions (#2861)", () path.join(localBin, "nemoclaw-codex-acp"), path.join(localLib, "sandbox-init.sh"), path.join(localLib, "generate-openclaw-config.py"), - path.join(localLib, "ws-proxy-fix.js"), pluginFile, nestedPluginFile, ]; @@ -295,15 +294,11 @@ describe("sandbox provisioning: copied OpenClaw helper permissions (#2861)", () const generatorMode = ( fs.statSync(path.join(localLib, "generate-openclaw-config.py")).mode & 0o777 ).toString(8); - const wsProxyMode = (fs.statSync(path.join(localLib, "ws-proxy-fix.js")).mode & 0o777).toString( - 8, - ); const pluginDirMode = (fs.statSync(pluginDir).mode & 0o777).toString(8); const pluginMode = (fs.statSync(pluginFile).mode & 0o777).toString(8); const nestedPluginDirMode = (fs.statSync(nestedPluginDir).mode & 0o777).toString(8); const nestedPluginMode = (fs.statSync(nestedPluginFile).mode & 0o777).toString(8); expect(generatorMode).toBe("755"); - expect(wsProxyMode).toBe("644"); expect(pluginDirMode).toBe("755"); expect(pluginMode).toBe("644"); expect(nestedPluginDirMode).toBe("755"); @@ -488,12 +483,12 @@ describe("Hermes sandbox provisioning", () => { expect(permissivePolicySrc).toContain("- /opt/hermes"); }); - it("allowlists the Discord sitecustomize preload dir so Python can load the facade shim", () => { + it("does not allowlist removed Hermes placeholder-normalization preload dirs", () => { const policySrc = fs.readFileSync(HERMES_POLICY, "utf-8"); const permissivePolicySrc = fs.readFileSync(HERMES_POLICY_PERMISSIVE, "utf-8"); - expect(policySrc).toContain("- /opt/nemoclaw-hermes-discord-preload"); - expect(permissivePolicySrc).toContain("- /opt/nemoclaw-hermes-discord-preload"); + expect(policySrc).not.toContain("- /opt/nemoclaw-hermes-discord-preload"); + expect(permissivePolicySrc).not.toContain("- /opt/nemoclaw-hermes-discord-preload"); }); }); diff --git a/test/seccomp-guard.test.ts b/test/seccomp-guard.test.ts index eefd912582..53a6f4ba48 100644 --- a/test/seccomp-guard.test.ts +++ b/test/seccomp-guard.test.ts @@ -69,7 +69,6 @@ describe("Seccomp guard preload", () => { '_NO_PROXY_VAL="localhost,127.0.0.1,::1,${PROXY_HOST}"', '_TOOL_REDIRECTS=()', '_PROXY_FIX_SCRIPT="/tmp/nemoclaw-http-proxy-fix.js"', - '_WS_FIX_SCRIPT="/nonexistent/ws-proxy-fix.js"', '_NEMOTRON_FIX_SCRIPT="/tmp/nemoclaw-nemotron-inference-fix.js"', "set +u", persistBlock, @@ -181,56 +180,6 @@ describe("Seccomp guard preload", () => { }); }); -describe("ws-proxy-fix Landlock mitigation", () => { - const src = fs.readFileSync(START_SCRIPT, "utf-8"); - - it("copies ws-proxy-fix.js from a Landlock-readable source into /tmp and registers it", () => { - const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-ws-fix-entrypoint-")); - const sourcePath = path.join(tempDir, "source-ws-proxy-fix.js"); - const runtimePath = path.join(tempDir, "runtime-ws-proxy-fix.js"); - const start = src.indexOf('_WS_FIX_SOURCE="/usr/local/lib/nemoclaw/preloads/ws-proxy-fix.js"'); - const end = src.indexOf("# ── Seccomp syscall guard", start); - if (start === -1 || end === -1 || end <= start) { - throw new Error("Expected ws-proxy-fix entrypoint block in scripts/nemoclaw-start.sh"); - } - const block = src - .slice(start, end) - .replace( - '_WS_FIX_SOURCE="/usr/local/lib/nemoclaw/preloads/ws-proxy-fix.js"', - `_WS_FIX_SOURCE=${JSON.stringify(sourcePath)}`, - ) - .replace( - '_WS_FIX_SCRIPT="/tmp/nemoclaw-ws-proxy-fix.js"', - `_WS_FIX_SCRIPT=${JSON.stringify(runtimePath)}`, - ); - const wrapper = [ - "#!/usr/bin/env bash", - "set -euo pipefail", - "emit_sandbox_sourced_file() { local target=\"$1\"; cat > \"$target\"; chmod 444 \"$target\"; }", - "NODE_OPTIONS='--require /already-loaded.js'", - block, - "printf 'NODE_OPTIONS=%s\\n' \"$NODE_OPTIONS\"", - "printf 'SCRIPT=%s\\n' \"$_WS_FIX_SCRIPT\"", - ].join("\n"); - const wrapperPath = path.join(tempDir, "run.sh"); - - try { - fs.writeFileSync(sourcePath, "// ws preload fixture\n"); - fs.writeFileSync(wrapperPath, wrapper, { mode: 0o700 }); - const result = spawnSync("bash", [wrapperPath], { encoding: "utf-8", timeout: 5000 }); - expect(result.status).toBe(0); - expect(result.stdout).toContain(`SCRIPT=${runtimePath}`); - expect(result.stdout).toContain("--require /already-loaded.js"); - expect(result.stdout).toContain(`--require ${runtimePath}`); - expect(fs.readFileSync(runtimePath, "utf-8")).toBe("// ws preload fixture\n"); - expect((fs.statSync(runtimePath).mode & 0o777).toString(8)).toBe("444"); - } finally { - fs.rmSync(tempDir, { recursive: true, force: true }); - } - }); - -}); - describe("Early entrypoint stderr capture", () => { const src = fs.readFileSync(START_SCRIPT, "utf-8"); diff --git a/test/service-env.test.ts b/test/service-env.test.ts index e241325505..e1dd6e6c08 100644 --- a/test/service-env.test.ts +++ b/test/service-env.test.ts @@ -263,7 +263,6 @@ describe("service environment", () => { '_NO_PROXY_VAL="localhost,127.0.0.1,::1,${PROXY_HOST}"', "_TOOL_REDIRECTS=()", `_AXIOS_FIX_SCRIPT="/nonexistent/axios-proxy-fix.js"`, - `_WS_FIX_SCRIPT="/nonexistent/ws-proxy-fix.js"`, // Simulate OpenShell injecting SSL_CERT_FILE and the entrypoint setting GIT_SSL_CAINFO `export SSL_CERT_FILE="${fakeCaBundle}"`, `export GIT_SSL_CAINFO="${fakeCaBundle}"`, @@ -303,7 +302,6 @@ describe("service environment", () => { '_NO_PROXY_VAL="localhost,127.0.0.1,::1,${PROXY_HOST}"', "_TOOL_REDIRECTS=()", `_AXIOS_FIX_SCRIPT="/nonexistent/axios-proxy-fix.js"`, - `_WS_FIX_SCRIPT="/nonexistent/ws-proxy-fix.js"`, // GIT_SSL_CAINFO intentionally NOT set "set +u # array expansion safe on macOS bash", persistBlock @@ -791,7 +789,6 @@ describe("service environment", () => { "NODE_USE_ENV_PROXY=1", "_TOOL_REDIRECTS=()", `_PROXY_FIX_SCRIPT="${fakeFixPath}"`, - `_WS_FIX_SCRIPT="/nonexistent/ws-proxy-fix.js"`, `_NEMOTRON_FIX_SCRIPT="/tmp/nemoclaw-nemotron-inference-fix.js"`, "set +u # array expansion safe on macOS bash", persistBlock @@ -835,7 +832,6 @@ describe("service environment", () => { // NODE_USE_ENV_PROXY intentionally NOT set "_TOOL_REDIRECTS=()", `_PROXY_FIX_SCRIPT="/tmp/nemoclaw-http-proxy-fix.js"`, - `_WS_FIX_SCRIPT="/nonexistent/ws-proxy-fix.js"`, `_NEMOTRON_FIX_SCRIPT="/tmp/nemoclaw-nemotron-inference-fix.js"`, "set +u # array expansion safe on macOS bash", persistBlock @@ -846,11 +842,10 @@ describe("service environment", () => { execFileSync("bash", [tmpFile], { encoding: "utf-8" }); const envFile = readFileSync(join(fakeDataDir, "proxy-env.sh"), "utf-8"); - // Proxy and ws fix preloads should NOT be injected when NODE_USE_ENV_PROXY - // is not 1 and ws fix script does not exist. The Nemotron inference fix is + // Proxy preloads should NOT be injected when NODE_USE_ENV_PROXY + // is not 1. The Nemotron inference fix is // unconditional (always needed regardless of proxy config). expect(envFile).not.toContain("http-proxy-fix"); - expect(envFile).not.toContain("ws-proxy-fix"); expect(envFile).toContain("nemotron-inference-fix"); } finally { try { @@ -861,200 +856,5 @@ describe("service environment", () => { } }); - it("NemoClaw#1570: proxy-env.sh includes ws-proxy-fix NODE_OPTIONS when fix script exists", () => { - const fakeDataDir = join(tmpdir(), `nemoclaw-ws-fix-test-${process.pid}`); - const fakeWsFixScript = join(fakeDataDir, "ws-proxy-fix.js"); - execFileSync("mkdir", ["-p", fakeDataDir]); - const tmpFile = join(tmpdir(), `nemoclaw-ws-fix-env-${process.pid}.sh`); - try { - const persistBlock = extractRuntimeShellEnvSnippet(); - const wrapper = [ - "#!/usr/bin/env bash", - "set -euo pipefail", - sandboxInitSource, - `PROXY_HOST="10.200.0.1"`, - `PROXY_PORT="3128"`, - `_PROXY_URL="http://\${PROXY_HOST}:\${PROXY_PORT}"`, - `_NO_PROXY_VAL="localhost,127.0.0.1,::1,\${PROXY_HOST}"`, - `_PROXY_FIX_SCRIPT="/tmp/nemoclaw-http-proxy-fix.js"`, - `_WS_FIX_SCRIPT="${fakeWsFixScript}"`, - `_NEMOTRON_FIX_SCRIPT="/tmp/nemoclaw-nemotron-inference-fix.js"`, - `_TOOL_REDIRECTS=()`, - "set +u # array expansion safe on macOS bash", - persistBlock - .trimEnd() - .replaceAll("/tmp/nemoclaw-proxy-env.sh", `${fakeDataDir}/proxy-env.sh`), - ].join("\n"); - writeFileSync(fakeWsFixScript, "// fake", { mode: 0o644 }); - writeFileSync(tmpFile, wrapper, { mode: 0o700 }); - execFileSync("bash", [tmpFile], { encoding: "utf-8" }); - - const envFile = readFileSync(join(fakeDataDir, "proxy-env.sh"), "utf-8"); - expect(envFile).toContain("NODE_OPTIONS"); - expect(envFile).toContain("--require"); - expect(envFile).toContain(fakeWsFixScript); - } finally { - try { - execFileSync("rm", ["-rf", fakeDataDir, tmpFile]); - } catch { - /* ignore */ - } - } - }); - - it("NemoClaw#1570: proxy-env.sh omits ws-proxy-fix when script does not exist", () => { - const fakeDataDir = join(tmpdir(), `nemoclaw-ws-noop-test-${process.pid}`); - execFileSync("mkdir", ["-p", fakeDataDir]); - const tmpFile = join(tmpdir(), `nemoclaw-ws-noop-env-${process.pid}.sh`); - try { - const persistBlock = extractRuntimeShellEnvSnippet(); - const wrapper = [ - "#!/usr/bin/env bash", - "set -euo pipefail", - sandboxInitSource, - `PROXY_HOST="10.200.0.1"`, - `PROXY_PORT="3128"`, - `_PROXY_URL="http://\${PROXY_HOST}:\${PROXY_PORT}"`, - `_NO_PROXY_VAL="localhost,127.0.0.1,::1,\${PROXY_HOST}"`, - `_PROXY_FIX_SCRIPT="/tmp/nemoclaw-http-proxy-fix.js"`, - `_WS_FIX_SCRIPT="/nonexistent/ws-proxy-fix.js"`, - `_NEMOTRON_FIX_SCRIPT="/tmp/nemoclaw-nemotron-inference-fix.js"`, - `_TOOL_REDIRECTS=()`, - "set +u # array expansion safe on macOS bash", - persistBlock - .trimEnd() - .replaceAll("/tmp/nemoclaw-proxy-env.sh", `${fakeDataDir}/proxy-env.sh`), - ].join("\n"); - writeFileSync(tmpFile, wrapper, { mode: 0o700 }); - execFileSync("bash", [tmpFile], { encoding: "utf-8" }); - - const envFile = readFileSync(join(fakeDataDir, "proxy-env.sh"), "utf-8"); - expect(envFile).not.toContain("ws-proxy-fix"); - } finally { - try { - execFileSync("rm", ["-rf", fakeDataDir, tmpFile]); - } catch { - /* ignore */ - } - } - }); - }); - - describe("ws-proxy-fix preload (issue #1570)", () => { - const wsFixPath = join(import.meta.dirname, "../nemoclaw-blueprint/scripts/ws-proxy-fix.js"); - - it("patches https.request when HTTPS_PROXY is set", () => { - const result = execFileSync( - "node", - ["--require", wsFixPath, "-e", "console.log(require('https').request.name)"], - { - encoding: "utf-8", - env: { ...process.env, HTTPS_PROXY: "http://10.200.0.1:3128" }, - }, - ).trim(); - expect(result).toBe("wsProxyFixedRequest"); - }); - - it("is a no-op when HTTPS_PROXY is unset", () => { - const env = { ...process.env }; - delete env.HTTPS_PROXY; - delete env.https_proxy; - const result = execFileSync( - "node", - ["--require", wsFixPath, "-e", "console.log(require('https').request.name)"], - { encoding: "utf-8", env }, - ).trim(); - expect(result).not.toBe("wsProxyFixedRequest"); - }); - - it("is idempotent — loading twice does not double-patch", () => { - const result = execFileSync( - "node", - [ - "--require", - wsFixPath, - "-e", - `require("${wsFixPath}"); console.log(require('https').request.name)`, - ], - { - encoding: "utf-8", - env: { ...process.env, HTTPS_PROXY: "http://10.200.0.1:3128" }, - }, - ).trim(); - expect(result).toBe("wsProxyFixedRequest"); - }); - - it("strips port from opts.host to avoid double-port CONNECT path", () => { - // When callers pass host:"gateway.discord.gg:443" instead of hostname, - // the CONNECT target must be "gateway.discord.gg:443" not - // "gateway.discord.gg:443:443". - const result = execFileSync( - "node", - [ - "--require", - wsFixPath, - "-e", - ` -const https = require("https"); -const http = require("http"); -// Intercept http.request to capture the CONNECT path, then abort immediately -http.request = function(opts) { - if (opts.method === "CONNECT") { - console.log(opts.path); - process.exit(0); - } - return http.__proto__.request.apply(this, arguments); -}; -const req = https.request({ - host: "gateway.discord.gg:443", - path: "/?v=10&encoding=json", - headers: { Connection: "Upgrade", Upgrade: "websocket", "Sec-WebSocket-Key": "dGVzdA==", "Sec-WebSocket-Version": "13" }, -}); -req.on("error", () => {}); -req.end(); - `, - ], - { - encoding: "utf-8", - env: { ...process.env, HTTPS_PROXY: "http://10.200.0.1:3128" }, - }, - ).trim(); - expect(result).toBe("gateway.discord.gg:443"); - expect(result).not.toContain("443:443"); - }); - - it("ignores non-Discord WebSocket upgrades", () => { - const result = execFileSync( - "node", - [ - "--require", - wsFixPath, - "-e", - ` -const https = require("https"); -const http = require("http"); -let sawConnect = false; -http.request = function(opts) { - if (opts.method === "CONNECT") sawConnect = true; - return http.__proto__.request.apply(this, arguments); -}; -const req = https.request({ - hostname: "echo.websocket.org", - path: "/", - headers: { Connection: "Upgrade", Upgrade: "websocket", "Sec-WebSocket-Key": "dGVzdA==", "Sec-WebSocket-Version": "13" }, -}); -req.on("error", () => {}); -req.destroy(); -console.log(sawConnect ? "CONNECT" : "NO_CONNECT"); - `, - ], - { - encoding: "utf-8", - env: { ...process.env, HTTPS_PROXY: "http://10.200.0.1:3128" }, - }, - ).trim(); - // Non-Discord host should NOT trigger the CONNECT tunnel - expect(result).toBe("NO_CONNECT"); - }); }); }); diff --git a/test/slack-token-rewriter-sync.test.ts b/test/slack-token-rewriter-sync.test.ts deleted file mode 100644 index 9278db5b71..0000000000 --- a/test/slack-token-rewriter-sync.test.ts +++ /dev/null @@ -1,73 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; -import { spawnSync } from "node:child_process"; -import { describe, it, expect } from "vitest"; - -const ROOT = path.join(import.meta.dirname, ".."); -const CANONICAL_REWRITER = path.join( - ROOT, - "nemoclaw-blueprint", - "scripts", - "slack-token-rewriter.js", -); -const START_SCRIPT = path.join(ROOT, "scripts", "nemoclaw-start.sh"); - -describe("slack-token-rewriter preload sync (#2085)", () => { - it("entrypoint emits byte-for-byte canonical rewriter and registers it in NODE_OPTIONS", () => { - const canonical = fs.readFileSync(CANONICAL_REWRITER, "utf-8"); - const startScript = fs.readFileSync(START_SCRIPT, "utf-8"); - const start = startScript.indexOf("# ── Slack token rewriter"); - const end = startScript.indexOf("# ── Slack secrets-on-disk tripwire", start); - if (start === -1 || end === -1 || end <= start) { - throw new Error("Expected Slack token rewriter entrypoint block in scripts/nemoclaw-start.sh"); - } - - const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-slack-rewriter-")); - const rewriterPath = path.join(tempDir, "slack-token-rewriter.js"); - const configPath = path.join(tempDir, "openclaw.json"); - fs.writeFileSync(configPath, JSON.stringify({ token: "xoxb-OPENSHELL-RESOLVE-ENV-SLACK_BOT_TOKEN" })); - const block = startScript - .slice(start, end) - .replace( - '_SLACK_REWRITER_SCRIPT="/tmp/nemoclaw-slack-token-rewriter.js"', - `_SLACK_REWRITER_SCRIPT=${JSON.stringify(rewriterPath)}`, - ) - .replace( - '_SLACK_REWRITER_SOURCE="/usr/local/lib/nemoclaw/preloads/slack-token-rewriter.js"', - `_SLACK_REWRITER_SOURCE=${JSON.stringify(CANONICAL_REWRITER)}`, - ) - .replace( - 'local config_file="/sandbox/.openclaw/openclaw.json"', - `local config_file=${JSON.stringify(configPath)}`, - ); - const wrapper = [ - "#!/usr/bin/env bash", - "set -euo pipefail", - "emit_sandbox_sourced_file() { local target=\"$1\"; cat > \"$target\"; chmod 444 \"$target\"; }", - "NODE_OPTIONS='--require /already-loaded.js'", - block, - "install_slack_token_rewriter", - "printf 'NODE_OPTIONS=%s\\n' \"$NODE_OPTIONS\"", - "printf 'SCRIPT=%s\\n' \"$_SLACK_REWRITER_SCRIPT\"", - ].join("\n"); - const wrapperPath = path.join(tempDir, "run.sh"); - - try { - fs.writeFileSync(wrapperPath, wrapper, { mode: 0o700 }); - const result = spawnSync("bash", [wrapperPath], { encoding: "utf-8", timeout: 5000 }); - expect(result.status).toBe(0); - expect(result.stdout).toContain(`SCRIPT=${rewriterPath}`); - expect(result.stdout).toContain("--require /already-loaded.js"); - expect(result.stdout).toContain(`--require ${rewriterPath}`); - const generated = fs.readFileSync(rewriterPath, "utf-8"); - expect(generated).toBe(canonical); - expect((fs.statSync(rewriterPath).mode & 0o777).toString(8)).toBe("444"); - } finally { - fs.rmSync(tempDir, { recursive: true, force: true }); - } - }); -}); diff --git a/test/slack-token-rewriter.test.ts b/test/slack-token-rewriter.test.ts deleted file mode 100644 index 145615a9da..0000000000 --- a/test/slack-token-rewriter.test.ts +++ /dev/null @@ -1,343 +0,0 @@ -// @ts-nocheck -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 -// -// Unit tests for nemoclaw-blueprint/scripts/slack-token-rewriter.js. -// -// Loads the canonical rewriter source into a function-scoped sandbox with -// stubbed http/https modules, then drives the wrapped methods through every -// signature shape Node accepts and asserts the placeholder is rewritten to -// the canonical openshell:resolve:env:VAR form before reaching the original -// request function. - -import fs from "node:fs"; -import path from "node:path"; -import { describe, it, expect, beforeEach, afterEach } from "vitest"; - -const CANONICAL_REWRITER = path.join( - import.meta.dirname, - "..", - "nemoclaw-blueprint", - "scripts", - "slack-token-rewriter.js", -); -const TOKEN_ENV_KEYS = ["SLACK_BOT_TOKEN", "SLACK_APP_TOKEN"]; -const ORIGINAL_TOKEN_ENV = new Map(TOKEN_ENV_KEYS.map((key) => [key, process.env[key]])); - -function clearTokenEnv() { - for (const key of TOKEN_ENV_KEYS) delete process.env[key]; -} - -function restoreTokenEnv() { - for (const key of TOKEN_ENV_KEYS) { - const original = ORIGINAL_TOKEN_ENV.get(key); - if (original === undefined) delete process.env[key]; - else process.env[key] = original; - } -} - -// Build fresh stub modules and load the rewriter on top of them. Returns the -// stub modules whose request/get methods have been monkey-patched by the -// rewriter, plus a `captured` array recording the args each wrapped call -// passed to the underlying (un-rewritten) implementation. -function loadRewriter() { - const src = fs.readFileSync(CANONICAL_REWRITER, "utf-8"); - const captured: { method: string; args: unknown[]; req: ReturnType }[] = []; - - const makeRequest = (args: unknown[]) => { - const headers = new Map(); - const calls: { method: string; args: unknown[] }[] = []; - const opts = args.find( - (arg) => arg && typeof arg === "object" && !Array.isArray(arg) && !(arg instanceof URL), - ) as { headers?: Record } | undefined; - if (opts?.headers) { - for (const [k, v] of Object.entries(opts.headers)) { - headers.set(k.toLowerCase(), String(v)); - } - } - return { - calls, - _header: null as string | null, - headersSent: false, - getHeader(name: string) { - return headers.get(name.toLowerCase()); - }, - setHeader(name: string, value: string) { - headers.set(name.toLowerCase(), String(value)); - }, - write(...writeArgs: unknown[]) { - calls.push({ method: "write", args: writeArgs }); - return true; - }, - end(...endArgs: unknown[]) { - calls.push({ method: "end", args: endArgs }); - return this; - }, - }; - }; - - const make = (label: string) => ({ - request(...args: unknown[]) { - const req = makeRequest(args); - captured.push({ method: `${label}.request`, args, req }); - return req; - }, - get(...args: unknown[]) { - const req = makeRequest(args); - captured.push({ method: `${label}.get`, args, req }); - return req; - }, - }); - - const http = make("http"); - const https = make("https"); - const fakeRequire = (name: string) => { - if (name === "http") return http; - if (name === "https") return https; - throw new Error(`unexpected require: ${name}`); - }; - - // Evaluate the rewriter source with a custom `require`. The rewriter is an - // IIFE that touches only `require`, `URL`, and built-in globals — running - // it inside a Function() body keeps the global URL constructor identity - // consistent with the test code, so `arg1 instanceof URL` works correctly. - new Function("require", src)(fakeRequire); - - return { http, https, captured }; -} - -describe("slack-token-rewriter: string rewriting", () => { - let mod: ReturnType; - beforeEach(() => { - clearTokenEnv(); - mod = loadRewriter(); - }); - afterEach(() => { - restoreTokenEnv(); - }); - - it("rewrites Bolt-shape placeholder in a string URL argument", () => { - mod.https.request( - "https://api.slack.com/api/auth.test?token=xoxb-OPENSHELL-RESOLVE-ENV-SLACK_BOT_TOKEN", - ); - const [arg0] = mod.captured[0].args; - expect(arg0).toBe( - "https://api.slack.com/api/auth.test?token=openshell:resolve:env:SLACK_BOT_TOKEN", - ); - }); - - it("rewrites Bolt-shape placeholder in a URL object", () => { - const url = new URL( - "https://api.slack.com/api/auth.test?token=xapp-OPENSHELL-RESOLVE-ENV-SLACK_APP_TOKEN", - ); - mod.https.request(url); - const [arg0] = mod.captured[0].args; - expect(arg0 instanceof URL).toBe(true); - expect((arg0 as URL).href).toContain("openshell:resolve:env:SLACK_APP_TOKEN"); - expect((arg0 as URL).href).not.toContain("OPENSHELL-RESOLVE-ENV-"); - }); - - it("rewrites options.path", () => { - const opts = { - hostname: "api.slack.com", - path: "/api/auth.test?token=xoxb-OPENSHELL-RESOLVE-ENV-SLACK_BOT_TOKEN", - }; - mod.https.request(opts); - expect(opts.path).toBe("/api/auth.test?token=openshell:resolve:env:SLACK_BOT_TOKEN"); - }); - - it("rewrites options.headers.Authorization (Bearer prefix)", () => { - const opts = { - hostname: "api.slack.com", - path: "/api/auth.test", - headers: { Authorization: "Bearer xoxb-OPENSHELL-RESOLVE-ENV-SLACK_BOT_TOKEN" }, - }; - mod.https.request(opts); - expect(opts.headers.Authorization).toBe("Bearer openshell:resolve:env:SLACK_BOT_TOKEN"); - }); - - it("rewrites to the revision-scoped OpenShell placeholder when present in env", () => { - process.env.SLACK_BOT_TOKEN = "openshell:resolve:env:v12_SLACK_BOT_TOKEN"; - const opts = { - hostname: "api.slack.com", - path: "/api/auth.test", - headers: { Authorization: "Bearer xoxb-OPENSHELL-RESOLVE-ENV-SLACK_BOT_TOKEN" }, - }; - mod.https.request(opts); - expect(opts.headers.Authorization).toBe( - "Bearer openshell:resolve:env:v12_SLACK_BOT_TOKEN", - ); - }); - - it("does not copy raw env token values into rewritten requests", () => { - process.env.SLACK_BOT_TOKEN = "xoxb-real-token-must-not-leak"; - const opts = { - hostname: "api.slack.com", - path: "/api/auth.test", - headers: { Authorization: "Bearer xoxb-OPENSHELL-RESOLVE-ENV-SLACK_BOT_TOKEN" }, - }; - mod.https.request(opts); - expect(opts.headers.Authorization).toBe("Bearer openshell:resolve:env:SLACK_BOT_TOKEN"); - }); - - it("rewrites lowercase header name", () => { - const opts = { - headers: { authorization: "Bearer xoxb-OPENSHELL-RESOLVE-ENV-SLACK_BOT_TOKEN" }, - }; - mod.https.request(opts); - expect(opts.headers.authorization).toBe("Bearer openshell:resolve:env:SLACK_BOT_TOKEN"); - }); - - it("rewrites array-valued header entries", () => { - const opts = { - headers: { - "X-Slack-Audit": [ - "Bearer xapp-OPENSHELL-RESOLVE-ENV-SLACK_APP_TOKEN", - "no-placeholder-here", - ], - }, - }; - mod.https.request(opts); - expect(opts.headers["X-Slack-Audit"][0]).toBe("Bearer openshell:resolve:env:SLACK_APP_TOKEN"); - expect(opts.headers["X-Slack-Audit"][1]).toBe("no-placeholder-here"); - }); - - it("leaves non-placeholder strings untouched (fast path)", () => { - const opts = { - hostname: "api.slack.com", - path: "/api/auth.test", - headers: { Authorization: "Bearer xoxb-real-1234567890-abcdef" }, - }; - mod.https.request(opts); - expect(opts.headers.Authorization).toBe("Bearer xoxb-real-1234567890-abcdef"); - expect(opts.path).toBe("/api/auth.test"); - }); -}); - -describe("slack-token-rewriter: identity and idempotence", () => { - it("preserves options object identity (axios reuses the headers object)", () => { - const mod = loadRewriter(); - const headers = { Authorization: "Bearer xoxb-OPENSHELL-RESOLVE-ENV-SLACK_BOT_TOKEN" }; - const opts = { hostname: "api.slack.com", path: "/", headers }; - mod.https.request(opts); - const [arg0] = mod.captured[0].args; - expect(arg0).toBe(opts); - expect((arg0 as { headers: object }).headers).toBe(headers); - }); - - it("is idempotent — replaying captured args produces the same canonical form", () => { - const mod = loadRewriter(); - const opts = { - headers: { Authorization: "Bearer xoxb-OPENSHELL-RESOLVE-ENV-SLACK_BOT_TOKEN" }, - }; - mod.https.request(opts); - const firstPass = opts.headers.Authorization; - mod.https.request(opts); - expect(opts.headers.Authorization).toBe(firstPass); - expect(opts.headers.Authorization).toBe("Bearer openshell:resolve:env:SLACK_BOT_TOKEN"); - }); -}); - -describe("slack-token-rewriter: request bodies", () => { - it("rewrites Buffer body chunks and adjusts Content-Length", () => { - const mod = loadRewriter(); - const original = Buffer.from("token=xoxb-OPENSHELL-RESOLVE-ENV-SLACK_BOT_TOKEN"); - const expected = "token=openshell:resolve:env:SLACK_BOT_TOKEN"; - const req = mod.https.request({ - headers: { "Content-Length": String(original.length) }, - }); - - req.write(original); - - const written = req.calls[0].args[0] as Buffer; - expect(Buffer.isBuffer(written)).toBe(true); - expect(written.toString("utf-8")).toBe(expected); - expect(req.getHeader("content-length")).toBe(String(Buffer.byteLength(expected))); - }); - - it("rewrites string body chunks passed to end()", () => { - const mod = loadRewriter(); - const original = "token=xapp-OPENSHELL-RESOLVE-ENV-SLACK_APP_TOKEN"; - const expected = "token=openshell:resolve:env:SLACK_APP_TOKEN"; - const req = mod.https.request({ - headers: { "content-length": String(Buffer.byteLength(original)) }, - }); - - req.end(original, "utf8"); - - expect(req.calls[0].args[0]).toBe(expected); - expect(req.getHeader("content-length")).toBe(String(Buffer.byteLength(expected))); - }); - - it("does not rewrite non-UTF-8 body chunks even if the marker bytes appear", () => { - const mod = loadRewriter(); - const original = Buffer.concat([ - Buffer.from([0xff, 0xfe, 0xfd]), - Buffer.from("xoxb-OPENSHELL-RESOLVE-ENV-SLACK_BOT_TOKEN"), - ]); - const req = mod.https.request({ - headers: { "content-length": String(original.length) }, - }); - - req.write(original); - - expect(req.calls[0].args[0]).toBe(original); - expect(req.getHeader("content-length")).toBe(String(original.length)); - }); -}); - -describe("slack-token-rewriter: every wrapped method", () => { - it("wraps http.request, http.get, https.request, and https.get", () => { - const mod = loadRewriter(); - const opts = () => ({ - headers: { Authorization: "Bearer xoxb-OPENSHELL-RESOLVE-ENV-SLACK_BOT_TOKEN" }, - }); - - const a = opts(); - mod.http.request(a); - expect(a.headers.Authorization).toBe("Bearer openshell:resolve:env:SLACK_BOT_TOKEN"); - - const b = opts(); - mod.http.get(b); - expect(b.headers.Authorization).toBe("Bearer openshell:resolve:env:SLACK_BOT_TOKEN"); - - const c = opts(); - mod.https.request(c); - expect(c.headers.Authorization).toBe("Bearer openshell:resolve:env:SLACK_BOT_TOKEN"); - - const d = opts(); - mod.https.get(d); - expect(d.headers.Authorization).toBe("Bearer openshell:resolve:env:SLACK_BOT_TOKEN"); - - expect(mod.captured.map((c) => c.method)).toEqual([ - "http.request", - "http.get", - "https.request", - "https.get", - ]); - }); -}); - -describe("slack-token-rewriter: signature shapes", () => { - it("supports request(url, options) — both args mutated", () => { - const mod = loadRewriter(); - const opts = { - headers: { Authorization: "Bearer xoxb-OPENSHELL-RESOLVE-ENV-SLACK_BOT_TOKEN" }, - }; - mod.https.request("https://api.slack.com/api/auth.test?t=xoxb-OPENSHELL-RESOLVE-ENV-X", opts); - const [arg0] = mod.captured[0].args; - expect(arg0).toContain("openshell:resolve:env:X"); - expect(opts.headers.Authorization).toBe("Bearer openshell:resolve:env:SLACK_BOT_TOKEN"); - }); - - it("supports request(url, callback) — callback is not treated as options", () => { - const mod = loadRewriter(); - const cb = () => { - /* noop */ - }; - mod.https.request("https://api.slack.com/?t=xoxb-OPENSHELL-RESOLVE-ENV-X", cb); - const args = mod.captured[0].args; - expect(args[0]).toContain("openshell:resolve:env:X"); - expect(args[1]).toBe(cb); - }); -}); diff --git a/test/validate-blueprint.test.ts b/test/validate-blueprint.test.ts index 548de6de04..b79f677537 100644 --- a/test/validate-blueprint.test.ts +++ b/test/validate-blueprint.test.ts @@ -50,6 +50,8 @@ type Endpoint = { enforcement?: string; access?: string; tls?: string; + websocket_credential_rewrite?: boolean; + request_body_credential_rewrite?: boolean; rules?: Rule[]; binaries?: Array<{ path: string }>; }; @@ -502,6 +504,81 @@ describe("huggingface preset", () => { }); }); +describe("messaging WebSocket presets", () => { + const DISCORD_PRESET_PATH = new URL( + "../nemoclaw-blueprint/policies/presets/discord.yaml", + import.meta.url, + ); + const SLACK_PRESET_PATH = new URL( + "../nemoclaw-blueprint/policies/presets/slack.yaml", + import.meta.url, + ); + + const presets = [ + { + name: "discord", + policyKey: "discord", + host: "gateway.discord.gg", + credentialRewrite: true, + data: loadYaml(DISCORD_PRESET_PATH), + }, + { + name: "slack", + policyKey: "slack", + host: "wss-primary.slack.com", + credentialRewrite: true, + data: loadYaml(SLACK_PRESET_PATH), + }, + { + name: "slack", + policyKey: "slack", + host: "wss-backup.slack.com", + credentialRewrite: true, + data: loadYaml(SLACK_PRESET_PATH), + }, + ]; + + for (const preset of presets) { + it(`${preset.name} ${preset.host} uses native WebSocket inspection`, () => { + const endpoints = preset.data.network_policies?.[preset.policyKey]?.endpoints ?? []; + const endpoint = endpoints.find((candidate) => candidate.host === preset.host); + expect(endpoint).toBeDefined(); + expect(endpoint).toMatchObject({ protocol: "websocket", enforcement: "enforce" }); + expect(endpoint).not.toHaveProperty("access"); + expect(endpoint).not.toHaveProperty("tls"); + expect(endpoint?.websocket_credential_rewrite === true).toBe(preset.credentialRewrite); + expect(endpoint?.rules).toEqual( + expect.arrayContaining([ + { allow: { method: "GET", path: "/**" } }, + { allow: { method: "WEBSOCKET_TEXT", path: "/**" } }, + ]), + ); + }); + } +}); + +describe("Slack REST credential rewrite", () => { + const SLACK_PRESET_PATH = new URL( + "../nemoclaw-blueprint/policies/presets/slack.yaml", + import.meta.url, + ); + const data = loadYaml(SLACK_PRESET_PATH); + const slackRestHosts = ["slack.com", "api.slack.com", "hooks.slack.com"]; + + for (const host of slackRestHosts) { + it(`${host} enables request-body credential rewrite`, () => { + const endpoints = data.network_policies?.slack?.endpoints ?? []; + const endpoint = endpoints.find((candidate) => candidate.host === host); + expect(endpoint).toBeDefined(); + expect(endpoint).toMatchObject({ + protocol: "rest", + enforcement: "enforce", + request_body_credential_rewrite: true, + }); + }); + } +}); + describe("npm preset", () => { // Regression #2767: npm/Yarn registry endpoints used `protocol: rest` // with only GET allowed. Node 22 undici issues HTTP CONNECT through diff --git a/test/validate-config-schemas.test.ts b/test/validate-config-schemas.test.ts index 3a0d6e61c0..7faeea1cf9 100644 --- a/test/validate-config-schemas.test.ts +++ b/test/validate-config-schemas.test.ts @@ -215,6 +215,67 @@ describe("sandbox-policy.schema.json", () => { }; expect(validate(bad)).toBe(false); }); + + it("accepts sandbox-policy native WebSocket text rules and credential rewrite", () => { + const valid = { + version: 1, + network_policies: { + test_service: { + name: "Test Service", + endpoints: [ + { + host: "gateway.example.com", + port: 443, + protocol: "websocket", + enforcement: "enforce", + websocket_credential_rewrite: true, + allowed_ips: ["10.0.0.0/8", "172.16.0.0/12"], + rules: [ + { allow: { method: "GET", path: "/**" } }, + { allow: { method: "WEBSOCKET_TEXT", path: "/**" } }, + ], + }, + ], + }, + }, + }; + expectValid(validate, valid, "websocket policy"); + }); + + it("accepts sandbox-policy request-body credential rewrite on REST endpoints", () => { + const valid = { + version: 1, + network_policies: { + slack: { + name: "Slack", + endpoints: [ + { + host: "api.slack.com", + port: 443, + protocol: "rest", + enforcement: "enforce", + request_body_credential_rewrite: true, + rules: [{ allow: { method: "POST", path: "/**" } }], + }, + ], + }, + }, + }; + expectValid(validate, valid, "rest body rewrite policy"); + }); + + it("rejects sandbox-policy endpoint with protocol websocket but no rules or access", () => { + const bad = { + version: 1, + network_policies: { + test_service: { + name: "Test Service", + endpoints: [{ host: "gateway.example.com", port: 443, protocol: "websocket" }], + }, + }, + }; + expect(validate(bad)).toBe(false); + }); }); // ── Policy presets ─────────────────────────────────────────────────────────── @@ -265,6 +326,67 @@ describe("policy-preset.schema.json", () => { }; expect(validate(bad)).toBe(false); }); + + it("accepts preset native WebSocket text rules and credential rewrite", () => { + const valid = { + preset: { name: "test", description: "test" }, + network_policies: { + test_service: { + name: "Test Service", + endpoints: [ + { + host: "gateway.example.com", + port: 443, + protocol: "websocket", + enforcement: "enforce", + websocket_credential_rewrite: true, + allowed_ips: ["10.0.0.0/8", "172.16.0.0/12"], + rules: [ + { allow: { method: "GET", path: "/**" } }, + { allow: { method: "WEBSOCKET_TEXT", path: "/**" } }, + ], + }, + ], + }, + }, + }; + expectValid(validate, valid, "websocket preset"); + }); + + it("accepts preset request-body credential rewrite on REST endpoints", () => { + const valid = { + preset: { name: "slack", description: "Slack" }, + network_policies: { + slack: { + name: "Slack", + endpoints: [ + { + host: "api.slack.com", + port: 443, + protocol: "rest", + enforcement: "enforce", + request_body_credential_rewrite: true, + rules: [{ allow: { method: "POST", path: "/**" } }], + }, + ], + }, + }, + }; + expectValid(validate, valid, "rest body rewrite preset"); + }); + + it("rejects preset endpoint with protocol websocket but no rules", () => { + const bad = { + preset: { name: "test", description: "test" }, + network_policies: { + test_service: { + name: "Test Service", + endpoints: [{ host: "gateway.example.com", port: 443, protocol: "websocket" }], + }, + }, + }; + expect(validate(bad)).toBe(false); + }); }); // ── OpenClaw plugin manifest ─────────────────────────────────────────────────