diff --git a/.github/actions/run-e2e-script-vpn/action.yaml b/.github/actions/run-e2e-script-vpn/action.yaml new file mode 100644 index 00000000000..3aa1d449109 --- /dev/null +++ b/.github/actions/run-e2e-script-vpn/action.yaml @@ -0,0 +1,138 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: Run VPN E2E script + +description: >- + Runs a repository E2E script from an already-checked-out workspace and uploads + configured failure artifacts plus optional sanitized trace summary artifacts. + +inputs: + script: + description: Repository-relative E2E script path. + required: true + working-directory: + description: Directory to run the script from. + required: false + default: . + artifact-name: + description: Failure artifact name. + required: true + artifact-path: + description: Newline-capable artifact path glob list. + required: true + always-artifact-name: + description: Optional sanitized trace artifact name to upload after every script run. + required: false + default: "" + always-artifact-path: + description: Optional legacy trace summary path; trusted code writes uploads to RUNNER_TEMP. + required: false + default: "" + always-artifact-trace-source-path: + description: Optional target-controlled trace path to reduce to a trusted timing-only summary before upload. + required: false + default: "" + +runs: + using: composite + steps: + - name: Run VPN E2E script + shell: bash + working-directory: ${{ inputs.working-directory }} + env: + E2E_SCRIPT: ${{ inputs.script }} + run: | + set -euo pipefail + + case "$E2E_SCRIPT" in + test/e2e-vpn/*.sh) ;; + *) + echo "::error::E2E script must match test/e2e-vpn/*.sh: $E2E_SCRIPT" >&2 + exit 1 + ;; + esac + + case "$E2E_SCRIPT" in + *..*|/*|*\"*|*\'*) + echo "::error::E2E script path contains unsafe characters: $E2E_SCRIPT" >&2 + exit 1 + ;; + esac + + if [ ! -f "$E2E_SCRIPT" ]; then + echo "::error::E2E script does not exist: $E2E_SCRIPT" >&2 + exit 1 + fi + + setsid bash "$E2E_SCRIPT" & + script_pid="$!" + + set +e + wait "$script_pid" + script_status="$?" + set -e + + # The target-ref script may start background processes with access to + # job secrets. Reap its process group before trusted sanitization/upload. + if kill -0 -- "-$script_pid" 2>/dev/null; then + kill -TERM -- "-$script_pid" 2>/dev/null || true + for _ in 1 2 3 4 5; do + if ! kill -0 -- "-$script_pid" 2>/dev/null; then + break + fi + sleep 1 + done + kill -KILL -- "-$script_pid" 2>/dev/null || true + fi + + exit "$script_status" + + - name: Upload E2E artifacts on failure + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ${{ inputs.artifact-name }} + path: ${{ inputs.artifact-path }} + if-no-files-found: ignore + + - name: Sanitize E2E trace artifacts + id: sanitize-trace-artifacts + if: always() && inputs.always-artifact-name != '' && inputs.always-artifact-path != '' && inputs.always-artifact-trace-source-path != '' + shell: bash + working-directory: ${{ inputs.working-directory }} + env: + E2E_TRACE_SOURCE_PATH: ${{ inputs.always-artifact-trace-source-path }} + run: | + set -euo pipefail + trusted_summary_dir="$(mktemp -d "${RUNNER_TEMP:-/tmp}/nemoclaw-trace-summary.XXXXXX")" + python3 "$GITHUB_ACTION_PATH/sanitize-trace-artifacts.py" \ + "$E2E_TRACE_SOURCE_PATH" \ + "$trusted_summary_dir" + + summary_file="$trusted_summary_dir/cloud-onboard-trace-timing-summary.json" + if [ -e "$summary_file" ]; then + if [ -L "$summary_file" ] || [ ! -f "$summary_file" ]; then + echo "::error::Sanitized trace summary must be a regular file: $summary_file" >&2 + exit 1 + fi + + unexpected="$( + find "$trusted_summary_dir" -mindepth 1 -maxdepth 1 \ + ! -name cloud-onboard-trace-timing-summary.json -print -quit + )" + if [ -n "$unexpected" ]; then + echo "::error::Unexpected file in trusted trace summary directory: $unexpected" >&2 + exit 1 + fi + fi + + printf 'summary-file=%s\n' "$summary_file" >> "$GITHUB_OUTPUT" + + - name: Upload E2E artifacts + if: always() && inputs.always-artifact-name != '' && inputs.always-artifact-path != '' && inputs.always-artifact-trace-source-path != '' && steps.sanitize-trace-artifacts.outcome == 'success' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ${{ inputs.always-artifact-name }} + path: ${{ steps.sanitize-trace-artifacts.outputs.summary-file }} + if-no-files-found: warn diff --git a/.github/actions/run-e2e-script-vpn/sanitize-trace-artifacts.py b/.github/actions/run-e2e-script-vpn/sanitize-trace-artifacts.py new file mode 100755 index 00000000000..e1d1ca41e70 --- /dev/null +++ b/.github/actions/run-e2e-script-vpn/sanitize-trace-artifacts.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Build a trusted timing-only trace artifact from target-ref trace output. + +The E2E script under test controls NEMOCLAW_TRACE_DIR, so the reusable trusted +workflow must never upload that raw directory from a secret-bearing job. This +helper reads candidate NemoClaw trace JSON files, validates the minimal timing +shape needed by the scorecard, and writes a single allowlisted summary that +contains no attributes, events, file names, prompts, environment dumps, or raw +error messages. +""" + +from __future__ import annotations + +import json +import math +import os +import re +import shutil +import sys +from pathlib import Path +from typing import Any + +SCHEMA_VERSION = "nemoclaw.trace_timing.v1" +OUTPUT_FILE = "cloud-onboard-trace-timing-summary.json" +ONBOARD_ROOT_SPAN = "nemoclaw.onboard" +ONBOARD_PHASE_PREFIX = "nemoclaw.onboard.phase." +MAX_JSON_FILES = 100 +MAX_JSON_BYTES = 2 * 1024 * 1024 +MAX_SLOWEST_SPANS = 10 +TRACE_ID_RE = re.compile(r"^[0-9a-f]{32}$") +STATUS_VALUES = {"OK", "ERROR", "UNSET"} + + +def finite_number(value: Any) -> float | None: + if isinstance(value, bool): + return None + try: + number = float(value) + except (TypeError, ValueError): + return None + if not math.isfinite(number) or number < 0: + return None + return number + + +def safe_status(value: Any) -> str: + return value if isinstance(value, str) and value in STATUS_VALUES else "UNSET" + + +def safe_span_name(value: Any) -> str | None: + if not isinstance(value, str): + return None + if value == ONBOARD_ROOT_SPAN or value.startswith(ONBOARD_PHASE_PREFIX): + return value + return None + + +def iter_json_files(source: Path) -> list[Path]: + if not source.exists(): + return [] + if source.is_file(): + return [source] if source.suffix == ".json" and not source.is_symlink() else [] + if not source.is_dir() or source.is_symlink(): + return [] + files: list[Path] = [] + for path in sorted(source.rglob("*.json")): + if path.is_file() and not path.is_symlink(): + files.append(path) + if len(files) >= MAX_JSON_FILES: + break + return files + + +def load_json(path: Path) -> Any | None: + try: + if path.stat().st_size > MAX_JSON_BYTES: + return None + return json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError): + return None + + +def first_dict(values: Any) -> dict[str, Any]: + if isinstance(values, list) and values and isinstance(values[0], dict): + return values[0] + return {} + + +def extract_spans(artifact: Any) -> list[dict[str, Any]]: + if not isinstance(artifact, dict): + return [] + resource = first_dict(artifact.get("resource_spans")) + scope = first_dict(resource.get("scope_spans")) + spans = scope.get("spans", []) + return [span for span in spans if isinstance(span, dict)] if isinstance(spans, list) else [] + + +def extract_candidate(artifact: Any) -> dict[str, Any] | None: + if not isinstance(artifact, dict): + return None + spans = extract_spans(artifact) + if not any(span.get("name") == ONBOARD_ROOT_SPAN for span in spans): + return None + + summary = artifact.get("summary") if isinstance(artifact.get("summary"), dict) else {} + total_ms = finite_number(summary.get("total_duration_ms")) + if total_ms is None: + return None + + phases: dict[str, float] = {} + for span in spans: + name = span.get("name") + duration_ms = finite_number(span.get("duration_ms")) + if isinstance(name, str) and name.startswith(ONBOARD_PHASE_PREFIX) and duration_ms is not None: + phases[name] = phases.get(name, 0.0) + duration_ms + + if not phases: + return None + + slowest_spans = [] + for span in summary.get("slowest_spans", []) if isinstance(summary.get("slowest_spans"), list) else []: + if not isinstance(span, dict): + continue + name = safe_span_name(span.get("name")) + duration_ms = finite_number(span.get("duration_ms")) + if name is None or duration_ms is None: + continue + slowest_spans.append( + { + "name": name, + "duration_ms": round(duration_ms, 3), + "status": safe_status(span.get("status")), + } + ) + if len(slowest_spans) >= MAX_SLOWEST_SPANS: + break + + trace_id = summary.get("trace_id") + return { + "schema_version": SCHEMA_VERSION, + "trace_id": trace_id if isinstance(trace_id, str) and TRACE_ID_RE.fullmatch(trace_id) else None, + "total_duration_ms": round(total_ms, 3), + "phases": {name: round(phases[name], 3) for name in sorted(phases)}, + "slowest_spans": slowest_spans, + } + + +def main(argv: list[str]) -> int: + if len(argv) != 3: + print("usage: sanitize-trace-artifacts.py ", file=sys.stderr) + return 2 + + source_input = Path(argv[1]).absolute() + if source_input.is_symlink(): + print("trace source must not be a symlink", file=sys.stderr) + return 2 + + source = source_input.resolve(strict=False) + output_dir = Path(argv[2]).absolute() + if source == output_dir.resolve(strict=False): + print("trace source and trusted output directory must be distinct", file=sys.stderr) + return 2 + + if output_dir.exists() or output_dir.is_symlink(): + if output_dir.is_symlink(): + output_dir.unlink() + elif output_dir.is_dir(): + shutil.rmtree(output_dir) + else: + output_dir.unlink() + output_dir.mkdir(parents=True, mode=0o700) + + candidates = [] + for json_file in iter_json_files(source): + artifact = load_json(json_file) + candidate = extract_candidate(artifact) + if candidate is not None: + candidates.append(candidate) + + if not candidates: + print("No valid NemoClaw onboard trace found; no timing summary emitted.") + return 0 + + selected = max(candidates, key=lambda item: item["total_duration_ms"]) + output = output_dir / OUTPUT_FILE + output.write_text(json.dumps(selected, indent=2, sort_keys=True) + "\n", encoding="utf-8") + os.chmod(output, 0o600) + print(f"Wrote trusted trace timing summary: {output}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv)) diff --git a/.github/workflows/e2e-script-vpn.yaml b/.github/workflows/e2e-script-vpn.yaml new file mode 100644 index 00000000000..df8d1660e9a --- /dev/null +++ b/.github/workflows/e2e-script-vpn.yaml @@ -0,0 +1,258 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: E2E / Script Runner VPN + +on: + workflow_call: + inputs: + ref: + description: Git ref or SHA to test. + required: true + type: string + script: + description: Repository-relative E2E script path. + required: true + type: string + runner: + description: GitHub Actions runner label. + required: false + type: string + default: linux-amd64-cpu4 + timeout_minutes: + description: Job timeout in minutes. + required: false + type: number + default: 45 + artifact_name: + description: Failure artifact name. + required: true + type: string + artifact_path: + description: Newline-capable failure artifact path glob list. + required: true + type: string + always_artifact_name: + description: Optional sanitized trace artifact name to upload after every script run. + required: false + type: string + default: "" + always_artifact_path: + description: Optional legacy trace summary path; trusted code writes uploads to RUNNER_TEMP. + required: false + type: string + default: "" + always_artifact_trace_source_path: + description: Optional target-controlled trace path to reduce to a trusted timing-only summary before upload. + required: false + type: string + default: "" + env_json: + description: JSON object of non-secret environment variables for the script. + required: false + type: string + default: "{}" + checked_out_ref_env: + description: Optional environment variable name to set to the checked-out commit SHA. + required: false + type: string + default: "" + nvidia_api_key: + description: Pass the hosted inference source secret as the CI custom endpoint credential. + required: false + type: boolean + default: false + brave_api_key: + description: Pass the BRAVE_API_KEY secret to the script. + required: false + type: boolean + default: false + github_token: + description: Pass github.token to the script as GITHUB_TOKEN. + required: false + type: boolean + default: false + messaging_live_secrets: + description: Pass optional live messaging provider secrets to the script. + required: false + type: boolean + default: false + secrets: + NVIDIA_API_KEY: + required: false + BRAVE_API_KEY: + required: false + DOCKERHUB_USERNAME: + required: false + DOCKERHUB_TOKEN: + required: false + TELEGRAM_BOT_TOKEN_REAL: + required: false + TELEGRAM_CHAT_ID_E2E: + required: false + DISCORD_BOT_TOKEN_REAL: + required: false + DISCORD_CHANNEL_ID_E2E: + required: false + SLACK_BOT_TOKEN_REAL: + required: false + SLACK_APP_TOKEN_REAL: + required: false + SLACK_CHANNEL_ID_E2E: + required: false + +permissions: + contents: read + +jobs: + run: + runs-on: ${{ inputs.runner }} + timeout-minutes: ${{ inputs.timeout_minutes }} + steps: + - name: Checkout target ref + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + ref: ${{ inputs.ref }} + path: repo + persist-credentials: false + + - name: Export checked-out ref environment + if: ${{ inputs.checked_out_ref_env != '' }} + env: + E2E_CHECKED_OUT_REF_ENV: ${{ inputs.checked_out_ref_env }} + shell: bash + run: | + set -euo pipefail + + if [[ ! "$E2E_CHECKED_OUT_REF_ENV" =~ ^[A-Z_][A-Z0-9_]*$ ]]; then + echo "::error::Invalid checked_out_ref_env variable name: $E2E_CHECKED_OUT_REF_ENV" >&2 + exit 1 + fi + + case "$E2E_CHECKED_OUT_REF_ENV" in + ACTIONS_*|GITHUB_*|INPUT_*|RUNNER_*|CI|HOME|PATH|PWD|SHELL) + echo "::error::Reserved checked_out_ref_env variable name: $E2E_CHECKED_OUT_REF_ENV" >&2 + exit 1 + ;; + esac + + printf '%s=%s\n' "$E2E_CHECKED_OUT_REF_ENV" "$(git -C repo rev-parse HEAD)" >> "$GITHUB_ENV" + + - name: Checkout workflow action + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + ref: ${{ github.ref }} + sparse-checkout: | + .github/actions/run-e2e-script-vpn + path: workflow-actions + persist-credentials: false + + - name: Export script environment + env: + E2E_ENV_JSON: ${{ inputs.env_json }} + shell: bash + run: | + python3 - <<'PY' + import json + import os + import re + import secrets + import sys + + values = json.loads(os.environ.get("E2E_ENV_JSON") or "{}") + if not isinstance(values, dict): + print("::error::env_json must be a JSON object", file=sys.stderr) + sys.exit(1) + + name_pattern = re.compile(r"^[A-Z_][A-Z0-9_]*$") + reserved_prefixes = ("ACTIONS_", "GITHUB_", "INPUT_", "RUNNER_") + reserved_names = {"CI", "HOME", "PATH", "PWD", "SHELL"} + + with open(os.environ["GITHUB_ENV"], "a", encoding="utf-8") as out: + for name, value in values.items(): + if not isinstance(name, str) or not name_pattern.fullmatch(name): + print(f"::error::Invalid env_json variable name: {name!r}", file=sys.stderr) + sys.exit(1) + if name in reserved_names or name.startswith(reserved_prefixes): + print(f"::error::Reserved env_json variable name: {name}", file=sys.stderr) + sys.exit(1) + + rendered = str(value) + if "\n" in rendered: + delimiter = f"EOF_{secrets.token_hex(16)}" + out.write(f"{name}<<{delimiter}\n{rendered}\n{delimiter}\n") + else: + out.write(f"{name}={rendered}\n") + PY + + - name: Authenticate to Docker Hub + if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.target_ref == '' }} + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + shell: bash + run: | + set -euo pipefail + if [[ -z "${DOCKERHUB_USERNAME}" || -z "${DOCKERHUB_TOKEN}" ]]; then + echo "::notice::Docker Hub credentials not configured; continuing with anonymous pulls." + exit 0 + fi + login_succeeded=0 + for attempt in 1 2 3; do + if echo "${DOCKERHUB_TOKEN}" | timeout 30s docker login docker.io --username "${DOCKERHUB_USERNAME}" --password-stdin; then + login_succeeded=1 + break + fi + if [[ "$attempt" -lt 3 ]]; then + echo "::warning::Docker Hub login attempt ${attempt} failed; retrying." + sleep $((attempt * 5)) + fi + done + if [[ "$login_succeeded" -ne 1 ]]; then + echo "::warning::Docker Hub login failed after 3 attempts; continuing with anonymous pulls." + fi + + - name: Export hosted CI inference environment + if: ${{ inputs.nvidia_api_key }} + env: + NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} + shell: bash + run: | + set -euo pipefail + + if [ -z "${NVIDIA_API_KEY:-}" ]; then + echo "::error::NVIDIA_API_KEY secret is required for hosted CI inference; it is withheld for workflow_dispatch target_ref runs." >&2 + exit 1 + fi + + { + printf 'NEMOCLAW_E2E_USE_HOSTED_INFERENCE=1\n' + printf 'NEMOCLAW_PROVIDER=custom\n' + printf 'NEMOCLAW_ENDPOINT_URL=https://inference.nvidia.com/v1\n' + printf 'NEMOCLAW_MODEL=nvidia/nvidia/nemotron-3-super-v3\n' + printf 'NEMOCLAW_COMPAT_MODEL=nvidia/nvidia/nemotron-3-super-v3\n' + printf 'NEMOCLAW_PREFERRED_API=openai-completions\n' + printf 'COMPATIBLE_API_KEY=%s\n' "${NVIDIA_API_KEY}" + } >> "$GITHUB_ENV" + + - name: Run E2E script + uses: ./workflow-actions/.github/actions/run-e2e-script-vpn + with: + working-directory: repo + script: ${{ inputs.script }} + artifact-name: ${{ inputs.artifact_name }} + artifact-path: ${{ inputs.artifact_path }} + always-artifact-name: ${{ inputs.always_artifact_name }} + always-artifact-path: ${{ inputs.always_artifact_path }} + always-artifact-trace-source-path: ${{ inputs.always_artifact_trace_source_path }} + env: + BRAVE_API_KEY: ${{ inputs.brave_api_key && secrets.BRAVE_API_KEY || '' }} + GITHUB_TOKEN: ${{ inputs.github_token && github.token || '' }} + NVIDIA_API_KEY: ${{ inputs.nvidia_api_key && secrets.NVIDIA_API_KEY || '' }} + TELEGRAM_BOT_TOKEN_REAL: ${{ inputs.messaging_live_secrets && secrets.TELEGRAM_BOT_TOKEN_REAL || '' }} + TELEGRAM_CHAT_ID_E2E: ${{ inputs.messaging_live_secrets && secrets.TELEGRAM_CHAT_ID_E2E || '' }} + DISCORD_BOT_TOKEN_REAL: ${{ inputs.messaging_live_secrets && secrets.DISCORD_BOT_TOKEN_REAL || '' }} + DISCORD_CHANNEL_ID_E2E: ${{ inputs.messaging_live_secrets && secrets.DISCORD_CHANNEL_ID_E2E || '' }} + SLACK_BOT_TOKEN_REAL: ${{ inputs.messaging_live_secrets && secrets.SLACK_BOT_TOKEN_REAL || '' }} + SLACK_APP_TOKEN_REAL: ${{ inputs.messaging_live_secrets && secrets.SLACK_APP_TOKEN_REAL || '' }} + SLACK_CHANNEL_ID_E2E: ${{ inputs.messaging_live_secrets && secrets.SLACK_CHANNEL_ID_E2E || '' }} diff --git a/.github/workflows/nightly-e2e-vpn-smoke.yaml b/.github/workflows/nightly-e2e-vpn-smoke.yaml new file mode 100644 index 00000000000..5f8ee638177 --- /dev/null +++ b/.github/workflows/nightly-e2e-vpn-smoke.yaml @@ -0,0 +1,47 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: E2E / Nightly VPN Smoke + +on: + push: + branches: + - add-vpn-nightly-e2e + +permissions: + contents: read + +concurrency: + group: nightly-e2e-vpn-smoke-${{ github.ref }} + cancel-in-progress: true + +jobs: + cloud-inference-e2e: + if: github.repository == 'NVIDIA/NemoClaw' + uses: ./.github/workflows/e2e-script-vpn.yaml + with: + ref: ${{ github.sha }} + script: test/e2e-vpn/test-cloud-inference-e2e.sh + runner: linux-amd64-cpu4 + timeout_minutes: 30 + artifact_name: "vpn-smoke-cloud-inference-log" + artifact_path: "/tmp/nemoclaw-e2e-cloud-inference-install.log" + env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_RECREATE_SANDBOX":"1","NEMOCLAW_SANDBOX_NAME":"e2e-vpn-smoke-cloud-inference"}' + nvidia_api_key: true + secrets: + NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} + + inference-routing-e2e: + if: github.repository == 'NVIDIA/NemoClaw' + uses: ./.github/workflows/e2e-script-vpn.yaml + with: + ref: ${{ github.sha }} + script: test/e2e-vpn/test-inference-routing.sh + runner: linux-amd64-cpu4 + timeout_minutes: 30 + artifact_name: "vpn-smoke-inference-routing-log" + artifact_path: "test-inference-routing-*.log" + env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_POLICY_TIER":"open"}' + nvidia_api_key: true + secrets: + NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} diff --git a/.github/workflows/nightly-e2e-vpn.yaml b/.github/workflows/nightly-e2e-vpn.yaml new file mode 100644 index 00000000000..1c222a93e2d --- /dev/null +++ b/.github/workflows/nightly-e2e-vpn.yaml @@ -0,0 +1,3107 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Nightly E2E tests: +# +# cloud-e2e Hosted inference (OpenAI-compatible endpoint) on ubuntu-latest. +# agent-turn-latency-e2e Times one real OpenClaw turn and one real Hermes +# turn through the configured hosted inference model. +# messaging-providers-e2e Validates messaging credential provider/placeholder/L7-proxy chain +# for Telegram + Discord + Slack. Uses fake tokens. Slack additionally +# exercises OpenShell provider-shaped alias resolution (#2085 follow-up). +# openclaw-slack-pairing-e2e +# Validates hermetic Slack Socket Mode pairing request approval across +# gateway and connect-shell OpenClaw state roots (#3730/#3737). +# openclaw-discord-pairing-e2e +# Validates hermetic Discord pairing request approval across +# gateway and connect-shell OpenClaw state roots (#4061). +# issue-4462-scope-upgrade-approval-e2e +# Validates real CLI scope-upgrade approval and confirms +# the approved agent run stays on gateway mode (#4462). +# issue-4462-gateway-pinned-approval-characterization-e2e +# Characterizes legacy gateway-pinned scope approval +# against a real sandbox, then recovers with the fix. +# messaging-compatible-endpoint-e2e +# Validates Telegram + OpenAI-compatible endpoint inference routing +# through inference.local with a hermetic local mock (#2766). +# kimi-inference-compat-e2e +# Validates Kimi K2.6 safe exec splitting through OpenClaw trajectories +# against public VPN NVIDIA inference, with a mock fallback (#2620). +# bedrock-runtime-compatible-anthropic-e2e +# Validates the silent Bedrock Runtime custom Anthropic endpoint path +# through a hermetic fake Bedrock Runtime host for OpenClaw and Hermes. +# token-rotation-e2e Validates that rotating a messaging token and re-running onboard +# propagates the new credential to the sandbox. Combined Telegram + +# Discord + Slack coverage with cross-talk assertions. See issue #1903. +# sandbox-survival-e2e Sandbox survival across gateway restarts (onboard, inference, +# gateway stop/start, verify sandbox + workspace + inference). +# openshell-gateway-upgrade-e2e +# Validates real v0.0.36 curl install upgrade into +# the current supported OpenShell with pre-upgrade backup, restored +# agent state, and the same agent type running. +# hermes-e2e Hermes Agent E2E — install → onboard --agent hermes → health +# probe → live inference. Validates the multi-agent architecture. +# hermes-dashboard-e2e Hermes Agent E2E with optional web dashboard enabled, +# validating API/dashboard forwards and host reachability. +# hermes-root-entrypoint-smoke-e2e +# Builds the real Hermes image and verifies root entrypoint startup, +# gateway-user execution, v0.14 layout repair, and PID migration. +# hermes-secret-boundary-e2e +# Builds Hermes default and managed-tool images, then verifies +# no raw secret-shaped values enter sandbox env/config. +# openclaw-onboard-security-posture-e2e +# Full OpenClaw onboard on a non-root host user +# with trusted rc-file and runtime guard assertions. +# hermes-onboard-security-posture-e2e +# Full Hermes onboard on a non-root host user +# with trusted rc-file and runtime guard assertions. +# hermes-inference-switch-e2e +# Switches a running Hermes sandbox with `nemohermes inference set` +# and verifies route, config.yaml, hashes, and live requests. +# hermes-anthropic-inference-switch-e2e +# Switches a running Hermes sandbox to a compatible +# Anthropic Messages provider and verifies agent traffic. +# hermes-discord-e2e Hermes Discord onboarding — validates the top-level Hermes +# Discord schema plus OpenShell placeholder/token isolation. +# hermes-slack-e2e Hermes Slack onboarding — validates the Hermes Slack policy, +# Slack providers, and OpenShell credential rewrite path. +# openclaw-inference-switch-e2e +# Switches a running OpenClaw sandbox with `nemoclaw inference set` +# and verifies route, openclaw.json, hashes, and live requests. +# openclaw-anthropic-inference-switch-e2e +# Switches a running OpenClaw sandbox to a compatible +# Anthropic Messages provider and verifies agent traffic. +# openclaw-skill-cli-e2e Validates workspace-installed OpenClaw skills survive sandbox +# lifecycle through OPENCLAW_HOME/STATE_DIR/WORKSPACE_DIR pinning +# (#4766 / #4709). Seven-phase deterministic skill-CLI exercise +# inside a real onboarded sandbox (install, list, info, check). +# channels-add-remove-e2e Telegram/Discord/Slack channel add/remove lifecycle plus +# gateway-credential reuse on rebuild (#4745 / #3895). Exercises +# the path where the host env credential is empty but the +# gateway already holds the provider credential. +# issue-4434-tui-unreachable-inference-e2e +# Recreates #4434's NVIDIA endpoint firewall block and verifies +# OpenClaw TUI shows a visible error and stops the active spinner. +# credential-migration-e2e Validates legacy ~/.nemoclaw/credentials.json migration to the +# OpenShell gateway, secure zero-fill on unlink, allowlist filter +# on non-credential env keys, and symlink-safe deletion. +# launchable-smoke-e2e Community install path (brev-launchable-ci-cpu.sh) on ubuntu-latest. +# gpu-e2e Local Ollama inference on an NVKS ephemeral GPU runner. +# gpu-double-onboard-e2e Ollama proxy token consistency after re-onboard (#2553). +# gpu-jetson-nvmap-e2e Jetson Orin /dev/nvmap CUDA usability + status proof (#4231). +# Gated behind vars.JETSON_E2E_ENABLED; needs a Jetson runner. +# concurrent-gateway-ports-e2e +# Two sandboxes coexisting on the same host with distinct +# NEMOCLAW_GATEWAY_PORT values; verifies per-instance +# gateway and dashboard segregation. +# notify-on-failure Auto-creates a GitHub issue when any E2E job fails. +# +# Runs directly on the runner (not inside Docker) because OpenShell bootstraps +# a K3s cluster inside a privileged Docker container — nesting would break networking. +# +# NVIDIA_API_KEY for hosted CI inference: +# - Repository secret: Settings → Secrets and variables → Actions → Repository secrets. +# - Environment secret: only available if the job sets `environment: `. +# (Storing the key under Environments / NVIDIA_API_KEY without `environment:` here leaves the +# variable empty in the job — repository secrets and environment secrets are separate.) +# Only runs on schedule and manual dispatch — never on PRs (secret protection). + +name: E2E / Nightly VPN +run-name: >- + ${{ github.event_name == 'workflow_dispatch' && inputs.advisor_dispatch_id != '' && format('E2E / Nightly VPN ({0})', inputs.advisor_dispatch_id) || 'E2E / Nightly VPN' }} + +on: + schedule: + - cron: "0 0 * * *" + workflow_dispatch: + inputs: + jobs: + description: >- + Comma-separated job names to run (empty = all). + Valid: cloud-e2e, cloud-onboard-e2e, cloud-inference-e2e, + cron-preflight-inference-local-e2e, + agent-turn-latency-e2e, skill-agent-e2e, openclaw-skill-cli-e2e, + docs-validation-e2e, messaging-providers-e2e, openclaw-slack-pairing-e2e, + openclaw-tui-chat-correlation-e2e, issue-4434-tui-unreachable-inference-e2e, + issue-3600-gpu-proof-optional-e2e, openclaw-discord-pairing-e2e, + issue-4462-scope-upgrade-approval-e2e, + issue-4462-gateway-pinned-approval-characterization-e2e, + messaging-compatible-endpoint-e2e, sessions-agents-cli-e2e, + channels-add-remove-e2e, channels-stop-start-openclaw-e2e, + channels-stop-start-hermes-e2e, brave-search-e2e, + common-egress-agent-e2e, kimi-inference-compat-e2e, + bedrock-runtime-compatible-anthropic-e2e, token-rotation-e2e, + sandbox-survival-e2e, issue-2478-crash-loop-recovery-e2e, + hermes-e2e, hermes-dashboard-e2e, hermes-root-entrypoint-smoke-e2e, + hermes-secret-boundary-e2e, + openclaw-onboard-security-posture-e2e, hermes-onboard-security-posture-e2e, + hermes-inference-switch-e2e, hermes-anthropic-inference-switch-e2e, + hermes-discord-e2e, hermes-slack-e2e, sandbox-operations-e2e, + inference-routing-e2e, openclaw-inference-switch-e2e, + openclaw-anthropic-inference-switch-e2e, + network-policy-e2e, state-backup-restore-e2e, tunnel-lifecycle-e2e, + diagnostics-e2e, credential-migration-e2e, snapshot-commands-e2e, + shields-config-e2e, rebuild-openclaw-e2e, + upgrade-stale-sandbox-e2e, openshell-gateway-upgrade-e2e, rebuild-hermes-e2e, + rebuild-hermes-stale-base-e2e, double-onboard-e2e, onboard-repair-e2e, + onboard-resume-e2e, onboard-negative-paths-e2e, runtime-overrides-e2e, + credential-sanitization-e2e, telegram-injection-e2e, overlayfs-autofix-e2e, + device-auth-health-e2e, launchable-smoke-e2e, gpu-e2e, gpu-double-onboard-e2e, + gpu-jetson-nvmap-e2e, concurrent-gateway-ports-e2e + required: false + type: string + default: "" + target_ref: + description: >- + Optional branch, ref, or SHA to test. When empty, tests run against + the workflow ref selected for the dispatch. Used by e2e-advisor + auto-dispatch so the trusted main workflow can test a PR head SHA. + required: false + type: string + default: "" + pr_number: + description: Optional PR number for selective-dispatch result comments. + required: false + type: string + default: "" + advisor_dispatch_id: + description: Optional correlation ID from e2e-advisor auto-dispatch. + required: false + type: string + default: "" + post_to_slack: + description: >- + Post the scorecard to a preview Slack channel on selective + dispatches (schedule and full runs always post). + required: false + type: boolean + default: false + +permissions: + contents: read + +concurrency: + group: nightly-e2e-vpn-${{ github.event_name }}-${{ github.event_name == 'workflow_dispatch' && format('{0}-{1}', github.ref, inputs.pr_number || 'manual') || 'schedule' }} + cancel-in-progress: true + +# Selective-dispatch contract: tools/e2e-advisor/dispatch.mts discovers +# dispatchable jobs by looking for each job's exact predicate shape below: +# github.event_name != 'workflow_dispatch' || inputs.jobs == '' || +# contains(format(',{0},', inputs.jobs), ',,') +# Keep this predicate format in sync with test/e2e-advisor-dispatch.test.ts if +# the workflow changes how individual jobs opt in to selective dispatch. +jobs: + cloud-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',cloud-e2e,')) + uses: ./.github/workflows/e2e-script-vpn.yaml + with: + ref: ${{ inputs.target_ref || github.ref }} + script: test/e2e-vpn/test-full-e2e.sh + artifact_name: "install-log" + artifact_path: "/tmp/nemoclaw-e2e-install.log" + env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_RECREATE_SANDBOX":"1","NEMOCLAW_SANDBOX_NAME":"e2e-nightly"}' + nvidia_api_key: true + github_token: true + secrets: &nightly-e2e-vpn-default-secrets + NVIDIA_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_API_KEY || '' }} + BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }} + DOCKERHUB_USERNAME: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.DOCKERHUB_USERNAME || '' }} + DOCKERHUB_TOKEN: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.DOCKERHUB_TOKEN || '' }} + cloud-onboard-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',cloud-onboard-e2e,')) + uses: ./.github/workflows/e2e-script-vpn.yaml + with: + ref: ${{ inputs.target_ref || github.ref }} + script: test/e2e-vpn/test-cloud-onboard-e2e.sh + artifact_name: "install-log-cloud-onboard" + artifact_path: "/tmp/nemoclaw-e2e-cloud-onboard-install.log" + always_artifact_name: "cloud-onboard-traces" + always_artifact_path: "/tmp/nemoclaw-trace-summary/" + always_artifact_trace_source_path: "/tmp/nemoclaw-traces/" + env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_POLICY_MODE":"custom","NEMOCLAW_POLICY_PRESETS":"npm,pypi","NEMOCLAW_RECREATE_SANDBOX":"1","NEMOCLAW_SANDBOX_NAME":"e2e-cloud-onboard","NEMOCLAW_TRACE_DIR":"/tmp/nemoclaw-traces"}' + checked_out_ref_env: "NEMOCLAW_PUBLIC_INSTALL_REF" + nvidia_api_key: true + github_token: true + secrets: *nightly-e2e-vpn-default-secrets + cloud-inference-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',cloud-inference-e2e,')) + uses: ./.github/workflows/e2e-script-vpn.yaml + with: + ref: ${{ inputs.target_ref || github.ref }} + script: test/e2e-vpn/test-cloud-inference-e2e.sh + timeout_minutes: 30 + artifact_name: "install-log-cloud-inference" + artifact_path: "/tmp/nemoclaw-e2e-cloud-inference-install.log" + env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_RECREATE_SANDBOX":"1","NEMOCLAW_SANDBOX_NAME":"e2e-cloud-inference"}' + nvidia_api_key: true + secrets: *nightly-e2e-vpn-default-secrets + cron-preflight-inference-local-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',cron-preflight-inference-local-e2e,')) + uses: ./.github/workflows/e2e-script-vpn.yaml + with: + ref: ${{ inputs.target_ref || github.ref }} + script: test/e2e-vpn/test-cron-preflight-inference-local-e2e.sh + timeout_minutes: 30 + artifact_name: "install-log-cron-preflight-inference-local" + artifact_path: "/tmp/nemoclaw-e2e-cron-preflight-install.log" + env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_RECREATE_SANDBOX":"1","NEMOCLAW_SANDBOX_NAME":"e2e-cron-preflight"}' + nvidia_api_key: true + secrets: *nightly-e2e-vpn-default-secrets + agent-turn-latency-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',agent-turn-latency-e2e,')) + uses: ./.github/workflows/e2e-script-vpn.yaml + with: + ref: ${{ inputs.target_ref || github.ref }} + script: test/e2e-vpn/test-agent-turn-latency-e2e.sh + timeout_minutes: 120 + artifact_name: "agent-turn-latency-logs" + artifact_path: | + /tmp/nemoclaw-e2e-openclaw-turn-latency-install.log + /tmp/nemoclaw-e2e-hermes-turn-latency-install.log + /tmp/nemoclaw-e2e-agent-turn-latency.json + env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_RECREATE_SANDBOX":"1"}' + nvidia_api_key: true + secrets: *nightly-e2e-vpn-default-secrets + skill-agent-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',skill-agent-e2e,')) + uses: ./.github/workflows/e2e-script-vpn.yaml + with: + ref: ${{ inputs.target_ref || github.ref }} + script: test/e2e-vpn/test-skill-agent-e2e.sh + timeout_minutes: 30 + artifact_name: "install-log-skill-agent" + artifact_path: "/tmp/nemoclaw-e2e-skill-agent-install.log" + env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_RECREATE_SANDBOX":"1","NEMOCLAW_SANDBOX_NAME":"e2e-skill-agent"}' + nvidia_api_key: true + secrets: *nightly-e2e-vpn-default-secrets + openclaw-skill-cli-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',openclaw-skill-cli-e2e,')) + uses: ./.github/workflows/e2e-script-vpn.yaml + with: + ref: ${{ inputs.target_ref || github.ref }} + script: test/e2e-vpn/test-openclaw-skill-cli-e2e.sh + timeout_minutes: 25 + artifact_name: "install-log-openclaw-skill-cli" + artifact_path: "/tmp/nemoclaw-e2e-openclaw-skill-cli-install.log" + env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_RECREATE_SANDBOX":"1","NEMOCLAW_SANDBOX_NAME":"e2e-openclaw-skill-cli"}' + nvidia_api_key: true + secrets: *nightly-e2e-vpn-default-secrets + docs-validation-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',docs-validation-e2e,')) + runs-on: linux-amd64-cpu4 + permissions: + contents: read + timeout-minutes: 15 + steps: + - &target-ref-checkout + name: Checkout + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + ref: ${{ inputs.target_ref || github.ref }} + persist-credentials: false + + - name: Setup Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 + with: + node-version: "22" + cache: npm + + - name: Install root dependencies + run: npm ci + + - name: Run docs validation E2E test + env: + CHECK_DOC_LINKS_REMOTE: "0" + run: bash test/e2e-vpn/test-docs-validation.sh + + - name: Upload docs validation artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: docs-validation-artifacts + path: e2e-artifacts/vitest/docs-validation/ + include-hidden-files: false + if-no-files-found: ignore + retention-days: 14 + + # ── Messaging Providers E2E ────────────────────────────────── + # Validates the full provider/placeholder/L7-proxy chain for token-backed + # messaging credentials, and the QR-only WhatsApp config/policy/no-provider + # path. Uses fake tokens by default — the L7 proxy rewrites placeholders and + # the real API returns 401, proving the chain works. See: PR #1081 + messaging-providers-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',messaging-providers-e2e,')) + uses: ./.github/workflows/e2e-script-vpn.yaml + with: + ref: ${{ inputs.target_ref || github.ref }} + script: test/e2e-vpn/test-messaging-providers.sh + timeout_minutes: 75 + artifact_name: "install-log-messaging-providers" + artifact_path: | + /tmp/nemoclaw-e2e-install.log + /tmp/nemoclaw-e2e-whatsapp-*.log + env_json: '{"DISCORD_BOT_TOKEN":"test-fake-discord-token-e2e","NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_POLICY_TIER":"open","NEMOCLAW_SANDBOX_NAME":"e2e-msg-provider","SLACK_APP_TOKEN":"xapp-fake-slack-app-token-e2e","SLACK_BOT_TOKEN":"xoxb-fake-slack-token-e2e","TELEGRAM_BOT_TOKEN":"test-fake-telegram-token-e2e"}' + nvidia_api_key: true + github_token: true + messaging_live_secrets: ${{ github.event_name != 'workflow_dispatch' || inputs.target_ref == '' }} + secrets: + NVIDIA_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_API_KEY || '' }} + BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }} + DOCKERHUB_USERNAME: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.DOCKERHUB_USERNAME || '' }} + DOCKERHUB_TOKEN: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.DOCKERHUB_TOKEN || '' }} + TELEGRAM_BOT_TOKEN_REAL: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.TELEGRAM_BOT_TOKEN_REAL || '' }} + TELEGRAM_CHAT_ID_E2E: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.TELEGRAM_CHAT_ID_E2E || '' }} + DISCORD_BOT_TOKEN_REAL: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.DISCORD_BOT_TOKEN_REAL || '' }} + DISCORD_CHANNEL_ID_E2E: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.DISCORD_CHANNEL_ID_E2E || '' }} + SLACK_BOT_TOKEN_REAL: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.SLACK_BOT_TOKEN_REAL || '' }} + SLACK_APP_TOKEN_REAL: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.SLACK_APP_TOKEN_REAL || '' }} + SLACK_CHANNEL_ID_E2E: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.SLACK_CHANNEL_ID_E2E || '' }} + openclaw-slack-pairing-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',openclaw-slack-pairing-e2e,')) + uses: ./.github/workflows/e2e-script-vpn.yaml + with: + ref: ${{ inputs.target_ref || github.ref }} + script: test/e2e-vpn/test-openclaw-slack-pairing.sh + artifact_name: "install-log-openclaw-slack-pairing" + artifact_path: "/tmp/nemoclaw-e2e-openclaw-slack-pairing-install.log" + env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_POLICY_TIER":"open","NEMOCLAW_SANDBOX_NAME":"e2e-openclaw-slack-pairing","SLACK_APP_TOKEN":"xapp-fake-slack-pairing-e2e","SLACK_BOT_TOKEN":"xoxb-fake-slack-pairing-e2e"}' + nvidia_api_key: true + github_token: true + secrets: *nightly-e2e-vpn-default-secrets + openclaw-tui-chat-correlation-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',openclaw-tui-chat-correlation-e2e,')) + runs-on: linux-amd64-cpu4 + timeout-minutes: 75 + steps: + - *target-ref-checkout + + # Authenticate Docker Hub pulls for sandbox image builds when CI + # credentials are configured. Withhold credentials from workflow_dispatch + # runs against an explicit target_ref because those can execute untrusted + # PR-head code checked out above. + - &dockerhub-auth-step + name: Authenticate to Docker Hub + if: ${{ github.event_name != 'workflow_dispatch' || inputs.target_ref == '' }} + env: + DOCKERHUB_USERNAME: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.DOCKERHUB_USERNAME || '' }} + DOCKERHUB_TOKEN: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.DOCKERHUB_TOKEN || '' }} + shell: bash + run: | + set -euo pipefail + if [[ -z "${DOCKERHUB_USERNAME}" || -z "${DOCKERHUB_TOKEN}" ]]; then + echo "::notice::Docker Hub credentials not configured; continuing with anonymous pulls." + exit 0 + fi + login_succeeded=0 + for attempt in 1 2 3; do + if echo "${DOCKERHUB_TOKEN}" | timeout 30s docker login docker.io --username "${DOCKERHUB_USERNAME}" --password-stdin; then + login_succeeded=1 + break + fi + if [[ "$attempt" -lt 3 ]]; then + echo "::warning::Docker Hub login attempt ${attempt} failed; retrying." + sleep $((attempt * 5)) + fi + done + if [[ "$login_succeeded" -ne 1 ]]; then + echo "::warning::Docker Hub login failed after 3 attempts; continuing with anonymous pulls." + fi + + - name: Resolve public install ref + id: public_install_ref + shell: bash + run: | + printf 'ref=%s\n' "$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + + - name: Setup Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: "22" + + - name: Install test dependencies + run: npm ci --include=dev + + - name: Run OpenClaw TUI chat correlation E2E test + env: + NVIDIA_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_API_KEY || '' }} + NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1" + NEMOCLAW_PROVIDER: custom + NEMOCLAW_ENDPOINT_URL: https://inference.nvidia.com/v1 + NEMOCLAW_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_COMPAT_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_PREFERRED_API: openai-completions + COMPATIBLE_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_API_KEY || '' }} + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_RECREATE_SANDBOX: "1" + NEMOCLAW_SANDBOX_NAME: "e2e-openclaw-tui-correlation" + NEMOCLAW_PUBLIC_INSTALL_REF: ${{ steps.public_install_ref.outputs.ref }} + GITHUB_TOKEN: ${{ github.token }} + run: bash test/e2e-vpn/test-openclaw-tui-chat-correlation.sh + + - name: Upload install log on failure + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: install-log-openclaw-tui-chat-correlation + path: /tmp/nemoclaw-e2e-openclaw-tui-correlation-install.log + if-no-files-found: ignore + + issue-4434-tui-unreachable-inference-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',issue-4434-tui-unreachable-inference-e2e,')) + runs-on: linux-amd64-cpu4 + timeout-minutes: 120 + steps: + - name: Checkout + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + # This privileged proof mutates host firewall state and receives + # NVIDIA_API_KEY. Keep the runner script from the trusted workflow ref; + # the product under test is selected separately via + # NEMOCLAW_PUBLIC_INSTALL_REF. + ref: ${{ github.ref }} + fetch-depth: 0 + persist-credentials: false + + - name: Resolve trusted public install ref + id: public_install_ref + shell: bash + env: + TARGET_REF: ${{ inputs.target_ref }} + run: | + set -euo pipefail + trusted_head="$(git rev-parse HEAD)" + ref="${TARGET_REF:-$trusted_head}" + if [[ ! "$ref" =~ ^[0-9a-f]{40}$ ]]; then + echo "::error::issue #4434 privileged E2E requires target_ref to be a full commit SHA" + exit 1 + fi + if ! git cat-file -e "${ref}^{commit}" 2>/dev/null; then + echo "::error::target_ref ${ref} is not present in the trusted workflow checkout" + exit 1 + fi + if ! git merge-base --is-ancestor "$ref" "$trusted_head"; then + echo "::error::target_ref ${ref} is not reachable from trusted workflow ref ${trusted_head}" + exit 1 + fi + printf 'ref=%s\n' "$ref" >> "$GITHUB_OUTPUT" + + - name: "Install issue #4434 test dependencies" + shell: bash + run: | + set -euo pipefail + sudo apt-get update + sudo apt-get install -y expect iptables + + - name: "Run issue #4434 TUI unreachable inference E2E test" + env: + NVIDIA_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_API_KEY || '' }} + NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1" + NEMOCLAW_PROVIDER: custom + NEMOCLAW_ENDPOINT_URL: https://inference.nvidia.com/v1 + NEMOCLAW_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_COMPAT_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_PREFERRED_API: openai-completions + COMPATIBLE_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_API_KEY || '' }} + NEMOCLAW_ISSUE_4434_LIVE: "1" + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_RECREATE_SANDBOX: "1" + NEMOCLAW_SANDBOX_NAME: "e2e-issue-4434-tui-unreachable" + NEMOCLAW_PUBLIC_INSTALL_REF: ${{ steps.public_install_ref.outputs.ref }} + GITHUB_TOKEN: ${{ github.token }} + run: bash test/e2e-vpn/test-issue-4434-tui-unreachable-inference.sh + + - name: "Sanitize issue #4434 logs on failure" + if: failure() + shell: bash + env: + NVIDIA_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_API_KEY || '' }} + GITHUB_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + for file in /tmp/nemoclaw-e2e-issue-4434-install.log /tmp/nemoclaw-issue-4434.*; do + [ -f "$file" ] || continue + if [ -n "${NVIDIA_API_KEY:-}" ]; then + perl -0pi -e 's/\Q$ENV{NVIDIA_API_KEY}\E/[REDACTED_NVIDIA_API_KEY]/g' "$file" + fi + if [ -n "${GITHUB_TOKEN:-}" ]; then + perl -0pi -e 's/\Q$ENV{GITHUB_TOKEN}\E/[REDACTED_GITHUB_TOKEN]/g' "$file" + fi + perl -0pi -e 's/nvapi-[A-Za-z0-9._-]+/[REDACTED_NVIDIA_API_KEY]/g; s/gh[pousr]_[A-Za-z0-9_]+/[REDACTED_GITHUB_TOKEN]/g' "$file" + done + + - name: "Upload issue #4434 logs on failure" + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: issue-4434-tui-unreachable-inference-logs + path: | + /tmp/nemoclaw-e2e-issue-4434-install.log + /tmp/nemoclaw-issue-4434.* + if-no-files-found: ignore + + # ── DGX Station GPU optional proof validation (#3600) ────────── + # CI cannot emulate GB300, but this guards the release-blocker mitigation: + # optional direct GPU proofs must not abort onboard before the fatal throw. + issue-3600-gpu-proof-optional-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',issue-3600-gpu-proof-optional-e2e,')) + runs-on: linux-amd64-cpu4 + timeout-minutes: 15 + steps: + - *target-ref-checkout + + - *dockerhub-auth-step + + - name: Setup Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: "22" + + - name: Install test dependencies + run: npm ci --include=dev + + - name: Verify optional GPU proof cannot abort onboard + run: npx vitest run src/lib/onboard/sandbox-gpu-preflight.test.ts --pool=forks -t "direct sandbox GPU proof" + + # ── OpenClaw Discord Pairing E2E (#4061) ────────────────────── + # Hermetic Discord Gateway placeholder rewrite proof, then connect-shell + # `openclaw pairing approve discord ` against shared state. + openclaw-discord-pairing-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',openclaw-discord-pairing-e2e,')) + uses: ./.github/workflows/e2e-script-vpn.yaml + with: + ref: ${{ inputs.target_ref || github.ref }} + script: test/e2e-vpn/test-openclaw-discord-pairing.sh + artifact_name: "install-log-openclaw-discord-pairing" + artifact_path: "/tmp/nemoclaw-e2e-openclaw-discord-pairing-install.log" + env_json: '{"DISCORD_BOT_TOKEN":"test-fake-discord-pairing-e2e","NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_POLICY_TIER":"open","NEMOCLAW_SANDBOX_NAME":"e2e-openclaw-discord-pairing"}' + nvidia_api_key: true + github_token: true + secrets: *nightly-e2e-vpn-default-secrets + # ── OpenClaw Scope-Upgrade Approval E2E (#4462) ──────────────── + # Positive proof: in a real sandbox, accept either a visible pending CLI + # scope upgrade or the fixed watcher's immediate approval, then confirm + # openclaw agent still uses the gateway path rather than embedded fallback. + issue-4462-scope-upgrade-approval-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',issue-4462-scope-upgrade-approval-e2e,')) + uses: ./.github/workflows/e2e-script-vpn.yaml + with: + ref: ${{ inputs.target_ref || github.ref }} + script: test/e2e-vpn/test-issue-4462-scope-upgrade-approval.sh + timeout_minutes: 60 + artifact_name: "issue-4462-scope-upgrade-approval-logs" + artifact_path: | + /tmp/nemoclaw-e2e-issue-4462-scope-upgrade-install.log + /tmp/nemoclaw-issue-4462-scope-upgrade-approval.log + /tmp/nemoclaw-issue-4462-scope-upgrade-agent.log + /tmp/nemoclaw-issue-4462-scope-upgrade-state.log + env_json: '{"NEMOCLAW_4462_MODE":"approval","NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_AUTO_PAIR_DEADLINE_SECS":"30","NEMOCLAW_AUTO_PAIR_FAST_DEADLINE_SECS":"3","NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS":"600","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_RECREATE_SANDBOX":"1","NEMOCLAW_SANDBOX_NAME":"e2e-issue-4462-scope-upgrade"}' + nvidia_api_key: true + github_token: false + secrets: *nightly-e2e-vpn-default-secrets + # ── OpenClaw Gateway-Pinned Approval Characterization (#4462) ── + # Diagnostic proof: in a real sandbox, wait for the fixed watcher to exit, + # force the legacy gateway-pinned approve path, record the observed + # OpenClaw outcome, and recover through the fixed proxy-env guard if needed. + issue-4462-gateway-pinned-approval-characterization-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',issue-4462-gateway-pinned-approval-characterization-e2e,')) + uses: ./.github/workflows/e2e-script-vpn.yaml + with: + ref: ${{ inputs.target_ref || github.ref }} + script: test/e2e-vpn/test-issue-4462-scope-upgrade-approval.sh + timeout_minutes: 60 + artifact_name: "issue-4462-gateway-pinned-approval-characterization-logs" + artifact_path: | + /tmp/nemoclaw-e2e-issue-4462-scope-upgrade-repro-install.log + /tmp/nemoclaw-issue-4462-scope-upgrade-repro-approval.log + /tmp/nemoclaw-issue-4462-scope-upgrade-repro-agent.log + /tmp/nemoclaw-issue-4462-scope-upgrade-repro-state.log + env_json: '{"NEMOCLAW_4462_AGENT_LOG":"/tmp/nemoclaw-issue-4462-scope-upgrade-repro-agent.log","NEMOCLAW_4462_APPROVAL_LOG":"/tmp/nemoclaw-issue-4462-scope-upgrade-repro-approval.log","NEMOCLAW_4462_AUTO_PAIR_DEADLINE_SECS":"12","NEMOCLAW_4462_AUTO_PAIR_FAST_DEADLINE_SECS":"1","NEMOCLAW_4462_AUTO_PAIR_RUN_TIMEOUT_SECS":"2","NEMOCLAW_4462_AUTO_PAIR_SLOW_INTERVAL_SECS":"1","NEMOCLAW_4462_INSTALL_LOG":"/tmp/nemoclaw-e2e-issue-4462-scope-upgrade-repro-install.log","NEMOCLAW_4462_MODE":"legacy-repro","NEMOCLAW_4462_STATE_LOG":"/tmp/nemoclaw-issue-4462-scope-upgrade-repro-state.log","NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_RECREATE_SANDBOX":"1","NEMOCLAW_SANDBOX_NAME":"e2e-issue-4462-scope-upgrade-repro"}' + nvidia_api_key: true + github_token: false + secrets: *nightly-e2e-vpn-default-secrets + messaging-compatible-endpoint-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',messaging-compatible-endpoint-e2e,')) + uses: ./.github/workflows/e2e-script-vpn.yaml + with: + ref: ${{ inputs.target_ref || github.ref }} + script: test/e2e-vpn/test-messaging-compatible-endpoint.sh + artifact_name: "install-log-messaging-compatible-endpoint" + artifact_path: "/tmp/nemoclaw-e2e-messaging-compatible-endpoint-install.log" + env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_SANDBOX_NAME":"e2e-msg-compat","TELEGRAM_ALLOWED_IDS":"123456789","TELEGRAM_BOT_TOKEN":"test-fake-telegram-token-e2e"}' + github_token: true + secrets: *nightly-e2e-vpn-default-secrets + # ── Sessions / agents CLI groups E2E ───────────────────────────────── + # Coverage for the `nemoclaw sessions` and `nemoclaw agents` + # CLI groups: openclaw-sessions passthrough (list/show/reset), agents + # add/delete lifecycle, and their related routing through the public-argv + # translator. The OpenShell sandbox is provisioned end-to-end so the + # groups are exercised against a live OpenClaw runtime. + # + # Secret-bearing target-ref execution — gating contract: + # * This workflow has no `pull_request` or `pull_request_target` trigger + # (see `on:` at the top of this file). The only ways to reach this job + # are the nightly `schedule:` (which checks out `github.ref` = + # default branch) and `workflow_dispatch:` (which requires repo write + # access to invoke). Fork PRs cannot trigger this job at all. + # * Therefore the `inputs.target_ref || github.ref` ref expression only + # runs target-ref code when one of: + # (a) `schedule:` fires against the default branch (no `target_ref` + # input is available on schedule events), or + # (b) a user with repo write access calls `workflow_dispatch` and + # chooses a `target_ref`. Both paths are "trusted-ref": the + # code reached at runtime has already passed maintainer review + # or is the default branch itself. + # * `NVIDIA_API_KEY` is the repo-scoped E2E credential — purposefully + # not a production key. It is wired only to the inference quota + # allocated to this repository's E2E lane, with no IAM / billing + # authority outside that quota. Treat exposure as "rotate at the + # quota boundary," not "rotate at the production boundary." Audit + # trail: `gh api repos/NVIDIA/NemoClaw/actions/secrets/NVIDIA_API_KEY` + # shows the secret scope. + # * The top-level `github.repository == 'NVIDIA/NemoClaw'` check + # additionally guards repo-forked schedules. + # * `github_token: false` (below) — the script does not need a GitHub + # token; keep it disabled unless a documented need appears. + # This shape matches every other `e2e-script.yaml` caller in this file; + # changing it for one job would diverge from the repo-wide pattern and + # should be done across the workflow, not in this PR. + sessions-agents-cli-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',sessions-agents-cli-e2e,')) + uses: ./.github/workflows/e2e-script-vpn.yaml + with: + ref: ${{ inputs.target_ref || github.ref }} + script: test/e2e-vpn/test-sessions-agents-cli.sh + timeout_minutes: 60 + artifact_name: "install-log-sessions-agents-cli" + artifact_path: | + /tmp/nemoclaw-e2e-install.log + env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_SANDBOX_NAME":"e2e-sessions-agents-cli"}' + nvidia_api_key: true + github_token: false + secrets: *nightly-e2e-vpn-default-secrets + channels-add-remove-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',channels-add-remove-e2e,')) + uses: ./.github/workflows/e2e-script-vpn.yaml + with: + ref: ${{ inputs.target_ref || github.ref }} + script: test/e2e-vpn/test-channels-add-remove.sh + timeout_minutes: 75 + artifact_name: "install-log-channels-add-remove" + artifact_path: | + /tmp/nemoclaw-e2e-install.log + /tmp/nc-add.log + /tmp/nc-remove.log + /tmp/nc-rebuild-add.log + /tmp/nc-rebuild-remove.log + env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_SANDBOX_NAME":"e2e-channels-add-remove","TELEGRAM_BOT_TOKEN":"test-fake-telegram-token-add-remove-e2e"}' + nvidia_api_key: true + github_token: true + secrets: *nightly-e2e-vpn-default-secrets + # ── Channels stop/start lifecycle E2E (#3462) ─────────────────────── + # Regression coverage for #3453 (stop must disable across rebuild), #3381 + # (start must re-attach from cached credentials). Sharded by agent so the + # OpenClaw and Hermes coverage can run in parallel. + channels-stop-start-openclaw-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',channels-stop-start-openclaw-e2e,')) + uses: ./.github/workflows/e2e-script-vpn.yaml + with: + ref: ${{ inputs.target_ref || github.ref }} + script: test/e2e-vpn/test-channels-stop-start.sh + timeout_minutes: 75 + artifact_name: "install-log-channels-stop-start-openclaw" + artifact_path: | + /tmp/nemoclaw-e2e-install.log + /tmp/nemoclaw-e2e-channels-*-install.log + /tmp/nc-channels-*.log + env_json: '{"DISCORD_ALLOWED_IDS":"1005536447329222676","DISCORD_BOT_TOKEN":"test-fake-discord-token-stop-start-e2e","DISCORD_REQUIRE_MENTION":"0","DISCORD_SERVER_ID":"1491590992753590594","NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_CHANNELS_STOP_START_AGENT":"openclaw","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_POLICY_TIER":"open","NEMOCLAW_SANDBOX_NAME":"e2e-channels-stop-start","SLACK_ALLOWED_USERS":"U0123456789,U09ABCDEFGH","SLACK_APP_TOKEN":"xapp-fake-slack-app-token-stop-start-e2e","SLACK_BOT_TOKEN":"xoxb-fake-slack-token-stop-start-e2e","TELEGRAM_ALLOWED_IDS":"123456789","TELEGRAM_BOT_TOKEN":"test-fake-telegram-token-stop-start-e2e","WECHAT_ACCOUNT_ID":"e2e-fake-account-stop-start","WECHAT_ALLOWED_IDS":"wxid_stopstart_operator","WECHAT_BASE_URL":"https://ilinkai.wechat.com","WECHAT_BOT_TOKEN":"test-fake-wechat-token-stop-start-e2e","WECHAT_USER_ID":"wxid_stopstart_operator"}' + nvidia_api_key: true + github_token: true + secrets: *nightly-e2e-vpn-default-secrets + channels-stop-start-hermes-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',channels-stop-start-hermes-e2e,')) + uses: ./.github/workflows/e2e-script-vpn.yaml + with: + ref: ${{ inputs.target_ref || github.ref }} + script: test/e2e-vpn/test-channels-stop-start.sh + timeout_minutes: 75 + artifact_name: "install-log-channels-stop-start-hermes" + artifact_path: | + /tmp/nemoclaw-e2e-install.log + /tmp/nemoclaw-e2e-channels-*-install.log + /tmp/nc-channels-*.log + env_json: '{"DISCORD_ALLOWED_IDS":"1005536447329222676","DISCORD_BOT_TOKEN":"test-fake-discord-token-stop-start-e2e","DISCORD_REQUIRE_MENTION":"0","DISCORD_SERVER_ID":"1491590992753590594","NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_CHANNELS_STOP_START_AGENT":"hermes","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_POLICY_TIER":"open","NEMOCLAW_SANDBOX_NAME":"e2e-channels-stop-start","SLACK_ALLOWED_USERS":"U0123456789,U09ABCDEFGH","SLACK_APP_TOKEN":"xapp-fake-slack-app-token-stop-start-e2e","SLACK_BOT_TOKEN":"xoxb-fake-slack-token-stop-start-e2e","TELEGRAM_ALLOWED_IDS":"123456789","TELEGRAM_BOT_TOKEN":"test-fake-telegram-token-stop-start-e2e","WECHAT_ACCOUNT_ID":"e2e-fake-account-stop-start","WECHAT_ALLOWED_IDS":"wxid_stopstart_operator","WECHAT_BASE_URL":"https://ilinkai.wechat.com","WECHAT_BOT_TOKEN":"test-fake-wechat-token-stop-start-e2e","WECHAT_USER_ID":"wxid_stopstart_operator"}' + nvidia_api_key: true + github_token: true + secrets: *nightly-e2e-vpn-default-secrets + brave-search-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',brave-search-e2e,')) + uses: ./.github/workflows/e2e-script-vpn.yaml + with: + ref: ${{ inputs.target_ref || github.ref }} + script: test/e2e-vpn/test-brave-search-e2e.sh + artifact_name: "install-log-brave-search" + artifact_path: "/tmp/nemoclaw-e2e-brave-search-onboard.log" + env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_SANDBOX_NAME":"e2e-brave-search"}' + brave_api_key: true + nvidia_api_key: true + github_token: true + secrets: *nightly-e2e-vpn-default-secrets + common-egress-agent-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',common-egress-agent-e2e,')) + uses: ./.github/workflows/e2e-script-vpn.yaml + with: + ref: ${{ inputs.target_ref || github.ref }} + script: test/e2e-vpn/test-common-egress-agent-e2e.sh + timeout_minutes: 120 + artifact_name: "common-egress-agent-e2e-logs" + artifact_path: "/tmp/nemoclaw-e2e-common-egress-*.log" + env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_RECREATE_SANDBOX":"1"}' + nvidia_api_key: true + github_token: true + secrets: *nightly-e2e-vpn-default-secrets + kimi-inference-compat-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',kimi-inference-compat-e2e,')) + runs-on: linux-amd64-cpu4 + timeout-minutes: 45 + steps: + - *target-ref-checkout + + - *dockerhub-auth-step + + - name: Run Kimi inference compatibility E2E test + env: + # Kimi uses the public VPN NVIDIA inference key intentionally. The script + # validates this nvapi-* key, then mirrors it only inside the process + # for the shared onboarding/provider-registration path. + NVIDIA_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_API_KEY || '' }} + NEMOCLAW_PROVIDER: cloud + NEMOCLAW_MODEL: moonshotai/kimi-k2.6 + NEMOCLAW_PREFERRED_API: openai-completions + NEMOCLAW_KIMI_USE_MOCK: "0" + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_SANDBOX_NAME: "e2e-kimi-compat" + GITHUB_TOKEN: ${{ github.token }} + run: bash test/e2e-vpn/test-kimi-inference-compat.sh + + - name: Sanitize Kimi logs on failure + if: failure() + shell: bash + env: + NVIDIA_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_API_KEY || '' }} + GITHUB_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + for file in \ + /tmp/nemoclaw-e2e-kimi-inference-compat-onboard.log \ + /tmp/nemoclaw-e2e-kimi-inference-compat-build.log \ + /tmp/nemoclaw-e2e-kimi-inference-compat-agent.log; do + [ -f "$file" ] || continue + if [ -n "${NVIDIA_API_KEY:-}" ]; then + perl -0pi -e 's/\Q$ENV{NVIDIA_API_KEY}\E/[REDACTED_NVIDIA_API_KEY]/g' "$file" + fi + if [ -n "${GITHUB_TOKEN:-}" ]; then + perl -0pi -e 's/\Q$ENV{GITHUB_TOKEN}\E/[REDACTED_GITHUB_TOKEN]/g' "$file" + fi + perl -0pi -e 's/nvapi-[A-Za-z0-9._-]+/[REDACTED_NVIDIA_API_KEY]/g; s/gh[pousr]_[A-Za-z0-9_]+/[REDACTED_GITHUB_TOKEN]/g' "$file" + done + + - name: Upload onboard log on failure + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: install-log-kimi-inference-compat + path: /tmp/nemoclaw-e2e-kimi-inference-compat-onboard.log + if-no-files-found: ignore + + - name: Upload build/setup log on failure + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: build-log-kimi-inference-compat + path: /tmp/nemoclaw-e2e-kimi-inference-compat-build.log + if-no-files-found: ignore + + - name: Upload agent log on failure + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: agent-log-kimi-inference-compat + path: /tmp/nemoclaw-e2e-kimi-inference-compat-agent.log + if-no-files-found: ignore + + # ── Bedrock Runtime compatible Anthropic endpoint (#3767) ───── + # Hermetic fake Bedrock Runtime endpoint path. The sandbox only sees + # inference.local; the host-side OpenShell provider owns the hidden adapter + # token and the upstream Bedrock bearer derived from the fake pasted key. + bedrock-runtime-compatible-anthropic-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',bedrock-runtime-compatible-anthropic-e2e,')) + runs-on: linux-amd64-cpu4 + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + agent: [openclaw, hermes] + steps: + - *target-ref-checkout + + - *dockerhub-auth-step + + - name: Run Bedrock Runtime compatible Anthropic E2E test + env: + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_RECREATE_SANDBOX: "1" + NEMOCLAW_AGENT: ${{ matrix.agent }} + NEMOCLAW_SANDBOX_NAME: e2e-bedrock-${{ matrix.agent }} + GITHUB_TOKEN: ${{ github.token }} + run: bash test/e2e-vpn/test-bedrock-runtime-compatible-anthropic.sh + + - name: Upload onboard log on failure + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: onboard-log-bedrock-runtime-compatible-anthropic-${{ matrix.agent }} + path: /tmp/nemoclaw-e2e-bedrock-runtime-${{ matrix.agent }}-onboard.log + if-no-files-found: ignore + + - name: Upload build/setup log on failure + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: build-log-bedrock-runtime-compatible-anthropic-${{ matrix.agent }} + path: /tmp/nemoclaw-e2e-bedrock-runtime-${{ matrix.agent }}-build.log + if-no-files-found: ignore + + - name: Upload fake Bedrock Runtime log on failure + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: mock-log-bedrock-runtime-compatible-anthropic-${{ matrix.agent }} + path: /tmp/nemoclaw-e2e-bedrock-runtime-${{ matrix.agent }}-mock.log + if-no-files-found: ignore + + - name: Upload Bedrock Runtime adapter log on failure + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: adapter-log-bedrock-runtime-compatible-anthropic-${{ matrix.agent }} + path: ~/.nemoclaw/bedrock-runtime-adapter.log + if-no-files-found: ignore + + # ── Token rotation (credential propagation to L7 proxy) ───── + # Validates that rotating a messaging token and re-running onboard + # propagates the new credential to the sandbox. Uses two fake tokens + # per provider (Telegram + Discord) to prove the sandbox is rebuilt on + # rotation and reused when unchanged. + # See: issue #1903 + token-rotation-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',token-rotation-e2e,')) + runs-on: linux-amd64-cpu4 + timeout-minutes: 45 + steps: + - *target-ref-checkout + + - *dockerhub-auth-step + + - name: Run token rotation E2E test + env: + NVIDIA_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_API_KEY || '' }} + NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1" + NEMOCLAW_PROVIDER: custom + NEMOCLAW_ENDPOINT_URL: https://inference.nvidia.com/v1 + NEMOCLAW_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_COMPAT_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_PREFERRED_API: openai-completions + COMPATIBLE_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_API_KEY || '' }} + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_POLICY_TIER: "open" + GITHUB_TOKEN: ${{ github.token }} + TELEGRAM_BOT_TOKEN_A: "test-fake-token-A-rotation-e2e" + TELEGRAM_BOT_TOKEN_B: "test-fake-token-B-rotation-e2e" + DISCORD_BOT_TOKEN_A: "discord-a" + DISCORD_BOT_TOKEN_B: "discord-b" + SLACK_BOT_TOKEN_A: "xoxb-fake-A-rotation-e2e" + SLACK_BOT_TOKEN_B: "xoxb-fake-B-rotation-e2e" + SLACK_APP_TOKEN_A: "xapp-fake-A-rotation-e2e" + SLACK_APP_TOKEN_B: "xapp-fake-B-rotation-e2e" + run: bash test/e2e-vpn/test-token-rotation.sh + + - name: Upload install log on failure + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: install-log-token-rotation + path: /tmp/nemoclaw-e2e-install.log + if-no-files-found: ignore + + # ── Sandbox survival (gateway restart recovery) ────────────── + sandbox-survival-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',sandbox-survival-e2e,')) + uses: ./.github/workflows/e2e-script-vpn.yaml + with: + ref: ${{ inputs.target_ref || github.ref }} + script: test/e2e-vpn/test-sandbox-survival.sh + timeout_minutes: 30 + artifact_name: "sandbox-survival-install-log" + artifact_path: "/tmp/nemoclaw-e2e-install.log" + env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_SANDBOX_NAME":"e2e-survival"}' + nvidia_api_key: true + github_token: true + secrets: *nightly-e2e-vpn-default-secrets + issue-2478-crash-loop-recovery-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',issue-2478-crash-loop-recovery-e2e,')) + uses: ./.github/workflows/e2e-script-vpn.yaml + with: + ref: ${{ inputs.target_ref || github.ref }} + script: test/e2e-vpn/test-issue-2478-crash-loop-recovery.sh + timeout_minutes: 30 + artifact_name: "issue-2478-crash-loop-recovery-install-log" + artifact_path: "/tmp/nemoclaw-e2e-install.log" + env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_E2E_USE_COMPAT_MOCK":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_SANDBOX_NAME":"e2e-2478"}' + github_token: true + secrets: *nightly-e2e-vpn-default-secrets + hermes-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',hermes-e2e,')) + uses: ./.github/workflows/e2e-script-vpn.yaml + with: + ref: ${{ inputs.target_ref || github.ref }} + script: test/e2e-vpn/test-hermes-e2e.sh + timeout_minutes: 60 + artifact_name: "hermes-e2e-install-log" + artifact_path: "/tmp/nemoclaw-e2e-hermes-install.log" + env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_AGENT":"hermes","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_RECREATE_SANDBOX":"1","NEMOCLAW_SANDBOX_NAME":"e2e-hermes"}' + nvidia_api_key: true + github_token: true + secrets: *nightly-e2e-vpn-default-secrets + hermes-dashboard-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',hermes-dashboard-e2e,')) + uses: ./.github/workflows/e2e-script-vpn.yaml + with: + ref: ${{ inputs.target_ref || github.ref }} + script: test/e2e-vpn/test-hermes-e2e.sh + timeout_minutes: 60 + artifact_name: "hermes-dashboard-e2e-install-log" + artifact_path: "/tmp/nemoclaw-e2e-hermes-install.log" + env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_AGENT":"hermes","NEMOCLAW_E2E_HERMES_DASHBOARD":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_RECREATE_SANDBOX":"1","NEMOCLAW_SANDBOX_NAME":"e2e-hermes-dashboard"}' + nvidia_api_key: true + github_token: true + secrets: *nightly-e2e-vpn-default-secrets + hermes-root-entrypoint-smoke-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',hermes-root-entrypoint-smoke-e2e,')) + uses: ./.github/workflows/e2e-script-vpn.yaml + with: + ref: ${{ inputs.target_ref || github.ref }} + script: test/e2e-vpn/test-hermes-root-entrypoint-smoke.sh + timeout_minutes: 45 + artifact_name: "hermes-root-entrypoint-smoke-log" + artifact_path: "/tmp/nemoclaw-hermes-root-entrypoint-smoke.log" + secrets: *nightly-e2e-vpn-default-secrets + hermes-secret-boundary-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',hermes-secret-boundary-e2e,')) + uses: ./.github/workflows/e2e-script-vpn.yaml + with: + ref: ${{ inputs.target_ref || github.ref }} + script: test/e2e-vpn/test-hermes-sandbox-secret-boundary.sh + timeout_minutes: 60 + artifact_name: "hermes-secret-boundary-log" + artifact_path: "/tmp/nemoclaw-hermes-sandbox-secret-boundary.log" + secrets: *nightly-e2e-vpn-default-secrets + openclaw-onboard-security-posture-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',openclaw-onboard-security-posture-e2e,')) + uses: ./.github/workflows/e2e-script-vpn.yaml + with: + ref: ${{ inputs.target_ref || github.ref }} + script: test/e2e-vpn/test-full-e2e.sh + timeout_minutes: 60 + artifact_name: "openclaw-onboard-security-posture-install-log" + artifact_path: "/tmp/nemoclaw-e2e-install.log" + env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_E2E_EXPECT_NON_ROOT_HOST":"1","NEMOCLAW_E2E_SECURITY_POSTURE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_RECREATE_SANDBOX":"1","NEMOCLAW_SANDBOX_NAME":"e2e-openclaw-security-posture"}' + nvidia_api_key: true + github_token: true + secrets: *nightly-e2e-vpn-default-secrets + hermes-onboard-security-posture-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',hermes-onboard-security-posture-e2e,')) + uses: ./.github/workflows/e2e-script-vpn.yaml + with: + ref: ${{ inputs.target_ref || github.ref }} + script: test/e2e-vpn/test-hermes-e2e.sh + timeout_minutes: 60 + artifact_name: "hermes-onboard-security-posture-install-log" + artifact_path: "/tmp/nemoclaw-e2e-hermes-install.log" + env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_AGENT":"hermes","NEMOCLAW_E2E_EXPECT_NON_ROOT_HOST":"1","NEMOCLAW_E2E_SECURITY_POSTURE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_RECREATE_SANDBOX":"1","NEMOCLAW_SANDBOX_NAME":"e2e-hermes-security-posture"}' + nvidia_api_key: true + github_token: true + secrets: *nightly-e2e-vpn-default-secrets + hermes-inference-switch-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',hermes-inference-switch-e2e,')) + uses: ./.github/workflows/e2e-script-vpn.yaml + with: + ref: ${{ inputs.target_ref || github.ref }} + script: test/e2e-vpn/test-hermes-inference-switch.sh + timeout_minutes: 60 + artifact_name: "hermes-inference-switch-install-log" + artifact_path: "/tmp/nemoclaw-e2e-hermes-inference-switch-install.log" + env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_AGENT":"hermes","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_RECREATE_SANDBOX":"1","NEMOCLAW_SANDBOX_NAME":"e2e-hermes-inference-switch"}' + nvidia_api_key: true + github_token: true + secrets: *nightly-e2e-vpn-default-secrets + hermes-anthropic-inference-switch-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',hermes-anthropic-inference-switch-e2e,')) + uses: ./.github/workflows/e2e-script-vpn.yaml + with: + ref: ${{ inputs.target_ref || github.ref }} + script: test/e2e-vpn/test-hermes-inference-switch.sh + timeout_minutes: 60 + artifact_name: "hermes-anthropic-inference-switch-install-log" + artifact_path: "/tmp/nemoclaw-e2e-hermes-inference-switch-install.log" + env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_AGENT":"hermes","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_RECREATE_SANDBOX":"1","NEMOCLAW_SANDBOX_NAME":"e2e-hermes-anthropic-inference-switch","NEMOCLAW_SWITCH_INFERENCE_API":"anthropic-messages","NEMOCLAW_SWITCH_MOCK_ANTHROPIC":"1","NEMOCLAW_SWITCH_MODEL":"mock-anthropic-model","NEMOCLAW_SWITCH_PROVIDER":"compatible-anthropic-endpoint"}' + nvidia_api_key: true + secrets: *nightly-e2e-vpn-default-secrets + hermes-discord-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',hermes-discord-e2e,')) + uses: ./.github/workflows/e2e-script-vpn.yaml + with: + ref: ${{ inputs.target_ref || github.ref }} + script: test/e2e-vpn/test-hermes-discord-e2e.sh + timeout_minutes: 60 + artifact_name: "hermes-discord-e2e-install-log" + artifact_path: "/tmp/nemoclaw-e2e-hermes-discord-install.log" + env_json: '{"DISCORD_ALLOWED_IDS":"1005536447329222676","DISCORD_BOT_TOKEN":"test-fake-discord-token-hermes-e2e","DISCORD_REQUIRE_MENTION":"0","DISCORD_SERVER_IDS":"1491590992753590594","NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_AGENT":"hermes","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_POLICY_TIER":"open","NEMOCLAW_RECREATE_SANDBOX":"1","NEMOCLAW_SANDBOX_NAME":"e2e-hermes-discord"}' + nvidia_api_key: true + github_token: true + secrets: *nightly-e2e-vpn-default-secrets + hermes-slack-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',hermes-slack-e2e,')) + uses: ./.github/workflows/e2e-script-vpn.yaml + with: + ref: ${{ inputs.target_ref || github.ref }} + script: test/e2e-vpn/test-hermes-slack-e2e.sh + runner: linux-amd64-cpu4 + timeout_minutes: 60 + artifact_name: "hermes-slack-e2e-install-log" + artifact_path: "/tmp/nemoclaw-e2e-hermes-slack-install.log" + env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_AGENT":"hermes","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_POLICY_TIER":"open","NEMOCLAW_RECREATE_SANDBOX":"1","NEMOCLAW_SANDBOX_NAME":"e2e-hermes-slack","SLACK_APP_TOKEN":"xapp-test-hermes-slack-app-token","SLACK_BOT_TOKEN":"xoxb-test-hermes-slack-token"}' + nvidia_api_key: true + github_token: true + secrets: *nightly-e2e-vpn-default-secrets + sandbox-operations-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',sandbox-operations-e2e,')) + runs-on: linux-amd64-cpu4 + timeout-minutes: 60 + steps: + - *target-ref-checkout + + - *dockerhub-auth-step + + - name: Start gateway log streamer (background) + run: | + # Diagnostic for NVIDIA/NemoClaw#2484: container log driver in + # openshell's k3s setup doesn't allow reading container stdio — + # only working path to /tmp/gateway.log is via SSH, which + # `nemoclaw logs` uses internally. + # + # Snapshot mode (not follow): every 10s, overwrite per-sandbox + # log file with the latest gateway log content. Bounded output + # (~62 lines per snapshot). When a sandbox is destroyed by the + # test, the file holds the final pre-destroy snapshot. + mkdir -p docker-logs + nohup bash -c ' + export PATH="$HOME/.local/bin:$PATH" + # Strategy: every 5s, snapshot each live sandbox via + # `docker exec openshell-cluster-nemoclaw kubectl ...`. This + # bypasses both per-pod networking (which has had connection- + # refused races for some sandboxes) and the host openshell + # client (which loses gateway metadata after TC-SBX-06s + # docker-kill). kubectl talks directly to k3s in the cluster + # container. + # + # Snapshot mode (overwrite per iteration), not live tail-F: + # the gateway-persistent.log file accumulates everything since + # boot (mirrored from /tmp/gateway.log by nemoclaw-start.sh), + # so a single full-cat at any point gives us complete history. + # Each iteration is short-lived so transient connection issues + # do not cause us to lose the entire stream. + # + # Also snapshot kubectl pod listing per iteration so we have + # the actual pod naming convention even if the cluster is + # destroyed by teardown later. + while sleep 5; do + if ! docker ps --format "{{.Names}}" 2>/dev/null | grep -q "^openshell-cluster-nemoclaw$"; then + continue + fi + docker exec openshell-cluster-nemoclaw kubectl get pods -A --no-headers >docker-logs/_pods.txt 2>&1 + registry="$HOME/.nemoclaw/sandboxes.json" + [ -f "$registry" ] || continue + live=$(jq -r ".sandboxes // {} | keys[]?" "$registry" 2>/dev/null) + for name in $live; do + case "$name" in + *[!a-z0-9_-]*|"") continue ;; + esac + # Find pod by sandbox name. openshell uses the sandbox + # name as the namespace and "agent" as the pod name. + # Try a few common patterns. + pod_match=$(awk -v n="$name" "\$1==n || \$2==n || \$1==\"sandbox-\" n || \$2==\"sandbox-\" n {print \$1\"/\"\$2; exit}" docker-logs/_pods.txt) + if [ -z "$pod_match" ]; then + # Fallback: any pod whose name contains the sandbox name + pod_match=$(awk -v n="$name" "index(\$2,n)>0 {print \$1\"/\"\$2; exit}" docker-logs/_pods.txt) + fi + if [ -z "$pod_match" ]; then continue; fi + pod_ns="${pod_match%%/*}" + pod_name="${pod_match##*/}" + docker exec openshell-cluster-nemoclaw kubectl exec -n "$pod_ns" "$pod_name" -- bash -c " + for f in /sandbox/.openclaw/logs/gateway-persistent.log /tmp/gateway.log /tmp/openclaw-*/openclaw-*.log; do + [ -f \"\$f\" ] || continue + printf \"\\n----- %s (size=%s) -----\\n\" \"\$f\" \"\$(stat -c%s \"\$f\" 2>/dev/null || echo ?)\" + cat -- \"\$f\" 2>/dev/null + done + " > "docker-logs/sandbox-${name}.log" 2>&1 + done + done + ' >/dev/null 2>&1 & + echo $! > /tmp/gateway-log-streamer.pid + + - name: Run sandbox operations E2E test + env: + NVIDIA_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_API_KEY || '' }} + NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1" + NEMOCLAW_PROVIDER: custom + NEMOCLAW_ENDPOINT_URL: https://inference.nvidia.com/v1 + NEMOCLAW_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_COMPAT_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_PREFERRED_API: openai-completions + COMPATIBLE_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_API_KEY || '' }} + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_POLICY_TIER: "open" + GITHUB_TOKEN: ${{ github.token }} + # Override the 1800s default in test/e2e-vpn/e2e-timeout.sh. Sandbox + # creation alone is ~14 min per sandbox in current CI conditions + # (build+upload to k3s gateway), and the test creates two — leaving + # the default 30-min budget completely consumed by setup with no + # room for the actual TC-SBX cases. The job-level timeout (60 min, + # set in `timeout-minutes` above) is the real upper bound. + NEMOCLAW_E2E_TIMEOUT_SECONDS: "2700" + run: bash test/e2e-vpn/test-sandbox-operations.sh + + - name: Stop gateway log streamer + if: always() + # Diagnostic step: never let `bash -e` kill the snapshot loop on a + # single command failure (openshell ssh-config, nemoclaw logs, etc. + # all routinely fail post-test depending on TC-SBX-06's docker-kill + # state). We log the failures inline and continue. + shell: bash --noprofile --norc -uo pipefail {0} + run: | + [ -f /tmp/gateway-log-streamer.pid ] && kill "$(cat /tmp/gateway-log-streamer.pid)" 2>/dev/null || true + # Kill any per-sandbox SSH+tail followers spawned by the streamer. + pkill -f 'tail -n \+1 -F /tmp/gateway.log' 2>/dev/null || true + pkill -f 'ssh.*openshell-' 2>/dev/null || true + sleep 2 + # Final snapshot: tail -F glob expands once at start, so log files + # for openclaw processes that ran as a different UID (creating new + # /tmp/openclaw-/ dirs mid-test) get missed. Re-glob now and + # append every openclaw log file from each live sandbox to the + # per-sandbox docker-logs file. + # + # Use `nemoclaw logs` (not raw openshell ssh-config + ssh) + # because nemoclaw handles SSH key/host setup and is robust to + # streamer race conditions. Tested working in TC-SBX-04. + export PATH="$HOME/.local/bin:$PATH" + echo "=== final-snapshot: PATH=$PATH" + echo "=== final-snapshot: nemoclaw=$(command -v nemoclaw)" + echo "=== final-snapshot: openshell=$(command -v openshell)" + # TC-SBX-06's docker kill of the gateway pod can leave openshell + # without an active gateway selected; re-select before the snapshot + # so `nemoclaw logs` and direct `openshell sandbox exec` both + # have a target. The select is best-effort — failure (e.g., gateway + # not yet recovered) just means we fall through to ssh-config-based + # capture below. + openshell gateway select nemoclaw 2>&1 | head -5 || true + openshell gateway list 2>&1 | head -10 || true + # NEW PATH: bypass the openshell client entirely. The + # openshell-cluster-nemoclaw docker container runs k3s with + # kubectl available inside. Even after TC-SBX-06's docker-kill, + # docker auto-restarts the container and k3s state survives via + # /var/lib/rancher/k3s. Use `docker exec ... kubectl` to read + # the persistent log directly from each sandbox pod, with no + # dependency on the host's openshell metadata. + echo "=== final-snapshot: docker containers:" + docker ps --format '{{.Names}}\t{{.Status}}' 2>&1 | head -10 + echo "=== final-snapshot: cluster pods:" + docker exec openshell-cluster-nemoclaw kubectl get pods -A --no-headers 2>&1 | head -20 + if [ -f "$HOME/.nemoclaw/sandboxes.json" ]; then + echo "=== final-snapshot: sandboxes.json contents:" + cat "$HOME/.nemoclaw/sandboxes.json" 2>&1 | head -30 + registry_keys=$(jq -r ".sandboxes // {} | keys[]?" "$HOME/.nemoclaw/sandboxes.json" 2>&1) + echo "=== final-snapshot: sandbox names from jq: '$registry_keys'" + for name in $registry_keys; do + case "$name" in *[!a-z0-9_-]*|"") echo "=== final-snapshot: skipping invalid name '$name'"; continue ;; esac + echo "=== final-snapshot: capturing logs for '$name'" + { + printf '\n\n===== FINAL SNAPSHOT: %s =====\n' "$name" + # FIRST attempt: docker exec into the cluster container and + # kubectl-exec into the sandbox pod. This works even when + # the host openshell client is broken post-TC-SBX-06 because + # docker (and k3s inside the cluster) survive the gateway + # docker-kill via auto-restart + persistent k3s state. + pod_ns_name=$(docker exec openshell-cluster-nemoclaw kubectl get pods -A --no-headers 2>/dev/null | awk -v n="$name" '$2==n {print $1"/"$2; exit}') + if [ -n "$pod_ns_name" ]; then + echo "(found pod $pod_ns_name for $name)" + pod_ns="${pod_ns_name%%/*}" + pod_name="${pod_ns_name##*/}" + k_out=$(mktemp) + docker exec openshell-cluster-nemoclaw kubectl exec -n "$pod_ns" "$pod_name" -- bash -c ' + for f in /sandbox/.openclaw/logs/gateway-persistent.log /tmp/gateway.log /tmp/openclaw-*/openclaw-*.log; do + [ -f "$f" ] || continue + printf "\n----- %s (size=%s) -----\n" "$f" "$(stat -c%s "$f" 2>/dev/null || echo ?)" + cat -- "$f" 2>/dev/null || true + done + ' >"$k_out" 2>&1 + k_rc=$? + echo "(kubectl exec rc=$k_rc size=$(wc -c <"$k_out"))" + tail -c 500000 "$k_out" + rm -f "$k_out" + else + echo "(no kubectl pod found matching '$name')" + fi + # Existing fallbacks (raw ssh + nemoclaw logs) preserved + # below in case the docker/kubectl path also fails — they + # provide complementary coverage during transient states. + ssh_cfg="/tmp/sshcfg-final-${name}.tmp" + if openshell sandbox ssh-config "$name" >"$ssh_cfg" 2>&1 && [ -s "$ssh_cfg" ]; then + ssh_out=$(mktemp) + ssh -F "$ssh_cfg" \ + -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 -o LogLevel=ERROR \ + "openshell-${name}" \ + 'for f in /sandbox/.openclaw/logs/gateway-persistent.log \ + /tmp/gateway.log \ + /tmp/openclaw-*/openclaw-*.log; do + [ -f "$f" ] || continue + printf "\n----- %s (size=%s) -----\n" "$f" "$(stat -c%s "$f" 2>/dev/null || echo ?)" + cat -- "$f" 2>/dev/null || true + done' >"$ssh_out" 2>&1 + ssh_rc=$? + tail -c 500000 "$ssh_out" + rm -f "$ssh_out" + [ "$ssh_rc" -eq 0 ] || echo "(direct ssh exited rc=$ssh_rc)" + else + echo "(openshell sandbox ssh-config failed for $name)" + # Fallback to nemoclaw logs (less reliable, but try anything) + if command -v nemoclaw >/dev/null 2>&1; then + nm_out=$(mktemp) + nemoclaw "$name" logs >"$nm_out" 2>&1 + echo "(nemoclaw logs rc=$? size=$(wc -c <"$nm_out"))" + tail -c 500000 "$nm_out" + rm -f "$nm_out" + fi + fi + rm -f "$ssh_cfg" + } >> "docker-logs/sandbox-${name}.log" + done + else + echo "=== final-snapshot: sandboxes.json not found at $HOME/.nemoclaw/sandboxes.json" + fi + # Cap each log file at 5MB by keeping only the last 5MB — useful + # content (real gateway events) is mixed throughout, so tail-trim + # is fine for diagnostic purposes. + for f in docker-logs/*.log; do + [ -f "$f" ] || continue + sz=$(stat -c%s "$f" 2>/dev/null || stat -f%z "$f" 2>/dev/null || echo 0) + if [ "$sz" -gt 5242880 ]; then + tail -c 5242880 "$f" > "${f}.tail" && mv "${f}.tail" "$f" + fi + done + ls -la docker-logs/ 2>&1 | head -20 || true + du -sh docker-logs/ 2>&1 || true + + - name: Upload sandbox gateway logs on failure + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: sandbox-operations-docker-logs + path: docker-logs/ + if-no-files-found: ignore + + - name: Upload test log on failure + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: sandbox-operations-test-log + path: test-sandbox-operations-*.log + if-no-files-found: ignore + + # ── Inference routing (credential isolation + error classification) ── + # TC-INF-05: real API key absent from sandbox env/process/filesystem + # TC-INF-06: invalid API key → classified credential error (PR-safe) + # TC-INF-07: unreachable endpoint → classified transport error (PR-safe) + inference-routing-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',inference-routing-e2e,')) + uses: ./.github/workflows/e2e-script-vpn.yaml + with: + ref: ${{ inputs.target_ref || github.ref }} + script: test/e2e-vpn/test-inference-routing.sh + timeout_minutes: 30 + artifact_name: "inference-routing-test-log" + artifact_path: "test-inference-routing-*.log" + env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_POLICY_TIER":"open"}' + nvidia_api_key: true + secrets: *nightly-e2e-vpn-default-secrets + openclaw-inference-switch-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',openclaw-inference-switch-e2e,')) + uses: ./.github/workflows/e2e-script-vpn.yaml + with: + ref: ${{ inputs.target_ref || github.ref }} + script: test/e2e-vpn/test-openclaw-inference-switch.sh + artifact_name: "openclaw-inference-switch-install-log" + artifact_path: "/tmp/nemoclaw-e2e-openclaw-inference-switch-install.log" + env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_RECREATE_SANDBOX":"1","NEMOCLAW_SANDBOX_NAME":"e2e-openclaw-inference-switch"}' + nvidia_api_key: true + github_token: true + secrets: *nightly-e2e-vpn-default-secrets + openclaw-anthropic-inference-switch-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',openclaw-anthropic-inference-switch-e2e,')) + uses: ./.github/workflows/e2e-script-vpn.yaml + with: + ref: ${{ inputs.target_ref || github.ref }} + script: test/e2e-vpn/test-openclaw-inference-switch.sh + artifact_name: "openclaw-anthropic-inference-switch-install-log" + artifact_path: "/tmp/nemoclaw-e2e-openclaw-inference-switch-install.log" + env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_AGENT":"openclaw","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_RECREATE_SANDBOX":"1","NEMOCLAW_SANDBOX_NAME":"e2e-openclaw-anthropic-inference-switch","NEMOCLAW_SWITCH_INFERENCE_API":"anthropic-messages","NEMOCLAW_SWITCH_MOCK_ANTHROPIC":"1","NEMOCLAW_SWITCH_MODEL":"mock-anthropic-model","NEMOCLAW_SWITCH_PROVIDER":"compatible-anthropic-endpoint"}' + nvidia_api_key: true + secrets: *nightly-e2e-vpn-default-secrets + network-policy-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',network-policy-e2e,')) + uses: ./.github/workflows/e2e-script-vpn.yaml + with: + ref: ${{ inputs.target_ref || github.ref }} + script: test/e2e-vpn/test-network-policy.sh + artifact_name: "network-policy-test-log" + artifact_path: "test-network-policy-*.log" + env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_POLICY_TIER":"restricted","NEMOCLAW_RECREATE_SANDBOX":"1"}' + nvidia_api_key: true + secrets: *nightly-e2e-vpn-default-secrets + state-backup-restore-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',state-backup-restore-e2e,')) + uses: ./.github/workflows/e2e-script-vpn.yaml + with: + ref: ${{ inputs.target_ref || github.ref }} + script: test/e2e-vpn/test-state-backup-restore.sh + timeout_minutes: 60 + artifact_name: "state-backup-restore-test-log" + artifact_path: "test-state-backup-restore-*.log" + env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1"}' + nvidia_api_key: true + secrets: *nightly-e2e-vpn-default-secrets + tunnel-lifecycle-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',tunnel-lifecycle-e2e,')) + uses: ./.github/workflows/e2e-script-vpn.yaml + with: + ref: ${{ inputs.target_ref || github.ref }} + script: test/e2e-vpn/test-tunnel-lifecycle.sh + timeout_minutes: 60 + artifact_name: "tunnel-lifecycle-test-log" + artifact_path: "test-tunnel-lifecycle-*.log" + env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1"}' + nvidia_api_key: true + secrets: *nightly-e2e-vpn-default-secrets + diagnostics-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',diagnostics-e2e,')) + uses: ./.github/workflows/e2e-script-vpn.yaml + with: + ref: ${{ inputs.target_ref || github.ref }} + script: test/e2e-vpn/test-diagnostics.sh + artifact_name: "diagnostics-test-log" + artifact_path: "test-diagnostics-*.log" + env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_RECREATE_SANDBOX":"1"}' + nvidia_api_key: true + secrets: *nightly-e2e-vpn-default-secrets + credential-migration-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',credential-migration-e2e,')) + runs-on: linux-amd64-cpu4 + permissions: + contents: read + timeout-minutes: 50 + steps: + - *target-ref-checkout + + - *dockerhub-auth-step + - name: Setup Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 + with: + node-version: "22" + cache: npm + + - name: Install root dependencies + run: npm ci + + - name: Build CLI + run: npm run build:cli + + - name: Run credential migration E2E test + # Trusted-code boundary: this job runs the checked-out target ref with + # NVIDIA_API_KEY because it validates live credential + # migration into the OpenShell gateway. The hosted service behind this + # repo-scoped secret is inference.nvidia.com, not legacy hosted inference + # Endpoints, so the test stages it as the custom provider's + # COMPATIBLE_API_KEY. Keep checkout credentials disabled, do not pass + # GITHUB_TOKEN, and rely on reviewed/maintainer-dispatched refs. + env: + NVIDIA_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_API_KEY || '' }} + NEMOCLAW_PROVIDER: custom + NEMOCLAW_ENDPOINT_URL: https://inference.nvidia.com/v1 + NEMOCLAW_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_COMPAT_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_PREFERRED_API: openai-completions + COMPATIBLE_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_API_KEY || '' }} + NEMOCLAW_SANDBOX_NAME: "e2e-cred-migration" + run: bash test/e2e-vpn/test-credential-migration.sh + + - name: Upload credential migration artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: credential-migration-artifacts + path: e2e-artifacts/vitest/credential-migration/ + include-hidden-files: false + if-no-files-found: ignore + retention-days: 14 + snapshot-commands-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',snapshot-commands-e2e,')) + uses: ./.github/workflows/e2e-script-vpn.yaml + with: + ref: ${{ inputs.target_ref || github.ref }} + script: test/e2e-vpn/test-snapshot-commands.sh + timeout_minutes: 30 + artifact_name: "snapshot-commands-install-log" + artifact_path: "/tmp/nemoclaw-e2e-install.log" + env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_SANDBOX_NAME":"e2e-snapshot"}' + nvidia_api_key: true + github_token: true + secrets: *nightly-e2e-vpn-default-secrets + shields-config-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',shields-config-e2e,')) + uses: ./.github/workflows/e2e-script-vpn.yaml + with: + ref: ${{ inputs.target_ref || github.ref }} + script: test/e2e-vpn/test-shields-config.sh + timeout_minutes: 30 + artifact_name: "shields-config-install-log" + artifact_path: "/tmp/nemoclaw-e2e-shields-install.log" + env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_SANDBOX_NAME":"e2e-shields"}' + nvidia_api_key: true + github_token: true + secrets: *nightly-e2e-vpn-default-secrets + rebuild-openclaw-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',rebuild-openclaw-e2e,')) + uses: ./.github/workflows/e2e-script-vpn.yaml + with: + ref: ${{ inputs.target_ref || github.ref }} + script: test/e2e-vpn/test-rebuild-openclaw.sh + timeout_minutes: 60 + artifact_name: "rebuild-openclaw-install-log" + artifact_path: "/tmp/nemoclaw-e2e-install.log" + env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_SANDBOX_NAME":"e2e-rebuild-oc"}' + nvidia_api_key: true + github_token: true + secrets: *nightly-e2e-vpn-default-secrets + upgrade-stale-sandbox-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',upgrade-stale-sandbox-e2e,')) + uses: ./.github/workflows/e2e-script-vpn.yaml + with: + ref: ${{ inputs.target_ref || github.ref }} + script: test/e2e-vpn/test-upgrade-stale-sandbox.sh + timeout_minutes: 60 + artifact_name: "upgrade-stale-sandbox-logs" + artifact_path: | + /tmp/nemoclaw-e2e-old-install.log + /tmp/nemoclaw-e2e-upgrade-install.log + env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_SANDBOX_NAME":"e2e-upgrade-stale"}' + nvidia_api_key: true + github_token: true + secrets: *nightly-e2e-vpn-default-secrets + openshell-gateway-upgrade-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',openshell-gateway-upgrade-e2e,')) + runs-on: linux-amd64-cpu4 + timeout-minutes: 60 + steps: + - *target-ref-checkout + + - *dockerhub-auth-step + + - name: Setup Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: "22" + + - name: Run OpenShell gateway upgrade E2E test + env: + GITHUB_TOKEN: ${{ github.token }} + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + run: bash test/e2e-vpn/test-openshell-gateway-upgrade.sh + + - name: Upload gateway upgrade logs on failure + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: openshell-gateway-upgrade-logs + path: | + /tmp/nemoclaw-e2e-openshell-gateway-upgrade.log + /tmp/nemoclaw-e2e-openshell-gateway-install.log + /tmp/nemoclaw-e2e-openshell-gateway-old-install.log + /tmp/nemoclaw-e2e-openshell-gateway-current-install.log + /tmp/nemoclaw-e2e-openshell-gateway-start.log + /tmp/nemoclaw-e2e-openshell-gateway-process.log + /tmp/nemoclaw-e2e-openshell-gateway-compatible-mock.log + if-no-files-found: ignore + + # ── Hermes rebuild upgrade E2E ────────────────────────────── + # Same upgrade scenario as OpenClaw but for Hermes Agent. + rebuild-hermes-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',rebuild-hermes-e2e,')) + uses: ./.github/workflows/e2e-script-vpn.yaml + with: + ref: ${{ inputs.target_ref || github.ref }} + script: test/e2e-vpn/test-rebuild-hermes.sh + timeout_minutes: 60 + artifact_name: "rebuild-hermes-install-log" + artifact_path: "/tmp/nemoclaw-e2e-install.log" + env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_AGENT":"hermes","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_SANDBOX_NAME":"e2e-rebuild-hm"}' + nvidia_api_key: true + github_token: true + secrets: *nightly-e2e-vpn-default-secrets + rebuild-hermes-stale-base-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',rebuild-hermes-stale-base-e2e,')) + uses: ./.github/workflows/e2e-script-vpn.yaml + with: + ref: ${{ inputs.target_ref || github.ref }} + script: test/e2e-vpn/test-rebuild-hermes.sh + timeout_minutes: 60 + artifact_name: "rebuild-hermes-stale-base-install-log" + artifact_path: "/tmp/nemoclaw-e2e-install.log" + env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_AGENT":"hermes","NEMOCLAW_HERMES_STALE_BASE_REBUILD_E2E":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_SANDBOX_NAME":"e2e-rebuild-hm-base"}' + nvidia_api_key: true + github_token: true + secrets: *nightly-e2e-vpn-default-secrets + double-onboard-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',double-onboard-e2e,')) + runs-on: linux-amd64-cpu4 + timeout-minutes: 90 + steps: + - *target-ref-checkout + + - *dockerhub-auth-step + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: 22 + cache: npm + + - name: Install root dependencies + run: npm ci --ignore-scripts + + - name: Build CLI + run: npm run build:cli + + - name: Install OpenShell CLI + run: env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN -u NVIDIA_API_KEY -u GITHUB_TOKEN bash scripts/install-openshell.sh + + - name: Run double onboard E2E test + env: + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + run: | + [ -f "$HOME/.bashrc" ] && source "$HOME/.bashrc" 2>/dev/null || true + export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" + [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" + [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]] && export PATH="$HOME/.local/bin:$PATH" + bash test/e2e-vpn/test-double-onboard.sh + - name: Upload test log on failure + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: double-onboard-test-log + path: test-double-onboard-*.log + if-no-files-found: ignore + + # ── Onboard Repair E2E ───────────────────────────────────── + onboard-repair-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',onboard-repair-e2e,')) + runs-on: linux-amd64-cpu4 + timeout-minutes: 60 + steps: + - *target-ref-checkout + + - *dockerhub-auth-step + - name: Install NemoClaw + env: + NVIDIA_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_API_KEY || '' }} + NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1" + NEMOCLAW_PROVIDER: custom + NEMOCLAW_ENDPOINT_URL: https://inference.nvidia.com/v1 + NEMOCLAW_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_COMPAT_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_PREFERRED_API: openai-completions + COMPATIBLE_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_API_KEY || '' }} + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + run: bash install.sh --non-interactive --yes-i-accept-third-party-software + - name: Run onboard repair E2E test + env: + NVIDIA_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_API_KEY || '' }} + NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1" + NEMOCLAW_PROVIDER: custom + NEMOCLAW_ENDPOINT_URL: https://inference.nvidia.com/v1 + NEMOCLAW_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_COMPAT_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_PREFERRED_API: openai-completions + COMPATIBLE_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_API_KEY || '' }} + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + run: | + [ -f "$HOME/.bashrc" ] && source "$HOME/.bashrc" 2>/dev/null || true + export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" + [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" + [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]] && export PATH="$HOME/.local/bin:$PATH" + bash test/e2e-vpn/test-onboard-repair.sh + - name: Upload test log on failure + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: onboard-repair-test-log + path: test-onboard-repair-*.log + if-no-files-found: ignore + + # ── Onboard Resume E2E ───────────────────────────────────── + onboard-resume-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',onboard-resume-e2e,')) + runs-on: linux-amd64-cpu4 + timeout-minutes: 60 + steps: + - *target-ref-checkout + + - *dockerhub-auth-step + - name: Install NemoClaw + env: + NVIDIA_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_API_KEY || '' }} + NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1" + NEMOCLAW_PROVIDER: custom + NEMOCLAW_ENDPOINT_URL: https://inference.nvidia.com/v1 + NEMOCLAW_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_COMPAT_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_PREFERRED_API: openai-completions + COMPATIBLE_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_API_KEY || '' }} + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + run: bash install.sh --non-interactive --yes-i-accept-third-party-software + - name: Run onboard resume E2E test + env: + NVIDIA_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_API_KEY || '' }} + NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1" + NEMOCLAW_PROVIDER: custom + NEMOCLAW_ENDPOINT_URL: https://inference.nvidia.com/v1 + NEMOCLAW_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_COMPAT_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_PREFERRED_API: openai-completions + COMPATIBLE_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_API_KEY || '' }} + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + run: | + [ -f "$HOME/.bashrc" ] && source "$HOME/.bashrc" 2>/dev/null || true + export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" + [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" + [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]] && export PATH="$HOME/.local/bin:$PATH" + bash test/e2e-vpn/test-onboard-resume.sh + - name: Upload test log on failure + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: onboard-resume-test-log + path: test-onboard-resume-*.log + if-no-files-found: ignore + + # -- Onboard Negative Paths E2E ------------------------------- + onboard-negative-paths-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',onboard-negative-paths-e2e,')) + runs-on: linux-amd64-cpu4 + timeout-minutes: 75 + steps: + - *target-ref-checkout + + - *dockerhub-auth-step + - name: Install NemoClaw + env: + NVIDIA_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_API_KEY || '' }} + NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1" + NEMOCLAW_PROVIDER: custom + NEMOCLAW_ENDPOINT_URL: https://inference.nvidia.com/v1 + NEMOCLAW_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_COMPAT_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_PREFERRED_API: openai-completions + COMPATIBLE_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_API_KEY || '' }} + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + run: bash install.sh --non-interactive --yes-i-accept-third-party-software + - name: Run onboard negative-path E2E test + env: + NVIDIA_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_API_KEY || '' }} + NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1" + NEMOCLAW_PROVIDER: custom + NEMOCLAW_ENDPOINT_URL: https://inference.nvidia.com/v1 + NEMOCLAW_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_COMPAT_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_PREFERRED_API: openai-completions + COMPATIBLE_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_API_KEY || '' }} + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + run: | + set -euo pipefail + [ -f "$HOME/.bashrc" ] && source "$HOME/.bashrc" 2>/dev/null || true + export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" + [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" + [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]] && export PATH="$HOME/.local/bin:$PATH" + bash test/e2e-vpn/test-onboard-negative-paths.sh + - name: Upload test log on failure + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: onboard-negative-paths-test-log + path: /tmp/nemoclaw-e2e-onboard-negative-paths.log + if-no-files-found: ignore + + # ── Runtime Overrides E2E ────────────────────────────────── + runtime-overrides-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',runtime-overrides-e2e,')) + runs-on: linux-amd64-cpu4 + timeout-minutes: 45 + steps: + - *target-ref-checkout + + - *dockerhub-auth-step + - name: Install NemoClaw + env: + NVIDIA_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_API_KEY || '' }} + NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1" + NEMOCLAW_PROVIDER: custom + NEMOCLAW_ENDPOINT_URL: https://inference.nvidia.com/v1 + NEMOCLAW_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_COMPAT_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_PREFERRED_API: openai-completions + COMPATIBLE_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_API_KEY || '' }} + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + run: bash install.sh --non-interactive --yes-i-accept-third-party-software + - name: Run runtime overrides E2E test + env: + NVIDIA_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_API_KEY || '' }} + NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1" + NEMOCLAW_PROVIDER: custom + NEMOCLAW_ENDPOINT_URL: https://inference.nvidia.com/v1 + NEMOCLAW_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_COMPAT_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_PREFERRED_API: openai-completions + COMPATIBLE_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_API_KEY || '' }} + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + run: | + [ -f "$HOME/.bashrc" ] && source "$HOME/.bashrc" 2>/dev/null || true + export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" + [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" + [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]] && export PATH="$HOME/.local/bin:$PATH" + bash test/e2e-vpn/test-runtime-overrides.sh + - name: Upload test log on failure + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: runtime-overrides-test-log + path: test-runtime-overrides-*.log + if-no-files-found: ignore + + # ── Credential Sanitization E2E ──────────────────────────── + # Requires a running sandbox. Bootstraps via install.sh then runs tests. + credential-sanitization-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',credential-sanitization-e2e,')) + runs-on: linux-amd64-cpu4 + timeout-minutes: 60 + steps: + - *target-ref-checkout + + - *dockerhub-auth-step + - name: Install NemoClaw and onboard sandbox + env: + NVIDIA_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_API_KEY || '' }} + NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1" + NEMOCLAW_PROVIDER: custom + NEMOCLAW_ENDPOINT_URL: https://inference.nvidia.com/v1 + NEMOCLAW_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_COMPAT_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_PREFERRED_API: openai-completions + COMPATIBLE_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_API_KEY || '' }} + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_SANDBOX_NAME: "e2e-test" + run: bash install.sh --non-interactive --yes-i-accept-third-party-software + - name: Run credential sanitization E2E test + env: + NVIDIA_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_API_KEY || '' }} + NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1" + NEMOCLAW_PROVIDER: custom + NEMOCLAW_ENDPOINT_URL: https://inference.nvidia.com/v1 + NEMOCLAW_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_COMPAT_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_PREFERRED_API: openai-completions + COMPATIBLE_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_API_KEY || '' }} + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_SANDBOX_NAME: "e2e-test" + run: | + # shellcheck source=/dev/null + [ -f "$HOME/.bashrc" ] && source "$HOME/.bashrc" 2>/dev/null || true + export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" + [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" + [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]] && export PATH="$HOME/.local/bin:$PATH" + bash test/e2e-vpn/test-credential-sanitization.sh + - name: Upload test log on failure + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: credential-sanitization-test-log + path: test-credential-sanitization-*.log + if-no-files-found: ignore + + # ── Telegram Injection E2E ───────────────────────────────── + # Requires a running sandbox. Bootstraps via install.sh then runs tests. + telegram-injection-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',telegram-injection-e2e,')) + runs-on: linux-amd64-cpu4 + timeout-minutes: 60 + steps: + - *target-ref-checkout + + - *dockerhub-auth-step + - name: Install NemoClaw and onboard sandbox + env: + NVIDIA_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_API_KEY || '' }} + NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1" + NEMOCLAW_PROVIDER: custom + NEMOCLAW_ENDPOINT_URL: https://inference.nvidia.com/v1 + NEMOCLAW_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_COMPAT_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_PREFERRED_API: openai-completions + COMPATIBLE_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_API_KEY || '' }} + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_SANDBOX_NAME: "e2e-test" + run: bash install.sh --non-interactive --yes-i-accept-third-party-software + - name: Run telegram injection E2E test + env: + NVIDIA_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_API_KEY || '' }} + NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1" + NEMOCLAW_PROVIDER: custom + NEMOCLAW_ENDPOINT_URL: https://inference.nvidia.com/v1 + NEMOCLAW_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_COMPAT_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_PREFERRED_API: openai-completions + COMPATIBLE_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_API_KEY || '' }} + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_SANDBOX_NAME: "e2e-test" + run: | + # shellcheck source=/dev/null + [ -f "$HOME/.bashrc" ] && source "$HOME/.bashrc" 2>/dev/null || true + export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" + [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" + [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]] && export PATH="$HOME/.local/bin:$PATH" + bash test/e2e-vpn/test-telegram-injection.sh + - name: Upload test log on failure + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: telegram-injection-test-log + path: test-telegram-injection-*.log + if-no-files-found: ignore + + # Remove this job — and the matching notify-on-failure entry — in the + # same PR that deletes cluster-image-patch.ts when the OpenShell + # roadmap migration off k3s (NVIDIA/OpenShell#873) lands. + # ── Docker 26+ overlayfs nested-mount auto-fix (#2481) ────── + # TEMPORARY: validates the auto-fix in src/lib/cluster-image-patch.ts. + overlayfs-autofix-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',overlayfs-autofix-e2e,')) + uses: ./.github/workflows/e2e-script-vpn.yaml + with: + ref: ${{ inputs.target_ref || github.ref }} + script: test/e2e-vpn/test-overlayfs-autofix.sh + artifact_name: "overlayfs-autofix-logs" + artifact_path: | + /tmp/nemoclaw-e2e-install.log + /tmp/nemoclaw-e2e-onboard-positive.log + /tmp/nemoclaw-e2e-onboard-negative.log + env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_SANDBOX_NAME":"e2e-overlayfs"}' + nvidia_api_key: true + github_token: true + secrets: *nightly-e2e-vpn-default-secrets + device-auth-health-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',device-auth-health-e2e,')) + uses: ./.github/workflows/e2e-script-vpn.yaml + with: + ref: ${{ inputs.target_ref || github.ref }} + script: test/e2e-vpn/test-device-auth-health.sh + timeout_minutes: 30 + artifact_name: "device-auth-health-install-log" + artifact_path: "/tmp/nemoclaw-e2e-health-install.log" + env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_RECREATE_SANDBOX":"1","NEMOCLAW_SANDBOX_NAME":"e2e-health-auth"}' + nvidia_api_key: true + github_token: true + secrets: *nightly-e2e-vpn-default-secrets + launchable-smoke-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',launchable-smoke-e2e,')) + runs-on: linux-amd64-cpu4 + timeout-minutes: 30 + steps: + - *target-ref-checkout + + - *dockerhub-auth-step + + - name: Run launchable install-flow smoke test + env: + NVIDIA_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_API_KEY || '' }} + NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1" + NEMOCLAW_PROVIDER: custom + NEMOCLAW_ENDPOINT_URL: https://inference.nvidia.com/v1 + NEMOCLAW_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_COMPAT_MODEL: nvidia/nvidia/nemotron-3-super-v3 + NEMOCLAW_PREFERRED_API: openai-completions + COMPATIBLE_API_KEY: ${{ (github.event_name != 'workflow_dispatch' || inputs.target_ref == '') && secrets.NVIDIA_API_KEY || '' }} + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_SANDBOX_NAME: "e2e-launchable" + NEMOCLAW_RECREATE_SANDBOX: "1" + SKIP_DOCKER_PULL: "1" + GITHUB_TOKEN: ${{ github.token }} + run: bash test/e2e-vpn/test-launchable-smoke.sh + + - name: Upload install log on failure + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: launchable-smoke-install-log + path: /tmp/nemoclaw-launchable-install.log + if-no-files-found: ignore + + - name: Upload onboard log on failure + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: launchable-smoke-onboard-log + path: /tmp/nemoclaw-launchable-onboard.log + if-no-files-found: ignore + + - name: Upload test log on failure + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: launchable-smoke-test-log + path: /tmp/nemoclaw-launchable-test.log + if-no-files-found: ignore + + # ── GPU E2E (Ollama local inference) ────────────────────────── + # Runs on an NVKS ephemeral GPU runner (RTX Pro 6000, 36 GB VRAM). + # Each job gets a fresh VM — no state leakage between runs. + gpu-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && + vars.GPU_E2E_ENABLED == 'true' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',gpu-e2e,')) + runs-on: linux-amd64-gpu-rtxpro6000-latest-1 + timeout-minutes: 30 + env: + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_SANDBOX_NAME: "e2e-gpu-ollama" + NEMOCLAW_RECREATE_SANDBOX: "1" + NEMOCLAW_PROVIDER: "ollama" + steps: + - *target-ref-checkout + + - *dockerhub-auth-step + + - name: Verify GPU availability + run: | + echo "=== GPU Info ===" + nvidia-smi + echo "" + echo "=== VRAM ===" + nvidia-smi --query-gpu=name,memory.total --format=csv,noheader + echo "" + echo "=== Docker ===" + docker info --format '{{.ServerVersion}}' + + - name: Run GPU E2E test (Ollama local inference) + run: bash test/e2e-vpn/test-gpu-e2e.sh + + - name: Upload install log on failure + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: gpu-e2e-install-log + path: /tmp/nemoclaw-gpu-e2e-install.log + if-no-files-found: ignore + + - name: Upload test log on failure + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: gpu-e2e-test-log + path: /tmp/nemoclaw-gpu-e2e-test.log + if-no-files-found: ignore + + # ── GPU Double-Onboard E2E (Ollama token consistency) ──────── + # Reproduces issue #2553: re-onboard with Ollama must not leave the + # proxy running with a different token than what's persisted to disk. + # Runs on its own ephemeral VM — no dependency on gpu-e2e. + gpu-double-onboard-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && + vars.GPU_E2E_ENABLED == 'true' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',gpu-double-onboard-e2e,')) + runs-on: linux-amd64-gpu-rtxpro6000-latest-1 + timeout-minutes: 30 + env: + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_SANDBOX_NAME: "e2e-gpu-double-onboard" + NEMOCLAW_RECREATE_SANDBOX: "1" + NEMOCLAW_PROVIDER: "ollama" + steps: + - *target-ref-checkout + + - *dockerhub-auth-step + + - name: Verify GPU availability + run: | + echo "=== GPU Info ===" + nvidia-smi + echo "" + echo "=== VRAM ===" + nvidia-smi --query-gpu=name,memory.total --format=csv,noheader + echo "" + echo "=== Docker ===" + docker info --format '{{.ServerVersion}}' + + - name: Run GPU double-onboard E2E test + run: bash test/e2e-vpn/test-gpu-double-onboard.sh + + - name: Upload install log on failure + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: gpu-double-onboard-install-log + path: /tmp/nemoclaw-gpu-double-onboard-install.log + if-no-files-found: ignore + + - name: Upload re-onboard log on failure + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: gpu-double-onboard-reonboard-log + path: /tmp/nemoclaw-gpu-double-onboard-reonboard.log + if-no-files-found: ignore + + - name: Upload test log on failure + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: gpu-double-onboard-test-log + path: /tmp/nemoclaw-gpu-double-onboard-test.log + if-no-files-found: ignore + + # ── Jetson nvmap GPU status E2E (#4231) ────────────────────── + # Reproduces the reporter's exact Jetson Orin workflow: onboard with GPU, + # then prove the sandbox user can open /dev/nvmap (CUDA cuInit(0)=0) and that + # `nemoclaw status` reports proven CUDA usability instead of a misleading + # bare "enabled". Requires a Jetson/Tegra (arm64 L4T) GPU runner, which the + # project does not yet host — so this job is gated behind + # `vars.JETSON_E2E_ENABLED` (unset by default → skipped) and an explicit + # runner label. When a Jetson runner is provisioned, set the variable and + # point `runs-on` at its label. The same fix has deterministic, hardware-free + # regression coverage in src/lib/onboard/docker-gpu-patch.test.ts. + gpu-jetson-nvmap-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && + vars.JETSON_E2E_ENABLED == 'true' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',gpu-jetson-nvmap-e2e,')) + runs-on: ${{ vars.JETSON_E2E_RUNNER_LABEL || 'linux-arm64-gpu-jetson-orin-latest-1' }} + timeout-minutes: 40 + env: + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_SANDBOX_NAME: "e2e-jetson-nvmap" + NEMOCLAW_RECREATE_SANDBOX: "1" + NEMOCLAW_PROVIDER: "ollama" + steps: + - *target-ref-checkout + + - *dockerhub-auth-step + + - name: Verify Jetson GPU availability + run: | + echo "=== Tegra release ===" + cat /etc/nv_tegra_release 2>/dev/null || echo "(no /etc/nv_tegra_release)" + echo "" + echo "=== /dev/nvmap ===" + ls -l /dev/nvmap 2>/dev/null || echo "(no /dev/nvmap)" + echo "" + echo "=== Docker ===" + docker info --format '{{.ServerVersion}}' + docker info --format '{{json .Runtimes}}' + + - name: Run Jetson nvmap GPU E2E test (#4231) + run: bash test/e2e-vpn/test-jetson-nvmap-gpu.sh + + - name: Upload install log on failure + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: gpu-jetson-nvmap-install-log + path: /tmp/nemoclaw-jetson-nvmap-e2e-install.log + if-no-files-found: ignore + + - name: Upload test log on failure + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: gpu-jetson-nvmap-test-log + path: /tmp/nemoclaw-jetson-nvmap-e2e-test.log + if-no-files-found: ignore + + concurrent-gateway-ports-e2e: + if: >- + github.repository == 'NVIDIA/NemoClaw' && + (github.event_name != 'workflow_dispatch' || + inputs.jobs == '' || + contains(format(',{0},', inputs.jobs), ',concurrent-gateway-ports-e2e,')) + runs-on: linux-amd64-cpu4 + timeout-minutes: 60 + env: + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_E2E_PHASE_TIMEOUT: "1200" + steps: + - *target-ref-checkout + - *dockerhub-auth-step + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: 22 + cache: npm + + - name: Install root dependencies + run: npm ci --ignore-scripts + + - name: Build CLI + run: npm run build:cli + + - name: Install OpenShell CLI + run: env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN -u NVIDIA_API_KEY -u GITHUB_TOKEN bash scripts/install-openshell.sh + + - name: Run concurrent gateway ports E2E test + run: | + [ -f "$HOME/.bashrc" ] && source "$HOME/.bashrc" 2>/dev/null || true + export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" + [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" + [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]] && export PATH="$HOME/.local/bin:$PATH" + bash test/e2e-vpn/test-concurrent-gateway-ports.sh + - name: Upload sandbox A onboard log on failure + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: concurrent-gateway-ports-sandbox-a-onboard-log + path: /tmp/e2e-cgp-a-onboard.log + if-no-files-found: ignore + - name: Upload sandbox B onboard log on failure + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: concurrent-gateway-ports-sandbox-b-onboard-log + path: /tmp/e2e-cgp-b-onboard.log + if-no-files-found: ignore + - name: Upload sandbox B destroy log on failure + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: concurrent-gateway-ports-sandbox-b-destroy-log + path: /tmp/e2e-cgp-b-destroy.log + if-no-files-found: ignore + + notify-on-failure: + runs-on: linux-amd64-cpu4 + needs: + [ + cloud-e2e, + cloud-onboard-e2e, + cloud-inference-e2e, + cron-preflight-inference-local-e2e, + agent-turn-latency-e2e, + skill-agent-e2e, + openclaw-skill-cli-e2e, + docs-validation-e2e, + messaging-providers-e2e, + openclaw-slack-pairing-e2e, + openclaw-tui-chat-correlation-e2e, + issue-4434-tui-unreachable-inference-e2e, + issue-3600-gpu-proof-optional-e2e, + openclaw-discord-pairing-e2e, + issue-4462-scope-upgrade-approval-e2e, + issue-4462-gateway-pinned-approval-characterization-e2e, + messaging-compatible-endpoint-e2e, + sessions-agents-cli-e2e, + channels-add-remove-e2e, + channels-stop-start-openclaw-e2e, + channels-stop-start-hermes-e2e, + brave-search-e2e, + common-egress-agent-e2e, + kimi-inference-compat-e2e, + bedrock-runtime-compatible-anthropic-e2e, + token-rotation-e2e, + sandbox-survival-e2e, + issue-2478-crash-loop-recovery-e2e, + hermes-e2e, + hermes-dashboard-e2e, + hermes-root-entrypoint-smoke-e2e, + hermes-secret-boundary-e2e, + openclaw-onboard-security-posture-e2e, + hermes-onboard-security-posture-e2e, + hermes-inference-switch-e2e, + hermes-anthropic-inference-switch-e2e, + hermes-discord-e2e, + hermes-slack-e2e, + sandbox-operations-e2e, + inference-routing-e2e, + openclaw-inference-switch-e2e, + openclaw-anthropic-inference-switch-e2e, + network-policy-e2e, + state-backup-restore-e2e, + tunnel-lifecycle-e2e, + diagnostics-e2e, + credential-migration-e2e, + snapshot-commands-e2e, + shields-config-e2e, + rebuild-openclaw-e2e, + upgrade-stale-sandbox-e2e, + openshell-gateway-upgrade-e2e, + rebuild-hermes-e2e, + rebuild-hermes-stale-base-e2e, + double-onboard-e2e, + onboard-repair-e2e, + onboard-resume-e2e, + onboard-negative-paths-e2e, + runtime-overrides-e2e, + credential-sanitization-e2e, + telegram-injection-e2e, + overlayfs-autofix-e2e, + device-auth-health-e2e, + launchable-smoke-e2e, + gpu-e2e, + gpu-double-onboard-e2e, + gpu-jetson-nvmap-e2e, + concurrent-gateway-ports-e2e, + ] + if: ${{ always() && github.event_name == 'schedule' && (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')) }} + permissions: + issues: write + steps: + - name: Create or update failure issue + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const title = 'Nightly E2E failed'; + + const needs = ${{ toJSON(needs) }}; + const failed = Object.entries(needs).filter(([, v]) => v.result === 'failure').map(([k]) => k); + const cancelled = Object.entries(needs).filter(([, v]) => v.result === 'cancelled').map(([k]) => k); + const summary = [ + failed.length ? `**Failed:** ${failed.join(', ')}` : '', + cancelled.length ? `**Cancelled:** ${cancelled.join(', ')}` : '', + ].filter(Boolean).join('\n'); + + const { data: existing } = await github.rest.issues.listForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + labels: 'CI/CD', + per_page: 100, + }); + const match = existing.find(i => !i.pull_request && i.title.startsWith(title)); + + if (match) { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: match.number, + body: `Failed again on ${new Date().toISOString().split('T')[0]}.\n\n**Run:** ${runUrl}\n${summary}\n**Artifacts:** Check the run artifacts for install/test logs (artifact names vary by job).`, + }); + } else { + await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title: `${title} — ${new Date().toISOString().split('T')[0]}`, + body: `The nightly E2E pipeline failed.\n\n**Run:** ${runUrl}\n${summary}\n**Artifacts:** Check the run artifacts for install/test logs (artifact names vary by job).`, + labels: ['bug', 'CI/CD'], + }); + } + + report-to-pr: + runs-on: linux-amd64-cpu4 + needs: + [ + cloud-e2e, + cloud-onboard-e2e, + cloud-inference-e2e, + cron-preflight-inference-local-e2e, + agent-turn-latency-e2e, + skill-agent-e2e, + openclaw-skill-cli-e2e, + docs-validation-e2e, + messaging-providers-e2e, + openclaw-slack-pairing-e2e, + openclaw-tui-chat-correlation-e2e, + issue-4434-tui-unreachable-inference-e2e, + issue-3600-gpu-proof-optional-e2e, + openclaw-discord-pairing-e2e, + issue-4462-scope-upgrade-approval-e2e, + issue-4462-gateway-pinned-approval-characterization-e2e, + messaging-compatible-endpoint-e2e, + sessions-agents-cli-e2e, + channels-add-remove-e2e, + channels-stop-start-openclaw-e2e, + channels-stop-start-hermes-e2e, + brave-search-e2e, + common-egress-agent-e2e, + kimi-inference-compat-e2e, + bedrock-runtime-compatible-anthropic-e2e, + token-rotation-e2e, + sandbox-survival-e2e, + issue-2478-crash-loop-recovery-e2e, + hermes-e2e, + hermes-dashboard-e2e, + hermes-root-entrypoint-smoke-e2e, + hermes-secret-boundary-e2e, + openclaw-onboard-security-posture-e2e, + hermes-onboard-security-posture-e2e, + hermes-inference-switch-e2e, + hermes-anthropic-inference-switch-e2e, + hermes-discord-e2e, + hermes-slack-e2e, + sandbox-operations-e2e, + inference-routing-e2e, + openclaw-inference-switch-e2e, + openclaw-anthropic-inference-switch-e2e, + network-policy-e2e, + state-backup-restore-e2e, + tunnel-lifecycle-e2e, + diagnostics-e2e, + credential-migration-e2e, + snapshot-commands-e2e, + shields-config-e2e, + rebuild-openclaw-e2e, + upgrade-stale-sandbox-e2e, + openshell-gateway-upgrade-e2e, + rebuild-hermes-e2e, + rebuild-hermes-stale-base-e2e, + double-onboard-e2e, + onboard-repair-e2e, + onboard-resume-e2e, + onboard-negative-paths-e2e, + runtime-overrides-e2e, + credential-sanitization-e2e, + telegram-injection-e2e, + overlayfs-autofix-e2e, + device-auth-health-e2e, + launchable-smoke-e2e, + gpu-e2e, + gpu-double-onboard-e2e, + gpu-jetson-nvmap-e2e, + concurrent-gateway-ports-e2e, + ] + if: ${{ always() && github.event_name == 'workflow_dispatch' }} + permissions: + issues: write + pull-requests: write + steps: + - name: Post E2E results to PR + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const needs = ${{ toJSON(needs) }}; + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const workflowBranch = context.ref.replace('refs/heads/', ''); + const targetRef = ${{ toJSON(inputs.target_ref) }} || ''; + const prNumberInput = ${{ toJSON(inputs.pr_number) }} || ''; + const displayRef = targetRef || workflowBranch; + const requestedJobs = ${{ toJSON(inputs.jobs) }} || ""; + + let prNumber = prNumberInput ? Number.parseInt(prNumberInput, 10) : undefined; + if (!prNumber) { + // Find open PR for this branch. This is the legacy manual-dispatch + // path where the workflow itself is dispatched on the PR branch. + const { data: prs } = await github.rest.pulls.list({ + owner: context.repo.owner, + repo: context.repo.repo, + head: `${context.repo.owner}:${workflowBranch}`, + state: 'open', + }); + + if (prs.length === 0) { + core.info(`No open PR found for branch ${workflowBranch} — skipping comment.`); + return; + } + + prNumber = prs[0].number; + } + + const requested = requestedJobs + .split(',') + .map((job) => job.trim()) + .filter(Boolean); + const requestedSet = new Set(requested); + + // Build results table. For selective dispatches, report only the + // requested jobs; otherwise the comment is dominated by expected skips. + const emoji = { success: '✅', failure: '❌', cancelled: '⚠️', skipped: '⏭️' }; + const allEntries = Object.entries(needs).sort(([a], [b]) => a.localeCompare(b)); + const missingRequested = requested.filter((job) => !(job in needs)); + const reportedEntries = requested.length + ? allEntries.filter(([name]) => requestedSet.has(name)) + : allEntries; + const rows = reportedEntries + .sort(([a], [b]) => a.localeCompare(b)) + .map(([name, { result }]) => `| ${name} | ${emoji[result] || '❓'} ${result} |`); + for (const name of missingRequested) { + rows.push(`| ${name} | ❓ not reported |`); + } + + const ran = reportedEntries.filter(([, v]) => v.result !== 'skipped'); + const passed = ran.filter(([, v]) => v.result === 'success'); + const failed = ran.filter(([, v]) => v.result === 'failure'); + const skipped = reportedEntries.filter(([, v]) => v.result === 'skipped'); + // Cancelled jobs (e.g. cancel-in-progress superseding an older run) + // are neither success, failure, nor skipped. Without a bucket for + // them they slipped through the status tally and the comment fell + // through to the default "✅ All requested jobs passed" — masking + // the fact that the run produced no signal at all. + const cancelled = ran.filter(([, v]) => v.result === 'cancelled'); + + const status = + failed.length > 0 || missingRequested.length > 0 + ? '❌ Some jobs failed' + : cancelled.length > 0 && passed.length === 0 + ? '⚠️ Run cancelled — no signal' + : cancelled.length > 0 && passed.length > 0 + ? '⚠️ Some jobs cancelled — partial pass' + : skipped.length > 0 && passed.length === 0 + ? '⚠️ No requested jobs ran' + : '✅ All requested jobs passed'; + + const body = [ + `### Selective E2E Results — ${status}`, + '', + `**Run:** [${context.runId}](${runUrl})`, + `**Target ref:** \`${displayRef}\``, + targetRef ? `**Workflow ref:** \`${workflowBranch}\`` : undefined, + requestedJobs ? `**Requested jobs:** \`${requestedJobs}\`` : '**Requested jobs:** all (no filter)', + `**Summary:** ${passed.length} passed, ${failed.length} failed, ${cancelled.length} cancelled, ${skipped.length} skipped`, + '', + '| Job | Result |', + '|-----|--------|', + ...rows, + '', + failed.length > 0 + ? `> **Failed jobs:** ${failed.map(([k]) => k).join(', ')}. Check [run artifacts](${runUrl}) for logs.` + : '', + missingRequested.length > 0 + ? `> **Missing requested jobs:** ${missingRequested.join(', ')}. The reporting workflow needs to include these jobs.` + : '', + ].filter((line) => line !== undefined).join('\n'); + + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + body, + }); + + # ── Nightly Scorecard ────────────────────────────────────────────────── + # Aggregates results into a scorecard published to $GITHUB_STEP_SUMMARY + # and optionally to Slack. Computes pass/fail/cancel breakdowns and + # cloud-onboard trace timing vs the latest prior-release run. Runs on: + # - schedule (cron) → Slack post (DAILY route) + # - workflow_dispatch full run → Slack post (FULLRUN route) + # - workflow_dispatch selective→ silent unless post_to_slack=true (preview) + scorecard: + runs-on: linux-amd64-cpu4 + needs: + [ + cloud-e2e, + cloud-onboard-e2e, + cloud-inference-e2e, + cron-preflight-inference-local-e2e, + agent-turn-latency-e2e, + skill-agent-e2e, + openclaw-skill-cli-e2e, + docs-validation-e2e, + messaging-providers-e2e, + openclaw-slack-pairing-e2e, + openclaw-tui-chat-correlation-e2e, + issue-4434-tui-unreachable-inference-e2e, + issue-3600-gpu-proof-optional-e2e, + openclaw-discord-pairing-e2e, + issue-4462-scope-upgrade-approval-e2e, + issue-4462-gateway-pinned-approval-characterization-e2e, + messaging-compatible-endpoint-e2e, + sessions-agents-cli-e2e, + channels-add-remove-e2e, + channels-stop-start-openclaw-e2e, + channels-stop-start-hermes-e2e, + brave-search-e2e, + common-egress-agent-e2e, + kimi-inference-compat-e2e, + bedrock-runtime-compatible-anthropic-e2e, + token-rotation-e2e, + sandbox-survival-e2e, + issue-2478-crash-loop-recovery-e2e, + hermes-e2e, + hermes-dashboard-e2e, + hermes-root-entrypoint-smoke-e2e, + hermes-secret-boundary-e2e, + openclaw-onboard-security-posture-e2e, + hermes-onboard-security-posture-e2e, + hermes-inference-switch-e2e, + hermes-anthropic-inference-switch-e2e, + hermes-discord-e2e, + hermes-slack-e2e, + sandbox-operations-e2e, + inference-routing-e2e, + openclaw-inference-switch-e2e, + openclaw-anthropic-inference-switch-e2e, + network-policy-e2e, + state-backup-restore-e2e, + tunnel-lifecycle-e2e, + diagnostics-e2e, + credential-migration-e2e, + snapshot-commands-e2e, + shields-config-e2e, + rebuild-openclaw-e2e, + upgrade-stale-sandbox-e2e, + openshell-gateway-upgrade-e2e, + rebuild-hermes-e2e, + rebuild-hermes-stale-base-e2e, + double-onboard-e2e, + onboard-repair-e2e, + onboard-resume-e2e, + onboard-negative-paths-e2e, + runtime-overrides-e2e, + credential-sanitization-e2e, + telegram-injection-e2e, + overlayfs-autofix-e2e, + device-auth-health-e2e, + launchable-smoke-e2e, + gpu-e2e, + gpu-double-onboard-e2e, + gpu-jetson-nvmap-e2e, + concurrent-gateway-ports-e2e, + ] + if: ${{ always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }} + permissions: + actions: read + steps: + - name: Checkout scorecard builder + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + sparse-checkout: | + scripts/scorecard + sparse-checkout-cone-mode: false + + - name: Generate nightly scorecard + id: scorecard + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + // ── Config ────────────────────────────────────────────── + const EXCLUDED_JOBS = new Set(['gpu-e2e', 'notify-on-failure', 'report-to-pr', 'scorecard']); + + // ── Helpers ───────────────────────────────────────────── + const path = require('path'); + const traceTiming = require(path.join( + process.env.GITHUB_WORKSPACE, + 'scripts/scorecard/analyze-trace-timing.ts', + )); + + function formatDate(date) { + return date.toLocaleDateString('en-US', { month: 'short', day: 'numeric' }); + } + + // ── Gather results from the current run's needs context ─ + const needs = ${{ toJSON(needs) }}; + const today = formatDate(new Date()); + const isDispatch = context.eventName === 'workflow_dispatch'; + const requestedJobsRaw = isDispatch ? (${{ toJSON(inputs.jobs) }} || '').trim() : ''; + const requestedJobs = requestedJobsRaw + ? requestedJobsRaw.split(',').map((name) => name.trim()).filter(Boolean) + : []; + const isSelectiveDispatch = isDispatch && requestedJobs.length > 0; + const runMode = isSelectiveDispatch + ? 'Selective dispatch' + : isDispatch + ? 'Manual full run' + : 'Scheduled full nightly'; + + // ── Canonical job set (API truth) ── + // Contract: + // - Source: listJobsForWorkflowRun (paginated). `needs` is + // insufficient because reusable-workflow inner jobs surface + // in the API as " / " but in `needs` as the + // plain caller ID, and wrapper-vs-inner conclusions can + // diverge on re-runs. + // - Authoritative fields: name (suffix-stripped), conclusion, + // status, html_url, run_attempt, completed_at. + // - Fallback on API throw: counts + names from `needs`, no + // URLs, may mis-classify wrapper-vs-inner mismatches. + // Degraded but still renders the scorecard. + // - Removable when `needs` exposes inner reusable-workflow + // conclusions + html_url natively (no GitHub feature yet). + const entries = Object.entries(needs).filter(([name]) => !EXCLUDED_JOBS.has(name)); + + let canonicalJobs = null; + try { + const apiJobs = await github.paginate( + github.rest.actions.listJobsForWorkflowRun, + { + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.runId, + per_page: 100, + }, + ); + const dedupedByName = new Map(); + for (const j of apiJobs) { + const name = j.name.replace(/ \/ [^/]+$/, ''); + if (EXCLUDED_JOBS.has(name)) continue; + // Deterministic: prefer higher run_attempt, then later + // completed_at as tiebreaker. Independent of API page order. + const existing = dedupedByName.get(name); + const isNewer = !existing + || (j.run_attempt ?? 0) > (existing.run_attempt ?? 0) + || ((j.run_attempt ?? 0) === (existing.run_attempt ?? 0) + && (j.completed_at ?? '') > (existing.completed_at ?? '')); + if (isNewer) dedupedByName.set(name, { ...j, name }); + } + canonicalJobs = [...dedupedByName.values()]; + } catch (e) { + const safeMsg = String(e.message ?? 'unknown').slice(0, 200); + core.warning(`Could not fetch jobs from API (status ${e.status ?? 'unknown'}); falling back to needs context. Reason: ${safeMsg}`); + } + + // ── Counts ── + let success = 0; + let failure = 0; + let cancelled = 0; + let skipped = 0; + + if (canonicalJobs) { + for (const j of canonicalJobs) { + if (j.conclusion === 'success') success++; + else if (j.conclusion === 'failure') failure++; + else if (j.conclusion === 'cancelled') cancelled++; + else if (j.conclusion === 'skipped' || j.status !== 'completed') skipped++; + // Catch-all for any other terminal conclusion (timed_out, + // action_required, neutral, stale, future values) → count as + // failure so `perfect` stays false on anomalies. + else failure++; + } + } else { + for (const [, { result }] of entries) { + if (result === 'success') success++; + else if (result === 'failure') failure++; + else if (result === 'cancelled') cancelled++; + else if (result === 'skipped') skipped++; + } + } + + const total = canonicalJobs ? canonicalJobs.length : entries.length; + const ran = total - skipped; + const perfect = failure === 0 && cancelled === 0 && ran > 0; + + // ── Failed jobs ── + const failedJobs = canonicalJobs + ? canonicalJobs + .filter((j) => j.conclusion === 'failure') + .map((j) => ({ name: j.name, url: j.html_url })) + .sort((a, b) => a.name.localeCompare(b.name)) + : entries + .filter(([, { result }]) => result === 'failure') + .map(([name]) => ({ name, url: null })) + .sort((a, b) => a.name.localeCompare(b.name)); + + const { traceTimingLine, traceSummaryLines } = + await traceTiming.buildTraceTimingResult({ github, context }); + + // ── Build scorecard ───────────────────────────────────── + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const lines = [ + `## 🌅 NemoClaw Nightly Scorecard — ${today}`, + '', + `**Run mode:** ${runMode}`, + ]; + + if (isSelectiveDispatch) { + lines.push(`**Requested jobs:** ${requestedJobs.map((name) => `\`${name}\``).join(', ')}`); + } + + lines.push( + `**Jobs run:** ${ran} of ${total}`, + ` ✅ ${success} passed`, + ` ❌ ${failure} failed`, + ` 🚫 ${cancelled} cancelled`, + ` ⏭️ ${skipped} skipped`, + ); + + if (failedJobs.length > 0) { + lines.push(''); + lines.push('**Failed jobs:**'); + for (const job of failedJobs) { + lines.push(job.url ? ` - [${job.name}](${job.url})` : ` - \`${job.name}\``); + } + } + + if (perfect) { + lines.push(''); + lines.push('🎉 **All jobs passed!**'); + } + + lines.push(''); + lines.push(traceTimingLine); + lines.push(...traceSummaryLines); + lines.push(''); + lines.push(`🔗 [Full run details](${runUrl})`); + + const scorecard = lines.join('\n'); + core.summary.addRaw(scorecard); + await core.summary.write(); + core.setOutput('scorecard', scorecard); + + // Structured data for the Slack step. + // Contract: scripts/scorecard/build-slack-blocks.ts + const actor = context.actor || ''; + core.setOutput('scorecardData', JSON.stringify({ + today, + runMode, + actor, + isSelectiveDispatch, + requestedJobs, + total, + ran, + success, + failure, + cancelled, + skipped, + perfect, + failedJobs, + traceTimingLine, + runUrl, + })); + + # ── Slack notification ──────────────────────────── + # Production routes (always post; runMode is computed by the generate step): + # "Scheduled full nightly" → SLACK_WEBHOOK_URL_DAILY + # "Manual full run" → SLACK_WEBHOOK_URL_FULLRUN + # + # Selective dispatch is skipped by default. Devs may opt into the + # preview channel via post_to_slack=true (SLACK_WEBHOOK_URL_PREVIEW). + - name: Post scorecard to Slack + if: ${{ steps.scorecard.outputs.scorecardData != '' }} + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + SLACK_WEBHOOK_URL_DAILY: ${{ secrets.SLACK_WEBHOOK_URL_DAILY }} + SLACK_WEBHOOK_URL_FULLRUN: ${{ secrets.SLACK_WEBHOOK_URL_FULLRUN }} + SLACK_WEBHOOK_URL_PREVIEW: ${{ secrets.SLACK_WEBHOOK_URL_PREVIEW }} + SCORECARD_DATA: ${{ steps.scorecard.outputs.scorecardData }} + POST_TO_SLACK: ${{ inputs.post_to_slack }} + with: + script: | + const path = require('path'); + const { + buildBlocks, + buildFallbackText, + getStatusColor, + getSlackChannel, + } = require( + path.join(process.env.GITHUB_WORKSPACE, 'scripts/scorecard/build-slack-blocks.ts'), + ); + + const data = JSON.parse(process.env.SCORECARD_DATA); + const channel = getSlackChannel(data); + + // Selective dispatches only post when the user explicitly opts in. + // Schedule + Manual full run always post. + if (channel === 'preview' && process.env.POST_TO_SLACK !== 'true') { + core.info('Selective dispatch without post_to_slack — skipping'); + return; + } + + const envByChannel = { + 'daily': 'SLACK_WEBHOOK_URL_DAILY', + 'fullrun': 'SLACK_WEBHOOK_URL_FULLRUN', + 'preview': 'SLACK_WEBHOOK_URL_PREVIEW', + }; + const webhookUrl = process.env[envByChannel[channel]]; + + if (!webhookUrl) { + core.info(`Slack webhook for "${channel}" not configured — skipping`); + return; + } + + // Legacy attachment wrapper enables the coloured left-edge bar. + const payload = { + text: buildFallbackText(data), + attachments: [ + { + color: getStatusColor(data), + blocks: buildBlocks(data), + }, + ], + }; + + const resp = await fetch(webhookUrl, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload), + }); + + if (!resp.ok) { + const errBody = (await resp.text()).slice(0, 100).replace(/\s+/g, ' '); + core.warning(`Slack webhook (${channel}) returned ${resp.status}. Reason (truncated): ${errBody}`); + } else { + core.info(`Scorecard posted to Slack (${channel})`); + } diff --git a/test/e2e-vpn/Dockerfile.full-e2e b/test/e2e-vpn/Dockerfile.full-e2e new file mode 100644 index 00000000000..daf4e53cfd9 --- /dev/null +++ b/test/e2e-vpn/Dockerfile.full-e2e @@ -0,0 +1,30 @@ +# hadolint global ignore=DL3008,DL4006,SC2086 +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y --no-install-recommends \ + curl git ca-certificates bash python3 sudo jq \ + && rm -rf /var/lib/apt/lists/* + +# Install Docker CLI only (NOT Docker daemon — will use host socket at runtime) +RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg && \ + echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] \ + https://download.docker.com/linux/ubuntu $(. /etc/os-release && echo $VERSION_CODENAME) stable" \ + > /etc/apt/sources.list.d/docker.list && \ + apt-get update && apt-get install -y --no-install-recommends docker-ce-cli && \ + rm -rf /var/lib/apt/lists/* + +# Create non-root testuser with sudo and docker group +RUN groupadd -f docker && \ + useradd -m -s /bin/bash -G docker testuser && \ + echo "testuser ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers + +# Copy repo +COPY . /workspace +RUN chown -R testuser:testuser /workspace + +USER testuser +WORKDIR /workspace + +ENTRYPOINT ["bash", "test/e2e-vpn/test-full-e2e.sh"] diff --git a/test/e2e-vpn/README.md b/test/e2e-vpn/README.md new file mode 100644 index 00000000000..c514941a5be --- /dev/null +++ b/test/e2e-vpn/README.md @@ -0,0 +1,50 @@ + + + +# NemoClaw E2E CI + +## Nightly Onboard Trace Timing + +The GitHub Actions workflow `.github/workflows/nightly-e2e.yaml` enables NemoClaw tracing for the `cloud-onboard-e2e` lane. +That lane is the current GitHub E2E trace-timing scope; other E2E lanes keep their existing failure-log artifacts until they opt into a trusted timing-summary artifact. +That job sets: + +```bash +NEMOCLAW_TRACE_DIR=/tmp/nemoclaw-traces +``` + +The reusable E2E runner does not upload `/tmp/nemoclaw-traces/` directly. +After the target-ref script finishes, trusted workflow code reads candidate trace JSON files from that target-controlled directory and writes a timing-only summary under `/tmp/nemoclaw-trace-summary/`. +Only that summary directory is uploaded after every run as the `cloud-onboard-traces` artifact. +Failure-only logs continue to use each job's normal `artifact_name` and `artifact_path`. +The uploaded timing summary keeps only the trace schema version, trace id, total duration, known `nemoclaw.onboard.phase.*` durations, and a bounded slowest-span timing list. +It omits raw attributes, events, prompts, environment values, file names, arbitrary files, and unrecognized trace fields. +NemoClaw also sanitizes trace files as they are written, but that in-process redaction is defense in depth rather than the artifact upload trust boundary. + +The nightly `scorecard` job reads the `cloud-onboard-traces` artifact, selects the trusted `nemoclaw.trace_timing.v1` summary JSON, and reports: + +- total onboard trace duration from `summary.total_duration_ms` +- top matching `nemoclaw.onboard.phase.*` duration changes in Slack +- a full phase timing table in the GitHub job summary +- deltas against the latest completed `nightly-e2e` run for the prior semver release tag's commit + +Phase deltas and the full summary table are reported only when the same trace span names exist in both runs. +If phase names change between runs, the scorecard reports only the total onboard duration change. +If the artifact, prior release tag, prior run, or matching trace data is unavailable, the scorecard keeps the nightly result best-effort and reports the missing comparison in the Slack summary instead of failing CI. + +## Slack Scorecard Configuration + +`nightly-e2e.yaml` posts the scorecard through repository Actions secrets: + +- `SLACK_WEBHOOK_URL_DAILY` for scheduled full nightly runs +- `SLACK_WEBHOOK_URL_FULLRUN` for manual full runs +- `SLACK_WEBHOOK_URL_PREVIEW` for selective dispatches when `post_to_slack=true` + +Scheduled nightly runs and manual full runs post the scorecard automatically. +Selective dispatches are silent by default and post only when `post_to_slack=true`, so developers can run targeted checks without notifying Slack. +The trace timing section is part of the same Slack scorecard message, but it stays compact: total duration, the three largest matching phase changes, and a pointer to the GitHub run summary for the full table. +The scorecard counts passed, failed, cancelled, and skipped jobs separately. +Runs with cancellations but no failures stay in the warning state instead of being reported as all passed, including mixed pass and cancelled selective dispatches. +Slack no longer includes the legacy `Trend` context; trace timing is the only duration comparison in the scorecard. +Slack does not post raw trace JSON, prompts, credentials, or environment values. +The uploaded artifact is the trusted timing-only summary, not the raw target-ref trace directory. diff --git a/test/e2e-vpn/e2e-cloud-experimental/check-docs.sh b/test/e2e-vpn/e2e-cloud-experimental/check-docs.sh new file mode 100755 index 00000000000..407a2dad572 --- /dev/null +++ b/test/e2e-vpn/e2e-cloud-experimental/check-docs.sh @@ -0,0 +1,1285 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Documentation checks (default: all): +# 1) Markdown/MDX links — local paths exist; optional curl for unique http(s) URLs. +# 2) CLI parity — `nemoclaw --help` vs ### `nemoclaw …` in docs/reference/commands.mdx. +# +# Usage (from repo root): +# test/e2e-vpn/e2e-cloud-experimental/check-docs.sh # both checks +# test/e2e-vpn/e2e-cloud-experimental/check-docs.sh --only-links +# test/e2e-vpn/e2e-cloud-experimental/check-docs.sh --only-cli +# test/e2e-vpn/e2e-cloud-experimental/check-docs.sh --local-only +# CHECK_DOC_LINKS_REMOTE=0 test/e2e-vpn/e2e-cloud-experimental/check-docs.sh +# test/e2e-vpn/e2e-cloud-experimental/check-docs.sh path/to/a.md path/to/b.mdx +# +# Environment: +# CHECK_DOC_LINKS_REMOTE If 0, skip http(s) probes for links check. +# CHECK_DOC_LINKS_VERBOSE If 1, log each URL during curl (same as --verbose). +# CHECK_DOC_LINKS_IGNORE_EXTRA Comma-separated extra http(s) URLs to skip curling (exact match, #fragment ignored). +# CHECK_DOC_LINKS_IGNORE_URL_REGEX If set, skip curl when the whole URL matches this ERE (bash [[ =~ ]]). +# NODE Node for CLI check (default: node). +# CURL curl binary (default: curl). + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(git -C "$SCRIPT_DIR" rev-parse --show-toplevel 2>/dev/null || true)" +if [[ -z "${REPO_ROOT:-}" ]]; then + REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" +fi +CURL="${CURL:-curl}" +NODE="${NODE:-node}" + +RUN_LINKS=1 +RUN_CLI=1 +RUN_INSTALL=1 +LOCAL_ONLY=0 +EXTRA_FILES=() +VERBOSE="${CHECK_DOC_LINKS_VERBOSE:-0}" +WITH_SKILLS=0 + +usage() { + cat <<'EOF' +Documentation checks: Markdown/MDX links + nemoclaw --help vs commands reference ++ install.sh --help vs canonical provider list. + +Usage: test/e2e-vpn/e2e-cloud-experimental/check-docs.sh [options] [extra.md/.mdx ...] + +Options: + --only-links Run only the Markdown/MDX link check. + --only-cli Run only the CLI help vs docs/reference/commands.mdx check + (includes both command-level and flag-level parity). + --only-install Run only the install.sh --help vs canonical provider check. + --local-only Do not curl http(s) URLs (same as CHECK_DOC_LINKS_REMOTE=0). + --with-skills Also scan .agents/skills/**/*.md (link check). + --verbose Log each URL while curling (link check). + -h, --help Show this help. + +Environment: CHECK_DOC_LINKS_REMOTE, CHECK_DOC_LINKS_VERBOSE, CHECK_DOC_LINKS_IGNORE_EXTRA, + CHECK_DOC_LINKS_IGNORE_URL_REGEX, NODE, CURL. +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --only-links) + RUN_CLI=0 + RUN_INSTALL=0 + shift + ;; + --only-cli) + RUN_LINKS=0 + RUN_INSTALL=0 + shift + ;; + --only-install) + RUN_LINKS=0 + RUN_CLI=0 + shift + ;; + --local-only) + LOCAL_ONLY=1 + shift + ;; + --with-skills) + WITH_SKILLS=1 + shift + ;; + --verbose) + VERBOSE=1 + shift + ;; + -h | --help) + usage + exit 0 + ;; + --) + shift + EXTRA_FILES+=("$@") + break + ;; + -*) + echo "check-docs: unknown option: $1" >&2 + usage >&2 + exit 2 + ;; + *) + EXTRA_FILES+=("$1") + shift + ;; + esac +done + +if [[ "$RUN_LINKS" -eq 0 && "$RUN_CLI" -eq 0 && "$RUN_INSTALL" -eq 0 ]]; then + echo "check-docs: use at least one of default (all), --only-links, --only-cli, or --only-install" >&2 + exit 2 +fi + +if [[ "$LOCAL_ONLY" -eq 1 ]]; then + CHECK_DOC_LINKS_REMOTE=0 +fi +CHECK_DOC_LINKS_REMOTE="${CHECK_DOC_LINKS_REMOTE:-1}" + +log() { + printf '%s\n' "check-docs: $*" +} + +# --- CLI: --help vs commands.mdx ------------------------------------------------ + +run_cli_check() { + local CLI_JS="$REPO_ROOT/bin/nemoclaw.js" + local COMMANDS_MD="$REPO_ROOT/docs/reference/commands.mdx" + + if [[ ! -f "$CLI_JS" ]]; then + echo "check-docs: [cli] missing $CLI_JS" >&2 + return 1 + fi + if [[ ! -f "$COMMANDS_MD" ]]; then + echo "check-docs: [cli] missing $COMMANDS_MD" >&2 + return 1 + fi + if ! command -v "$NODE" >/dev/null 2>&1; then + echo "check-docs: [cli] '$NODE' not found" >&2 + return 1 + fi + + local _tmp + _tmp="$(mktemp -d)" + local _cli_home="$_tmp/home" + mkdir -p "$_cli_home/.nemoclaw" + cat >"$_cli_home/.nemoclaw/sandboxes.json" <<'JSON' +{"defaultSandbox":"placeholder-sandbox","sandboxes":{"placeholder-sandbox":{"name":"placeholder-sandbox"}}} +JSON + + log "[cli] comparing: $NODE bin/nemoclaw.js --dump-commands" + # shellcheck disable=SC2016 + # log text: backticks are documentation markers, not command substitution + log '[cli] vs: docs/reference/commands.mdx (### `nemoclaw …` or `$$nemoclaw …` headings)' + + log "[cli] phase 1/2: dump canonical command list from registry" + if ! HOME="$_cli_home" "$NODE" "$CLI_JS" --dump-commands >"$_tmp/help.txt" 2>"$_tmp/help.err"; then + cat "$_tmp/help.err" >&2 + rm -rf "$_tmp" + return 1 + fi + LC_ALL=C sort -u -o "$_tmp/help.txt" "$_tmp/help.txt" + + local _n_help + _n_help="$(wc -l <"$_tmp/help.txt" | tr -d " ")" + log "[cli] phase 1: extracted ${_n_help} unique command line(s) from --dump-commands" + + # shellcheck disable=SC2016 + # log text: backticks are documentation markers, not command substitution + log '[cli] phase 2/2: extract ### `nemoclaw …` / `$$nemoclaw …` headings from commands reference' + # Allow optional MyST suffix on the same line, e.g. ### `nemoclaw onboard` {#anchor}. + # Preserve placeholders that are part of the canonical help signature, but + # keep accepting docs-only suffixes such as `snapshot restore [selector]`. + grep -E '^### `(\$\$)?nemoclaw ' "$COMMANDS_MD" | LC_ALL=C perl -CS -ne ' + BEGIN { + my $help_path = shift @ARGV; + open my $help_fh, "<", $help_path or die "open help list: $!"; + while (my $line = <$help_fh>) { + chomp $line; + $help{$line} = 1; + } + close $help_fh; + } + if (/^### `([^`]+)`\s*(?:\{[^}]+\})?\s*$/) { + my $c = $1; + $c =~ s/^\$\$nemoclaw\b/nemoclaw/; + $c =~ s/\s+$//; + while (!$help{$c}) { + my $changed = 0; + $changed ||= ($c =~ s/\s*\[[^\]]*\]\s*$//); + $changed ||= ($c =~ s/\s+<[^>]+>\s*$//); + $c =~ s/\s+$//; + last unless $changed; + } + print "$c\n"; + } + ' "$_tmp/help.txt" | LC_ALL=C sort -u >"$_tmp/doc.txt" + + local _n_doc + _n_doc="$(wc -l <"$_tmp/doc.txt" | tr -d " ")" + log "[cli] phase 2: extracted ${_n_doc} heading(s) from ${COMMANDS_MD#"$REPO_ROOT"/}" + + if ! cmp -s "$_tmp/help.txt" "$_tmp/doc.txt"; then + echo "check-docs: [cli] mismatch between --help and $COMMANDS_MD" >&2 + echo "" >&2 + echo "Only in --help (add ### to commands.mdx or fix help):" >&2 + comm -23 "$_tmp/help.txt" "$_tmp/doc.txt" | sed 's/^/ /' >&2 || true + echo "" >&2 + echo "Only in commands.mdx (add to help() in bin/nemoclaw.js or fix heading):" >&2 + comm -13 "$_tmp/help.txt" "$_tmp/doc.txt" | sed 's/^/ /' >&2 || true + rm -rf "$_tmp" + return 1 + fi + + log "[cli] command-level parity OK (${_n_help} nemoclaw command(s))" + + # ── Phase 3/3: flag-level parity (NemoClaw#3224) ────────────────────────── + # For each command, run its `--help`, extract every long-form flag mentioned, + # and confirm each appears within that command's own section in + # commands.mdx (between its `### \`nemoclaw \`` heading and the next + # ### heading). Two help formats coexist: oclif global commands use a + # USAGE/FLAGS layout; `nemoclaw ...` commands use a custom + # Options: section. Greping the full help output handles both formats. + # Section-scoped grep avoids false negatives where a flag like `--yes` + # appears in many sections but is missing from the one being audited. + # Word-boundary regex avoids false positives where `--yes` is contained + # in `--yes-i-accept-third-party-software`. Skips global -h/--help/--version. + # + # The check runs with an isolated HOME that contains a fake + # `placeholder-sandbox` registry entry. That keeps CI deterministic and lets + # sandbox-scoped commands print `--help` without touching the user's real + # ~/.nemoclaw state. + log "[cli] phase 3/3: flag-level parity" + + # Awk extractor: print lines belonging to the section whose heading + # canonicalizes to after the same trailing-placeholder strip phase 2 + # applies (`### \`nemoclaw foo \`` → `nemoclaw foo`). Stops at the + # next ### heading. MyST anchors after the closing backtick are tolerated. + extract_md_section() { + local cmd="$1" + local md="$2" + LC_ALL=C awk -v target="$cmd" ' + # End the section when a new top-level heading appears (h1, h2, or + # h3). h4+ are kept since they are sub-sections of the same command. + # Explicit alternation since traditional awk treats `{n,m}` literally. + in_sec && /^(# |## |### )/ { exit } + /^### `/ { + line = $0 + sub(/^### `/, "", line) + bt = index(line, "`") + if (bt > 0) { + cand = substr(line, 1, bt - 1) + sub(/^\$\$nemoclaw/, "nemoclaw", cand) + sub(/[[:space:]]+$/, "", cand) + if (cand == target) { + in_sec = 1 + next + } + while (sub(/[[:space:]]*\[[^]]*\][[:space:]]*$/, "", cand)) {} + while (sub(/[[:space:]]+<[^>]+>[[:space:]]*$/, "", cand)) {} + sub(/[[:space:]]+$/, "", cand) + if (cand == target) { + in_sec = 1 + next + } + } + } + in_sec { print } + ' "$md" + } + + extract_help_flags() { + printf '%s\n' "$1" | LC_ALL=C perl -CS -ne ' + sub emit_flags { + my ($s) = @_; + while ($s =~ /--(?:\[no-\])?([a-z][a-z0-9-]+)/g) { + my $flag = $1; + my $matched = $&; + print "--$flag\n"; + print "--no-$flag\n" if $matched =~ /^\Q--[no-]\E/; + } + } + + if (/^\s*Usage:\s*(.*)$/i) { + $mode = "usage"; + emit_flags($1); + next; + } + if (/^\s*USAGE\s*$/) { + $mode = "usage"; + next; + } + if (/^\s*(FLAGS|GLOBAL FLAGS|Options):?\s*$/i) { + $mode = "flags"; + next; + } + if (/^\s*(ARGUMENTS|DESCRIPTION|EXAMPLES)\s*$/i || /^\s*$/) { + $mode = ""; + next; + } + emit_flags($_) if $mode; + ' | LC_ALL=C sort -u + } + + local _flag_drift=0 + while IFS= read -r cmd_line || [[ -n "$cmd_line" ]]; do + [[ -z "$cmd_line" ]] && continue + # Skip "command-line variant" entries like `nemoclaw onboard --from` + # — those describe a flagged invocation of a parent command (here + # `nemoclaw onboard`) that is iterated separately. Re-invoking them + # with `--help` would just trigger flag-value parsing errors. + case "$cmd_line" in *" --"*) continue ;; esac + # `--dump-commands` lines start with `nemoclaw `; strip that since we + # re-invoke via `node bin/nemoclaw.js`. Then replace with a + # sandbox name that passes name validation (lowercase, starts with + # letter, only letters/digits/hyphens — underscores are rejected). + local invoke + invoke="${cmd_line#nemoclaw }" + invoke="${invoke///placeholder-sandbox}" + # Read into an array so each space-separated token is a distinct argv + # element to node — avoids SC2086 and any quoting surprises. + local -a _invoke_args + read -ra _invoke_args <<<"$invoke" + # Redirect stdin to /dev/null. The outer `while read` is consuming + # `$_tmp/help.txt` via `done <` redirection; any inner command that + # touches stdin (some node startup paths do) would eat subsequent + # lines, silently truncating the iteration. Negative-tested by + # mutating commands.mdx and confirming drift is now reported. + # + # Capture exit code separately so a real failure (broken command path, + # crashed loader, etc.) propagates instead of being swallowed by + # `|| true`. + local _help_text _help_err _help_rc=0 + _help_err="$(mktemp)" + _help_text="$(HOME="$_cli_home" "$NODE" "$CLI_JS" "${_invoke_args[@]}" --help "$_help_err")" || _help_rc=$? + if [[ "$_help_rc" -ne 0 ]]; then + cat "$_help_err" >&2 + rm -f "$_help_err" + rm -rf "$_tmp" + return 1 + fi + rm -f "$_help_err" + [[ -z "$_help_text" ]] && continue + + local _flags + _flags="$(extract_help_flags "$_help_text")" + [[ -z "$_flags" ]] && continue + + local _section + _section="$(extract_md_section "$cmd_line" "$COMMANDS_MD")" + if [[ -z "$_section" ]]; then + # Phase 2 already enforces the heading exists; if the section is + # somehow empty here, fall back to the full doc rather than skipping. + _section="$(cat "$COMMANDS_MD")" + fi + + while IFS= read -r flag; do + [[ -z "$flag" ]] && continue + case "$flag" in --help | --version) continue ;; esac + # Word-boundary regex: treat letters/digits/_/- as continuation chars + # so `--yes` does not match inside `--yes-i-accept-third-party-software`. + local _pat="(^|[^a-zA-Z0-9_-])${flag}([^a-zA-Z0-9_-]|$)" + if ! grep -qE -- "$_pat" <<<"$_section"; then + echo "check-docs: [cli] flag $flag (from \`$cmd_line --help\`) not in '$cmd_line' section of $COMMANDS_MD" >&2 + _flag_drift=1 + fi + done <<<"$_flags" + + # Reverse direction: extract long flags mentioned in the doc section + # and confirm each appears in the actual --help. Catches stale docs + # (flag removed from CLI but still listed in commands.mdx). + # + # Scoping rule: inside fenced code blocks (where USAGE lines live like + # `[--non-interactive]`), any `--foo` counts. Outside fences, only + # backtick-bounded `\`--foo\`` mentions count, so prose references to + # other tools (e.g. `\`openshell gateway start --recreate\``) don't get + # mistaken for nemoclaw flag documentation. + local _doc_flags + _doc_flags="$( + printf '%s\n' "$_section" \ + | LC_ALL=C perl -CS -ne ' + if (/^```/) { $in_fence = !$in_fence; next; } + if ($in_fence) { + while (/--([a-z][a-z0-9-]+)/g) { print "--$1\n"; } + } else { + while (/`--([a-z][a-z0-9-]+)/g) { print "--$1\n"; } + } + ' \ + | grep -vxE -- '--help|--version' \ + | LC_ALL=C sort -u || true + )" + while IFS= read -r flag; do + [[ -z "$flag" ]] && continue + if ! grep -qxF -- "$flag" <<<"$_flags"; then + echo "check-docs: [cli] flag $flag documented under \`$cmd_line\` but absent from \`$cmd_line --help\`" >&2 + _flag_drift=1 + fi + done <<<"$_doc_flags" + done <"$_tmp/help.txt" + + if [[ "$_flag_drift" -ne 0 ]]; then + rm -rf "$_tmp" + return 1 + fi + + log "[cli] flag-level parity OK" + while IFS= read -r line || [[ -n "$line" ]]; do + [[ -z "$line" ]] && continue + log "[cli] $line" + done <"$_tmp/help.txt" + log "[cli] done." + rm -rf "$_tmp" + return 0 +} + +# --- Install: install.sh --help vs canonical provider list (NemoClaw#3224) ---- + +run_install_check() { + # Two installer entry points need to stay in sync with the canonical + # provider list: + # 1. install.sh (bootstrap_usage) — what users see via `curl | bash --help` + # 2. scripts/install.sh — what the bootstrap sources locally; what users + # see when they run `bash install.sh --help` from a clone + local BOOTSTRAP_SH="$REPO_ROOT/install.sh" + local PAYLOAD_SH="$REPO_ROOT/scripts/install.sh" + + # The providers list has moved between layouts; tolerate both the legacy + # flat path and the post-refactor layered path. + local PROVIDERS_TS="" + for _candidate in \ + "$REPO_ROOT/src/lib/onboard/providers.ts" \ + "$REPO_ROOT/src/lib/onboard-providers.ts"; do + if [[ -f "$_candidate" ]]; then + PROVIDERS_TS="$_candidate" + break + fi + done + + if [[ ! -f "$BOOTSTRAP_SH" ]]; then + echo "check-docs: [install] missing $BOOTSTRAP_SH" >&2 + return 1 + fi + if [[ -z "$PROVIDERS_TS" ]]; then + echo "check-docs: [install] could not locate onboard providers TS source" >&2 + return 1 + fi + + log "[install] comparing: NEMOCLAW_PROVIDER values in install.sh + scripts/install.sh" + log "[install] vs: ${PROVIDERS_TS#"$REPO_ROOT"/} canonical 'Valid values' list" + + # The canonical values live in a single error-message line that lists every + # accepted NEMOCLAW_PROVIDER input. Extract the comma-separated payload. + local _canonical + _canonical="$(grep -oE 'Valid values: [^"]+' "$PROVIDERS_TS" | head -1 | sed 's/^Valid values: //')" + if [[ -z "$_canonical" ]]; then + echo "check-docs: [install] could not locate canonical provider list in $PROVIDERS_TS" >&2 + return 1 + fi + + # Extract the NEMOCLAW_PROVIDER usage block from each script (the printf + # lines starting at NEMOCLAW_PROVIDER through the next NEMOCLAW_ entry or + # blank-line printf), then verify each canonical value appears within that + # block. Grepping the whole script would match unrelated mentions of + # `gemini` / `ollama` in helper text, prompts, etc. + # + # Skip the install-helper / wizard-only keys (install-vllm, install-ollama, + # install-windows-ollama, start-windows-ollama). They are option keys the + # interactive wizard exposes, not values a user is expected to set + # NEMOCLAW_PROVIDER to from the installer entrypoint. + extract_provider_block() { + # Order matters: check the boundary BEFORE printing so the next + # NEMOCLAW_* printf line (e.g. NEMOCLAW_POLICY_MODE) does not bleed + # into the block. `custom` is both a canonical provider value and a + # POLICY_MODE token, so a leaked POLICY line would falsely make it + # appear that `custom` is documented even after removal. + awk ' + /printf .*NEMOCLAW_PROVIDER/ { in_block = 1; print; next } + in_block && /printf .*NEMOCLAW_/ && !/NEMOCLAW_PROVIDER/ { + in_block = 0 + } + in_block { print } + ' "$1" + } + + local _bootstrap_block _payload_block _drift=0 + _bootstrap_block="$(extract_provider_block "$BOOTSTRAP_SH")" + if [[ -z "$_bootstrap_block" ]]; then + echo "check-docs: [install] no NEMOCLAW_PROVIDER block found in $BOOTSTRAP_SH" >&2 + return 1 + fi + if [[ -f "$PAYLOAD_SH" ]]; then + _payload_block="$(extract_provider_block "$PAYLOAD_SH")" + fi + + # Tokenize each block into the discrete provider identifiers it mentions + # so we can exact-match (not substring-match) against the canonical list. + # Substring matching would let `anthropic` falsely pass when only + # `anthropicCompatible` appears. + # The pattern allows camelCase since `anthropicCompatible` is canonical. + # `\n` literals in printf strings are stripped first so tokens at line + # ends (e.g. `routed\n"`) reduce to the bare identifier. + tokenize_provider_block() { + # Drop `(aliases: cloud -> build, ...)` lines (alias keys aren't + # canonical providers and would falsely fail the bidirectional check) + # and the shell tokens `printf` / `NEMOCLAW_PROVIDER` that appear + # because the block opens with a `printf " NEMOCLAW_PROVIDER ..."` + # line. Both filters exist solely to clean up tokenization artifacts; + # they don't relax the actual provider-name check. + printf '%s\n' "$1" \ + | grep -v '(aliases:' \ + | sed 's/\\n//g' \ + | tr '"`,()|' '\n' \ + | awk '{ for (i = 1; i <= NF; i++) print $i }' \ + | grep -E '^[a-zA-Z][a-zA-Z0-9-]*$' \ + | grep -vxE 'printf|NEMOCLAW_PROVIDER' \ + | LC_ALL=C sort -u + } + + local _bootstrap_values _payload_values="" + _bootstrap_values="$(tokenize_provider_block "$_bootstrap_block")" + if [[ -n "${_payload_block:-}" ]]; then + _payload_values="$(tokenize_provider_block "$_payload_block")" + fi + + IFS=',' read -ra _values <<<"$_canonical" + for _raw in "${_values[@]}"; do + local v + v="$(echo "$_raw" | tr -d '[:space:]')" + [[ -z "$v" ]] && continue + case "$v" in install-* | start-windows-ollama) continue ;; esac + if ! grep -qxF -- "$v" <<<"$_bootstrap_values"; then + echo "check-docs: [install] provider \"$v\" canonical but absent from $BOOTSTRAP_SH bootstrap_usage" >&2 + _drift=1 + fi + if [[ -n "$_payload_values" ]] && ! grep -qxF -- "$v" <<<"$_payload_values"; then + echo "check-docs: [install] provider \"$v\" canonical but absent from $PAYLOAD_SH usage()" >&2 + _drift=1 + fi + done + + # Reverse direction: tokens appearing in either install help block but + # not on the canonical list mean the script is advertising a provider + # that the CLI no longer accepts. Build the canonical set with the same + # exemptions used above. + local _canonical_values + _canonical_values="$( + printf '%s\n' "$_canonical" \ + | tr ',' '\n' \ + | sed 's/[[:space:]]//g' \ + | grep -vxE 'install-.*|start-windows-ollama' \ + | grep -E '^[a-zA-Z][a-zA-Z0-9-]*$' \ + | LC_ALL=C sort -u + )" + while IFS= read -r v; do + [[ -z "$v" ]] && continue + if ! grep -qxF -- "$v" <<<"$_canonical_values"; then + echo "check-docs: [install] provider \"$v\" appears in $BOOTSTRAP_SH bootstrap_usage but is not canonical" >&2 + _drift=1 + fi + done <<<"$_bootstrap_values" + if [[ -n "$_payload_values" ]]; then + while IFS= read -r v; do + [[ -z "$v" ]] && continue + if ! grep -qxF -- "$v" <<<"$_canonical_values"; then + echo "check-docs: [install] provider \"$v\" appears in $PAYLOAD_SH usage() but is not canonical" >&2 + _drift=1 + fi + done <<<"$_payload_values" + fi + + local COMMANDS_REF="$REPO_ROOT/docs/reference/commands.mdx" + if [[ ! -f "$COMMANDS_REF" ]]; then + echo "check-docs: [install] missing $COMMANDS_REF" >&2 + return 1 + fi + + local _doc_provider_row _doc_provider_values + _doc_provider_row="$(grep -F "| \`NEMOCLAW_PROVIDER\` |" "$COMMANDS_REF" || true)" + if [[ -z "$_doc_provider_row" ]]; then + echo "check-docs: [install] no NEMOCLAW_PROVIDER row found in ${COMMANDS_REF#"$REPO_ROOT"/}" >&2 + _drift=1 + else + _doc_provider_values="$( + printf '%s\n' "$_doc_provider_row" \ + | awk -F '|' '{ print $3 }' \ + | grep -oE "\`[a-zA-Z][a-zA-Z0-9-]*\`" \ + | tr -d '`' \ + | grep -vxE 'install-.*|start-windows-ollama' \ + | LC_ALL=C sort -u + )" + while IFS= read -r v; do + [[ -z "$v" ]] && continue + if ! grep -qxF -- "$v" <<<"$_doc_provider_values"; then + echo "check-docs: [install] provider \"$v\" canonical but absent from ${COMMANDS_REF#"$REPO_ROOT"/} NEMOCLAW_PROVIDER row" >&2 + _drift=1 + fi + done <<<"$_canonical_values" + while IFS= read -r v; do + [[ -z "$v" ]] && continue + if ! grep -qxF -- "$v" <<<"$_canonical_values"; then + echo "check-docs: [install] provider \"$v\" appears in ${COMMANDS_REF#"$REPO_ROOT"/} NEMOCLAW_PROVIDER row but is not canonical" >&2 + _drift=1 + fi + done <<<"$_doc_provider_values" + fi + + if [[ "$_drift" -ne 0 ]]; then + return 1 + fi + + log "[install] parity OK" + log "[install] done." + return 0 +} + +# --- Markdown links ------------------------------------------------------------- + +collect_default_docs() { + local f + for f in \ + "$REPO_ROOT/README.md" \ + "$REPO_ROOT/CONTRIBUTING.md" \ + "$REPO_ROOT/docs/CONTRIBUTING.md" \ + "$REPO_ROOT/SECURITY.md" \ + "$REPO_ROOT/spark-install.md" \ + "$REPO_ROOT/CODE_OF_CONDUCT.md" \ + "$REPO_ROOT/.github/PULL_REQUEST_TEMPLATE.md"; do + [[ -f "$f" ]] && printf '%s\n' "$f" + done + if [[ -d "$REPO_ROOT/docs" ]]; then + find "$REPO_ROOT/docs" -type f \( -name '*.md' -o -name '*.mdx' \) | LC_ALL=C sort + fi + if [[ "$WITH_SKILLS" -eq 1 && -d "$REPO_ROOT/.agents/skills" ]]; then + find "$REPO_ROOT/.agents/skills" -type f -name '*.md' | LC_ALL=C sort + fi +} + +extract_targets() { + LC_ALL=C perl -CS -ne ' + if ($in_fence) { + if (/^\s*(`{3,}|~{3,})(.*)$/) { + my $fence = $1; + my $rest = $2; + my $char = substr($fence, 0, 1); + my $length = length($fence); + if ($char eq $fch && $length >= $flen && $rest =~ /^\s*$/) { + ($in_fence, $fch, $flen) = (0, "", 0); + } + } + next; + } + + my $line = $.; + my $text = $_; + my $visible = ""; + + while (length $text) { + if ($in_comment) { + if ($text =~ s/^(.*?)-->//s) { + $in_comment = 0; + next; + } + $text = ""; + next; + } + + if ($text =~ s/^(.*?)/) { + die "malformed HTML comment\n"; + } + + $visible .= $text; + last; + } + + if ($visible =~ /^\s*(`{3,}|~{3,})(.*)$/) { + my $fence = $1; + my $char = substr($fence, 0, 1); + my $length = length($fence); + ($in_fence, $fch, $flen) = (1, $char, $length); + next; + } + + my $scan = $visible; + $scan =~ s/`[^`]*`//g; + while ($scan =~ /\!?\[[^\]]*\]\(([^)\s]+)(?:\s+["'"'"'][^)"'"'"']*["'"'"'])?\)/g) { print $line . "\t" . $1 . "\n"; } + while ($scan =~ /<(https?:[^>\s]+)>/g) { print $line . "\t" . $1 . "\n"; } + while ($scan =~ /\bhref=(["'"'"'])([^"'"'"'\s]+)\1/g) { print $line . "\t" . $2 . "\n"; } + END { + die "malformed HTML comment\n" if $in_comment; + } + ' -- "$1" +} + +FERN_ROUTE_INDEX_LOADED=0 +FERN_ROUTE_INDEX="" + +load_fern_route_index() { + [[ "$FERN_ROUTE_INDEX_LOADED" -eq 1 ]] && return 0 + FERN_ROUTE_INDEX_LOADED=1 + + local nav_yml="${CHECK_DOCS_FERN_NAV_YML:-$REPO_ROOT/docs/index.yml}" + [[ -f "$nav_yml" ]] || return 0 + if ! command -v "$NODE" >/dev/null 2>&1; then + return 0 + fi + + # Build a lightweight route index from Fern navigation without requiring npm + # dependencies. Each emitted row is: TAB . + # The parser intentionally handles the subset used by docs/index.yml: + # variants, nested sections with slugs, and pages/sections with path+slug. + local _fern_route_index_err + _fern_route_index_err="$(mktemp)" + if ! FERN_ROUTE_INDEX="$( + "$NODE" - "$nav_yml" <<'NODE' 2>"$_fern_route_index_err" +const fs = require("node:fs"); +const navPath = process.argv[2]; +const lines = fs.readFileSync(navPath, "utf8").split(/\r?\n/); + +let variant = ""; +let stack = []; +let current = null; +const rows = []; + +function clean(value) { + let out = value.trim(); + const hash = out.indexOf(" #"); + if (hash >= 0) out = out.slice(0, hash).trim(); + if ((out.startsWith('"') && out.endsWith('"')) || (out.startsWith("'") && out.endsWith("'"))) { + out = out.slice(1, -1); + } + return out; +} + +function maybeEmit(item) { + if (!item || item.emitted || !variant || !item.path || !item.slug || item.indent <= 6) return; + const route = ["user-guide", variant, ...item.parent, item.slug].join("/"); + rows.push(`${item.path}\t${route}`); + const sourcePath = agentVariantSourcePath(item.path); + if (sourcePath && sourcePath !== item.path) { + rows.push(`${sourcePath}\t${route}`); + } + item.emitted = true; +} + +function agentVariantSourcePath(navPath) { + const match = navPath.match(/^_build\/agent-variants\/(.+)\.(?:openclaw|hermes)\.generated\.mdx$/); + return match ? `${match[1]}.mdx` : null; +} + +function maybePushSection(item) { + if (!item || item.pushed || item.type !== "section" || !item.slug || item.indent <= 6) return; + stack.push({ indent: item.indent, slug: item.slug }); + item.pushed = true; +} + +for (const line of lines) { + const itemMatch = line.match(/^(\s*)-\s+(page|section|link|title):/); + if (itemMatch) { + const indent = itemMatch[1].length; + const type = itemMatch[2]; + while (stack.length && stack[stack.length - 1].indent >= indent) stack.pop(); + if (indent === 6 && type === "title") { + variant = ""; + stack = []; + } + current = { + indent, + type, + parent: stack.map((part) => part.slug), + path: "", + slug: "", + emitted: false, + pushed: false, + }; + continue; + } + + const propMatch = line.match(/^(\s*)(path|slug):\s*(.+?)\s*$/); + if (!propMatch || !current) continue; + const indent = propMatch[1].length; + if (indent !== current.indent + 2) continue; + + const key = propMatch[2]; + const value = clean(propMatch[3]); + if (current.indent === 6 && key === "slug") { + variant = value; + stack = []; + continue; + } + if (key === "path") current.path = value; + if (key === "slug") current.slug = value; + maybeEmit(current); + maybePushSection(current); +} + +if (rows.length === 0) { + throw new Error(`no Fern routes found in ${navPath}`); +} +process.stdout.write(rows.join("\n")); +NODE + )"; then + echo "check-docs: [links] failed to parse Fern navigation ${nav_yml#"$REPO_ROOT"/}: $(tr '\n' ' ' <"$_fern_route_index_err" | sed 's/[[:space:]]\+/ /g; s/^ //; s/ $//')" >&2 + rm -f "$_fern_route_index_err" + return 1 + fi + rm -f "$_fern_route_index_err" +} + +normalize_fern_route_path() { + local input="$1" part + input="${input#/}" + case "$input" in + nemoclaw/latest/*) input="${input#nemoclaw/latest/}" ;; + nemoclaw/*) input="${input#nemoclaw/}" ;; + latest/*) input="${input#latest/}" ;; + esac + input="${input%.mdx}" + input="${input%.md}" + + local -a parts=() out=() + local IFS='/' + read -r -a parts <<<"$input" + unset IFS + for part in "${parts[@]}"; do + case "$part" in + "" | .) ;; + ..) + if [[ "${#out[@]}" -eq 0 ]]; then + return 1 + fi + unset 'out[${#out[@]}-1]' + ;; + *) out+=("$part") ;; + esac + done + + local joined + joined="$( + IFS=/ + printf '%s' "${out[*]}" + )" + printf '%s' "$joined" +} + +fern_route_exists() { + local route="$1" candidate + if ! load_fern_route_index; then + return 3 + fi + [[ -n "$FERN_ROUTE_INDEX" ]] || return 1 + + route="$(normalize_fern_route_path "$route")" || return 1 + local -a candidates=("$route") + case "$route" in + openclaw) + candidates+=("user-guide/openclaw/home") + ;; + hermes) + candidates+=("user-guide/hermes/home") + ;; + user-guide/openclaw | user-guide/hermes) + candidates+=("$route/home") + ;; + openclaw/* | hermes/*) + candidates+=("user-guide/$route") + ;; + user-guide/*) ;; + about/* | get-started/* | inference/* | manage-sandboxes/* | network-policy/* | deployment/* | monitoring/* | security/* | reference/* | resources/*) + candidates+=("user-guide/openclaw/$route") + ;; + esac + if [[ "$route" == get-started/quickstart-hermes ]]; then + candidates+=("user-guide/hermes/get-started/quickstart-hermes") + elif [[ "$route" == get-started/hermes/* ]]; then + candidates+=("user-guide/hermes/get-started/${route#get-started/hermes/}") + fi + + local _source indexed_route + for candidate in "${candidates[@]}"; do + while IFS=$'\t' read -r _source indexed_route || [[ -n "${indexed_route:-}" ]]; do + [[ "$indexed_route" == "$candidate" ]] && return 0 + done <<<"$FERN_ROUTE_INDEX" + done + return 1 +} + +fern_relative_ref_exists() { + local md_path="$1" stripped="$2" + local abs_md="$md_path" source_rel current route + [[ "$abs_md" == /* ]] || abs_md="$REPO_ROOT/$abs_md" + case "$abs_md" in + "$REPO_ROOT/docs/"*) source_rel="${abs_md#"$REPO_ROOT/docs/"}" ;; + *) return 1 ;; + esac + + if ! load_fern_route_index; then + return 3 + fi + [[ -n "$FERN_ROUTE_INDEX" ]] || return 1 + + while IFS=$'\t' read -r _source current || [[ -n "${current:-}" ]]; do + [[ "$_source" == "$source_rel" ]] || continue + route="${current%/*}/$stripped" + local _fern_rc + set +e + fern_route_exists "$route" + _fern_rc=$? + set -e + if [[ "$_fern_rc" -eq 0 ]]; then + return 0 + elif [[ "$_fern_rc" -eq 3 ]]; then + return 3 + fi + done <<<"$FERN_ROUTE_INDEX" + return 1 +} + +source_ref_exists() { + local base_dir="$1" stripped="$2" candidate + local -a candidates=("$stripped") + if [[ "$stripped" == */ ]]; then + candidates+=("${stripped}index.mdx" "${stripped}index.md") + else + candidates+=("$stripped.mdx" "$stripped.md" "$stripped/index.mdx" "$stripped/index.md") + fi + + for candidate in "${candidates[@]}"; do + if (cd "$base_dir" && [[ -e "$candidate" ]]); then + return 0 + fi + done + return 1 +} + +site_source_ref_exists() { + local stripped="$1" + local site_path="${stripped#/}" + local -a site_paths=("$site_path") + case "$site_path" in + nemoclaw/latest/*) site_paths+=("${site_path#nemoclaw/latest/}") ;; + nemoclaw/*) site_paths+=("${site_path#nemoclaw/}") ;; + latest/*) site_paths+=("${site_path#latest/}") ;; + esac + case "$site_path" in + user-guide/openclaw/*) site_paths+=("${site_path#user-guide/openclaw/}") ;; + user-guide/hermes/*) site_paths+=("${site_path#user-guide/hermes/}") ;; + openclaw/*) site_paths+=("${site_path#openclaw/}") ;; + hermes/*) site_paths+=("${site_path#hermes/}") ;; + esac + + local route_path + for route_path in "${site_paths[@]}"; do + if source_ref_exists "$REPO_ROOT/docs" "$route_path"; then + return 0 + fi + done + return 1 +} + +has_markdown_extension() { + case "$1" in + *.md | *.mdx) return 0 ;; + *) return 1 ;; + esac +} + +check_local_ref() { + local md_path="$1" line_no="$2" target="$3" + local stripped + + stripped="${target%%\#*}" + stripped="${stripped%%\?*}" + + [[ -z "$stripped" ]] && return 0 + [[ "$stripped" == mailto:* ]] && return 0 + [[ "$stripped" == tel:* ]] && return 0 + [[ "$stripped" == javascript:* ]] && return 0 + + if [[ "$stripped" == http://* || "$stripped" == https://* ]]; then + return 2 + fi + if [[ "$stripped" == *://* ]]; then + return 0 + fi + + if [[ "$stripped" == /* ]]; then + local _fern_rc + set +e + fern_route_exists "$stripped" + _fern_rc=$? + set -e + if [[ "$_fern_rc" -eq 0 ]] && has_markdown_extension "$stripped"; then + echo "check-docs: [links] route-style link should omit .md/.mdx extension in $md_path:$line_no -> $target" >&2 + return 1 + fi + if [[ "$_fern_rc" -eq 0 ]]; then + return 0 + elif [[ "$_fern_rc" -eq 3 ]]; then + return 1 + fi + if site_source_ref_exists "$stripped"; then + return 0 + fi + echo "check-docs: [links] broken site route in $md_path:$line_no -> $target" >&2 + return 1 + fi + + local _fern_relative_rc + set +e + fern_relative_ref_exists "$md_path" "$stripped" + _fern_relative_rc=$? + set -e + if [[ "$_fern_relative_rc" -eq 0 ]] && has_markdown_extension "$stripped"; then + echo "check-docs: [links] route-style link should omit .md/.mdx extension in $md_path:$line_no -> $target" >&2 + return 1 + fi + if [[ "$_fern_relative_rc" -eq 0 ]]; then + return 0 + elif [[ "$_fern_relative_rc" -eq 3 ]]; then + return 1 + fi + if source_ref_exists "$(dirname "$md_path")" "$stripped"; then + return 0 + fi + echo "check-docs: [links] broken local link in $md_path:$line_no -> $target" >&2 + return 1 +} + +check_remote_url() { + local url="$1" + if ! command -v "$CURL" >/dev/null 2>&1; then + echo "check-docs: [links] curl not found; cannot verify $url" >&2 + return 1 + fi + if ! "$CURL" -fsS -L -o /dev/null \ + --connect-timeout 12 --max-time 35 \ + -A 'NemoClaw-doc-link-check/1.0 (+https://github.com/NVIDIA/NemoClaw)' \ + "$url" 2>/dev/null; then + echo "check-docs: [links] unreachable URL: $url" >&2 + return 1 + fi + return 0 +} + +# Normalized form: strip #fragment and trailing slash for ignore-list comparison. +normalize_url_for_ignore_match() { + local u="$1" + u="${u%%\#*}" + u="${u%/}" + printf '%s' "$u" +} + +# Built-in skip list: pages that often fail in CI (bot wall, redirects, or flaky) but are non-critical for doc correctness. +check_docs_default_ignored_urls() { + printf '%s\n' \ + 'https://github.com/NVIDIA/NemoClaw/commits/main' \ + 'https://github.com/NVIDIA/NemoClaw/pulls?q=is%3Apr+is%3Amerged' \ + 'https://github.com/NVIDIA/NemoClaw/pulls?q=is:pr+is:merged' \ + 'https://github.com/openclaw/openclaw/issues/49950' +} + +url_should_skip_remote_probe() { + local url="$1" + local nu ign _re + nu="$(normalize_url_for_ignore_match "$url")" + + while IFS= read -r ign || [[ -n "${ign:-}" ]]; do + [[ -z "${ign:-}" ]] && continue + [[ "$(normalize_url_for_ignore_match "$ign")" == "$nu" ]] && return 0 + done < <(check_docs_default_ignored_urls) + + if [[ -n "${CHECK_DOC_LINKS_IGNORE_EXTRA:-}" ]]; then + local -a _extra_parts=() + local IFS=',' + read -ra _extra_parts <<<"${CHECK_DOC_LINKS_IGNORE_EXTRA}" + unset IFS + for ign in "${_extra_parts[@]}"; do + ign="${ign#"${ign%%[![:space:]]*}"}" + ign="${ign%"${ign##*[![:space:]]}"}" + [[ -z "$ign" ]] && continue + [[ "$(normalize_url_for_ignore_match "$ign")" == "$nu" ]] && return 0 + done + fi + + if [[ -n "${CHECK_DOC_LINKS_IGNORE_URL_REGEX:-}" ]]; then + _re="${CHECK_DOC_LINKS_IGNORE_URL_REGEX}" + [[ "$url" =~ $_re ]] && return 0 + fi + + return 1 +} + +run_links_check() { + local -a DOC_FILES + if [[ ${#EXTRA_FILES[@]} -gt 0 ]]; then + DOC_FILES=("${EXTRA_FILES[@]}") + else + DOC_FILES=() + while IFS= read -r _docf || [[ -n "${_docf:-}" ]]; do + [[ -z "${_docf:-}" ]] && continue + DOC_FILES+=("$_docf") + done < <(collect_default_docs | LC_ALL=C sort -u) + fi + + if [[ ${#DOC_FILES[@]} -eq 0 ]]; then + echo "check-docs: [links] no documentation files to scan under $REPO_ROOT" >&2 + return 1 + fi + + log "[links] repository root: $REPO_ROOT" + if [[ "$WITH_SKILLS" -eq 1 ]]; then + log "[links] scope: default doc set + .agents/skills/**/*.md" + else + log "[links] scope: README, CONTRIBUTING, SECURITY, spark-install, CODE_OF_CONDUCT, .github PR template, docs/**/*.{md,mdx}" + fi + if [[ "$CHECK_DOC_LINKS_REMOTE" != 0 ]]; then + log "[links] remote: curl unique http(s) targets (disable: CHECK_DOC_LINKS_REMOTE=0 or --local-only)" + log "[links] remote: built-in skip list for flaky/GitHub pages (override: CHECK_DOC_LINKS_IGNORE_EXTRA, CHECK_DOC_LINKS_IGNORE_URL_REGEX)" + else + log "[links] remote: skipped (local paths only)" + fi + log "[links] Markdown file(s) (${#DOC_FILES[@]}):" + local md + for md in "${DOC_FILES[@]}"; do + case "$md" in + "$REPO_ROOT"/*) log "[links] ${md#"$REPO_ROOT"/}" ;; + *) log "[links] $md" ;; + esac + done + + local failures=0 + declare -a REMOTE_URLS=() + + log "[links] phase 1/2: local file targets and Fern routes for [](url) / ![]() / (code fences skipped)" + for md in "${DOC_FILES[@]}"; do + if [[ ! -f "$md" ]]; then + echo "check-docs: [links] missing file: $md" >&2 + failures=1 + continue + fi + local target rc + local _targets_output _targets_err + _targets_err="$(mktemp)" + if ! _targets_output="$(extract_targets "$md" 2>"$_targets_err")"; then + echo "check-docs: [links] malformed HTML comment in $md: $(tr '\n' ' ' <"$_targets_err" | sed 's/[[:space:]]\+/ /g; s/^ //; s/ $//')" >&2 + rm -f "$_targets_err" + failures=1 + continue + fi + rm -f "$_targets_err" + local line_no + while IFS=$'\t' read -r line_no target || [[ -n "${target:-}" ]]; do + [[ -z "$target" ]] && continue + set +e + check_local_ref "$md" "$line_no" "$target" + rc=$? + set -e + if [[ "$rc" -eq 0 ]]; then + continue + elif [[ "$rc" -eq 2 ]]; then + REMOTE_URLS+=("$target") + else + failures=1 + fi + done <<<"$_targets_output" + done + + if [[ "$failures" -ne 0 ]]; then + log "[links] phase 1 failed" + return 1 + fi + log "[links] phase 1 OK (local paths and Fern routes resolve)" + + local _n_raw _deduped _unique _i _u url + _n_raw="${#REMOTE_URLS[@]}" + _deduped="" + if [[ ${#REMOTE_URLS[@]} -gt 0 ]]; then + _deduped="$(printf '%s\n' "${REMOTE_URLS[@]}" | LC_ALL=C sort -u)" + fi + _unique="$(printf '%s\n' "${REMOTE_URLS[@]}" | LC_ALL=C sort -u | grep -c . || true)" + log "[links] http(s): ${_n_raw} reference(s) → ${_unique} unique URL(s)" + if [[ -n "$_deduped" ]]; then + log "[links] unique http(s) URL(s) (alphabetically):" + while IFS= read -r _u || [[ -n "${_u:-}" ]]; do + [[ -z "${_u:-}" ]] && continue + log "[links] ${_u}" + done <<<"$_deduped" + fi + + if [[ "$CHECK_DOC_LINKS_REMOTE" != 0 ]]; then + if [[ -n "$_deduped" ]]; then + local _probe_list="" _skip_count=0 _probe_n=0 + while IFS= read -r url || [[ -n "${url:-}" ]]; do + [[ -z "${url:-}" ]] && continue + if url_should_skip_remote_probe "$url"; then + log "[links] skipped (ignore list): ${url}" + _skip_count=$((_skip_count + 1)) + else + _probe_list+="${url}"$'\n' + fi + done <<<"$_deduped" + _probe_n="$(printf '%s\n' "$_probe_list" | grep -c . || true)" + if [[ "$_skip_count" -gt 0 ]]; then + log "[links] phase 2/2: curl ${_probe_n} URL(s), ${_skip_count} skipped (GET, -L, fail 4xx/5xx)" + else + log "[links] phase 2/2: curl ${_probe_n} URL(s) (GET, -L, fail 4xx/5xx)" + fi + _i=0 + while IFS= read -r url || [[ -n "${url:-}" ]]; do + [[ -z "${url:-}" ]] && continue + _i=$((_i + 1)) + if [[ "$VERBOSE" -eq 1 ]]; then + log "[links] [${_i}/${_probe_n}] ${url}" + fi + if ! check_remote_url "$url"; then + failures=1 + fi + done <<<"$_probe_list" + else + log "[links] phase 2/2: no http(s) links" + fi + else + if [[ -n "$_deduped" ]]; then + log "[links] phase 2/2: skipped ${_unique} URL(s) (local-only)" + else + log "[links] phase 2/2: skipped (no http(s) links)" + fi + fi + + if [[ "$failures" -ne 0 ]]; then + log "[links] phase 2 failed" + return 1 + fi + if [[ "$CHECK_DOC_LINKS_REMOTE" != 0 ]] && [[ ${_unique:-0} -gt 0 ]]; then + log "[links] phase 2 OK (${_unique} unique http(s); probed those not in ignore list)" + fi + log "[links] summary: ${#DOC_FILES[@]} file(s), local OK$( + [[ "$CHECK_DOC_LINKS_REMOTE" != 0 ]] && [[ ${_unique:-0} -gt 0 ]] && printf ', %s remote OK' "${_unique}" + )$( + [[ "$CHECK_DOC_LINKS_REMOTE" == 0 ]] && [[ ${_unique:-0} -gt 0 ]] && printf ' (%s remote not checked)' "${_unique}" + )" + log "[links] done." + return 0 +} + +# --- main --------------------------------------------------------------------- + +_planned=() +[[ "$RUN_CLI" -eq 1 ]] && _planned+=("[cli]") +[[ "$RUN_INSTALL" -eq 1 ]] && _planned+=("[install]") +[[ "$RUN_LINKS" -eq 1 ]] && _planned+=("[links]") +log "running: ${_planned[*]}" +unset _planned + +if [[ "$RUN_CLI" -eq 1 ]]; then + if ! run_cli_check; then + exit 1 + fi +fi + +if [[ "$RUN_INSTALL" -eq 1 ]]; then + if ! run_install_check; then + exit 1 + fi +fi + +if [[ "$RUN_LINKS" -eq 1 ]]; then + if ! run_links_check; then + exit 1 + fi +fi + +log "all requested checks passed." +exit 0 diff --git a/test/e2e-vpn/e2e-cloud-experimental/checks/02-inference-local-http.sh b/test/e2e-vpn/e2e-cloud-experimental/checks/02-inference-local-http.sh new file mode 100755 index 00000000000..c247ba734dc --- /dev/null +++ b/test/e2e-vpn/e2e-cloud-experimental/checks/02-inference-local-http.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Case: inside sandbox, https://inference.local responds (HTTP 200). +# Pattern aligned with test-full-e2e.sh (ssh via openshell sandbox ssh-config). + +set -euo pipefail + +SANDBOX_NAME="${SANDBOX_NAME:-${NEMOCLAW_SANDBOX_NAME:-e2e-cloud-experimental}}" + +die() { + printf '%s\n' "02-inference-local-http: FAIL: $*" >&2 + exit 1 +} + +ssh_config="$(mktemp)" +trap 'rm -f "$ssh_config"' EXIT + +openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config" 2>/dev/null \ + || die "openshell sandbox ssh-config failed for '${SANDBOX_NAME}'" + +TIMEOUT_CMD="" +command -v timeout >/dev/null 2>&1 && TIMEOUT_CMD="timeout 90" +command -v gtimeout >/dev/null 2>&1 && TIMEOUT_CMD="gtimeout 90" + +# GET /v1/models — lightweight 200 check (no API key in curl; gateway routes inference) +ssh_host="openshell-${SANDBOX_NAME}" +curl_inner='curl -sS -o /dev/null -w "%{http_code}" --max-time 60 https://inference.local/v1/models' + +set +e +# stderr from ssh can include host-key noise; curl -w code should be the only stdout line +http_code=$( + $TIMEOUT_CMD ssh -F "$ssh_config" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o LogLevel=ERROR \ + "$ssh_host" \ + "$curl_inner" 2>/dev/null +) +ssh_rc=$? +set -e +http_code="$(echo "$http_code" | tr -d '\r' | tail -n 1)" + +[ "$ssh_rc" -eq 0 ] || die "ssh/curl failed (exit $ssh_rc): ${http_code:0:200}" +[ "$http_code" = "200" ] || die "expected HTTP 200 from https://inference.local/v1/models, got '${http_code}'" + +printf '%s\n' "02-inference-local-http: OK (HTTP 200 on /v1/models)" +exit 0 diff --git a/test/e2e-vpn/e2e-cloud-experimental/checks/03-security-checks.sh b/test/e2e-vpn/e2e-cloud-experimental/checks/03-security-checks.sh new file mode 100755 index 00000000000..40c257bc211 --- /dev/null +++ b/test/e2e-vpn/e2e-cloud-experimental/checks/03-security-checks.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Case: host-side security checks (add sections here as the suite grows). +# +# Current: +# - VDR3 #13: cloud API token env var must not appear in `ps` (full value or env-style argv assignment leak). +# +# We avoid grepping the live secret on the command line (that would leak the key into ps). + +set -euo pipefail + +# The caller can point this check at the active hosted-inference credential. +_api_key_env_name="${NEMOCLAW_E2E_CLOUD_API_KEY_ENV:-NVIDIA_API_KEY}" +if [[ ! "$_api_key_env_name" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]]; then + printf '%s\n' "03-security-checks: FAIL: invalid cloud API token env var name: ${_api_key_env_name}" >&2 + exit 1 +fi +: "${!_api_key_env_name:?cloud API token env var must be set (export before running)}" + +die() { + printf '%s\n' "03-security-checks: FAIL: $*" >&2 + exit 1 +} + +# ── VDR3 #13: API key not in ps ───────────────────────────────────── +ps_lines=$( (ps auxww 2>/dev/null || ps auxeww 2>/dev/null || ps aux 2>/dev/null) || true) +[ -n "$ps_lines" ] || die "api-key-in-ps: could not capture ps output" + +_api_key_value="${!_api_key_env_name}" +while IFS= read -r line; do + case "$line" in + *"$_api_key_value"*) die "api-key-in-ps: full API key material appears in ps output" ;; + esac +done <<<"$ps_lines" + +# argv-style leak: NAME=. The caller can override or +# disable this marker with NEMOCLAW_E2E_CLOUD_API_KEY_ARGV_PREFIX. +_key_argv_prefix_marker="${NEMOCLAW_E2E_CLOUD_API_KEY_ARGV_PREFIX:-}" +if [ -z "${NEMOCLAW_E2E_CLOUD_API_KEY_ARGV_PREFIX+x}" ]; then + _key_argv_prefix_marker="$(printf '%.6s' "$_api_key_value")" +fi +if [ -n "$_key_argv_prefix_marker" ]; then + _key_argv_needle="${_api_key_env_name}=${_key_argv_prefix_marker}" + while IFS= read -r line; do + case "$line" in + *"${_key_argv_needle}"*) die "api-key-in-ps: env-style API key argv leak in ps" ;; + esac + done <<<"$ps_lines" +fi + +printf '%s\n' "03-security-checks: OK (api-key-in-ps)" +exit 0 diff --git a/test/e2e-vpn/e2e-cloud-experimental/checks/04-landlock-readonly.sh b/test/e2e-vpn/e2e-cloud-experimental/checks/04-landlock-readonly.sh new file mode 100755 index 00000000000..ae529114031 --- /dev/null +++ b/test/e2e-vpn/e2e-cloud-experimental/checks/04-landlock-readonly.sh @@ -0,0 +1,117 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Case: Landlock filesystem enforcement (#804). +# +# These checks run INSIDE a real OpenShell sandbox where Landlock is active. +# They verify that the kernel enforces the filesystem policy: /sandbox and +# /sandbox/.openclaw are writable (mutable default), trusted shell startup +# files remain read-only, system paths are read-only, and /tmp is writable. +# +# The Docker-only e2e tests (test/e2e-gateway-isolation.sh) cover DAC +# enforcement but cannot exercise Landlock. This script closes that gap. +# +# Prerequisites: +# - openshell on PATH, sandbox exists and is Ready +# - SANDBOX_NAME set (default: e2e-cloud-experimental) + +set -euo pipefail + +SANDBOX_NAME="${SANDBOX_NAME:-${NEMOCLAW_SANDBOX_NAME:-e2e-cloud-experimental}}" + +die() { + printf '%s\n' "04-landlock-readonly: FAIL: $*" >&2 + exit 1 +} +ok() { printf '%s\n' "04-landlock-readonly: OK ($*)"; } +info() { printf '%s\n' "04-landlock-readonly: $*"; } + +PASSED=0 +FAILED=0 + +pass() { + ok "$1" + PASSED=$((PASSED + 1)) +} +fail_test() { + printf '%s\n' "04-landlock-readonly: FAIL: $1" >&2 + FAILED=$((FAILED + 1)) +} + +# Helper: run a command inside the sandbox via openshell +sandbox_exec() { + openshell sandbox exec --name "$SANDBOX_NAME" -- bash -c "$1" 2>&1 +} + +info "Running Landlock filesystem checks in sandbox: $SANDBOX_NAME" + +# ── 1: CAN create files in /sandbox (include_workdir: true) ─────── +info "1. Can create files in /sandbox (home is writable)" +OUT=$(sandbox_exec "touch /sandbox/landlock-test && echo OK || echo FAILED" || true) +if echo "$OUT" | grep -q "OK"; then + pass "sandbox home is Landlock writable" +else + fail_test "/sandbox is NOT writable under Landlock: $OUT" +fi + +# ── 2: Cannot modify trusted shell startup files ───────────────── +info "2. Cannot modify .bashrc/.profile (trusted startup snippets)" +OUT=$(sandbox_exec "echo '# test' >> /sandbox/.bashrc 2>&1 || echo BASHRC_BLOCKED; sed -i '/^# test$/d' /sandbox/.bashrc 2>/dev/null || true; echo '# test' >> /sandbox/.profile 2>&1 || echo PROFILE_BLOCKED; sed -i '/^# test$/d' /sandbox/.profile 2>/dev/null || true" || true) +if echo "$OUT" | grep -q "BASHRC_BLOCKED" && echo "$OUT" | grep -q "PROFILE_BLOCKED"; then + pass ".bashrc/.profile remain read-only while home is mutable" +else + fail_test ".bashrc/.profile should be read-only trusted startup files: $OUT" +fi + +# ── 3: CAN write to .openclaw (mutable default) ────────────────── +info "3. Can create files in .openclaw (mutable default)" +OUT=$(sandbox_exec "touch /sandbox/.openclaw/landlock-test && echo OK || echo FAILED" || true) +if echo "$OUT" | grep -q "OK"; then + pass ".openclaw dir is writable in mutable-default mode" +else + fail_test ".openclaw dir is NOT writable under Landlock: $OUT" +fi + +# ── 4: Cannot write to /usr (system path read-only) ────────────── +info "4. Cannot write to /usr (system path read-only)" +OUT=$(sandbox_exec "touch /usr/landlock-test 2>&1 || echo BLOCKED" || true) +if echo "$OUT" | grep -qi "BLOCKED\|Permission denied\|Read-only\|EACCES"; then + pass "/usr is Landlock read-only" +else + fail_test "/usr is writable under Landlock: $OUT" +fi + +# ── 5: Cannot write to /etc (system path read-only) ────────────── +info "5. Cannot write to /etc (system path read-only)" +OUT=$(sandbox_exec "touch /etc/landlock-test 2>&1 || echo BLOCKED" || true) +if echo "$OUT" | grep -qi "BLOCKED\|Permission denied\|Read-only\|EACCES"; then + pass "/etc is Landlock read-only" +else + fail_test "/etc is writable under Landlock: $OUT" +fi + +# ── 6: CAN write to .nemoclaw/state (Landlock read_write via parent) ─ +info "6. Can write to .nemoclaw/state (Landlock read_write)" +OUT=$(sandbox_exec "touch /sandbox/.nemoclaw/state/landlock-test && echo OK || echo FAILED" || true) +if echo "$OUT" | grep -q "OK"; then + pass ".nemoclaw/state is writable under Landlock" +else + fail_test ".nemoclaw/state is NOT writable under Landlock: $OUT" +fi + +# ── 7: CAN write to /tmp (Landlock read_write) ─────────────────── +info "7. Can write to /tmp (Landlock read_write)" +OUT=$(sandbox_exec "touch /tmp/landlock-test && echo OK || echo FAILED" || true) +if echo "$OUT" | grep -q "OK"; then + pass "/tmp is writable under Landlock" +else + fail_test "/tmp is NOT writable under Landlock: $OUT" +fi + +# ── Cleanup test artifacts ──────────────────────────────────────── +sandbox_exec "sed -i '/^# test$/d' /sandbox/.bashrc /sandbox/.profile 2>/dev/null || true; rm -f /sandbox/landlock-test /sandbox/.openclaw/landlock-test /sandbox/.nemoclaw/state/landlock-test /usr/landlock-test /etc/landlock-test /tmp/landlock-test 2>/dev/null" || true + +# ── Summary ─────────────────────────────────────────────────────── +printf '%s\n' "04-landlock-readonly: $PASSED passed, $FAILED failed" +[ "$FAILED" -eq 0 ] || exit 1 diff --git a/test/e2e-vpn/e2e-cloud-experimental/checks/05-deepagents-code-landlock-readonly.sh b/test/e2e-vpn/e2e-cloud-experimental/checks/05-deepagents-code-landlock-readonly.sh new file mode 100755 index 00000000000..069ce7cd8be --- /dev/null +++ b/test/e2e-vpn/e2e-cloud-experimental/checks/05-deepagents-code-landlock-readonly.sh @@ -0,0 +1,72 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Case: Deep Agents Code Landlock policy behavior (#4861). +# +# These checks run INSIDE a real OpenShell sandbox where Landlock is active. +# They are intentionally skipped for non-Deep Agents Code sandboxes so the +# shared cloud checks can continue to validate OpenClaw/Hermes sandboxes. + +set -euo pipefail + +SANDBOX_NAME="${SANDBOX_NAME:-${NEMOCLAW_SANDBOX_NAME:-e2e-cloud-onboard}}" +PREFIX="05-deepagents-code-landlock-readonly" + +ok() { printf '%s\n' "${PREFIX}: OK ($*)"; } +info() { printf '%s\n' "${PREFIX}: $*"; } +fail_test() { + printf '%s\n' "${PREFIX}: FAIL: $1" >&2 + FAILED=$((FAILED + 1)) +} +pass() { + ok "$1" + PASSED=$((PASSED + 1)) +} + +sandbox_exec() { + openshell sandbox exec --name "$SANDBOX_NAME" -- bash -c "$1" 2>&1 +} + +PASSED=0 +FAILED=0 + +if ! sandbox_exec "test -d /sandbox/.deepagents && command -v dcode >/dev/null 2>&1" >/dev/null; then + info "SKIP: sandbox '${SANDBOX_NAME}' is not a Deep Agents Code sandbox" + exit 0 +fi + +info "Running Deep Agents Code Landlock checks in sandbox: $SANDBOX_NAME" + +OUT=$(sandbox_exec "touch /sandbox/.deepagents/deepagents-landlock-test && echo OK || echo FAILED" || true) +if echo "$OUT" | grep -q "OK"; then + pass "/sandbox/.deepagents is writable for Deep Agents state" +else + fail_test "/sandbox/.deepagents is NOT writable under Landlock: $OUT" +fi + +OUT=$(sandbox_exec "touch /usr/deepagents-landlock-test 2>&1 || echo BLOCKED" || true) +if echo "$OUT" | grep -qi "BLOCKED\|Permission denied\|Read-only\|EACCES"; then + pass "/usr is Landlock read-only for Deep Agents Code" +else + fail_test "/usr is writable under the Deep Agents Code policy: $OUT" +fi + +OUT=$(sandbox_exec "touch /etc/deepagents-landlock-test 2>&1 || echo BLOCKED" || true) +if echo "$OUT" | grep -qi "BLOCKED\|Permission denied\|Read-only\|EACCES"; then + pass "/etc is Landlock read-only for Deep Agents Code" +else + fail_test "/etc is writable under the Deep Agents Code policy: $OUT" +fi + +OUT=$(sandbox_exec "touch /tmp/deepagents-landlock-test && echo OK || echo FAILED" || true) +if echo "$OUT" | grep -q "OK"; then + pass "/tmp is writable for Deep Agents temporary files" +else + fail_test "/tmp is NOT writable under Landlock: $OUT" +fi + +sandbox_exec "rm -f /sandbox/.deepagents/deepagents-landlock-test /usr/deepagents-landlock-test /etc/deepagents-landlock-test /tmp/deepagents-landlock-test 2>/dev/null || true" || true + +printf '%s\n' "${PREFIX}: $PASSED passed, $FAILED failed" +[ "$FAILED" -eq 0 ] || exit 1 diff --git a/test/e2e-vpn/e2e-cloud-experimental/checks/06-deepagents-code-python-egress.sh b/test/e2e-vpn/e2e-cloud-experimental/checks/06-deepagents-code-python-egress.sh new file mode 100755 index 00000000000..1495004f890 --- /dev/null +++ b/test/e2e-vpn/e2e-cloud-experimental/checks/06-deepagents-code-python-egress.sh @@ -0,0 +1,90 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Case: Deep Agents Code Python egress boundary (#4861). +# +# Deep Agents Code network traffic is attributed to the Python interpreter by +# OpenShell. This live check documents the supported boundary: arbitrary Python +# may use only the hosts explicitly present in policy-additions.yaml, while +# optional Tavily, LangSmith, MCP, and arbitrary hosts remain denied until a +# user adds explicit policy. + +set -euo pipefail + +SANDBOX_NAME="${SANDBOX_NAME:-${NEMOCLAW_SANDBOX_NAME:-e2e-cloud-onboard}}" +PREFIX="06-deepagents-code-python-egress" + +ok() { printf '%s\n' "${PREFIX}: OK ($*)"; } +info() { printf '%s\n' "${PREFIX}: $*"; } +fail_test() { + printf '%s\n' "${PREFIX}: FAIL: $1" >&2 + FAILED=$((FAILED + 1)) +} +pass() { + ok "$1" + PASSED=$((PASSED + 1)) +} + +sandbox_exec() { + openshell sandbox exec --name "$SANDBOX_NAME" -- bash -c "$1" 2>&1 +} + +python_probe() { + local url="$1" + sandbox_exec "python3 - ${url@Q} <<'PY' +import sys +import urllib.request +url = sys.argv[1] +try: + with urllib.request.urlopen(url, timeout=8) as response: + print(f'REACHED:{response.status}') +except Exception as exc: + print(f'BLOCKED:{type(exc).__name__}:{exc}') +PY +" +} + +expect_reached() { + local label="$1" + local url="$2" + local output + output="$(python_probe "$url")" + if echo "$output" | grep -q "REACHED:"; then + pass "arbitrary Python can reach approved ${label} host" + else + fail_test "arbitrary Python could not reach approved ${label} host: $output" + fi +} + +expect_blocked() { + local label="$1" + local url="$2" + local output + output="$(python_probe "$url")" + if echo "$output" | grep -q "BLOCKED:" && ! echo "$output" | grep -q "REACHED:"; then + pass "arbitrary Python cannot reach ${label} without explicit policy" + else + fail_test "arbitrary Python reached ${label} unexpectedly: $output" + fi +} + +PASSED=0 +FAILED=0 + +if ! sandbox_exec "test -d /sandbox/.deepagents && command -v dcode >/dev/null 2>&1" >/dev/null; then + info "SKIP: sandbox '${SANDBOX_NAME}' is not a Deep Agents Code sandbox" + exit 0 +fi + +info "Running Deep Agents Code arbitrary-Python egress checks in sandbox: $SANDBOX_NAME" + +expect_reached "GitHub" "https://api.github.com/" +expect_reached "PyPI" "https://pypi.org/" +expect_blocked "Tavily" "https://api.tavily.com/" +expect_blocked "LangSmith" "https://api.smith.langchain.com/" +expect_blocked "MCP hosts" "https://modelcontextprotocol.io/" +expect_blocked "unapproved hosts" "https://example.com/" + +printf '%s\n' "${PREFIX}: $PASSED passed, $FAILED failed" +[ "$FAILED" -eq 0 ] || exit 1 diff --git a/test/e2e-vpn/e2e-cloud-experimental/cleanup.sh b/test/e2e-vpn/e2e-cloud-experimental/cleanup.sh new file mode 100755 index 00000000000..961d4966f80 --- /dev/null +++ b/test/e2e-vpn/e2e-cloud-experimental/cleanup.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Shared teardown for e2e-cloud-experimental (extracted from test-e2e-cloud-experimental.sh Phase 0 + Phase 6). +# +# Destroys nemoclaw sandbox, OpenShell sandbox, port 18789 forward, and nemoclaw gateway. +# +# Usage: +# SANDBOX_NAME=my-sbx bash test/e2e-vpn/e2e-cloud-experimental/cleanup.sh +# SANDBOX_NAME=my-sbx bash test/e2e-vpn/e2e-cloud-experimental/cleanup.sh --verify +# +# Environment: +# SANDBOX_NAME or NEMOCLAW_SANDBOX_NAME — default: e2e-cloud-experimental +# +# Modes: +# (default) — destroy only (best-effort; always exits 0) +# --verify — destroy then assert sandbox is gone from openshell get + nemoclaw list (exits 1 on failure) + +set -uo pipefail + +pass() { printf '\033[32m PASS: %s\033[0m\n' "$1"; } +fail() { printf '\033[31m FAIL: %s\033[0m\n' "$1"; } +skip() { printf '\033[33m SKIP: %s\033[0m\n' "$1"; } +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } + +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-${SANDBOX_NAME:-e2e-cloud-experimental}}" +VERIFY=0 +if [ "${1:-}" = "--verify" ]; then + VERIFY=1 +fi + +info "e2e-cloud-experimental cleanup: sandbox='${SANDBOX_NAME}' (verify=${VERIFY})" + +if command -v nemoclaw >/dev/null 2>&1; then + nemoclaw "$SANDBOX_NAME" destroy --yes 2>/dev/null || true +fi +if command -v openshell >/dev/null 2>&1; then + openshell sandbox delete "$SANDBOX_NAME" 2>/dev/null || true + openshell forward stop 18789 2>/dev/null || true + openshell gateway destroy -g nemoclaw 2>/dev/null || true +fi + +if [ "$VERIFY" != "1" ]; then + pass "Cleanup destroy complete (no --verify)" + exit 0 +fi + +# ── Post-teardown checks (Phase 6 parity) ── +if command -v openshell >/dev/null 2>&1; then + if openshell sandbox get "$SANDBOX_NAME" >/dev/null 2>&1; then + fail "openshell sandbox get '${SANDBOX_NAME}' still succeeds after cleanup" + exit 1 + fi + pass "openshell: sandbox '${SANDBOX_NAME}' no longer visible to sandbox get" +else + skip "openshell not on PATH — skipped sandbox get check after cleanup" +fi + +if command -v nemoclaw >/dev/null 2>&1; then + set +e + list_out=$(nemoclaw list 2>&1) + list_rc=$? + set -uo pipefail + if [ "$list_rc" -eq 0 ]; then + if echo "$list_out" | grep -Fq " ${SANDBOX_NAME}"; then + fail "nemoclaw list still lists '${SANDBOX_NAME}' after destroy" + exit 1 + fi + pass "nemoclaw list: '${SANDBOX_NAME}' removed from registry" + else + skip "nemoclaw list failed after cleanup — could not verify registry (exit $list_rc)" + fi +else + skip "nemoclaw not on PATH — skipped list check after cleanup" +fi + +pass "Cleanup + verify complete" +exit 0 diff --git a/test/e2e-vpn/e2e-cloud-experimental/expect-interactive-install.sh b/test/e2e-vpn/e2e-cloud-experimental/expect-interactive-install.sh new file mode 100755 index 00000000000..0ba4d15c337 --- /dev/null +++ b/test/e2e-vpn/e2e-cloud-experimental/expect-interactive-install.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Thin wrapper: real logic lives in test/e2e-vpn/test-e2e-cloud-experimental.sh (Phase 3 expect branch). +# +# Prereq: repo checkout at cwd or run from repo; NVIDIA_API_KEY for cloud onboard unless creds on disk. +# +# Usage (full suite; Phase 3 is interactive by default in test-e2e-cloud-experimental.sh — this wrapper is optional): +# NVIDIA_API_KEY=nvapi-... bash test/e2e-vpn/e2e-cloud-experimental/expect-interactive-install.sh +# +# Offline expect-only smoke: +# DEMO_FAKE_ONLY=1 bash test/e2e-vpn/e2e-cloud-experimental/expect-interactive-install.sh +# +# Optional env: INTERACTIVE_SANDBOX_NAME (default: e2e-expect-demo), INTERACTIVE_* sends, +# NEMOCLAW_INSTALL_SCRIPT_URL, NEMOCLAW_CLOUD_EXPERIMENTAL_MODEL, etc. — see test-e2e-cloud-experimental.sh header. + +set -euo pipefail + +_root="$(cd "$(dirname "$0")/../../.." && pwd)" +cd "$_root" + +if [[ "${DEMO_FAKE_ONLY:-0}" == "1" ]]; then + exec bash test/e2e-vpn/test-e2e-cloud-experimental.sh +fi + +export RUN_E2E_CLOUD_EXPERIMENTAL_INTERACTIVE_INSTALL=1 # redundant with script default; keeps intent explicit +exec bash test/e2e-vpn/test-e2e-cloud-experimental.sh diff --git a/test/e2e-vpn/e2e-cloud-experimental/features/skill/add-sandbox-skill.sh b/test/e2e-vpn/e2e-cloud-experimental/features/skill/add-sandbox-skill.sh new file mode 100755 index 00000000000..19b81e3abaa --- /dev/null +++ b/test/e2e-vpn/e2e-cloud-experimental/features/skill/add-sandbox-skill.sh @@ -0,0 +1,177 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Add one skill into a target sandbox and verify it can be queried back. +# +# Usage examples (from repo root): +# SANDBOX_NAME=e2e-cloud-experimental \ +# SKILL_ID=demo-skill \ +# SKILL_DESCRIPTION="Demo skill from e2e helper" \ +# SKILL_BODY="## Demo\nThis is a smoke skill." \ +# bash test/e2e-vpn/e2e-cloud-experimental/features/skill/add-sandbox-skill.sh +# +# SANDBOX_NAME=e2e-cloud-experimental \ +# SKILL_ID=demo-skill \ +# SKILL_FILE=/absolute/path/to/SKILL.md \ +# bash test/e2e-vpn/e2e-cloud-experimental/features/skill/add-sandbox-skill.sh +# +# If SKILL_FILE / SKILL_BODY are omitted, script renders a template file: +# test/e2e-vpn/e2e-cloud-experimental/fixtures/skill-smoke-template.SKILL.md +# +# After deploy, optional: run one agent turn to prove the skill is used: +# NVIDIA_API_KEY=nvapi-... SANDBOX_NAME=... SKILL_ID=... bash test/e2e-vpn/e2e-cloud-experimental/features/skill/verify-sandbox-skill-via-agent.sh +# +# Exit code: +# 0 = add + query succeeded +# 1 = failure + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +SANDBOX_NAME="${SANDBOX_NAME:-${NEMOCLAW_SANDBOX_NAME:-}}" +SKILL_ID="${SKILL_ID:-}" +DEFAULT_SKILL_DESCRIPTION="E2E smoke skill injected into sandbox for read/write validation." +SKILL_DESCRIPTION="${SKILL_DESCRIPTION:-$DEFAULT_SKILL_DESCRIPTION}" +SKILL_BODY="${SKILL_BODY:-}" +SKILL_FILE="${SKILL_FILE:-}" +SKILL_TEMPLATE_FILE="${SKILL_TEMPLATE_FILE:-${SCRIPT_DIR}/fixtures/skill-smoke-template.SKILL.md}" +# NemoClaw state lives under /sandbox/.openclaw; OpenClaw CLI inside the sandbox uses ~/.openclaw +# (typically /home/sandbox/.openclaw). Deploy to both so `openclaw agent` can read managed skills. +SKILL_ROOT="${SKILL_ROOT:-/sandbox/.openclaw/skills}" + +die() { + printf '%s\n' "add-sandbox-skill: FAIL: $*" >&2 + exit 1 +} +ok() { printf '%s\n' "add-sandbox-skill: OK: $*"; } +info() { printf '%s\n' "add-sandbox-skill: INFO: $*"; } + +[ -n "$SANDBOX_NAME" ] || die "set SANDBOX_NAME (or NEMOCLAW_SANDBOX_NAME)" +[ -n "$SKILL_ID" ] || die "set SKILL_ID (e.g. demo-skill)" +case "$SKILL_ID" in + *[!A-Za-z0-9._-]* | "") die "SKILL_ID may only contain [A-Za-z0-9._-]" ;; +esac + +if [ -n "$SKILL_FILE" ] && [ -n "$SKILL_BODY" ]; then + die "use either SKILL_FILE or SKILL_BODY, not both" +fi + +if [ -n "$SKILL_FILE" ]; then + [ -f "$SKILL_FILE" ] || die "SKILL_FILE not found: $SKILL_FILE" + payload_source="$SKILL_FILE" + cleanup_payload="" +else + payload_source="$(mktemp)" + cleanup_payload="$payload_source" + if [ -z "$SKILL_BODY" ]; then + [ -f "$SKILL_TEMPLATE_FILE" ] || die "SKILL_TEMPLATE_FILE not found: $SKILL_TEMPLATE_FILE" + command -v python3 >/dev/null 2>&1 || die "python3 not on PATH (needed for template rendering)" + SKILL_ID="$SKILL_ID" SKILL_DESCRIPTION="$SKILL_DESCRIPTION" SKILL_TEMPLATE_FILE="$SKILL_TEMPLATE_FILE" python3 -c ' +from pathlib import Path +import os + +tpl = Path(os.environ["SKILL_TEMPLATE_FILE"]).read_text(encoding="utf-8") +tpl = tpl.replace("__SKILL_ID__", os.environ["SKILL_ID"]) +tpl = tpl.replace("__SKILL_DESCRIPTION__", os.environ["SKILL_DESCRIPTION"]) +print(tpl, end="") +' >"$payload_source" + else + { + printf '%s\n' "---" + printf 'name: "%s"\n' "$SKILL_ID" + printf 'description: "%s"\n' "$SKILL_DESCRIPTION" + printf '%s\n' "---" + printf '\n' + printf '%s\n' "$SKILL_BODY" + } >"$payload_source" + fi +fi + +ssh_config="$(mktemp)" +remote_script="$(mktemp)" +trap 'rm -f "${cleanup_payload:-}" "$ssh_config" "$remote_script"' EXIT + +command -v openshell >/dev/null 2>&1 || die "openshell not on PATH" +openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config" 2>/dev/null \ + || die "openshell sandbox ssh-config failed for '${SANDBOX_NAME}'" + +remote_skill_dir="${SKILL_ROOT%/}/${SKILL_ID}" +remote_skill_file="${remote_skill_dir}/SKILL.md" + +info "Copying skill payload to sandbox '${SANDBOX_NAME}'..." +set +e +upload_out=$( + ssh -F "$ssh_config" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" \ + "cat > '/tmp/${SKILL_ID}.md'" <"$payload_source" 2>&1 +) +upload_rc=$? +set -e +[ "$upload_rc" -eq 0 ] || die "ssh payload upload failed (exit ${upload_rc}): ${upload_out:0:300}" + +cat >"$remote_script" <<'EOF' +set -e +skill_dir="$1" +skill_file="$2" +temp_file="$3" + +mkdir -p "$skill_dir" +cp "$temp_file" "$skill_file" + +# Mirror into $HOME/.openclaw/skills so OpenClaw tools resolve the same SKILL.md (see agent ENOENT on /home/sandbox/.openclaw/skills/...). +skill_id="$(basename "$skill_dir")" +home_root="${HOME:-/home/sandbox}" +home_skill_dir="${home_root}/.openclaw/skills/${skill_id}" +home_skill_file="${home_skill_dir}/SKILL.md" +mkdir -p "$home_skill_dir" +cp "$temp_file" "$home_skill_file" + +rm -f "$temp_file" + +if [ ! -f "$skill_file" ]; then + echo "WRITE_FAILED" + exit 2 +fi + +if grep -q '^name:' "$skill_file"; then + : +else + echo "MISSING_NAME" + exit 3 +fi + +echo "QUERY_PATH=$skill_file" +echo "HOME_QUERY_PATH=$home_skill_file" +echo "QUERY_HEAD_BEGIN" +sed -n '1,20p' "$skill_file" +echo "QUERY_HEAD_END" +EOF + +TIMEOUT_CMD="" +command -v timeout >/dev/null 2>&1 && TIMEOUT_CMD="timeout 60" +command -v gtimeout >/dev/null 2>&1 && TIMEOUT_CMD="gtimeout 60" + +set +e +query_out=$( + $TIMEOUT_CMD ssh -F "$ssh_config" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" \ + "sh -s -- '$remote_skill_dir' '$remote_skill_file' '/tmp/${SKILL_ID}.md'" <"$remote_script" 2>&1 +) +query_rc=$? +set -e + +[ "$query_rc" -eq 0 ] || die "remote add/query failed (exit ${query_rc}): ${query_out:0:300}" +echo "$query_out" | grep -q "QUERY_PATH=${remote_skill_file}" || die "did not find query path marker" +echo "$query_out" | grep -q "HOME_QUERY_PATH=" || die "did not find HOME_QUERY_PATH marker" + +ok "skill added and queryable at ${remote_skill_file}" +printf '%s\n' "$query_out" diff --git a/test/e2e-vpn/e2e-cloud-experimental/features/skill/fixtures/skill-smoke-template.SKILL.md b/test/e2e-vpn/e2e-cloud-experimental/features/skill/fixtures/skill-smoke-template.SKILL.md new file mode 100644 index 00000000000..cc68bf4fa34 --- /dev/null +++ b/test/e2e-vpn/e2e-cloud-experimental/features/skill/fixtures/skill-smoke-template.SKILL.md @@ -0,0 +1,31 @@ +--- +name: "__SKILL_ID__" +description: "__SKILL_DESCRIPTION__" +--- + +# E2E smoke skill fixture + +## Purpose + +This fixture verifies that skill content can be written into the target sandbox +and queried back immediately. + +## Use when + +The user asks to verify the **skill-smoke-fixture** / **E2E smoke skill** / **read/write validation** skill, +or asks for the **agent verification token** from this skill. + +## Usage + +- Treat this as an E2E smoke-test fixture. +- Do not rely on it for production behavior. + +## Expected Output + +When queried, this file should exist and include this section. + +## Agent verification + +When this skill is applied, if the user asks for the verification token, respond with **only** this exact string (no quotes, no extra words): + +`SKILL_SMOKE_VERIFY_K9X2` diff --git a/test/e2e-vpn/e2e-cloud-experimental/features/skill/lib/README.md b/test/e2e-vpn/e2e-cloud-experimental/features/skill/lib/README.md new file mode 100644 index 00000000000..84961e56b74 --- /dev/null +++ b/test/e2e-vpn/e2e-cloud-experimental/features/skill/lib/README.md @@ -0,0 +1,8 @@ +# E2E helper libraries (e2e-cloud-experimental) + +| File | Role | +|------|------| +| `validate_repo_skills.sh` | Ensures `.agents/skills/*/SKILL.md` has YAML frontmatter (`name`, `description`) and a non-trivial body. Stdlib only. | +| `validate_sandbox_openclaw_skills.sh` | SSH: `/sandbox/.openclaw` + `openclaw.json` required; prints `SKILLS_SUBDIR=present` or `absent`. | + +Used by **`test/e2e-vpn/test-e2e-cloud-experimental.sh`** (Phase 5c). diff --git a/test/e2e-vpn/e2e-cloud-experimental/features/skill/lib/validate_repo_skills.sh b/test/e2e-vpn/e2e-cloud-experimental/features/skill/lib/validate_repo_skills.sh new file mode 100755 index 00000000000..1821662a4a1 --- /dev/null +++ b/test/e2e-vpn/e2e-cloud-experimental/features/skill/lib/validate_repo_skills.sh @@ -0,0 +1,159 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Validate Cursor/agent skills under .agents/skills//SKILL.md (YAML frontmatter + body). +# Bash-only counterpart to the former validate_repo_skills.py. + +set -euo pipefail + +usage() { + printf 'Usage: %s [--repo DIR]\n' "$(basename "$0")" >&2 + exit 2 +} + +REPO=$(pwd) +while [[ $# -gt 0 ]]; do + case "$1" in + --repo) + [[ $# -ge 2 ]] || usage + REPO=$(cd "$2" && pwd) + shift 2 + ;; + -h | --help) usage ;; + *) usage ;; + esac +done + +SKILLS_ROOT="${REPO}/.agents/skills" +if [[ ! -d "$SKILLS_ROOT" ]]; then + printf 'validate_repo_skills: FAIL: missing directory %s\n' "$SKILLS_ROOT" >&2 + exit 1 +fi + +paths=() +while IFS= read -r p; do + [[ -n "$p" ]] && paths+=("$p") +done < <(find "$SKILLS_ROOT" -mindepth 2 -maxdepth 2 -name SKILL.md -print | LC_ALL=C sort) + +if [[ ${#paths[@]} -eq 0 ]]; then + printf 'validate_repo_skills: FAIL: no SKILL.md under %s\n' "$SKILLS_ROOT" >&2 + exit 1 +fi + +# Extract first line matching "^key:" and return the value part (trim + strip one pair of quotes). +extract_scalar() { + local fm=$1 key=$2 + local line val + line=$(printf '%s\n' "$fm" | grep -m1 "^${key}:" || true) + [[ -n "$line" ]] || { + printf '' + return 0 + } + val="${line#*:}" + val="${val#"${val%%[![:space:]]*}"}" + val="${val%"${val##*[![:space:]]}"}" + case $val in + \"*) + val="${val#\"}" + val="${val%\"}" + ;; + \'*) + val="${val#\'}" + val="${val%\'}" + ;; + esac + val="${val#"${val%%[![:space:]]*}"}" + val="${val%"${val##*[![:space:]]}"}" + printf '%s' "$val" +} + +# Length of body after leading/trailing whitespace (matches Python strip semantics via awk). +body_stripped_len() { + local body=$1 + printf '%s' "$body" | awk '{ r = r $0 "\n" } + END { + sub(/^[[:space:]]+/, "", r) + sub(/[[:space:]]+$/, "", r) + print length(r) + }' +} + +validate_skill_file() { + local path=$1 + local rel=$2 + local failed=0 + local raw fm body name desc blen state line + + if ! raw=$(cat "$path"); then + printf '%s: FAIL\n - cannot read file\n' "$rel" >&2 + return 1 + fi + + if [[ ! "$raw" == ---* ]]; then + printf '%s: FAIL\n - missing or invalid YAML frontmatter (expected --- ... ---)\n' "$rel" >&2 + return 1 + fi + + fm="" + body="" + state=0 + while IFS= read -r line || [[ -n "$line" ]]; do + if [[ "$line" == "---" ]]; then + if ((state == 0)); then + state=1 + continue + fi + if ((state == 1)); then + state=2 + continue + fi + fi + if ((state == 1)); then + fm+="${line}"$'\n' + elif ((state == 2)); then + body+="${line}"$'\n' + fi + done <<<"$raw" + + if ((state != 2)); then + printf '%s: FAIL\n - missing or invalid YAML frontmatter (expected --- ... ---)\n' "$rel" >&2 + return 1 + fi + + name=$(extract_scalar "$fm" "name") + desc=$(extract_scalar "$fm" "description") + if [[ -z "$name" ]]; then + printf '%s: FAIL\n - frontmatter missing non-empty '\''name:'\''\n' "$rel" >&2 + failed=1 + fi + if [[ -z "$desc" ]]; then + printf '%s: FAIL\n - frontmatter missing non-empty '\''description:'\''\n' "$rel" >&2 + failed=1 + fi + + blen=$(body_stripped_len "$body") + if ((blen < 20)); then + printf '%s: FAIL\n - body too short after frontmatter (expected real SKILL content)\n' "$rel" >&2 + failed=1 + fi + + ((failed == 0)) +} + +failed_any=0 +for p in "${paths[@]}"; do + rel=${p#"${REPO}/"} + if validate_skill_file "$p" "$rel"; then + printf '%s: OK\n' "$rel" + else + failed_any=1 + fi +done + +if ((failed_any)); then + exit 1 +fi + +printf 'validate_repo_skills: %d skill(s) OK\n' "${#paths[@]}" +exit 0 diff --git a/test/e2e-vpn/e2e-cloud-experimental/features/skill/lib/validate_sandbox_openclaw_skills.sh b/test/e2e-vpn/e2e-cloud-experimental/features/skill/lib/validate_sandbox_openclaw_skills.sh new file mode 100755 index 00000000000..b9d31a1ff24 --- /dev/null +++ b/test/e2e-vpn/e2e-cloud-experimental/features/skill/lib/validate_sandbox_openclaw_skills.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# OpenClaw skill-related layout inside the NemoClaw sandbox (after migrate). +# - Requires migrated state at /sandbox/.openclaw (openclaw.json). +# - /sandbox/.openclaw/skills is optional (host snapshot may omit it); prints status for the caller. +# +# Usage: +# SANDBOX_NAME=my-sbx bash test/e2e-vpn/e2e-cloud-experimental/features/skill/lib/validate_sandbox_openclaw_skills.sh +# Exit: +# 0 — state dir + config OK (stdout: SKILLS_SUBDIR=present|absent) +# 1 — ssh/openshell failure or missing required paths + +set -euo pipefail + +SANDBOX_NAME="${SANDBOX_NAME:-${NEMOCLAW_SANDBOX_NAME:-e2e-cloud-experimental}}" + +die() { + printf '%s\n' "validate_sandbox_openclaw_skills: FAIL: $*" >&2 + exit 1 +} + +ssh_config="$(mktemp)" +trap 'rm -f "$ssh_config"' EXIT + +openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config" 2>/dev/null \ + || die "openshell sandbox ssh-config failed for '${SANDBOX_NAME}'" + +TIMEOUT_CMD="" +command -v timeout >/dev/null 2>&1 && TIMEOUT_CMD="timeout 60" +command -v gtimeout >/dev/null 2>&1 && TIMEOUT_CMD="gtimeout 60" + +ssh_host="openshell-${SANDBOX_NAME}" +remote='set -e +if [ ! -d /sandbox/.openclaw ]; then echo "MISSING_STATE_DIR"; exit 2; fi +if [ ! -f /sandbox/.openclaw/openclaw.json ]; then echo "MISSING_CONFIG"; exit 3; fi +if [ -d /sandbox/.openclaw/skills ]; then echo "SKILLS_SUBDIR=present"; else echo "SKILLS_SUBDIR=absent"; fi +exit 0' + +set +e +out=$( + $TIMEOUT_CMD ssh -F "$ssh_config" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o LogLevel=ERROR \ + "$ssh_host" \ + "$remote" 2>/dev/null +) +rc=$? +set -e + +out="$(echo "$out" | tr -d '\r' | tail -n 5)" + +[ "$rc" -eq 0 ] || die "ssh failed (exit $rc): ${out:0:200}" + +case "$out" in + *MISSING_STATE_DIR*) die "/sandbox/.openclaw missing inside sandbox" ;; + *MISSING_CONFIG*) die "/sandbox/.openclaw/openclaw.json missing inside sandbox" ;; + *SKILLS_SUBDIR=present*) + printf '%s\n' "$out" + exit 0 + ;; + *SKILLS_SUBDIR=absent*) + printf '%s\n' "$out" + exit 0 + ;; + *) die "unexpected remote output: ${out:0:200}" ;; +esac diff --git a/test/e2e-vpn/e2e-cloud-experimental/features/skill/verify-sandbox-skill-via-agent.sh b/test/e2e-vpn/e2e-cloud-experimental/features/skill/verify-sandbox-skill-via-agent.sh new file mode 100755 index 00000000000..402cc93faea --- /dev/null +++ b/test/e2e-vpn/e2e-cloud-experimental/features/skill/verify-sandbox-skill-via-agent.sh @@ -0,0 +1,114 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Run one openclaw agent turn inside the sandbox and check the reply for the +# skill verification token (proves the skill content was available to the agent). +# +# Prereq: skill deployed with test/e2e-vpn/e2e-cloud-experimental/fixtures/skill-smoke-template.SKILL.md +# (includes SKILL_SMOKE_VERIFY_K9X2). Re-run add-sandbox-skill.sh after template updates. +# +# Usage (from repo root): +# NVIDIA_API_KEY=nvapi-... SANDBOX_NAME=test01 SKILL_ID=skill-smoke-fixture \ +# bash test/e2e-vpn/e2e-cloud-experimental/features/skill/verify-sandbox-skill-via-agent.sh +# +# Optional: +# SKILL_VERIFY_PROMPT — override user message (still must elicit VERIFY_TOKEN in practice) +# VERIFY_TOKEN — default SKILL_SMOKE_VERIFY_K9X2 +# SKILL_VERIFY_SESSION_ID — default unique id (time + RANDOMs) to avoid jsonl.lock collisions +# SKILL_VERIFY_NO_CLEAR_LOCK=1 — do not rm stale .jsonl.lock for this session before agent (debug only) +# OPENCLAW_AGENT_PREFIX — default "nemoclaw-start" (run before openclaw agent, same as telegram-bridge) + +set -euo pipefail + +SANDBOX_NAME="${SANDBOX_NAME:-${NEMOCLAW_SANDBOX_NAME:-}}" +SKILL_ID="${SKILL_ID:-skill-smoke-fixture}" +VERIFY_TOKEN="${VERIFY_TOKEN:-SKILL_SMOKE_VERIFY_K9X2}" +OPENCLAW_AGENT_PREFIX="${OPENCLAW_AGENT_PREFIX:-nemoclaw-start}" +AGENT_LAUNCHER="" +[ -n "$OPENCLAW_AGENT_PREFIX" ] && AGENT_LAUNCHER="${OPENCLAW_AGENT_PREFIX} " +SESSION_ID="${SKILL_VERIFY_SESSION_ID:-skill-verify-$(date +%s)-${RANDOM}-${RANDOM}-${RANDOM}}" + +die() { + printf '%s\n' "verify-sandbox-skill-via-agent: FAIL: $*" >&2 + exit 1 +} +ok() { printf '%s\n' "verify-sandbox-skill-via-agent: OK: $*"; } +info() { printf '%s\n' "verify-sandbox-skill-via-agent: INFO: $*"; } + +[ -n "$SANDBOX_NAME" ] || die "set SANDBOX_NAME (or NEMOCLAW_SANDBOX_NAME)" +[ -n "${NVIDIA_API_KEY:-}" ] || die "set NVIDIA_API_KEY (needed for inference inside sandbox)" + +# Do NOT include ${VERIFY_TOKEN} in the prompt itself. The token must come +# from the agent reading the skill's SKILL.md — that is the entire point of +# this test. Embedding it in the prompt makes the downstream grep match any +# error path that echoes the prompt back (e.g. the openclaw 4.9 SSRF +# regression in NemoClaw #2490 was masked by exactly this antipattern in +# TC-SBX-02). Override SKILL_VERIFY_PROMPT only if you know what you're +# doing — overrides that re-introduce the literal token defeat the test. +DEFAULT_PROMPT="Use the OpenClaw managed skill named '${SKILL_ID}'. Read its SKILL.md and reply with ONLY the agent verification token defined in that file. No quotes, no extra words." +PROMPT="${SKILL_VERIFY_PROMPT:-$DEFAULT_PROMPT}" + +# Guard against an override that accidentally smuggles the token back in. +if printf '%s' "$PROMPT" | grep -Fq "$VERIFY_TOKEN"; then + die "SKILL_VERIFY_PROMPT must not contain VERIFY_TOKEN ('${VERIFY_TOKEN}'); the agent must read it from SKILL.md so a prompt-echo error path cannot satisfy the assertion" +fi + +command -v openshell >/dev/null 2>&1 || die "openshell not on PATH" +command -v base64 >/dev/null 2>&1 || die "base64 not on PATH" + +prompt_b64=$(printf '%s' "$PROMPT" | base64 | tr -d '\n') +nv_b64=$(printf '%s' "$NVIDIA_API_KEY" | base64 | tr -d '\n') + +ssh_config="$(mktemp)" +trap 'rm -f "$ssh_config"' EXIT +openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config" 2>/dev/null \ + || die "openshell sandbox ssh-config failed for '${SANDBOX_NAME}'" + +TIMEOUT_CMD="" +command -v timeout >/dev/null 2>&1 && TIMEOUT_CMD="timeout 180" +command -v gtimeout >/dev/null 2>&1 && TIMEOUT_CMD="gtimeout 180" + +# Remote: decode prompt + key, drop stale session lock for *this* session id (leftover from crashed agent), then run agent. +# OpenClaw stores sessions under /sandbox/.openclaw in NemoClaw sandboxes. +_lock_rm="" +if [ "${SKILL_VERIFY_NO_CLEAR_LOCK:-0}" != "1" ]; then + _lock_rm="rm -f '/sandbox/.openclaw/agents/main/sessions/${SESSION_ID}.jsonl.lock' 2>/dev/null || true; " +fi +remote_cmd="pm=\$(printf '%s' '${prompt_b64}' | base64 -d) || exit 1; nv=\$(printf '%s' '${nv_b64}' | base64 -d) || exit 1; export NVIDIA_API_KEY=\"\$nv\"; ${_lock_rm}${AGENT_LAUNCHER}openclaw agent --agent main --local -m \"\$pm\" --session-id '${SESSION_ID}'" + +info "Running openclaw agent in sandbox '${SANDBOX_NAME}' (session ${SESSION_ID})..." + +set +e +raw_out=$( + $TIMEOUT_CMD ssh -T -F "$ssh_config" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" \ + "$remote_cmd" 2>&1 +) +agent_rc=$? +set -e + +printf '\n%s\n' "--- agent stdout/stderr (trimmed for display) ---" +printf '%s' "$raw_out" | tail -c 12000 +printf '\n%s\n' "--- end ---" + +# Fail closed on provider/transport errors so a coincidental token match +# (e.g. someone overrode SKILL_VERIFY_PROMPT to embed the token, or the +# token leaked into a stack trace via the skill manifest path) cannot mask +# an SSRF block, transport reset, or gateway error. See NemoClaw #2490. +if printf '%s' "$raw_out" | grep -qiE "SsrFBlockedError|Blocked hostname|Blocked: resolves to|transport error|provider error|ECONNREFUSED|EAI_AGAIN|gateway unavailable"; then + die "agent failed before completing turn — provider/transport error in output (exit ${agent_rc}). Session: ${SESSION_ID}" +fi + +# Collapse newlines so a model-wrapped token (e.g. "SKILL_SMOKE_VER\nIFY_K9X2") still matches. +collapsed_out=$(printf '%s' "$raw_out" | tr -d '\n\r') +if printf '%s' "$collapsed_out" | grep -Fq "$VERIFY_TOKEN"; then + ok "agent output contains ${VERIFY_TOKEN}" + exit 0 +fi + +die "token ${VERIFY_TOKEN} not found in agent output (ssh/agent exit ${agent_rc}). Hints: session file locked → stale .jsonl.lock (this script clears it for the chosen session id) or kill stuck openclaw in sandbox; [tools] ENOENT on skills → re-run add-sandbox-skill.sh. Session was: ${SESSION_ID}" diff --git a/test/e2e-vpn/e2e-cloud-experimental/openclaw-tui-in-sandbox.sh b/test/e2e-vpn/e2e-cloud-experimental/openclaw-tui-in-sandbox.sh new file mode 100755 index 00000000000..5f2c288ebf6 --- /dev/null +++ b/test/e2e-vpn/e2e-cloud-experimental/openclaw-tui-in-sandbox.sh @@ -0,0 +1,86 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# shellcheck disable=SC2016 +# expect(1) Tcl: $ and {...} are Tcl, not bash expansion +# +# OpenClaw TUI flow in one command (local / interactive). +# +# Automated CI-style smoke (finite expect, no `interact`) runs as Phase 5e inside: +# test/e2e-vpn/test-e2e-cloud-experimental.sh +# +# default: use `expect` to run `nemoclaw connect`, then send `openclaw tui` +# manual: pass --manual to only run `nemoclaw connect` +# +# Usage: +# bash test/e2e-vpn/e2e-cloud-experimental/openclaw-tui-in-sandbox.sh +# bash test/e2e-vpn/e2e-cloud-experimental/openclaw-tui-in-sandbox.sh my-sandbox +# bash test/e2e-vpn/e2e-cloud-experimental/openclaw-tui-in-sandbox.sh --manual +# +# Optional env: +# OPENCLAW_TUI_AUTO_MESSAGE default: 你好 +# OPENCLAW_TUI_SEND_DELAY_SEC default: 3 + +set -euo pipefail + +MANUAL_MODE=0 +CLI_SANDBOX_NAME="" +for arg in "$@"; do + case "$arg" in + --manual) MANUAL_MODE=1 ;; + *) + if [ -z "$CLI_SANDBOX_NAME" ]; then + CLI_SANDBOX_NAME="$arg" + else + echo "ERROR: unexpected extra argument: $arg" >&2 + exit 1 + fi + ;; + esac +done +SANDBOX_NAME="${CLI_SANDBOX_NAME:-${SANDBOX_NAME:-${NEMOCLAW_SANDBOX_NAME:-e2e-cloud-experimental}}}" + +if ! command -v nemoclaw >/dev/null 2>&1; then + echo "ERROR: nemoclaw not on PATH." >&2 + exit 1 +fi + +if [ "$MANUAL_MODE" -eq 1 ]; then + exec nemoclaw "$SANDBOX_NAME" connect +fi + +printf '%s\n' \ + "Connecting to sandbox '${SANDBOX_NAME}' and launching openclaw tui..." \ + "After TUI opens, send your message (e.g. 你好)." \ + "" + +if command -v expect >/dev/null 2>&1; then + AUTO_MESSAGE="${OPENCLAW_TUI_AUTO_MESSAGE:-你好}" + SEND_DELAY_SEC="${OPENCLAW_TUI_SEND_DELAY_SEC:-3}" + exec env \ + NEMOCLAW_TUI_SANDBOX_NAME="$SANDBOX_NAME" \ + OPENCLAW_TUI_AUTO_MESSAGE="$AUTO_MESSAGE" \ + OPENCLAW_TUI_SEND_DELAY_SEC="$SEND_DELAY_SEC" \ + expect -c ' + set timeout -1 + set sandbox $env(NEMOCLAW_TUI_SANDBOX_NAME) + set auto_msg $env(OPENCLAW_TUI_AUTO_MESSAGE) + set send_delay $env(OPENCLAW_TUI_SEND_DELAY_SEC) + spawn nemoclaw $sandbox connect + expect { + -re {[$#>] $} { + send "openclaw tui\r" + sleep $send_delay + send -- "$auto_msg\r" + interact + } + timeout { puts "Timed out waiting for sandbox shell prompt."; exit 1 } + eof { exit 1 } + } + ' +fi + +echo "WARN: expect not found; falling back to manual connect." >&2 +echo "After entering sandbox, run: openclaw tui" >&2 +exec nemoclaw "$SANDBOX_NAME" connect diff --git a/test/e2e-vpn/e2e-cloud-experimental/skip/README.md b/test/e2e-vpn/e2e-cloud-experimental/skip/README.md new file mode 100644 index 00000000000..3f75c638079 --- /dev/null +++ b/test/e2e-vpn/e2e-cloud-experimental/skip/README.md @@ -0,0 +1,10 @@ +# Opt-in checks (`skip/`) + +Scripts here are **not** picked up by `test/e2e-vpn/test-e2e-cloud-experimental.sh` (only `checks/*.sh` runs in Phase 5). + +Use when a check is useful but flaky, slow, or environment-specific — run manually: + +```bash +export SANDBOX_NAME=… +bash test/e2e-vpn/e2e-cloud-experimental/skip/05-network-policy.sh +``` diff --git a/test/e2e-vpn/e2e-cloud-experimental/test-inference-local-chat.sh b/test/e2e-vpn/e2e-cloud-experimental/test-inference-local-chat.sh new file mode 100755 index 00000000000..f7de5a73072 --- /dev/null +++ b/test/e2e-vpn/e2e-cloud-experimental/test-inference-local-chat.sh @@ -0,0 +1,141 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Demo: POST /v1/chat/completions to https://inference.local from *inside* the sandbox (SSH). +# Same idea as test-e2e-cloud-experimental.sh Phase 5b — use this to verify dialogue without the full suite. +# +# Prerequisites: +# - openshell on PATH, sandbox exists and is Ready +# - Inference already configured for that sandbox (after onboard) +# - python3 on host (JSON parse) +# +# Environment (defaults match e2e-cloud-experimental): +# SANDBOX_NAME or NEMOCLAW_SANDBOX_NAME — default: e2e-cloud-experimental +# CLOUD_EXPERIMENTAL_MODEL or NEMOCLAW_CLOUD_EXPERIMENTAL_MODEL / NEMOCLAW_SCENARIO_A_MODEL +# CHAT_USER_MESSAGE — optional override for the user message (default asks for PONG) +# DEMO_CHAT_MAX_DISPLAY_CHARS — max chars of assistant text to print (default: 12000; 0 = unlimited) +# DEMO_CHAT_SHOW_RAW_JSON=1 — also print raw response body (can be large) +# +# Usage (from repo root): +# bash test/e2e-vpn/demo-inference-local-chat.sh +# +# Exit: 0 = assistant text contains PONG (case-insensitive); 1 = failure + +set -euo pipefail + +SANDBOX_NAME="${SANDBOX_NAME:-${NEMOCLAW_SANDBOX_NAME:-e2e-cloud-experimental}}" +CLOUD_EXPERIMENTAL_MODEL="${CLOUD_EXPERIMENTAL_MODEL:-${NEMOCLAW_CLOUD_EXPERIMENTAL_MODEL:-${NEMOCLAW_SCENARIO_A_MODEL:-nvidia/nemotron-3-super-120b-a12b}}}" +CHAT_USER_MESSAGE="${CHAT_USER_MESSAGE:-Reply with exactly one word: PONG}" + +die() { + printf '\033[31m[demo-chat] FAIL:\033[0m %s\n' "$*" >&2 + exit 1 +} +ok() { printf '\033[32m[demo-chat] OK:\033[0m %s\n' "$*"; } + +DEMO_CHAT_MAX_DISPLAY_CHARS="${DEMO_CHAT_MAX_DISPLAY_CHARS:-12000}" + +print_assistant_text() { + local text=$1 + local n=${#text} + local max="$DEMO_CHAT_MAX_DISPLAY_CHARS" + if [ "$max" = "0" ] || [ "$n" -le "$max" ]; then + printf '%s\n' "$text" + return + fi + printf '%s' "${text:0:max}" + printf '\n[demo-chat] … truncated for display (%d chars total, DEMO_CHAT_MAX_DISPLAY_CHARS=%s)\n' "$n" "$max" +} + +parse_chat_content() { + python3 -c " +import json, sys +try: + r = json.load(sys.stdin) + c = r['choices'][0]['message'] + # Some gateways put interim/final text in \"reasoning\" while content is null + content = c.get('content') or c.get('reasoning_content') or c.get('reasoning') or '' + print(content.strip()) +except Exception as e: + print(f'PARSE_ERROR: {e}', file=sys.stderr) + sys.exit(1) +" +} + +command -v python3 >/dev/null 2>&1 || die "python3 not on PATH" +command -v openshell >/dev/null 2>&1 || die "openshell not on PATH" + +printf '[demo-chat] sandbox=%s\n' "$SANDBOX_NAME" + +payload=$( + CLOUD_EXPERIMENTAL_MODEL="$CLOUD_EXPERIMENTAL_MODEL" \ + CHAT_USER_MESSAGE="$CHAT_USER_MESSAGE" \ + python3 -c " +import json, os +print(json.dumps({ + 'model': os.environ['CLOUD_EXPERIMENTAL_MODEL'], + 'messages': [{'role': 'user', 'content': os.environ['CHAT_USER_MESSAGE']}], + 'max_tokens': 100, +})) +" +) || die "could not build JSON payload" + +printf '\n\033[1;36m--- Request ---\033[0m\n' +printf ' URL (inside sandbox): https://inference.local/v1/chat/completions\n' +printf ' model: %s\n' "$CLOUD_EXPERIMENTAL_MODEL" +printf ' user message:\n' +printf '%s\n' "$CHAT_USER_MESSAGE" | sed 's/^/ | /' +printf ' JSON body:\n' +printf '%s\n' "$payload" | python3 -m json.tool 2>/dev/null | sed 's/^/ /' || printf ' %s\n' "$payload" +printf '\n' + +TIMEOUT_CMD="" +command -v timeout >/dev/null 2>&1 && TIMEOUT_CMD="timeout 120" +command -v gtimeout >/dev/null 2>&1 && TIMEOUT_CMD="gtimeout 120" + +ssh_config="$(mktemp)" +trap 'rm -f "$ssh_config"' EXIT + +openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config" 2>/dev/null \ + || die "openshell sandbox ssh-config failed for '${SANDBOX_NAME}'" + +set +e +out=$( + $TIMEOUT_CMD ssh -F "$ssh_config" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" \ + "curl -sS --max-time 90 https://inference.local/v1/chat/completions -H 'Content-Type: application/json' -d $(printf '%q' "$payload")" \ + 2>&1 +) +rc=$? +set -e + +[ "$rc" -eq 0 ] || die "ssh/curl exit $rc — ${out:0:500}" +[ -n "$out" ] || die "empty response" + +chat_text=$(printf '%s' "$out" | parse_chat_content 2>/dev/null) || chat_text="" + +printf '\033[1;36m--- Assistant text (parsed: content | reasoning_content | reasoning) ---\033[0m\n' +if [ -n "$chat_text" ]; then + print_assistant_text "$chat_text" +else + printf ' (empty — see raw JSON below if enabled)\n' +fi +printf '\n' + +if [ "${DEMO_CHAT_SHOW_RAW_JSON:-}" = "1" ]; then + printf '\033[1;36m--- Raw JSON response ---\033[0m\n' + printf '%s\n' "$out" | python3 -m json.tool 2>/dev/null || printf '%s\n' "$out" + printf '\n' +fi + +if echo "$chat_text" | grep -qi "PONG"; then + ok "assistant text contains PONG (see above)" + exit 0 +fi + +die "expected PONG in assistant text (parsed block above); raw (first 800 chars): ${out:0:800}" diff --git a/test/e2e-vpn/e2e-cloud-experimental/test-port8080-conflict.sh b/test/e2e-vpn/e2e-cloud-experimental/test-port8080-conflict.sh new file mode 100755 index 00000000000..44e29b1baae --- /dev/null +++ b/test/e2e-vpn/e2e-cloud-experimental/test-port8080-conflict.sh @@ -0,0 +1,154 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Port 8080 conflict during nemoclaw onboard (VDR3 #5) +# +# OPTIONAL / standalone — not invoked by test-e2e-cloud-experimental.sh. Run manually or from +# a separate CI job when you want to validate preflight port checks. +# +# Expects a working NemoClaw/OpenShell install from a prior onboard (gateway may +# hold 8080). Destroys the nemoclaw gateway, binds a dummy listener on 8080, runs +# nemoclaw onboard --non-interactive, asserts preflight fails with +# "Port 8080 is not available", then restores gateway (+ optional re-onboard). +# +# Exit codes: +# 0 — success +# 1 — failure +# 2 — skipped (no python3/python to bind 8080) +# +# Environment (typical): +# NEMOCLAW_SANDBOX_NAME — default: e2e-cloud-experimental +# NEMOCLAW_NON_INTERACTIVE — should be 1 (onboard non-interactive) +# NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 — required for non-interactive onboard/re-onboard +# NVIDIA_API_KEY — required if onboard reaches cloud inference (restore path) +# +# Usage: +# NEMOCLAW_NON_INTERACTIVE=1 NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 NVIDIA_API_KEY=nvapi-... bash test/e2e-vpn/e2e-cloud-experimental/test-port8080-conflict.sh + +set -uo pipefail + +PASS() { printf '\033[32m [port8080] PASS: %s\033[0m\n' "$1"; } +FAIL() { printf '\033[31m [port8080] FAIL: %s\033[0m\n' "$1"; } +INFO() { printf '\033[1;34m [port8080] INFO:\033[0m %s\n' "$1"; } + +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-cloud-experimental}" + +if ! command -v nemoclaw >/dev/null 2>&1; then + FAIL "nemoclaw not on PATH" + exit 1 +fi +if ! command -v openshell >/dev/null 2>&1; then + FAIL "openshell not on PATH" + exit 1 +fi + +if ! command -v python3 >/dev/null 2>&1 && ! command -v python >/dev/null 2>&1; then + INFO "python3/python not found — cannot bind port 8080 for this test" + exit 2 +fi + +PYHTTP="python3" +command -v python3 >/dev/null 2>&1 || PYHTTP="python" + +INFO "Stopping nemoclaw gateway so we can bind a non-OpenShell process on 8080..." +openshell forward stop 18789 2>/dev/null || true +openshell gateway destroy -g nemoclaw 2>/dev/null || true +sleep 3 + +INFO "Starting dummy HTTP listener on 127.0.0.1:8080..." +$PYHTTP -m http.server 8080 --bind 127.0.0.1 >/dev/null 2>&1 & +occupier_pid=$! +sleep 1 +if ! kill -0 "$occupier_pid" 2>/dev/null; then + FAIL "Dummy listener on 8080 did not stay running (pid ${occupier_pid})" + exit 1 +fi +if ! curl -sf --max-time 2 "http://127.0.0.1:8080/" >/dev/null 2>&1; then + kill "$occupier_pid" 2>/dev/null || true + wait "$occupier_pid" 2>/dev/null || true + FAIL "Could not reach dummy server on 127.0.0.1:8080" + exit 1 +fi +PASS "Port 8080 occupied by dummy process (PID ${occupier_pid})" + +P4_LOG="$(mktemp)" +INFO "Running nemoclaw onboard --non-interactive (expect preflight to fail on port 8080)..." +set +e +NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 nemoclaw onboard --non-interactive >"$P4_LOG" 2>&1 +p4_exit=$? +set -euo pipefail +p4_out="$(cat "$P4_LOG")" +rm -f "$P4_LOG" + +kill "$occupier_pid" 2>/dev/null || true +wait "$occupier_pid" 2>/dev/null || true + +if [ "$p4_exit" -eq 0 ]; then + FAIL "Expected nemoclaw onboard to exit non-zero when 8080 is taken (got 0)" + exit 1 +fi +PASS "nemoclaw onboard exited non-zero (${p4_exit}) with 8080 blocked" + +if echo "$p4_out" | grep -Fq "Port 8080 is not available"; then + PASS "Onboard output reports Port 8080 is not available (VDR3 #5)" +else + FAIL "Expected 'Port 8080 is not available' in onboard output" + exit 1 +fi + +# #2497: the preflight error must surface the env-var override so users +# can rerun with a different port instead of being blocked indefinitely. +if echo "$p4_out" | grep -Fq "NEMOCLAW_GATEWAY_PORT= nemoclaw onboard"; then + PASS "Onboard error surfaces NEMOCLAW_GATEWAY_PORT override hint (#2497)" +else + FAIL "Expected NEMOCLAW_GATEWAY_PORT override hint in onboard output (#2497)" + exit 1 +fi + +INFO "Restoring nemoclaw gateway for subsequent phases..." +if ! openshell gateway start --name nemoclaw 2>&1; then + FAIL "openshell gateway start --name nemoclaw failed after port test" + exit 1 +fi +gw_ok=0 +for _i in 1 2 3 4 5 6 7 8 9 10; do + if openshell status 2>&1 | grep -q "Connected"; then + gw_ok=1 + break + fi + sleep 2 +done +if [ "$gw_ok" -ne 1 ]; then + FAIL "Gateway did not become healthy (openshell status) after restore" + exit 1 +fi +PASS "Gateway restored and reports Connected" + +openshell forward start --background 18789 "$SANDBOX_NAME" 2>/dev/null || true + +if openshell sandbox get "$SANDBOX_NAME" >/dev/null 2>&1; then + PASS "Sandbox '${SANDBOX_NAME}' present after gateway restore" +else + INFO "Sandbox missing after gateway destroy/recreate — re-onboarding with NEMOCLAW_RECREATE_SANDBOX=1..." + P4R_LOG="$(mktemp)" + set +e + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 NEMOCLAW_RECREATE_SANDBOX=1 nemoclaw onboard --non-interactive >"$P4R_LOG" 2>&1 + p4r_exit=$? + set -euo pipefail + if [ "$p4r_exit" -ne 0 ]; then + FAIL "Re-onboard after port test failed (exit $p4r_exit); log: ${P4R_LOG}" + exit 1 + fi + rm -f "$P4R_LOG" + openshell forward start --background 18789 "$SANDBOX_NAME" 2>/dev/null || true + if openshell sandbox get "$SANDBOX_NAME" >/dev/null 2>&1; then + PASS "Sandbox '${SANDBOX_NAME}' recreated after port test" + else + FAIL "Sandbox '${SANDBOX_NAME}' still missing after re-onboard" + exit 1 + fi +fi + +PASS "Port 8080 conflict subtest complete" +exit 0 diff --git a/test/e2e-vpn/e2e-timeout.sh b/test/e2e-vpn/e2e-timeout.sh new file mode 100755 index 00000000000..4270d615a19 --- /dev/null +++ b/test/e2e-vpn/e2e-timeout.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# ============================================================================= +# e2e-timeout.sh — shared timeout detection, self-wrap, and run_with_timeout +# +# Source this file near the top of any E2E test script, AFTER `set -euo pipefail` +# (or `set -uo pipefail`) and BEFORE any commands that need timeout protection. +# +# Required before sourcing: +# NEMOCLAW_E2E_DEFAULT_TIMEOUT — per-script default (seconds). Falls back to +# $NEMOCLAW_E2E_TIMEOUT_SECONDS if set by the +# caller, or this default if not. +# +# Exported after sourcing: +# TIMEOUT_CMD — "timeout", "gtimeout", or "" (empty when bypassed) +# run_with_timeout() — helper: run_with_timeout [args...] +# +# Environment knobs (set by caller / CI): +# NEMOCLAW_E2E_NO_TIMEOUT=1 — skip the self-wrap AND run commands bare +# NEMOCLAW_E2E_TIMEOUT_SECONDS — override the per-script default +# NEMOCLAW_E2E_TIMEOUT_WRAPPED=1 — (internal) prevents recursive exec +# ============================================================================= + +# ── Detect timeout binary ──────────────────────────────────────────────────── +TIMEOUT_CMD="" +if command -v timeout >/dev/null 2>&1; then + TIMEOUT_CMD="timeout" +elif command -v gtimeout >/dev/null 2>&1; then + TIMEOUT_CMD="gtimeout" +fi + +# ── Self-wrap the calling script under the overall timeout ──────────────────── +if [ "${NEMOCLAW_E2E_NO_TIMEOUT:-0}" != "1" ] && [ "${NEMOCLAW_E2E_TIMEOUT_WRAPPED:-0}" != "1" ]; then + TIMEOUT_SECONDS="${NEMOCLAW_E2E_TIMEOUT_SECONDS:-${NEMOCLAW_E2E_DEFAULT_TIMEOUT:-900}}" + if [ -n "$TIMEOUT_CMD" ]; then + export NEMOCLAW_E2E_TIMEOUT_WRAPPED=1 + # Re-exec the *calling* script (not this helper) under $TIMEOUT_CMD. + # $0 and $@ are inherited from the caller because this file is sourced. + exec "$TIMEOUT_CMD" -s TERM "$TIMEOUT_SECONDS" "$0" "$@" + else + echo "ERROR: 'timeout' not found. Install coreutils (macOS: 'brew install coreutils')" >&2 + echo " or bypass with NEMOCLAW_E2E_NO_TIMEOUT=1" >&2 + exit 127 + fi +fi + +# ── Per-command timeout helper ──────────────────────────────────────────────── +# Usage: run_with_timeout [args...] +# +# Runs under $TIMEOUT_CMD when timeouts are enabled; runs it bare +# when NEMOCLAW_E2E_NO_TIMEOUT=1. Avoids the foot-gun where an empty +# $TIMEOUT_CMD turns `$TIMEOUT_CMD 60 ssh …` into `60 ssh …`. +run_with_timeout() { + local seconds="$1" + shift + if [ "${NEMOCLAW_E2E_NO_TIMEOUT:-0}" != "1" ] && [ -n "$TIMEOUT_CMD" ]; then + "$TIMEOUT_CMD" "$seconds" "$@" + else + "$@" + fi +} diff --git a/test/e2e-vpn/lib/anthropic-switch-provider.sh b/test/e2e-vpn/lib/anthropic-switch-provider.sh new file mode 100755 index 00000000000..d1a68183298 --- /dev/null +++ b/test/e2e-vpn/lib/anthropic-switch-provider.sh @@ -0,0 +1,197 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Shared helpers for inference-switch E2Es that need a compatible Anthropic +# Messages provider. The mock provider runs on the host; agents still reach it +# only through OpenShell-managed inference.local. + +ANTHROPIC_SWITCH_MOCK_PID="" +ANTHROPIC_SWITCH_MOCK_LOG="${ANTHROPIC_SWITCH_MOCK_LOG:-/tmp/nemoclaw-e2e-anthropic-switch-provider.log}" + +parse_anthropic_content() { + python3 -c ' +import json, sys +try: + r = json.load(sys.stdin) + parts = r.get("content") or [] + text = [] + for part in parts: + if isinstance(part, dict) and isinstance(part.get("text"), str): + text.append(part["text"]) + print(" ".join(text).strip()) +except Exception as e: + print(f"PARSE_ERROR: {e}", file=sys.stderr) + sys.exit(1) +' +} + +start_mock_anthropic_switch_provider() { + local port="${SWITCH_MOCK_PORT:-18766}" + local host="${SWITCH_MOCK_HOST:-host.openshell.internal}" + local health_url="http://127.0.0.1:${port}/health" + SWITCH_ENDPOINT_URL="${SWITCH_ENDPOINT_URL:-http://${host}:${port}}" + export SWITCH_ENDPOINT_URL + + python3 - "$port" >"$ANTHROPIC_SWITCH_MOCK_LOG" 2>&1 <<'PY' & +import json +import sys +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.parse import urlparse + +port = int(sys.argv[1]) + +class Handler(BaseHTTPRequestHandler): + def log_message(self, fmt, *args): + sys.stderr.write((fmt % args) + "\n") + + def _json(self, status, payload): + body = json.dumps(payload).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def _sse(self, events): + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Cache-Control", "no-cache") + self.end_headers() + for name, payload in events: + self.wfile.write(("event: " + name + "\n").encode("utf-8")) + self.wfile.write(("data: " + json.dumps(payload) + "\n\n").encode("utf-8")) + self.wfile.flush() + + def do_GET(self): + path = urlparse(self.path).path + if path == "/health": + self._json(200, {"ok": True}) + return + if path in ("/v1/models", "/v1/models/mock-anthropic-model"): + self._json(200, {"data": [{"id": "mock-anthropic-model"}]}) + return + self._json(404, {"error": "not found", "path": path}) + + def do_POST(self): + path = urlparse(self.path).path + length = int(self.headers.get("Content-Length") or "0") + raw = self.rfile.read(length) if length else b"{}" + try: + payload = json.loads(raw.decode("utf-8") or "{}") + except Exception: + payload = {} + if path != "/v1/messages": + self._json(404, {"error": "unexpected path", "path": path}) + return + model = payload.get("model") or "mock-anthropic-model" + if payload.get("stream") is True: + message = { + "id": "msg_mock", + "type": "message", + "role": "assistant", + "model": model, + "content": [], + "stop_reason": None, + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 0}, + } + self._sse([ + ("message_start", {"type": "message_start", "message": message}), + ( + "content_block_start", + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + ), + ( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "PONG"}}, + ), + ("content_block_stop", {"type": "content_block_stop", "index": 0}), + ( + "message_delta", + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": {"output_tokens": 1}, + }, + ), + ("message_stop", {"type": "message_stop"}), + ]) + return + self._json(200, { + "id": "msg_mock", + "type": "message", + "role": "assistant", + "model": model, + "content": [{"type": "text", "text": "PONG"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 1, "output_tokens": 1}, + }) + +ThreadingHTTPServer(("0.0.0.0", port), Handler).serve_forever() +PY + ANTHROPIC_SWITCH_MOCK_PID=$! + + local attempt=1 + while [ "$attempt" -le 5 ]; do + if curl -sf --max-time 2 "$health_url" >/dev/null 2>&1; then + pass "Mock Anthropic Messages provider is listening on ${SWITCH_ENDPOINT_URL}" + return 0 + fi + attempt=$((attempt + 1)) + sleep 1 + done + + fail "Mock Anthropic Messages provider did not start; log: ${ANTHROPIC_SWITCH_MOCK_LOG}" + return 1 +} + +stop_mock_anthropic_switch_provider() { + if [ -n "${ANTHROPIC_SWITCH_MOCK_PID:-}" ]; then + kill "$ANTHROPIC_SWITCH_MOCK_PID" >/dev/null 2>&1 || true + wait "$ANTHROPIC_SWITCH_MOCK_PID" >/dev/null 2>&1 || true + ANTHROPIC_SWITCH_MOCK_PID="" + fi +} + +ensure_compatible_anthropic_switch_provider() { + if [ "${SWITCH_PROVIDER:-}" != "compatible-anthropic-endpoint" ]; then + return 0 + fi + if [ "${SWITCH_INFERENCE_API:-}" != "anthropic-messages" ]; then + return 0 + fi + + if [ "${SWITCH_MOCK_ANTHROPIC:-}" = "1" ]; then + start_mock_anthropic_switch_provider || return 1 + export COMPATIBLE_ANTHROPIC_API_KEY="${COMPATIBLE_ANTHROPIC_API_KEY:-test-compatible-anthropic-key}" + fi + + if [ -z "${SWITCH_ENDPOINT_URL:-}" ]; then + fail "NEMOCLAW_SWITCH_ENDPOINT_URL is required for compatible Anthropic inference switches" + return 1 + fi + if [ -z "${COMPATIBLE_ANTHROPIC_API_KEY:-}" ]; then + fail "COMPATIBLE_ANTHROPIC_API_KEY is required for compatible Anthropic inference switches" + return 1 + fi + + if openshell provider get -g nemoclaw compatible-anthropic-endpoint >/dev/null 2>&1; then + if ! openshell provider update -g nemoclaw compatible-anthropic-endpoint \ + --credential COMPATIBLE_ANTHROPIC_API_KEY \ + --config "ANTHROPIC_BASE_URL=${SWITCH_ENDPOINT_URL}" >/dev/null; then + fail "Failed to update OpenShell provider compatible-anthropic-endpoint" + return 1 + fi + else + if ! openshell provider create -g nemoclaw \ + --name compatible-anthropic-endpoint \ + --type anthropic \ + --credential COMPATIBLE_ANTHROPIC_API_KEY \ + --config "ANTHROPIC_BASE_URL=${SWITCH_ENDPOINT_URL}" >/dev/null; then + fail "Failed to create OpenShell provider compatible-anthropic-endpoint" + return 1 + fi + fi + pass "OpenShell provider compatible-anthropic-endpoint is registered for ${SWITCH_ENDPOINT_URL}" +} diff --git a/test/e2e-vpn/lib/ci-compatible-inference.sh b/test/e2e-vpn/lib/ci-compatible-inference.sh new file mode 100755 index 00000000000..0233b902de4 --- /dev/null +++ b/test/e2e-vpn/lib/ci-compatible-inference.sh @@ -0,0 +1,112 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# VPN-only hosted inference shim: live E2E lanes use the repository's +# NVIDIA_API_KEY secret against the OpenAI-compatible endpoint at +# inference.nvidia.com. Keep this helper in test/e2e-vpn so the existing +# E2E suite remains unchanged. + +NEMOCLAW_E2E_COMPATIBLE_INFERENCE_MODEL_DEFAULT="nvidia/nvidia/nemotron-3-super-v3" +NEMOCLAW_E2E_HOSTED_INFERENCE_PROVIDER_DEFAULT="compatible-endpoint" + +nemoclaw_e2e_using_compatible_inference() { + return 0 +} + +nemoclaw_e2e_configure_compatible_inference() { + if ! nemoclaw_e2e_using_compatible_inference; then + return 0 + fi + + if [ -z "${NVIDIA_API_KEY:-}" ]; then + echo "ERROR: NVIDIA_API_KEY is required for hosted CI inference" >&2 + return 1 + fi + + export NEMOCLAW_PROVIDER="${NEMOCLAW_PROVIDER:-custom}" + export NEMOCLAW_ENDPOINT_URL="${NEMOCLAW_ENDPOINT_URL:-https://inference.nvidia.com/v1}" + export NEMOCLAW_MODEL="${NEMOCLAW_MODEL:-${NEMOCLAW_CLOUD_EXPERIMENTAL_MODEL:-$NEMOCLAW_E2E_COMPATIBLE_INFERENCE_MODEL_DEFAULT}}" + export NEMOCLAW_COMPAT_MODEL="${NEMOCLAW_COMPAT_MODEL:-$NEMOCLAW_MODEL}" + export NEMOCLAW_PREFERRED_API="${NEMOCLAW_PREFERRED_API:-openai-completions}" + export COMPATIBLE_API_KEY="$NVIDIA_API_KEY" +} + +nemoclaw_e2e_hosted_inference_key() { + printf '%s' "${NVIDIA_API_KEY:-}" +} + +nemoclaw_e2e_hosted_inference_base_url() { + printf '%s' "${NEMOCLAW_ENDPOINT_URL:-https://inference.nvidia.com/v1}" +} + +nemoclaw_e2e_expected_route_provider() { + printf '%s' "$NEMOCLAW_E2E_HOSTED_INFERENCE_PROVIDER_DEFAULT" +} + +nemoclaw_e2e_strip_ansi() { + if command -v perl >/dev/null 2>&1; then + perl -pe 's/\x1b\][^\a]*(?:\a|\x1b\\)//g; s/\x1b\[[0-9;?]*[ -\/]*[@-~]//g' + else + sed -E $'s/\x1B\\[[0-9;?]*[ -\\/]*[@-~]//g' + fi +} + +nemoclaw_e2e_inference_output_matches() { + local output="$1" + local provider="$2" + local model="${3:-}" + local plain + + plain="$(printf '%s' "$output" | nemoclaw_e2e_strip_ansi)" + grep -Eqi "Provider:[[:space:]]*${provider}" <<<"$plain" || return 1 + [ -z "$model" ] || grep -Fq "$model" <<<"$plain" +} + +nemoclaw_e2e_note_pass() { + if declare -F pass >/dev/null 2>&1; then + pass "$@" + else + printf 'PASS: %s\n' "$*" + fi +} + +nemoclaw_e2e_note_fail() { + if declare -F fail >/dev/null 2>&1; then + fail "$@" + else + printf 'ERROR: %s\n' "$*" >&2 + fi +} + +nemoclaw_e2e_hosted_inference_model() { + printf '%s' "${NEMOCLAW_MODEL:-${NEMOCLAW_CLOUD_EXPERIMENTAL_MODEL:-$NEMOCLAW_E2E_COMPATIBLE_INFERENCE_MODEL_DEFAULT}}" +} + +nemoclaw_e2e_probe_hosted_inference() { + local base_url status + base_url="$(nemoclaw_e2e_hosted_inference_base_url)" + + # This preflight is a network/TLS reachability check only. Do not spend an + # inference request here: full parallel nightly runs can otherwise burn CI + # quota or trip HTTP 429 before the scenario reaches the behavior under test. + # In compatible mode, NEMOCLAW_ENDPOINT_URL is a trusted repo-controlled CI + # input from nightly workflow env_json; this probe intentionally validates + # only TCP/TLS/HTTP reachability for that base URL, not provider semantics. + # Onboarding still performs the authenticated model/API validation with + # redaction and retries. + status=$(curl -sS --connect-timeout 10 --max-time 20 -o /dev/null -w "%{http_code}" "$base_url" 2>/dev/null) || return $? + [ -n "$status" ] && [ "$status" != "000" ] +} + +nemoclaw_e2e_require_hosted_inference_key() { + local key + key="$(nemoclaw_e2e_hosted_inference_key)" + + if [ -n "$key" ]; then + nemoclaw_e2e_note_pass "NVIDIA_API_KEY is set for VPN hosted CI inference" + else + nemoclaw_e2e_note_fail "NVIDIA_API_KEY not set - required for VPN hosted CI inference" + return 1 + fi +} diff --git a/test/e2e-vpn/lib/cloudflared-version-resolver.sh b/test/e2e-vpn/lib/cloudflared-version-resolver.sh new file mode 100755 index 00000000000..64e88aca36b --- /dev/null +++ b/test/e2e-vpn/lib/cloudflared-version-resolver.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Invalid state: Cloudflare's APT retention can drop a previously pinned +# cloudflared version while newer signed packages remain available. The source +# boundary is only Cloudflare's GPG-signed APT metadata; do not scrape a second +# source to recover old pins. The source-fix constraint is to choose an +# available signed package above the floor, or use CLOUDFLARED_VERSION for exact +# repro. Regression coverage lives in test/cloudflared-version-resolver.test.ts. +# Remove this resolver once Cloudflare retains pinned versions long enough for +# nightly E2E, or CI caches a vetted package. + +CLOUDFLARED_DEFAULT_MIN_VERSION="${CLOUDFLARED_DEFAULT_MIN_VERSION:-2026.5.1}" + +# Return success when the candidate is syntactically valid for dpkg comparison. +cloudflared_is_debian_version() { + local version="${1:-}" + [[ "$version" =~ ^[0-9][0-9A-Za-z.+:~-]*$ ]] || return 1 + dpkg --compare-versions "$version" eq "$version" >/dev/null 2>&1 +} + +# Choose the newest signed Cloudflare APT version that satisfies the floor. +cloudflared_resolve_package_version() { + local available_versions="${1:-}" + local min_version="${2:-${CLOUDFLARED_MIN_VERSION:-$CLOUDFLARED_DEFAULT_MIN_VERSION}}" + local override_version="${3:-${CLOUDFLARED_VERSION:-}}" + + # Emergency repro knob: install the exact requested version and let APT report + # unavailable overrides, rather than silently substituting another package. + # Still validate Debian-version syntax before the sudo apt install boundary. + if [[ -n "$override_version" ]]; then + if ! cloudflared_is_debian_version "$override_version"; then + printf 'ERROR: invalid CLOUDFLARED_VERSION %q\n' "$override_version" >&2 + return 1 + fi + printf '%s\n' "$override_version" + return 0 + fi + + if [[ -z "$available_versions" ]]; then + printf 'ERROR: no cloudflared versions available in Cloudflare APT repo\n' >&2 + return 1 + fi + + if ! cloudflared_is_debian_version "$min_version"; then + printf 'ERROR: invalid CLOUDFLARED_MIN_VERSION %q\n' "$min_version" >&2 + return 1 + fi + + local version best_version="" + while IFS= read -r version; do + [[ -z "$version" ]] && continue + if ! cloudflared_is_debian_version "$version"; then + printf 'ERROR: invalid cloudflared version from Cloudflare APT repo: %q\n' "$version" >&2 + return 1 + fi + if dpkg --compare-versions "$version" ge "$min_version"; then + if [[ -z "$best_version" ]] || dpkg --compare-versions "$version" gt "$best_version"; then + best_version="$version" + fi + fi + done <<<"$available_versions" + + if [[ -z "$best_version" ]]; then + printf 'ERROR: no cloudflared version in Cloudflare APT repo meets minimum %s\n' "$min_version" >&2 + return 1 + fi + + printf '%s\n' "$best_version" +} diff --git a/test/e2e-vpn/lib/discord-gateway-proof.sh b/test/e2e-vpn/lib/discord-gateway-proof.sh new file mode 100755 index 00000000000..18e9358b348 --- /dev/null +++ b/test/e2e-vpn/lib/discord-gateway-proof.sh @@ -0,0 +1,450 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Shared hermetic Discord Gateway helpers for messaging E2E scripts. + +append_exit_trap_for_fake_discord_gateway() { + local command="$1" + local existing + existing="$(trap -p EXIT | sed "s/^trap -- '//;s/' EXIT$//")" + trap ''"${existing:+$existing; }$command"'' EXIT +} + +cleanup_fake_discord_gateway() { + if [ -n "${FAKE_DISCORD_GATEWAY_CONTAINER:-}" ]; then + docker rm -f "$FAKE_DISCORD_GATEWAY_CONTAINER" >/dev/null 2>&1 || true + fi + if [ -n "${FAKE_DISCORD_GATEWAY_PID:-}" ]; then + kill "$FAKE_DISCORD_GATEWAY_PID" 2>/dev/null || true + wait "$FAKE_DISCORD_GATEWAY_PID" 2>/dev/null || true + fi + if [ -n "${FAKE_DISCORD_GATEWAY_DIR:-}" ]; then + rm -rf "$FAKE_DISCORD_GATEWAY_DIR" 2>/dev/null || true + fi +} + +start_fake_discord_gateway() { + local expected_token="$1" + mkdir -p "$REPO/.tmp" + FAKE_DISCORD_GATEWAY_DIR="$(mktemp -d "$REPO/.tmp/fake-discord.XXXXXX")" + FAKE_DISCORD_GATEWAY_PORT_FILE="$FAKE_DISCORD_GATEWAY_DIR/port" + FAKE_DISCORD_GATEWAY_CAPTURE_FILE="$FAKE_DISCORD_GATEWAY_DIR/capture.jsonl" + FAKE_DISCORD_GATEWAY_CONTAINER="nemoclaw-fake-discord-$$-$RANDOM" + FAKE_DISCORD_GATEWAY_HOST="host.docker.internal" + : >"$FAKE_DISCORD_GATEWAY_CAPTURE_FILE" + + if ! docker run -d --rm \ + --name "$FAKE_DISCORD_GATEWAY_CONTAINER" \ + -p 0:8080 \ + -e FAKE_DISCORD_GATEWAY_PORT=8080 \ + -e FAKE_DISCORD_GATEWAY_EXPECTED_TOKEN="$expected_token" \ + -e FAKE_DISCORD_GATEWAY_PORT_FILE=/tmp/fake-discord/port \ + -e FAKE_DISCORD_GATEWAY_CAPTURE_FILE=/tmp/fake-discord/capture.jsonl \ + -v "$FAKE_DISCORD_GATEWAY_DIR:/tmp/fake-discord" \ + -v "$REPO/test/e2e-vpn/lib:/opt/nemoclaw-e2e:ro" \ + node:22-bookworm-slim \ + node /opt/nemoclaw-e2e/fake-discord-gateway.cjs \ + >"$FAKE_DISCORD_GATEWAY_DIR/container.id" 2>"$FAKE_DISCORD_GATEWAY_DIR/server.log"; then + cat "$FAKE_DISCORD_GATEWAY_DIR/server.log" >&2 || true + return 1 + fi + append_exit_trap_for_fake_discord_gateway cleanup_fake_discord_gateway + + for _ in $(seq 1 50); do + if [ -s "$FAKE_DISCORD_GATEWAY_PORT_FILE" ]; then + local published_port + published_port="$(docker port "$FAKE_DISCORD_GATEWAY_CONTAINER" 8080/tcp 2>/dev/null | head -1 | sed 's/.*://')" + if [ -n "$published_port" ]; then + # Exported for callers that source this helper and apply policy/probes after startup. + export FAKE_DISCORD_GATEWAY_PORT + FAKE_DISCORD_GATEWAY_PORT="$published_port" + return 0 + fi + fi + if ! docker inspect "$FAKE_DISCORD_GATEWAY_CONTAINER" >/dev/null 2>&1; then + docker logs "$FAKE_DISCORD_GATEWAY_CONTAINER" >&2 || true + cat "$FAKE_DISCORD_GATEWAY_DIR/server.log" >&2 || true + return 1 + fi + sleep 0.1 + done + cat "$FAKE_DISCORD_GATEWAY_DIR/server.log" >&2 || true + return 1 +} + +fake_discord_gateway_allowed_ip_options() { + printf '%s' 'allowed-ip=10.0.0.0/8,allowed-ip=172.16.0.0/12,allowed-ip=192.168.0.0/16' +} + +check_fake_discord_gateway_rewrite_capture() { + local capture_file="$1" + local expected_token="$2" + + node - "$capture_file" "$expected_token" <<'NODE' +const fs = require("fs"); +const [file, expected] = process.argv.slice(2); +let serialized; +let rows; +try { + serialized = fs.readFileSync(file, "utf8"); + rows = serialized + .trim() + .split(/\n+/) + .filter(Boolean) + .map((line) => JSON.parse(line)); +} catch { + console.log("CAPTURE_PARSE_ERROR"); + process.exit(8); +} + +const identify = rows.filter((row) => row.event === "identify").at(-1); +if (!identify) { + console.log("NO_IDENTIFY"); + process.exit(2); +} +if (identify.tokenMatchesExpected !== true) { + console.log("BAD_TOKEN_REWRITE"); + process.exit(3); +} +if (identify.tokenLooksPlaceholder) { + console.log("PLACEHOLDER_LEAK"); + process.exit(4); +} +if (Object.prototype.hasOwnProperty.call(identify, "token")) { + console.log("RAW_TOKEN_CAPTURED"); + process.exit(5); +} +if (serialized.includes(expected)) { + console.log("RAW_TOKEN_LEAK"); + process.exit(6); +} +if (serialized.includes("openshell:resolve:env:")) { + console.log("PLACEHOLDER_LEAK"); + process.exit(7); +} +console.log("OK"); +NODE +} + +apply_fake_discord_gateway_policy() { + local sandbox_name="$1" + local port="$2" + local host="${FAKE_DISCORD_GATEWAY_HOST:-host.openshell.internal}" + local allowed_ip_options + allowed_ip_options="$(fake_discord_gateway_allowed_ip_options)" + openshell policy update "$sandbox_name" \ + --add-endpoint "${host}:${port}:read-write:websocket:enforce:websocket-credential-rewrite,${allowed_ip_options}" \ + --add-allow "${host}:${port}:GET:/**" \ + --add-allow "${host}:${port}:WEBSOCKET_TEXT:/**" \ + --binary /usr/local/bin/node \ + --binary /usr/bin/node \ + --binary /usr/local/bin/python3 \ + --binary /usr/bin/python3 \ + --binary /opt/hermes/.venv/bin/python \ + --wait +} + +run_fake_discord_gateway_node_client() { + local port="$1" + local identify_token="$2" + local proxy_url="${3:-}" + local host="${FAKE_DISCORD_GATEWAY_HOST:-host.openshell.internal}" + local proxy_env="" + if [ -n "$proxy_url" ]; then + printf -v proxy_env ' FAKE_DISCORD_GATEWAY_PROXY_URL=%q' "$proxy_url" + fi + sandbox_exec_stdin "FAKE_DISCORD_GATEWAY_CLIENT_HOST='$host' FAKE_DISCORD_GATEWAY_CLIENT_PORT='$port' FAKE_DISCORD_GATEWAY_IDENTIFY_TOKEN='$identify_token'$proxy_env node - 2>&1" <<'NODE' +const crypto = require("crypto"); +const net = require("net"); + +const host = process.env.FAKE_DISCORD_GATEWAY_CLIENT_HOST || "host.openshell.internal"; +const port = Number(process.env.FAKE_DISCORD_GATEWAY_CLIENT_PORT); +const identifyToken = process.env.FAKE_DISCORD_GATEWAY_IDENTIFY_TOKEN; +const proxyUrl = process.env.FAKE_DISCORD_GATEWAY_PROXY_URL || process.env.HTTP_PROXY || process.env.http_proxy || ""; +const results = []; + +function proxyTarget() { + if (!proxyUrl) return null; + try { + const parsed = new URL(proxyUrl); + if (parsed.protocol !== "http:") return null; + return { + host: parsed.hostname, + port: Number(parsed.port || "80"), + }; + } catch { + return null; + } +} + +function finish(message) { + if (message) results.push(message); + console.log(results.join("\n")); + process.exit(0); +} + +function encodeClientText(payload) { + const body = Buffer.from(payload, "utf8"); + const mask = crypto.randomBytes(4); + const masked = Buffer.alloc(body.length); + for (let i = 0; i < body.length; i += 1) masked[i] = body[i] ^ mask[i % 4]; + if (body.length < 126) { + return Buffer.concat([Buffer.from([0x81, 0x80 | body.length]), mask, masked]); + } + const header = Buffer.alloc(4); + header[0] = 0x81; + header[1] = 0x80 | 126; + header.writeUInt16BE(body.length, 2); + return Buffer.concat([header, mask, masked]); +} + +function encodeClientClose(code) { + const body = Buffer.alloc(2); + body.writeUInt16BE(code, 0); + const mask = crypto.randomBytes(4); + for (let i = 0; i < body.length; i += 1) body[i] ^= mask[i % 4]; + return Buffer.concat([Buffer.from([0x88, 0x80 | 2]), mask, body]); +} + +function decodeFrame(buffer) { + if (buffer.length < 2) return null; + const opcode = buffer[0] & 0x0f; + let payloadLength = buffer[1] & 0x7f; + let offset = 2; + if (payloadLength === 126) { + if (buffer.length < 4) return null; + payloadLength = buffer.readUInt16BE(2); + offset = 4; + } else if (payloadLength === 127) { + if (buffer.length < 10) return null; + payloadLength = Number(buffer.readBigUInt64BE(2)); + offset = 10; + } + if (buffer.length < offset + payloadLength) return null; + return { + opcode, + payload: buffer.slice(offset, offset + payloadLength), + totalLength: offset + payloadLength, + }; +} + +const proxy = proxyTarget(); +const socket = proxy + ? net.createConnection({ host: proxy.host, port: proxy.port }) + : net.createConnection({ host, port }); +const timer = setTimeout(() => { + try { socket.destroy(); } catch {} + finish("TIMEOUT"); +}, 20000); + +let handshake = Buffer.alloc(0); +let framed = Buffer.alloc(0); +let upgraded = false; +let sawReady = false; + +socket.on("connect", () => { + const key = crypto.randomBytes(16).toString("base64"); + const requestTarget = proxy + ? `http://${host}:${port}/gateway?v=10&encoding=json` + : "/gateway?v=10&encoding=json"; + socket.write([ + `GET ${requestTarget} HTTP/1.1`, + `Host: ${host}:${port}`, + "Upgrade: websocket", + "Connection: Upgrade", + `Sec-WebSocket-Key: ${key}`, + "Sec-WebSocket-Version: 13", + "\r\n", + ].join("\r\n")); +}); + +socket.on("data", (chunk) => { + if (!upgraded) { + handshake = Buffer.concat([handshake, chunk]); + const end = handshake.indexOf("\r\n\r\n"); + if (end === -1) return; + const statusLine = handshake.slice(0, end).toString("latin1").split("\r\n")[0] || ""; + if (!statusLine.includes("101")) { + clearTimeout(timer); + finish(`HTTP_${statusLine}`); + } + upgraded = true; + results.push("UPGRADE"); + framed = Buffer.concat([framed, handshake.slice(end + 4)]); + } else { + framed = Buffer.concat([framed, chunk]); + } + + while (framed.length > 0) { + const frame = decodeFrame(framed); + if (!frame) break; + framed = framed.slice(frame.totalLength); + if (frame.opcode === 1) { + const message = JSON.parse(frame.payload.toString("utf8")); + if (message.op === 10) { + results.push("HELLO"); + socket.write(encodeClientText(JSON.stringify({ + op: 2, + d: { + token: identifyToken, + intents: 0, + properties: { os: "linux", browser: "nemoclaw-e2e", device: "nemoclaw-e2e" }, + }, + }))); + results.push( + identifyToken.includes("openshell:resolve:env:") + ? "IDENTIFY_SENT_PLACEHOLDER" + : "IDENTIFY_SENT_NON_PLACEHOLDER", + ); + } else if (message.op === 0 && message.t === "READY") { + sawReady = true; + results.push("READY"); + socket.write(encodeClientText(JSON.stringify({ op: 1, d: message.s ?? null }))); + } else if (message.op === 11) { + results.push("HEARTBEAT_ACK"); + socket.write(encodeClientClose(1000)); + clearTimeout(timer); + finish(); + } + } else if (frame.opcode === 8) { + const code = frame.payload.length >= 2 ? frame.payload.readUInt16BE(0) : 0; + clearTimeout(timer); + finish(`CLOSE_${code}`); + } + } +}); + +socket.on("error", (error) => { + clearTimeout(timer); + finish(`ERROR ${error.message}`); +}); +socket.on("close", () => { + clearTimeout(timer); + if (!sawReady) finish("CLOSED"); +}); +NODE +} + +run_fake_discord_gateway_python_client() { + local port="$1" + local host="${FAKE_DISCORD_GATEWAY_HOST:-host.openshell.internal}" + sandbox_exec_stdin "FAKE_DISCORD_GATEWAY_CLIENT_HOST='$host' FAKE_DISCORD_GATEWAY_CLIENT_PORT='$port' /opt/hermes/.venv/bin/python - 2>&1" <<'PY' +import asyncio +import inspect +import os +from pathlib import Path + +try: + import aiohttp + import discord + from discord.http import DiscordClientWebSocketResponse + from yarl import URL +except Exception as exc: + print(f"IMPORT_DISCORD_FAILED {type(exc).__name__}: {exc}") + raise SystemExit(0) + + +def read_env_token(): + env_text = Path("/sandbox/.hermes/.env").read_text(encoding="utf-8") + for line in env_text.splitlines(): + if line.startswith("DISCORD_BOT_TOKEN="): + return line.split("=", 1)[1] + raise RuntimeError("missing DISCORD_BOT_TOKEN in /sandbox/.hermes/.env") + + +def note_heartbeat_ack(ws, results, previous_ack=None): + keep_alive = getattr(ws, "_keep_alive", None) + if keep_alive is None: + return False + current_ack = getattr(keep_alive, "_last_ack", None) + latency = getattr(keep_alive, "latency", float("inf")) + if previous_ack is not None and current_ack == previous_ack: + return False + if latency == float("inf"): + return False + if "HEARTBEAT_ACK" not in results: + results.append("HEARTBEAT_ACK") + return True + + +async def wait_for_ready(ws, results): + for _ in range(20): + await ws.poll_event() + note_heartbeat_ack(ws, results) + if getattr(ws, "session_id", None): + results.append("READY") + return + raise AssertionError("timed out waiting for READY") + + +async def wait_for_heartbeat_ack(ws, results): + if "HEARTBEAT_ACK" in results: + return + keep_alive = getattr(ws, "_keep_alive", None) + previous_ack = getattr(keep_alive, "_last_ack", None) + for _ in range(20): + await ws.poll_event() + if note_heartbeat_ack(ws, results, previous_ack): + return + raise AssertionError("timed out waiting for HEARTBEAT_ACK") + + +async def main(): + port = int(os.environ["FAKE_DISCORD_GATEWAY_CLIENT_PORT"]) + host = os.environ.get("FAKE_DISCORD_GATEWAY_CLIENT_HOST", "host.openshell.internal") + token = read_env_token() + results = [] + client = discord.Client(intents=discord.Intents.none()) + setup = getattr(client, "_async_setup_hook", None) + if setup is not None: + await setup() + client.http.token = token + client.http.proxy = os.environ.get("HTTP_PROXY") or os.environ.get("http_proxy") + client.http.proxy_auth = None + if getattr(client.http, "connector", None) is discord.utils.MISSING: + client.http.connector = aiohttp.TCPConnector(limit=0) + setattr( + client.http, + "_HTTPClient__session", + aiohttp.ClientSession( + connector=client.http.connector, + ws_response_class=DiscordClientWebSocketResponse, + trace_configs=None, + cookie_jar=aiohttp.DummyCookieJar(), + ), + ) + client.http._global_over = asyncio.Event() + client.http._global_over.set() + try: + from_client = discord.gateway.DiscordWebSocket.from_client + kwargs = {"gateway": URL(f"ws://{host}:{port}/gateway")} + params = inspect.signature(from_client).parameters + if "initial" in params: + kwargs["initial"] = False + if "compress" in params: + kwargs["compress"] = False + elif "zlib" in params: + kwargs["zlib"] = False + ws = await from_client(client, **kwargs) + results.append("UPGRADE") + results.append("HELLO") + if "openshell:resolve:env:" in token: + results.append("IDENTIFY_SENT_PLACEHOLDER") + await wait_for_ready(ws, results) + await ws.send_as_json({"op": 1, "d": ws.sequence}) + await wait_for_heartbeat_ack(ws, results) + close = getattr(ws, "close", None) + if close is not None: + await close(code=1000) + finally: + await client.close() + print("\n".join(results)) + + +try: + asyncio.run(main()) +except Exception as exc: + print(f"ERROR {type(exc).__name__}: {exc}") +PY +} diff --git a/test/e2e-vpn/lib/discord-rest-policy-proof.sh b/test/e2e-vpn/lib/discord-rest-policy-proof.sh new file mode 100755 index 00000000000..7c5fc1f32e6 --- /dev/null +++ b/test/e2e-vpn/lib/discord-rest-policy-proof.sh @@ -0,0 +1,491 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Shared hermetic Discord REST helpers for policy/binary-whitelist E2E checks. + +append_exit_trap_for_fake_discord_rest_api() { + local command="$1" + local existing + existing="$(trap -p EXIT | sed "s/^trap -- '//;s/' EXIT$//")" + trap ''"${existing:+$existing; }$command"'' EXIT +} + +cleanup_fake_discord_rest_api() { + if [ -n "${FAKE_DISCORD_REST_CONTAINER:-}" ]; then + docker rm -f "$FAKE_DISCORD_REST_CONTAINER" >/dev/null 2>&1 || true + fi + if [ -n "${FAKE_DISCORD_REST_DIR:-}" ]; then + rm -rf "$FAKE_DISCORD_REST_DIR" 2>/dev/null || true + fi +} + +cleanup_fake_discord_message_api() { + if [ -n "${FAKE_DISCORD_MESSAGE_API_CONTAINER:-}" ]; then + docker rm -f "$FAKE_DISCORD_MESSAGE_API_CONTAINER" >/dev/null 2>&1 || true + fi + if [ -n "${FAKE_DISCORD_MESSAGE_API_DIR:-}" ]; then + rm -rf "$FAKE_DISCORD_MESSAGE_API_DIR" 2>/dev/null || true + fi +} + +start_fake_discord_rest_api() { + if ! command -v openssl >/dev/null 2>&1; then + echo "openssl is required for fake Discord REST TLS cert generation" >&2 + return 1 + fi + + mkdir -p "$REPO/.tmp" + FAKE_DISCORD_REST_DIR="$(mktemp -d "$REPO/.tmp/fake-discord-rest.XXXXXX")" + FAKE_DISCORD_REST_PORT_FILE="$FAKE_DISCORD_REST_DIR/port" + FAKE_DISCORD_REST_CAPTURE_FILE="$FAKE_DISCORD_REST_DIR/capture.jsonl" + FAKE_DISCORD_REST_KEY_PATH="$FAKE_DISCORD_REST_DIR/key.pem" + FAKE_DISCORD_REST_CERT_PATH="$FAKE_DISCORD_REST_DIR/cert.pem" + FAKE_DISCORD_REST_CONTAINER="nemoclaw-fake-discord-rest-$$-$RANDOM" + FAKE_DISCORD_REST_HOST="host.docker.internal" + : >"$FAKE_DISCORD_REST_CAPTURE_FILE" + append_exit_trap_for_fake_discord_rest_api cleanup_fake_discord_rest_api + + if ! openssl req -x509 -newkey rsa:2048 \ + -keyout "$FAKE_DISCORD_REST_KEY_PATH" \ + -out "$FAKE_DISCORD_REST_CERT_PATH" \ + -days 7 \ + -nodes \ + -subj "/CN=host.docker.internal" \ + -addext "subjectAltName=DNS:host.docker.internal,DNS:host.openshell.internal" \ + >/dev/null 2>&1; then + echo "failed to generate fake Discord REST TLS certificate" >&2 + return 1 + fi + + if ! docker run -d --rm \ + --name "$FAKE_DISCORD_REST_CONTAINER" \ + -p 0:8443 \ + -e FAKE_DISCORD_REST_PORT=8443 \ + -e FAKE_DISCORD_REST_KEY_PATH=/tmp/fake-discord-rest/key.pem \ + -e FAKE_DISCORD_REST_CERT_PATH=/tmp/fake-discord-rest/cert.pem \ + -e FAKE_DISCORD_REST_PORT_FILE=/tmp/fake-discord-rest/port \ + -e FAKE_DISCORD_REST_CAPTURE_FILE=/tmp/fake-discord-rest/capture.jsonl \ + -v "$FAKE_DISCORD_REST_DIR:/tmp/fake-discord-rest" \ + -v "$REPO/test/e2e-vpn/lib:/opt/nemoclaw-e2e:ro" \ + node:22-bookworm-slim \ + node /opt/nemoclaw-e2e/fake-discord-rest-api.cjs \ + >"$FAKE_DISCORD_REST_DIR/container.id" 2>"$FAKE_DISCORD_REST_DIR/server.log"; then + cat "$FAKE_DISCORD_REST_DIR/server.log" >&2 || true + return 1 + fi + + for _ in $(seq 1 50); do + if [ -s "$FAKE_DISCORD_REST_PORT_FILE" ]; then + local published_port + published_port="$(docker port "$FAKE_DISCORD_REST_CONTAINER" 8443/tcp 2>/dev/null | head -1 | sed 's/.*://')" + if [ -n "$published_port" ]; then + export FAKE_DISCORD_REST_PORT + FAKE_DISCORD_REST_PORT="$published_port" + return 0 + fi + fi + if ! docker inspect "$FAKE_DISCORD_REST_CONTAINER" >/dev/null 2>&1; then + docker logs "$FAKE_DISCORD_REST_CONTAINER" >&2 || true + cat "$FAKE_DISCORD_REST_DIR/server.log" >&2 || true + return 1 + fi + sleep 0.1 + done + cat "$FAKE_DISCORD_REST_DIR/server.log" >&2 || true + return 1 +} + +start_fake_discord_message_api() { + local token="$1" + mkdir -p "$REPO/.tmp" + FAKE_DISCORD_MESSAGE_API_DIR="$(mktemp -d "$REPO/.tmp/fake-discord-message.XXXXXX")" + FAKE_DISCORD_MESSAGE_API_PORT_FILE="$FAKE_DISCORD_MESSAGE_API_DIR/port" + FAKE_DISCORD_MESSAGE_API_CAPTURE_FILE="$FAKE_DISCORD_MESSAGE_API_DIR/capture.jsonl" + FAKE_DISCORD_MESSAGE_API_CONTAINER="nemoclaw-fake-discord-message-$$-$RANDOM" + FAKE_DISCORD_MESSAGE_API_HOST="host.docker.internal" + : >"$FAKE_DISCORD_MESSAGE_API_CAPTURE_FILE" + + if ! docker run -d --rm \ + --name "$FAKE_DISCORD_MESSAGE_API_CONTAINER" \ + -p 0:8080 \ + -e FAKE_DISCORD_MESSAGE_API_PORT=8080 \ + -e FAKE_DISCORD_MESSAGE_API_EXPECTED_TOKEN="$token" \ + -e FAKE_DISCORD_MESSAGE_API_PORT_FILE=/tmp/fake-discord-message/port \ + -e FAKE_DISCORD_MESSAGE_API_CAPTURE_FILE=/tmp/fake-discord-message/capture.jsonl \ + -v "$FAKE_DISCORD_MESSAGE_API_DIR:/tmp/fake-discord-message" \ + -v "$REPO/test/e2e-vpn/lib:/opt/nemoclaw-e2e:ro" \ + node:22-bookworm-slim \ + node /opt/nemoclaw-e2e/fake-discord-message-api.cjs \ + >"$FAKE_DISCORD_MESSAGE_API_DIR/container.id" 2>"$FAKE_DISCORD_MESSAGE_API_DIR/server.log"; then + cat "$FAKE_DISCORD_MESSAGE_API_DIR/server.log" >&2 || true + return 1 + fi + append_exit_trap_for_fake_discord_rest_api cleanup_fake_discord_message_api + + for _ in $(seq 1 50); do + if [ -s "$FAKE_DISCORD_MESSAGE_API_PORT_FILE" ]; then + local published_port + published_port="$(docker port "$FAKE_DISCORD_MESSAGE_API_CONTAINER" 8080/tcp 2>/dev/null | head -1 | sed 's/.*://')" + if [ -n "$published_port" ]; then + export FAKE_DISCORD_MESSAGE_API_PORT + FAKE_DISCORD_MESSAGE_API_PORT="$published_port" + return 0 + fi + fi + if ! docker inspect "$FAKE_DISCORD_MESSAGE_API_CONTAINER" >/dev/null 2>&1; then + docker logs "$FAKE_DISCORD_MESSAGE_API_CONTAINER" >&2 || true + cat "$FAKE_DISCORD_MESSAGE_API_DIR/server.log" >&2 || true + return 1 + fi + sleep 0.1 + done + cat "$FAKE_DISCORD_MESSAGE_API_DIR/server.log" >&2 || true + return 1 +} + +apply_fake_discord_rest_policy() { + local sandbox_name="$1" + local port="$2" + local host="${FAKE_DISCORD_REST_HOST:-host.openshell.internal}" + local preset_file + preset_file="$FAKE_DISCORD_REST_DIR/policy.yaml" + + cat >"$preset_file" </tmp/nemoclaw-fake-discord-rest-policy.log 2>&1 || { + cat /tmp/nemoclaw-fake-discord-rest-policy.log >&2 || true + return 1 + } +} + +fake_discord_message_api_allowed_ip_options() { + printf '%s' 'allowed-ip=10.0.0.0/8,allowed-ip=172.16.0.0/12,allowed-ip=192.168.0.0/16' +} + +apply_fake_discord_message_api_policy() { + local sandbox_name="$1" + local port="$2" + local host="${FAKE_DISCORD_MESSAGE_API_HOST:-host.openshell.internal}" + local allowed_ip_options + allowed_ip_options="$(fake_discord_message_api_allowed_ip_options)" + openshell policy update "$sandbox_name" \ + --add-endpoint "${host}:${port}:read-write:rest:enforce:request-body-credential-rewrite,${allowed_ip_options}" \ + --add-allow "${host}:${port}:GET:/**" \ + --add-allow "${host}:${port}:POST:/**" \ + --binary /usr/local/bin/node \ + --binary /usr/bin/node \ + --wait +} + +run_fake_discord_plugin_send_proof() { + local port="$1" + local channel_id="$2" + local message="$3" + local host="${FAKE_DISCORD_MESSAGE_API_HOST:-host.openshell.internal}" + local message_b64 + message_b64=$(printf '%s' "$message" | base64 | tr -d '\n') + + sandbox_exec_stdin "FAKE_DISCORD_MESSAGE_API_HOST='$host' FAKE_DISCORD_MESSAGE_API_PORT='$port' FAKE_DISCORD_MESSAGE_CHANNEL_ID='$channel_id' FAKE_DISCORD_MESSAGE_TEXT_B64='$message_b64' node --preserve-symlinks --input-type=module - 2>&1" <<'NODE' +import { execFileSync } from "node:child_process"; +import fs from "node:fs"; +import http from "node:http"; +import { createRequire } from "node:module"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +function fail(message) { + console.error(message); + process.exit(1); +} + +function decodeBase64(value) { + return Buffer.from(value || "", "base64").toString("utf8"); +} + +function addPathWalk(candidates, seen, start) { + if (!start) return; + let current = path.resolve(start); + for (let depth = 0; depth < 8; depth += 1) { + if (!seen.has(current)) { + seen.add(current); + candidates.push(current); + } + const parent = path.dirname(current); + if (parent === current) break; + current = parent; + } +} + +function resolveDiscordSendApiPath() { + const require = createRequire(import.meta.url); + const candidates = []; + const seen = new Set(); + const add = (candidate) => { + if (candidate && !seen.has(candidate)) { + seen.add(candidate); + candidates.push(candidate); + } + }; + + for (const base of [process.cwd(), "/sandbox", "/usr/local/lib/node_modules", "/tmp/npm-global/lib/node_modules"]) { + try { + add(path.join(path.dirname(require.resolve("@openclaw/discord/package.json", { paths: [base] })), "dist/runtime-api.send.js")); + } catch {} + try { + add(path.join(path.dirname(require.resolve("openclaw/package.json", { paths: [base] })), "dist/extensions/discord/runtime-api.send.js")); + } catch {} + } + + try { + const globalRoot = execFileSync("npm", ["root", "-g"], { encoding: "utf8" }).trim(); + add(path.join(globalRoot, "@openclaw/discord/dist/runtime-api.send.js")); + add(path.join(globalRoot, "openclaw/dist/extensions/discord/runtime-api.send.js")); + } catch {} + + try { + const openclawBin = execFileSync("sh", ["-lc", "command -v openclaw || true"], { encoding: "utf8" }).trim(); + if (openclawBin) { + const realBin = execFileSync("readlink", ["-f", openclawBin], { encoding: "utf8" }).trim(); + const walk = []; + const walkSeen = new Set(); + addPathWalk(walk, walkSeen, path.dirname(realBin)); + for (const root of walk) { + add(path.join(root, "node_modules/@openclaw/discord/dist/runtime-api.send.js")); + add(path.join(root, "dist/extensions/discord/runtime-api.send.js")); + } + } + } catch {} + + try { + const searchRoots = ["/usr/local", "/tmp/npm-global", "/sandbox"].filter((root) => fs.existsSync(root)); + if (searchRoots.length) { + const discovered = execFileSync("find", [ + ...searchRoots, + "(", + "-path", + "*/node_modules/@openclaw/discord/dist/runtime-api.send.js", + "-o", + "-path", + "*/node_modules/openclaw/dist/extensions/discord/runtime-api.send.js", + ")", + "-print", + "-quit", + ], { encoding: "utf8" }).trim(); + add(discovered); + } + } catch {} + + for (const candidate of candidates) { + if (candidate && fs.existsSync(candidate)) return candidate; + } + return null; +} + +function requestFakeDiscord(method, apiPath, body, token) { + const payload = body === undefined ? "" : JSON.stringify(body.body ?? body); + const options = { + hostname: process.env.FAKE_DISCORD_MESSAGE_API_HOST || "host.openshell.internal", + port: Number(process.env.FAKE_DISCORD_MESSAGE_API_PORT), + path: `/api/v10${apiPath}`, + method, + headers: { + Authorization: `Bot ${token}`, + "Content-Type": "application/json", + "Content-Length": Buffer.byteLength(payload), + "User-Agent": "nemoclaw-openclaw-discord-plugin-e2e", + }, + }; + return new Promise((resolve, reject) => { + const req = http.request(options, (res) => { + let responseBody = ""; + res.on("data", (chunk) => { + responseBody += chunk; + }); + res.on("end", () => { + let parsed = {}; + try { + parsed = responseBody ? JSON.parse(responseBody) : {}; + } catch (error) { + reject(new Error(`invalid JSON from fake Discord: ${error.message}: ${responseBody}`)); + return; + } + if (res.statusCode < 200 || res.statusCode >= 300) { + const err = new Error(`fake Discord returned HTTP ${res.statusCode}`); + err.status = res.statusCode; + err.rawError = parsed; + reject(err); + return; + } + resolve(parsed); + }); + }); + req.on("error", reject); + req.setTimeout(30000, () => { + req.destroy(new Error("fake Discord message API timed out")); + }); + if (payload) req.write(payload); + req.end(); + }); +} + +const sendApiPath = resolveDiscordSendApiPath(); +if (!sendApiPath) fail("could not find installed OpenClaw Discord runtime-api.send.js"); + +const { sendMessageDiscord } = await import(pathToFileURL(sendApiPath).href); +if (typeof sendMessageDiscord !== "function") fail("installed Discord runtime API does not export sendMessageDiscord"); + +const cfg = JSON.parse(fs.readFileSync("/sandbox/.openclaw/openclaw.json", "utf8")); +const account = cfg.channels?.discord?.accounts?.default; +if (!account?.token) fail("missing channels.discord.accounts.default.token in openclaw.json"); + +const channelId = process.env.FAKE_DISCORD_MESSAGE_CHANNEL_ID || "420000000000000123"; +const text = decodeBase64(process.env.FAKE_DISCORD_MESSAGE_TEXT_B64); +const token = account.token; +const rest = { + get: (apiPath) => requestFakeDiscord("GET", apiPath, undefined, token), + post: (apiPath, data) => requestFakeDiscord("POST", apiPath, data, token), + patch: (apiPath, data) => requestFakeDiscord("PATCH", apiPath, data, token), + put: (apiPath, data) => requestFakeDiscord("PUT", apiPath, data, token), + delete: (apiPath, data) => requestFakeDiscord("DELETE", apiPath, data, token), +}; + +const result = await sendMessageDiscord(`channel:${channelId}`, text, { + cfg, + accountId: "default", + rest, +}); + +console.log(JSON.stringify({ + ok: true, + proof: "openclaw-discord-runtime-send", + channelId: result.channelId ?? channelId, + messageId: result.messageId ?? result.platformMessageIds?.[0] ?? null, +})); +NODE +} + +check_fake_discord_message_capture() { + local expected_channel="$1" + local expected_text="$2" + node - "$FAKE_DISCORD_MESSAGE_API_CAPTURE_FILE" "$expected_channel" "$expected_text" <<'NODE' +const fs = require("fs"); +const [file, expectedChannel, expectedText] = process.argv.slice(2); +const rows = fs + .readFileSync(file, "utf8") + .trim() + .split(/\n+/) + .filter(Boolean) + .map((line) => JSON.parse(line)) + .filter((row) => row.event === "request" && row.method === "POST" && row.path.endsWith("/messages")); +const last = rows.at(-1); +if (!last) { + console.log("NO_MESSAGE_REQUEST"); + process.exit(2); +} +if (last.tokenMatchesExpected !== true) { + console.log("BAD_TOKEN_REWRITE"); + process.exit(3); +} +if (last.tokenLooksPlaceholder) { + console.log("PLACEHOLDER_LEAK"); + process.exit(4); +} +if (last.channelId !== expectedChannel) { + console.log(`BAD_CHANNEL ${last.channelId}`); + process.exit(5); +} +if (last.content !== expectedText) { + console.log(`BAD_TEXT ${last.content}`); + process.exit(6); +} +console.log("OK"); +NODE +} + +run_fake_discord_rest_node_request() { + local port="$1" + local path="$2" + local host="${FAKE_DISCORD_REST_HOST:-host.openshell.internal}" + sandbox_exec_stdin "FAKE_DISCORD_REST_HOST='$host' FAKE_DISCORD_REST_PORT='$port' FAKE_DISCORD_REST_PATH='$path' node - 2>&1" <<'NODE' +const https = require("https"); + +const options = { + hostname: process.env.FAKE_DISCORD_REST_HOST || "host.openshell.internal", + port: Number(process.env.FAKE_DISCORD_REST_PORT), + path: process.env.FAKE_DISCORD_REST_PATH || "/api/v10/gateway", + method: "GET", + rejectUnauthorized: false, + headers: { "User-Agent": "nemoclaw-e2e-node" }, +}; + +const req = https.request(options, (res) => { + let body = ""; + res.on("data", (chunk) => { + body += chunk; + }); + res.on("end", () => { + console.log(`${res.statusCode} ${body.slice(0, 300)}`); + }); +}); + +req.on("error", (error) => { + console.log(`ERROR: ${error.message}`); +}); +req.setTimeout(30000, () => { + req.destroy(); + console.log("TIMEOUT"); +}); +req.end(); +NODE +} + +run_fake_discord_rest_curl_request() { + local port="$1" + local host="${FAKE_DISCORD_REST_HOST:-host.openshell.internal}" + sandbox_exec "set +e +rm -f /tmp/nemoclaw-fake-discord-curl.err /tmp/nemoclaw-fake-discord-curl.body +curl -k -v --max-time 15 https://$host:$port/api/v10/gateway \ + -A nemoclaw-e2e-curl \ + -o /tmp/nemoclaw-fake-discord-curl.body \ + 2>/tmp/nemoclaw-fake-discord-curl.err +rc=\$? +printf 'RC=%s\n' \"\$rc\" +grep -E 'Uses proxy|CONNECT .* HTTP|HTTP/1\\.[01] 403|CONNECT tunnel failed|Connection established|policy_denied|Forbidden' /tmp/nemoclaw-fake-discord-curl.err /tmp/nemoclaw-fake-discord-curl.body 2>/dev/null || true +" 2>/dev/null || true +} + +fake_discord_rest_capture_counts() { + node - "$FAKE_DISCORD_REST_CAPTURE_FILE" <<'NODE' +const fs = require("fs"); +const file = process.argv[2]; +const rows = fs.readFileSync(file, "utf8").trim().split(/\n+/).filter(Boolean).map((line) => JSON.parse(line)); +const requests = rows.filter((row) => row.event === "request"); +const node = requests.filter((row) => String(row.userAgent || "").includes("nemoclaw-e2e-node")).length; +const curl = requests.filter((row) => String(row.userAgent || "").includes("nemoclaw-e2e-curl")).length; +console.log(`requests=${requests.length} node=${node} curl=${curl}`); +NODE +} diff --git a/test/e2e-vpn/lib/fake-discord-gateway.cjs b/test/e2e-vpn/lib/fake-discord-gateway.cjs new file mode 100755 index 00000000000..cc2fbe15026 --- /dev/null +++ b/test/e2e-vpn/lib/fake-discord-gateway.cjs @@ -0,0 +1,218 @@ +#!/usr/bin/env node +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +"use strict"; + +const crypto = require("crypto"); +const fs = require("fs"); +const net = require("net"); + +const host = process.env.FAKE_DISCORD_GATEWAY_HOST || "0.0.0.0"; +const port = Number(process.env.FAKE_DISCORD_GATEWAY_PORT || "0"); +const portFile = process.env.FAKE_DISCORD_GATEWAY_PORT_FILE || ""; +const captureFile = process.env.FAKE_DISCORD_GATEWAY_CAPTURE_FILE || ""; +const expectedToken = process.env.FAKE_DISCORD_GATEWAY_EXPECTED_TOKEN || ""; + +if (!expectedToken) { + console.error("FAKE_DISCORD_GATEWAY_EXPECTED_TOKEN is required"); + process.exit(2); +} + +function record(event) { + if (!captureFile) return; + fs.appendFileSync(captureFile, `${JSON.stringify({ at: Date.now(), ...event })}\n`); +} + +function encodeText(payload) { + const body = Buffer.from(payload, "utf8"); + if (body.length < 126) { + return Buffer.concat([Buffer.from([0x81, body.length]), body]); + } + if (body.length <= 0xffff) { + const header = Buffer.alloc(4); + header[0] = 0x81; + header[1] = 126; + header.writeUInt16BE(body.length, 2); + return Buffer.concat([header, body]); + } + const header = Buffer.alloc(10); + header[0] = 0x81; + header[1] = 127; + header.writeBigUInt64BE(BigInt(body.length), 2); + return Buffer.concat([header, body]); +} + +function encodeClose(code) { + const body = Buffer.alloc(2); + body.writeUInt16BE(code, 0); + return Buffer.from([0x88, body.length, ...body]); +} + +function decodeFrame(buffer) { + if (buffer.length < 2) return null; + const opcode = buffer[0] & 0x0f; + const masked = (buffer[1] & 0x80) !== 0; + let payloadLength = buffer[1] & 0x7f; + let offset = 2; + + if (payloadLength === 126) { + if (buffer.length < 4) return null; + payloadLength = buffer.readUInt16BE(2); + offset = 4; + } else if (payloadLength === 127) { + if (buffer.length < 10) return null; + payloadLength = Number(buffer.readBigUInt64BE(2)); + offset = 10; + } + + let mask; + if (masked) { + if (buffer.length < offset + 4) return null; + mask = buffer.slice(offset, offset + 4); + offset += 4; + } + if (buffer.length < offset + payloadLength) return null; + + const payload = Buffer.from(buffer.slice(offset, offset + payloadLength)); + if (masked && mask) { + for (let i = 0; i < payload.length; i += 1) { + payload[i] ^= mask[i % 4]; + } + } + return { + opcode, + payload, + totalLength: offset + payloadLength, + }; +} + +function sendJson(socket, payload) { + socket.write(encodeText(JSON.stringify(payload))); +} + +function handleGatewayMessage(socket, payload) { + let message; + try { + message = JSON.parse(payload.toString("utf8")); + } catch (error) { + record({ event: "malformed_text", error: error.message }); + socket.write(encodeClose(4002)); + socket.end(); + return; + } + + if (message.op === 2) { + const token = message && message.d && message.d.token; + record({ + event: "identify", + tokenMatchesExpected: token === expectedToken, + tokenLooksPlaceholder: typeof token === "string" && token.includes("openshell:resolve:env:"), + }); + if (token !== expectedToken) { + socket.write(encodeClose(4004)); + socket.end(); + return; + } + sendJson(socket, { + op: 0, + t: "READY", + s: 1, + d: { + session_id: "fake-discord-gateway-session", + resume_gateway_url: "ws://host.openshell.internal/gateway", + user: { + id: "0", + username: "nemoclaw-fake-gateway", + discriminator: "0000", + avatar: null, + bot: true, + }, + guilds: [], + }, + }); + return; + } + + if (message.op === 1) { + record({ event: "heartbeat", d: message.d ?? null }); + sendJson(socket, { op: 11, d: null }); + return; + } + + record({ event: "gateway_message", op: message.op ?? null }); +} + +const server = net.createServer((socket) => { + let handshake = Buffer.alloc(0); + let framed = Buffer.alloc(0); + let upgraded = false; + + socket.on("data", (chunk) => { + if (!upgraded) { + handshake = Buffer.concat([handshake, chunk]); + const end = handshake.indexOf("\r\n\r\n"); + if (end === -1) return; + + const request = handshake.slice(0, end).toString("latin1"); + const requestLine = request.split("\r\n")[0] || ""; + const keyLine = request + .split("\r\n") + .find((line) => line.toLowerCase().startsWith("sec-websocket-key:")); + const key = keyLine ? keyLine.slice(keyLine.indexOf(":") + 1).trim() : ""; + if (!key) { + socket.end("HTTP/1.1 400 Bad Request\r\nConnection: close\r\n\r\n"); + return; + } + + const accept = crypto + .createHash("sha1") + .update(`${key}258EAFA5-E914-47DA-95CA-C5AB0DC85B11`) + .digest("base64"); + socket.write( + [ + "HTTP/1.1 101 Switching Protocols", + "Upgrade: websocket", + "Connection: Upgrade", + `Sec-WebSocket-Accept: ${accept}`, + "\r\n", + ].join("\r\n"), + ); + upgraded = true; + record({ event: "upgrade", requestLine }); + sendJson(socket, { op: 10, d: { heartbeat_interval: 30000 } }); + framed = Buffer.concat([framed, handshake.slice(end + 4)]); + } else { + framed = Buffer.concat([framed, chunk]); + } + + while (framed.length > 0) { + const frame = decodeFrame(framed); + if (!frame) break; + framed = framed.slice(frame.totalLength); + if (frame.opcode === 0x1) { + handleGatewayMessage(socket, frame.payload); + } else if (frame.opcode === 0x8) { + socket.write(encodeClose(1000)); + socket.end(); + } else if (frame.opcode === 0x9) { + socket.write(Buffer.from([0x8a, 0x00])); + } + } + }); +}); + +server.listen(port, host, () => { + const address = server.address(); + if (portFile) { + fs.writeFileSync(portFile, `${address.port}\n`, { mode: 0o600 }); + } + record({ event: "listening", host, port: address.port }); +}); + +for (const signal of ["SIGTERM", "SIGINT"]) { + process.on(signal, () => { + server.close(() => process.exit(0)); + setTimeout(() => process.exit(0), 1000).unref(); + }); +} diff --git a/test/e2e-vpn/lib/fake-discord-message-api.cjs b/test/e2e-vpn/lib/fake-discord-message-api.cjs new file mode 100755 index 00000000000..a99475e48fa --- /dev/null +++ b/test/e2e-vpn/lib/fake-discord-message-api.cjs @@ -0,0 +1,157 @@ +#!/usr/bin/env node +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +"use strict"; + +const fs = require("fs"); +const http = require("http"); + +const host = process.env.FAKE_DISCORD_MESSAGE_API_HOST || "0.0.0.0"; +const rawPort = process.env.FAKE_DISCORD_MESSAGE_API_PORT || "0"; +const port = Number(rawPort); +const portFile = process.env.FAKE_DISCORD_MESSAGE_API_PORT_FILE || ""; +const captureFile = process.env.FAKE_DISCORD_MESSAGE_API_CAPTURE_FILE || ""; +const expectedToken = process.env.FAKE_DISCORD_MESSAGE_API_EXPECTED_TOKEN || ""; +const MAX_BODY_BYTES = 1024 * 1024; + +if (!Number.isInteger(port) || port < 0 || port > 65535) { + console.error(`FAKE_DISCORD_MESSAGE_API_PORT must be an integer between 0 and 65535 (received: ${rawPort})`); + process.exit(2); +} + +if (!expectedToken) { + console.error("FAKE_DISCORD_MESSAGE_API_EXPECTED_TOKEN is required"); + process.exit(2); +} + +function record(event) { + if (!captureFile) return; + fs.appendFileSync(captureFile, `${JSON.stringify({ at: Date.now(), ...event })}\n`); +} + +function tokenFromAuthorization(value) { + const raw = String(value || ""); + if (raw.length < 4 || raw.slice(0, 3).toLowerCase() !== "bot") return raw; + const next = raw.charCodeAt(3); + if (next !== 0x20 && next !== 0x09) return raw; + let index = 4; + while (index < raw.length) { + const code = raw.charCodeAt(index); + if (code !== 0x20 && code !== 0x09) break; + index += 1; + } + return raw.slice(index); +} + +function tokenLooksPlaceholder(value) { + return typeof value === "string" && value.includes("openshell:resolve:env:"); +} + +function writeJson(res, status, body) { + res.writeHead(status, { "content-type": "application/json" }); + res.end(JSON.stringify(body)); +} + +function parseJson(body) { + try { + return JSON.parse(body || "{}"); + } catch { + return {}; + } +} + +const server = http.createServer((req, res) => { + const chunks = []; + let bodyBytes = 0; + let bodyTooLarge = false; + req.on("data", (chunk) => { + if (bodyTooLarge) return; + bodyBytes += chunk.length; + if (bodyBytes > MAX_BODY_BYTES) { + bodyTooLarge = true; + record({ event: "request-too-large", method: req.method, path: req.url || "/", bodyBytes }); + writeJson(res, 413, { message: "payload too large", code: 413 }); + req.destroy(); + return; + } + chunks.push(chunk); + }); + + req.on("end", () => { + if (bodyTooLarge) return; + const body = Buffer.concat(chunks).toString("utf8"); + const url = new URL(req.url || "/", "http://fake-discord.local"); + const token = tokenFromAuthorization(req.headers.authorization); + const tokenMatchesExpected = token === expectedToken; + const messageMatch = /^\/api\/v10\/channels\/([^/]+)\/messages$/.exec(url.pathname); + const channelMatch = /^\/api\/v10\/channels\/([^/]+)$/.exec(url.pathname); + const parsed = parseJson(body); + const content = typeof parsed.content === "string" ? parsed.content : ""; + + record({ + event: "request", + method: req.method, + path: url.pathname, + tokenMatchesExpected, + tokenLooksPlaceholder: tokenLooksPlaceholder(token), + authorizationPresent: Boolean(req.headers.authorization), + authorizationRedacted: true, + bodyRedacted: true, + channelId: messageMatch?.[1] || channelMatch?.[1] || "", + content, + contentLength: content.length, + }); + + if (!tokenMatchesExpected) { + writeJson(res, 401, { message: "401: Unauthorized", code: 0 }); + return; + } + + if (req.method === "GET" && channelMatch) { + writeJson(res, 200, { + id: channelMatch[1], + type: 0, + name: "nemoclaw-e2e", + }); + return; + } + + if (req.method === "POST" && messageMatch) { + writeJson(res, 200, { + id: "420000000000000001", + channel_id: messageMatch[1], + content, + timestamp: new Date().toISOString(), + author: { + id: "420000000000000000", + username: "NemoClaw E2E", + bot: true, + }, + }); + return; + } + + writeJson(res, 404, { message: "Unknown Endpoint", code: 10001 }); + }); +}); + +server.on("error", (error) => { + record({ event: "server_error", error: error.message }); + console.error(error.stack || error.message); +}); + +server.listen(port, host, () => { + const address = server.address(); + if (portFile) { + fs.writeFileSync(portFile, `${address.port}\n`, { mode: 0o600 }); + } + record({ event: "listening", host, port: address.port }); +}); + +for (const signal of ["SIGTERM", "SIGINT"]) { + process.on(signal, () => { + server.close(() => process.exit(0)); + setTimeout(() => process.exit(0), 1000).unref(); + }); +} diff --git a/test/e2e-vpn/lib/fake-discord-rest-api.cjs b/test/e2e-vpn/lib/fake-discord-rest-api.cjs new file mode 100755 index 00000000000..b9b6532883d --- /dev/null +++ b/test/e2e-vpn/lib/fake-discord-rest-api.cjs @@ -0,0 +1,75 @@ +#!/usr/bin/env node +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +"use strict"; + +const fs = require("fs"); +const https = require("https"); + +const host = process.env.FAKE_DISCORD_REST_HOST || "0.0.0.0"; +const port = Number(process.env.FAKE_DISCORD_REST_PORT || "0"); +const keyPath = process.env.FAKE_DISCORD_REST_KEY_PATH || ""; +const certPath = process.env.FAKE_DISCORD_REST_CERT_PATH || ""; +const portFile = process.env.FAKE_DISCORD_REST_PORT_FILE || ""; +const captureFile = process.env.FAKE_DISCORD_REST_CAPTURE_FILE || ""; + +if (!keyPath || !certPath) { + console.error("FAKE_DISCORD_REST_KEY_PATH and FAKE_DISCORD_REST_CERT_PATH are required"); + process.exit(2); +} + +function record(event) { + if (!captureFile) return; + fs.appendFileSync(captureFile, `${JSON.stringify({ at: Date.now(), ...event })}\n`); +} + +const server = https.createServer( + { + key: fs.readFileSync(keyPath), + cert: fs.readFileSync(certPath), + }, + (req, res) => { + const chunks = []; + req.on("data", (chunk) => chunks.push(chunk)); + req.on("end", () => { + const body = Buffer.concat(chunks).toString("utf8"); + record({ + event: "request", + method: req.method, + url: req.url, + userAgent: req.headers["user-agent"] || "", + bodyLength: body.length, + }); + + if (req.url === "/api/v10/gateway") { + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ url: "wss://gateway.discord.gg" })); + return; + } + + res.writeHead(200, { "Content-Type": "text/plain" }); + res.end("fake discord cdn ok\n"); + }); + }, +); + +server.on("error", (error) => { + record({ event: "server_error", error: error.message }); + console.error(error.stack || error.message); +}); + +server.listen(port, host, () => { + const address = server.address(); + if (portFile) { + fs.writeFileSync(portFile, `${address.port}\n`, { mode: 0o600 }); + } + record({ event: "listening", host, port: address.port }); +}); + +for (const signal of ["SIGTERM", "SIGINT"]) { + process.on(signal, () => { + server.close(() => process.exit(0)); + setTimeout(() => process.exit(0), 1000).unref(); + }); +} diff --git a/test/e2e-vpn/lib/fake-openai-compatible-api.mts b/test/e2e-vpn/lib/fake-openai-compatible-api.mts new file mode 100755 index 00000000000..1aaa8f156e2 --- /dev/null +++ b/test/e2e-vpn/lib/fake-openai-compatible-api.mts @@ -0,0 +1,194 @@ +#!/usr/bin/env -S node --experimental-strip-types +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { appendFileSync, writeFileSync } from "node:fs"; +import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; + +type JsonObject = Record; + +const host = process.env.NEMOCLAW_FAKE_OPENAI_HOST || "127.0.0.1"; +const port = Number(process.env.NEMOCLAW_FAKE_OPENAI_PORT || "0"); +const portFile = process.env.NEMOCLAW_FAKE_OPENAI_PORT_FILE || ""; +const logFile = process.env.NEMOCLAW_FAKE_OPENAI_LOG_FILE || ""; +const requestsFile = process.env.NEMOCLAW_FAKE_OPENAI_REQUESTS_FILE || ""; +const model = process.env.NEMOCLAW_FAKE_OPENAI_MODEL || "test-model"; +const apiKey = process.env.NEMOCLAW_FAKE_OPENAI_API_KEY || ""; +const requireAuth = process.env.NEMOCLAW_FAKE_OPENAI_REQUIRE_AUTH === "1"; +const chatContent = process.env.NEMOCLAW_FAKE_OPENAI_CHAT_CONTENT || "ok"; +const responseText = process.env.NEMOCLAW_FAKE_OPENAI_RESPONSE_TEXT || chatContent; + +function log(message: string): void { + if (logFile) { + appendFileSync(logFile, `${message}\n`); + return; + } + console.log(message); +} + +function recordRequest(entry: JsonObject): void { + if (!requestsFile) return; + appendFileSync(requestsFile, `${JSON.stringify(entry)}\n`); +} + +function sendJson(res: ServerResponse, status: number, payload: unknown): void { + const body = JSON.stringify(payload); + res.writeHead(status, { + "Content-Type": "application/json", + "Content-Length": Buffer.byteLength(body), + }); + res.end(body); +} + +function sendChatSse(res: ServerResponse, content: string): void { + const chunk = JSON.stringify({ + id: "chatcmpl-fake-openai-compatible", + object: "chat.completion.chunk", + choices: [{ index: 0, delta: { role: "assistant", content }, finish_reason: null }], + }); + const doneChunk = JSON.stringify({ + id: "chatcmpl-fake-openai-compatible", + object: "chat.completion.chunk", + choices: [{ index: 0, delta: {}, finish_reason: "stop" }], + }); + const body = `data: ${chunk}\n\ndata: ${doneChunk}\n\ndata: [DONE]\n\n`; + res.writeHead(200, { + "Content-Type": "text/event-stream", + "Content-Length": Buffer.byteLength(body), + }); + res.end(body); +} + +function sendResponseSse(res: ServerResponse, text: string): void { + const body = [ + "event: response.output_text.delta", + `data: ${JSON.stringify({ delta: text })}`, + "", + "event: response.completed", + "data: {}", + "", + ].join("\n"); + res.writeHead(200, { + "Content-Type": "text/event-stream", + "Content-Length": Buffer.byteLength(body), + }); + res.end(body); +} + +function isAuthOk(req: IncomingMessage): boolean { + if (!requireAuth) return true; + return req.headers.authorization === `Bearer ${apiKey}`; +} + +function requestPath(req: IncomingMessage): string { + return new URL(req.url || "/", "http://fake-openai-compatible.local").pathname; +} + +function readBody(req: IncomingMessage): Promise { + return new Promise((resolve) => { + const chunks: Buffer[] = []; + req.on("data", (chunk: Buffer) => chunks.push(chunk)); + req.on("end", () => resolve(Buffer.concat(chunks))); + }); +} + +function parseJsonBody(raw: Buffer): JsonObject { + if (raw.length === 0) return {}; + try { + const parsed = JSON.parse(raw.toString("utf8")); + return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : {}; + } catch { + return {}; + } +} + +const server = createServer(async (req, res) => { + const path = requestPath(req); + + if (req.method === "GET" && ["/v1/models", "/models"].includes(path)) { + log(`GET ${path}`); + recordRequest({ method: "GET", path, bodyBytes: 0 }); + sendJson(res, 200, { object: "list", data: [{ id: model, object: "model" }] }); + return; + } + + const raw = await readBody(req); + const payload = parseJsonBody(raw); + const auth = isAuthOk(req) ? "ok" : "missing"; + recordRequest({ + method: req.method || "GET", + path, + bodyBytes: raw.length, + auth, + model: payload.model, + stream: Boolean(payload.stream), + }); + + if (req.method === "POST" && ["/v1/chat/completions", "/chat/completions"].includes(path)) { + log( + `POST ${path} auth=${auth} model=${String(payload.model || "")} stream=${Boolean(payload.stream)}`, + ); + if (!isAuthOk(req)) { + sendJson(res, 401, { error: { message: "missing bearer credential" } }); + return; + } + if (payload.stream) { + sendChatSse(res, chatContent); + return; + } + sendJson(res, 200, { + id: "chatcmpl-fake-openai-compatible", + object: "chat.completion", + choices: [ + { + index: 0, + message: { role: "assistant", content: chatContent }, + finish_reason: "stop", + }, + ], + }); + return; + } + + if (req.method === "POST" && ["/v1/responses", "/responses"].includes(path)) { + log(`POST ${path} auth=${auth} stream=${Boolean(payload.stream)}`); + if (!isAuthOk(req)) { + sendJson(res, 401, { error: { message: "missing bearer credential" } }); + return; + } + if (payload.stream) { + sendResponseSse(res, responseText); + return; + } + sendJson(res, 200, { + id: "resp-fake-openai-compatible", + object: "response", + output: [ + { + type: "message", + role: "assistant", + content: [{ type: "output_text", text: responseText }], + }, + ], + }); + return; + } + + sendJson(res, 404, { error: { message: "not found" } }); +}); + +server.listen(port, host, () => { + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("fake OpenAI-compatible server did not bind to a TCP port"); + } + if (portFile) writeFileSync(portFile, String(address.port)); + log(`READY host=${host} port=${address.port} model=${model}`); +}); + +for (const signal of ["SIGINT", "SIGTERM"] as const) { + process.on(signal, () => { + server.close(() => process.exit(0)); + setTimeout(() => process.exit(0), 500).unref(); + }); +} diff --git a/test/e2e-vpn/lib/fake-slack-api.cjs b/test/e2e-vpn/lib/fake-slack-api.cjs new file mode 100755 index 00000000000..30b72a19eff --- /dev/null +++ b/test/e2e-vpn/lib/fake-slack-api.cjs @@ -0,0 +1,310 @@ +#!/usr/bin/env node +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +"use strict"; + +const fs = require("fs"); +const crypto = require("crypto"); +const http = require("http"); + +const host = process.env.FAKE_SLACK_API_HOST || "0.0.0.0"; +const rawPort = process.env.FAKE_SLACK_API_PORT || "0"; +const port = Number(rawPort); +const portFile = process.env.FAKE_SLACK_API_PORT_FILE || ""; +const captureFile = process.env.FAKE_SLACK_API_CAPTURE_FILE || ""; +const expectedBotToken = process.env.FAKE_SLACK_API_EXPECTED_BOT_TOKEN || ""; +const expectedAppToken = process.env.FAKE_SLACK_API_EXPECTED_APP_TOKEN || ""; +const socketUserId = process.env.FAKE_SLACK_API_SOCKET_USER_ID || "U3730E2E"; +const socketChannelId = process.env.FAKE_SLACK_API_SOCKET_CHANNEL_ID || "D3730E2E"; +const socketTeamId = process.env.FAKE_SLACK_API_SOCKET_TEAM_ID || "T3730E2E"; +const MAX_BODY_BYTES = 1024 * 1024; +const WS_GUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"; + +if (!Number.isInteger(port) || port < 0 || port > 65535) { + console.error(`FAKE_SLACK_API_PORT must be an integer between 0 and 65535 (received: ${rawPort})`); + process.exit(2); +} + +if (!expectedBotToken || !expectedAppToken) { + console.error("FAKE_SLACK_API_EXPECTED_BOT_TOKEN and FAKE_SLACK_API_EXPECTED_APP_TOKEN are required"); + process.exit(2); +} + +function record(event) { + if (!captureFile) return; + fs.appendFileSync(captureFile, `${JSON.stringify({ at: Date.now(), ...event })}\n`); +} + +function expectedTokenForPath(pathname) { + if (pathname === "/api/apps.connections.open") return expectedAppToken; + return expectedBotToken; +} + +function tokenLooksPlaceholder(value) { + return ( + typeof value === "string" && + (value.includes("openshell:resolve:env:") || value.includes("OPENSHELL-RESOLVE-ENV-")) + ); +} + +function slackResponseFor(pathname, authAccepted, message = {}) { + if (pathname === "/api/chat.postMessage") { + return { + status: 200, + body: authAccepted + ? { + ok: true, + channel: message.channel || socketChannelId, + ts: "1710000000.000200", + message: { + type: "message", + channel: message.channel || socketChannelId, + text: message.text || "", + ts: "1710000000.000200", + ...(message.threadTs ? { thread_ts: message.threadTs } : {}), + }, + } + : { + ok: false, + error: "bad_auth", + endpoint: pathname, + }, + }; + } + if (!authAccepted) { + return { status: 401, body: { ok: false, error: "bad_auth", endpoint: pathname } }; + } + return { status: 200, body: { ok: false, error: "invalid_auth", endpoint: pathname } }; +} + +function encodeServerText(payload) { + const body = Buffer.from(payload, "utf8"); + if (body.length < 126) { + return Buffer.concat([Buffer.from([0x81, body.length]), body]); + } + if (body.length < 65536) { + const header = Buffer.alloc(4); + header[0] = 0x81; + header[1] = 126; + header.writeUInt16BE(body.length, 2); + return Buffer.concat([header, body]); + } + const header = Buffer.alloc(10); + header[0] = 0x81; + header[1] = 127; + header.writeBigUInt64BE(BigInt(body.length), 2); + return Buffer.concat([header, body]); +} + +function decodeClientFrame(buffer) { + if (buffer.length < 2) return null; + const opcode = buffer[0] & 0x0f; + const masked = (buffer[1] & 0x80) !== 0; + let payloadLength = buffer[1] & 0x7f; + let offset = 2; + if (payloadLength === 126) { + if (buffer.length < 4) return null; + payloadLength = buffer.readUInt16BE(2); + offset = 4; + } else if (payloadLength === 127) { + if (buffer.length < 10) return null; + payloadLength = Number(buffer.readBigUInt64BE(2)); + offset = 10; + } + const maskOffset = offset; + if (masked) offset += 4; + if (buffer.length < offset + payloadLength) return null; + const payload = Buffer.from(buffer.slice(offset, offset + payloadLength)); + if (masked) { + const mask = buffer.slice(maskOffset, maskOffset + 4); + for (let i = 0; i < payload.length; i += 1) { + payload[i] ^= mask[i % 4]; + } + } + return { + opcode, + payload, + totalLength: offset + payloadLength, + }; +} + +function sendSocketModeEvent(socket) { + const envelope = { + envelope_id: "slack-e2e-envelope-3730", + type: "events_api", + accepts_response_payload: true, + payload: { + token: "verification-token", + team_id: socketTeamId, + api_app_id: "A3730E2E", + type: "event_callback", + event_id: "Ev3730E2E", + event_time: Math.floor(Date.now() / 1000), + authorizations: [{ team_id: socketTeamId, user_id: "UOPENCLAWBOT", is_bot: true }], + event: { + type: "message", + channel_type: "im", + channel: socketChannelId, + user: socketUserId, + text: "pair me", + ts: `${Math.floor(Date.now() / 1000)}.000000`, + }, + }, + }; + socket.write(encodeServerText(JSON.stringify(envelope))); + record({ event: "websocket-event-sent", path: "/socket-mode", envelopeId: envelope.envelope_id }); +} + +const server = http.createServer((req, res) => { + const chunks = []; + let bodyBytes = 0; + let bodyTooLarge = false; + req.on("data", (chunk) => { + if (bodyTooLarge) return; + bodyBytes += chunk.length; + if (bodyBytes > MAX_BODY_BYTES) { + bodyTooLarge = true; + record({ + event: "request-too-large", + method: req.method, + path: new URL(req.url || "/", "http://fake-slack.local").pathname, + bodyBytes, + }); + res.writeHead(413, { "content-type": "application/json" }); + res.end(JSON.stringify({ ok: false, error: "payload_too_large" })); + req.destroy(); + return; + } + chunks.push(chunk); + }); + req.on("end", () => { + if (bodyTooLarge) return; + const body = Buffer.concat(chunks).toString("utf8"); + const pathname = new URL(req.url || "/", "http://fake-slack.local").pathname; + const authorization = req.headers.authorization || ""; + const expectedToken = expectedTokenForPath(pathname); + const expectedAuthorization = `Bearer ${expectedToken}`; + const bodyParams = new URLSearchParams(body); + const bodyToken = bodyParams.get("token") || ""; + const channel = bodyParams.get("channel") || ""; + const text = bodyParams.get("text") || ""; + const threadTs = bodyParams.get("thread_ts") || ""; + const tokenMatchesExpected = authorization === expectedAuthorization; + const bodyMatchesExpected = bodyToken === expectedToken; + const authAccepted = tokenMatchesExpected && bodyMatchesExpected; + const requestTokenLooksPlaceholder = + tokenLooksPlaceholder(authorization) || tokenLooksPlaceholder(body); + + record({ + event: "request", + method: req.method, + path: pathname, + tokenMatchesExpected, + bodyMatchesExpected, + tokenLooksPlaceholder: requestTokenLooksPlaceholder, + authorizationPresent: Boolean(authorization), + bodyTokenPresent: Boolean(bodyToken), + authorizationRedacted: true, + bodyRedacted: true, + ...(pathname === "/api/chat.postMessage" + ? { + channel, + text, + textLength: text.length, + threadTs, + } + : {}), + }); + + const response = slackResponseFor(pathname, authAccepted, { channel, text, threadTs }); + res.writeHead(response.status, { + "content-type": "application/json", + }); + res.end(JSON.stringify(response.body)); + }); +}); + +server.on("upgrade", (req, socket) => { + const pathname = new URL(req.url || "/", "http://fake-slack.local").pathname; + if (pathname !== "/socket-mode") { + socket.destroy(); + return; + } + + const key = req.headers["sec-websocket-key"]; + if (typeof key !== "string" || !key) { + socket.destroy(); + return; + } + + const accept = crypto.createHash("sha1").update(`${key}${WS_GUID}`).digest("base64"); + socket.write( + [ + "HTTP/1.1 101 Switching Protocols", + "Upgrade: websocket", + "Connection: Upgrade", + `Sec-WebSocket-Accept: ${accept}`, + "\r\n", + ].join("\r\n"), + ); + record({ event: "websocket-upgrade", path: pathname }); + + let buffer = Buffer.alloc(0); + let sentEvent = false; + + socket.on("data", (chunk) => { + buffer = Buffer.concat([buffer, chunk]); + while (buffer.length > 0) { + const frame = decodeClientFrame(buffer); + if (!frame) break; + buffer = buffer.slice(frame.totalLength); + if (frame.opcode === 8) { + socket.end(); + return; + } + if (frame.opcode !== 1) continue; + const text = frame.payload.toString("utf8"); + let token = ""; + let messageType = ""; + let envelopeId = ""; + try { + const parsed = JSON.parse(text); + token = typeof parsed.token === "string" ? parsed.token : ""; + messageType = typeof parsed.type === "string" ? parsed.type : ""; + envelopeId = typeof parsed.envelope_id === "string" ? parsed.envelope_id : ""; + } catch { + // Capture classification below is still useful for malformed frames. + } + record({ + event: "websocket-message", + path: pathname, + messageType, + tokenMatchesExpected: token === expectedAppToken, + tokenLooksPlaceholder: tokenLooksPlaceholder(text), + textRedacted: true, + }); + if (!sentEvent) { + sentEvent = true; + sendSocketModeEvent(socket); + } else if (envelopeId === "slack-e2e-envelope-3730") { + socket.end(); + } + } + }); +}); + +server.listen(port, host, () => { + const address = server.address(); + if (portFile) { + fs.writeFileSync(portFile, `${address.port}\n`, { mode: 0o600 }); + } + record({ event: "listening", host, port: address.port }); +}); + +for (const signal of ["SIGTERM", "SIGINT"]) { + process.on(signal, () => { + server.close(() => process.exit(0)); + setTimeout(() => process.exit(0), 1000).unref(); + }); +} diff --git a/test/e2e-vpn/lib/fake-telegram-api.cjs b/test/e2e-vpn/lib/fake-telegram-api.cjs new file mode 100755 index 00000000000..022756010af --- /dev/null +++ b/test/e2e-vpn/lib/fake-telegram-api.cjs @@ -0,0 +1,156 @@ +#!/usr/bin/env node +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +"use strict"; + +const fs = require("fs"); +const http = require("http"); + +const host = process.env.FAKE_TELEGRAM_API_HOST || "0.0.0.0"; +const rawPort = process.env.FAKE_TELEGRAM_API_PORT || "0"; +const port = Number(rawPort); +const portFile = process.env.FAKE_TELEGRAM_API_PORT_FILE || ""; +const captureFile = process.env.FAKE_TELEGRAM_API_CAPTURE_FILE || ""; +const expectedToken = process.env.FAKE_TELEGRAM_API_EXPECTED_TOKEN || ""; +const MAX_BODY_BYTES = 1024 * 1024; + +if (!Number.isInteger(port) || port < 0 || port > 65535) { + console.error(`FAKE_TELEGRAM_API_PORT must be an integer between 0 and 65535 (received: ${rawPort})`); + process.exit(2); +} + +if (!expectedToken) { + console.error("FAKE_TELEGRAM_API_EXPECTED_TOKEN is required"); + process.exit(2); +} + +function record(event) { + if (!captureFile) return; + fs.appendFileSync(captureFile, `${JSON.stringify({ at: Date.now(), ...event })}\n`); +} + +function tokenLooksPlaceholder(value) { + return typeof value === "string" && value.includes("openshell:resolve:env:"); +} + +function readFields(req, body) { + const contentType = String(req.headers["content-type"] || ""); + if (contentType.includes("application/json")) { + try { + return JSON.parse(body || "{}"); + } catch { + return {}; + } + } + const params = new URLSearchParams(body); + return Object.fromEntries(params.entries()); +} + +function writeJson(res, status, body) { + res.writeHead(status, { "content-type": "application/json" }); + res.end(JSON.stringify(body)); +} + +const server = http.createServer((req, res) => { + const chunks = []; + let bodyBytes = 0; + let bodyTooLarge = false; + req.on("data", (chunk) => { + if (bodyTooLarge) return; + bodyBytes += chunk.length; + if (bodyBytes > MAX_BODY_BYTES) { + bodyTooLarge = true; + record({ event: "request-too-large", method: req.method, path: req.url || "/", bodyBytes }); + writeJson(res, 413, { ok: false, error_code: 413, description: "payload too large" }); + req.destroy(); + return; + } + chunks.push(chunk); + }); + + req.on("end", () => { + if (bodyTooLarge) return; + const body = Buffer.concat(chunks).toString("utf8"); + const url = new URL(req.url || "/", "http://fake-telegram.local"); + const match = /^\/bot([^/]+)\/([^/?]+)$/.exec(url.pathname); + const token = match?.[1] || ""; + const endpoint = match?.[2] || ""; + const fields = readFields(req, body); + const tokenMatchesExpected = token === expectedToken; + + record({ + event: "request", + method: req.method, + path: url.pathname, + endpoint, + tokenMatchesExpected, + tokenLooksPlaceholder: tokenLooksPlaceholder(token), + tokenRedacted: true, + chatId: fields.chat_id ? String(fields.chat_id) : "", + text: fields.text ? String(fields.text) : "", + textLength: fields.text ? String(fields.text).length : 0, + }); + + if (!match) { + writeJson(res, 404, { ok: false, error_code: 404, description: "not found" }); + return; + } + + if (!tokenMatchesExpected) { + writeJson(res, 401, { ok: false, error_code: 401, description: "Unauthorized" }); + return; + } + + if (endpoint === "getMe") { + writeJson(res, 200, { + ok: true, + result: { + id: 420000001, + is_bot: true, + first_name: "NemoClaw E2E", + username: "nemoclaw_e2e_bot", + }, + }); + return; + } + + if (endpoint === "sendMessage") { + writeJson(res, 200, { + ok: true, + result: { + message_id: 4201, + date: Math.floor(Date.now() / 1000), + chat: { + id: Number(fields.chat_id) || String(fields.chat_id || ""), + type: "private", + }, + text: String(fields.text || ""), + }, + }); + return; + } + + writeJson(res, 200, { ok: true, result: true }); + }); +}); + +server.on("error", (error) => { + record({ event: "server_error", error: error.message }); + console.error(error.stack || error.message); +}); + +server.listen(port, host, () => { + const address = server.address(); + if (portFile) { + fs.writeFileSync(portFile, `${address.port}\n`, { mode: 0o600 }); + } + record({ event: "listening", host, port: address.port }); +}); + +for (const signal of ["SIGTERM", "SIGINT"]) { + process.on(signal, () => { + server.close(() => process.exit(0)); + setTimeout(() => process.exit(0), 1000).unref(); + }); +} diff --git a/test/e2e-vpn/lib/inference-switch-retry.sh b/test/e2e-vpn/lib/inference-switch-retry.sh new file mode 100755 index 00000000000..753971314e5 --- /dev/null +++ b/test/e2e-vpn/lib/inference-switch-retry.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Shared retry helpers for inference-switch E2Es. These tests still verify the +# final OpenShell route, sandbox config, and live inference after this helper +# returns. The --no-verify fallback is only used after verified route-setting +# attempts fail with transient upstream/network symptoms. + +is_transient_inference_set_failure() { + grep -qiE 'timed? out|timeout|ETIMEDOUT|ECONNRESET|EAI_AGAIN|ENOTFOUND|failed to connect|error sending request|failed to verify inference endpoint|502|503|504|temporar' <<<"$1" +} + +log_inference_switch_retry_info() { + if declare -F info >/dev/null 2>&1; then + info "$1" + else + printf '\033[1;34m [info]\033[0m %s\n' "$1" + fi +} + +run_inference_set_with_retry() { + local attempts="${NEMOCLAW_SWITCH_SET_ATTEMPTS:-3}" + if ! [[ "$attempts" =~ ^[1-9][0-9]*$ ]]; then + printf 'Invalid NEMOCLAW_SWITCH_SET_ATTEMPTS=%s; expected a positive integer.\n' "$attempts" >&2 + return 2 + fi + if [ "$#" -eq 0 ]; then + printf 'run_inference_set_with_retry requires an inference set command.\n' >&2 + return 2 + fi + + local attempt rc output fallback_output + local -a command=("$@") + for ((attempt = 1; attempt <= attempts; attempt++)); do + output=$("${command[@]}" 2>&1) + rc=$? + if [ "$rc" -eq 0 ]; then + printf '%s\n' "$output" + return 0 + fi + + if ! is_transient_inference_set_failure "$output" || [ "$attempt" -ge "$attempts" ]; then + if is_transient_inference_set_failure "$output"; then + log_inference_switch_retry_info "Verified inference switch failed after ${attempts} transient attempt(s); retrying with --no-verify before live route checks..." + fallback_output=$("${command[@]}" --no-verify 2>&1) + rc=$? + printf '%s\n%s\n' "$output" "$fallback_output" + return "$rc" + fi + printf '%s\n' "$output" + return "$rc" + fi + + log_inference_switch_retry_info "Verified inference switch attempt ${attempt}/${attempts} hit a transient failure; retrying..." + sleep $((attempt * 5)) + done + + printf 'Inference switch retry loop completed without running an attempt.\n' >&2 + return 1 +} diff --git a/test/e2e-vpn/lib/install-path-refresh.sh b/test/e2e-vpn/lib/install-path-refresh.sh new file mode 100755 index 00000000000..36c855bb1bb --- /dev/null +++ b/test/e2e-vpn/lib/install-path-refresh.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Shared install-path-refresh helper for e2e test scripts. Meant to be sourced; +# the shebang and executable bit satisfy repo shell-file conventions. +# +# Why: install.sh places the openshell/nemoclaw binaries under ~/.local/bin. +# Sourcing ~/.bashrc on GitHub runners triggers nvm.sh, which rebuilds $PATH +# from scratch and drops ~/.local/bin — so a post-install `command -v +# nemoclaw` check fails with "nemoclaw not found". This helper centralises +# the recovery so every e2e test script applies the same guard. +# +# Usage: +# . "$(dirname "${BASH_SOURCE[0]}")/lib/install-path-refresh.sh" +# +# # After running install.sh, reload the shell profile and pick up the +# # binaries it installed: +# nemoclaw_refresh_install_env +# +# # If you only need to defensively ensure ~/.local/bin is on PATH: +# nemoclaw_ensure_local_bin_on_path + +# Prepend ~/.local/bin to PATH if it exists and isn't already there. +nemoclaw_ensure_local_bin_on_path() { + if [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]]; then + export PATH="$HOME/.local/bin:$PATH" + fi +} + +# Source ~/.bashrc (best-effort) and then ensure ~/.local/bin is on PATH. +# Needed after running install.sh because nvm.sh (loaded via .bashrc) rebuilds +# PATH from scratch and can drop the directory where install.sh places the +# openshell/nemoclaw binaries. +nemoclaw_refresh_install_env() { + if [ -f "$HOME/.bashrc" ]; then + # shellcheck source=/dev/null + source "$HOME/.bashrc" 2>/dev/null || true + fi + nemoclaw_ensure_local_bin_on_path +} diff --git a/test/e2e-vpn/lib/openai-compatible-api-proof.sh b/test/e2e-vpn/lib/openai-compatible-api-proof.sh new file mode 100755 index 00000000000..d1c813fbce7 --- /dev/null +++ b/test/e2e-vpn/lib/openai-compatible-api-proof.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +start_fake_openai_compatible_api() { + local script_dir server_script port_file ready_host public_host + script_dir="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" + server_script="${script_dir}/fake-openai-compatible-api.mts" + port_file="${FAKE_OPENAI_PORT_FILE:-$(mktemp)}" + ready_host="${FAKE_OPENAI_READY_HOST:-127.0.0.1}" + + : "${FAKE_OPENAI_HOST:=127.0.0.1}" + : "${FAKE_OPENAI_PORT:=0}" + : "${FAKE_OPENAI_MODEL:=test-model}" + : "${FAKE_OPENAI_LOG:=$(mktemp)}" + + rm -f "$port_file" + : >"$FAKE_OPENAI_LOG" + + NEMOCLAW_FAKE_OPENAI_HOST="$FAKE_OPENAI_HOST" \ + NEMOCLAW_FAKE_OPENAI_PORT="$FAKE_OPENAI_PORT" \ + NEMOCLAW_FAKE_OPENAI_PORT_FILE="$port_file" \ + NEMOCLAW_FAKE_OPENAI_LOG_FILE="$FAKE_OPENAI_LOG" \ + NEMOCLAW_FAKE_OPENAI_MODEL="$FAKE_OPENAI_MODEL" \ + NEMOCLAW_FAKE_OPENAI_API_KEY="${FAKE_OPENAI_API_KEY:-}" \ + NEMOCLAW_FAKE_OPENAI_REQUIRE_AUTH="${FAKE_OPENAI_REQUIRE_AUTH:-0}" \ + NEMOCLAW_FAKE_OPENAI_CHAT_CONTENT="${FAKE_OPENAI_CHAT_CONTENT:-ok}" \ + NEMOCLAW_FAKE_OPENAI_RESPONSE_TEXT="${FAKE_OPENAI_RESPONSE_TEXT:-${FAKE_OPENAI_CHAT_CONTENT:-ok}}" \ + node --experimental-strip-types "$server_script" & + FAKE_OPENAI_PID="$!" + + for _ in $(seq 1 "${FAKE_OPENAI_READY_ATTEMPTS:-30}"); do + if [ -s "$port_file" ]; then + FAKE_OPENAI_PORT="$(cat "$port_file")" + if curl -sf "http://${ready_host}:${FAKE_OPENAI_PORT}/v1/models" >/dev/null 2>&1; then + public_host="${FAKE_OPENAI_PUBLIC_HOST:-$FAKE_OPENAI_HOST}" + if [ "$public_host" = "0.0.0.0" ]; then + public_host="127.0.0.1" + fi + FAKE_OPENAI_BASE_URL="http://${public_host}:${FAKE_OPENAI_PORT}/v1" + rm -f "$port_file" + export FAKE_OPENAI_BASE_URL FAKE_OPENAI_PID FAKE_OPENAI_PORT + return 0 + fi + fi + sleep 1 + done + + stop_fake_openai_compatible_api + rm -f "$port_file" + return 1 +} + +stop_fake_openai_compatible_api() { + if [ -n "${FAKE_OPENAI_PID:-}" ] && kill -0 "$FAKE_OPENAI_PID" 2>/dev/null; then + kill "$FAKE_OPENAI_PID" 2>/dev/null || true + wait "$FAKE_OPENAI_PID" 2>/dev/null || true + fi + FAKE_OPENAI_PID="" +} diff --git a/test/e2e-vpn/lib/openclaw-agent-json.py b/test/e2e-vpn/lib/openclaw-agent-json.py new file mode 100755 index 00000000000..c99e9f5a168 --- /dev/null +++ b/test/e2e-vpn/lib/openclaw-agent-json.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Extract text payloads from `openclaw agent --json` output. + +OpenClaw has emitted both of these envelopes across recent versions: + + {"result": {"payloads": [{"text": "..."}]}} + {"payloads": [{"text": "..."}]} + +The E2E smoke checks only need the joined assistant text. Invalid JSON is a +real harness failure and exits nonzero; valid JSON with no text prints nothing. +""" + +from __future__ import annotations + +import json +import sys +from typing import Any + + +def _payloads(doc: Any) -> list[Any]: + if not isinstance(doc, dict): + return [] + top_level = doc.get("payloads") + if isinstance(top_level, list): + return top_level + result = doc.get("result") + if isinstance(result, dict) and isinstance(result.get("payloads"), list): + return result["payloads"] + return [] + + +def _load_agent_json_docs(text: str) -> list[Any]: + try: + doc = json.loads(text) + except json.JSONDecodeError: + pass + else: + return doc if isinstance(doc, list) else [doc] + + decoder = json.JSONDecoder() + docs: list[Any] = [] + index = 0 + while index < len(text): + start = text.find("{", index) + if start < 0: + break + try: + doc, end = decoder.raw_decode(text[start:]) + except json.JSONDecodeError: + index = start + 1 + continue + docs.append(doc) + index = start + end + if docs: + return docs + raise json.JSONDecodeError("no JSON object found", text, 0) + + +def main() -> int: + raw = sys.stdin.read() + try: + docs = _load_agent_json_docs(raw) + except json.JSONDecodeError as err: + print(f"invalid JSON: {err}", file=sys.stderr) + return 1 + + parts = [ + payload["text"] + for doc in docs + for payload in _payloads(doc) + if isinstance(payload, dict) and isinstance(payload.get("text"), str) + ] + print("\n".join(parts)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/test/e2e-vpn/lib/openclaw-json.sh b/test/e2e-vpn/lib/openclaw-json.sh new file mode 100755 index 00000000000..8f17bab69fa --- /dev/null +++ b/test/e2e-vpn/lib/openclaw-json.sh @@ -0,0 +1,90 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Extract human-readable assistant text from `openclaw agent --json` output. +# OpenClaw's JSON envelope has moved between result.payloads[] and top-level +# payloads[]; keep E2E assertions focused on visible reply text instead of one +# exact envelope shape. This also tolerates wrapper output before the JSON blob +# but intentionally ignores metadata fields so IDs, durations, session names, +# and model/provider details cannot satisfy reply assertions. +parse_openclaw_agent_text() { + python3 -c ' +import json +import sys + +raw = sys.stdin.read() +if not raw.strip(): + sys.exit(0) + +parts = [] +visited = set() + +TEXT_KEYS = {"text", "content", "reasoning_content"} +CONTAINER_KEYS = { + "result", "payloads", "payload", "messages", "choices", "response", + "data", "output", "outputs", "items", "segments", "delta", +} + + +def add(value): + if isinstance(value, str) and value.strip(): + parts.append(value.strip()) + + +def collect(value): + value_id = id(value) + if value_id in visited: + return + visited.add(value_id) + + if isinstance(value, str): + add(value) + return + if isinstance(value, list): + for item in value: + collect(item) + return + if not isinstance(value, dict): + return + + for key in TEXT_KEYS: + add(value.get(key)) + + # OpenAI-style choices can nest assistant text under message/delta objects. + for choice in value.get("choices") or []: + if isinstance(choice, dict): + collect(choice.get("message")) + collect(choice.get("delta")) + add(choice.get("text")) + + for key in CONTAINER_KEYS: + if key in value: + collect(value[key]) + + +def collect_from_doc(doc): + if isinstance(doc, dict) and isinstance(doc.get("result"), dict): + collect(doc["result"]) + else: + collect(doc) + +try: + collect_from_doc(json.loads(raw)) +except Exception: + decoder = json.JSONDecoder() + for idx, char in enumerate(raw): + if char != "{": + continue + try: + doc, _end = decoder.raw_decode(raw[idx:]) + except Exception: + continue + before = len(parts) + collect_from_doc(doc) + if len(parts) > before: + break + +print("\n".join(parts)) +' +} diff --git a/test/e2e-vpn/lib/sandbox-teardown.sh b/test/e2e-vpn/lib/sandbox-teardown.sh new file mode 100755 index 00000000000..9beca2271b4 --- /dev/null +++ b/test/e2e-vpn/lib/sandbox-teardown.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Shared sandbox-teardown helper for e2e test scripts. Meant to be sourced; +# the shebang and executable bit satisfy repo shell-file conventions. +# +# Why: the nightly Brev launchable is reused across runs, and any test that +# exits before cleaning up its sandbox leaves a dangling k8s pod + netns + +# volume behind. Over time these accumulate and can push subsequent runs into +# "sandbox already exists but is not ready" states that block onboard. +# +# Usage (place after SANDBOX_NAME is defined): +# . "$(dirname "${BASH_SOURCE[0]}")/lib/sandbox-teardown.sh" +# register_sandbox_for_teardown "$SANDBOX_NAME" +# +# Multiple sandboxes: call register_sandbox_for_teardown once per sandbox. +# +# Local-dev escape hatch: set NEMOCLAW_E2E_KEEP_SANDBOX=1 to skip the destroy +# on exit so the sandbox survives for post-mortem inspection. + +_NEMOCLAW_TEARDOWN_SANDBOXES=() + +register_sandbox_for_teardown() { + local name="${1:-}" + [[ -z "$name" ]] && return 0 + _NEMOCLAW_TEARDOWN_SANDBOXES+=("$name") +} + +_nemoclaw_sandbox_teardown() { + # Run on script EXIT — destroys every registered sandbox. + # + # Intentionally does NOT unlink ~/.nemoclaw/onboard.lock: that lock is + # global and ownership-aware (acquireOnboardLock in src/lib/onboard-session.ts + # verifies PID liveness and inode before cleaning up a stale lock), so an + # unconditional rm here could unlink a concurrent run's live lock on a + # shared machine. A crashed process leaves a stale lock that the next + # onboard cleans up automatically. + if [[ "${NEMOCLAW_E2E_KEEP_SANDBOX:-}" = "1" ]]; then + return 0 + fi + set +e + local sbx + for sbx in "${_NEMOCLAW_TEARDOWN_SANDBOXES[@]}"; do + nemoclaw "$sbx" destroy --yes >/dev/null 2>&1 + done + set -e +} + +trap _nemoclaw_sandbox_teardown EXIT diff --git a/test/e2e-vpn/lib/security-posture-assertions.sh b/test/e2e-vpn/lib/security-posture-assertions.sh new file mode 100755 index 00000000000..d4f73e0f681 --- /dev/null +++ b/test/e2e-vpn/lib/security-posture-assertions.sh @@ -0,0 +1,214 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Shared assertions for full onboard tests that need to prove the Linux +# Docker-driver security posture that caught the Hermes rc-file startup bug. +# The caller provides the e2e `section`, `info`, `pass`, and `fail` functions. + +security_posture_sandbox_exec() { + local sandbox_name="$1" + local remote_cmd="$2" + openshell sandbox exec --name "$sandbox_name" -- sh -lc "$remote_cmd" 2>&1 +} + +security_posture_cap_absent() { + local cap_hex="$1" + local bit="$2" + local cap_name="$3" + local context="$4" + local cap_val + + cap_val=$((16#$cap_hex)) + if [ $(((cap_val >> bit) & 1)) -eq 0 ]; then + pass "${context}: ${cap_name} absent from CapBnd (0x${cap_hex})" + else + fail "${context}: ${cap_name} still present in CapBnd (0x${cap_hex})" + fi +} + +security_posture_assert_host_user() { + if [ "${NEMOCLAW_E2E_EXPECT_NON_ROOT_HOST:-}" != "1" ]; then + return 0 + fi + + local uid gid + uid="$(id -u)" + gid="$(id -g)" + if [ "$uid" -eq 0 ]; then + fail "Host test process is running as root; expected a non-root host user" + else + pass "Host test process is non-root (uid=${uid}, gid=${gid})" + fi +} + +security_posture_dangerous_caps_present() { + local cap_hex="$1" + local val entry bit name present_caps="" + + val=$((16#$cap_hex)) + for entry in \ + "21:CAP_SYS_ADMIN" \ + "19:CAP_SYS_PTRACE" \ + "13:CAP_NET_RAW" \ + "10:CAP_NET_BIND_SERVICE" \ + "1:CAP_DAC_OVERRIDE"; do + bit="${entry%%:*}" + name="${entry#*:}" + if [ $(((val >> bit) & 1)) -ne 0 ]; then + present_caps="${present_caps:+$present_caps,}$name" + fi + done + printf '%s\n' "$present_caps" +} + +security_posture_assert_entrypoint_process() { + local sandbox_name="$1" + local out cap_bnd cap_eff no_new_privs entry_uid present_caps + + out="$(security_posture_sandbox_exec "$sandbox_name" 'grep -E "^(Uid|Gid|CapBnd|CapEff|NoNewPrivs):" /proc/1/status 2>/dev/null || true')" || true + info "PID 1 status: ${out//$'\n'/; }" + entry_uid="$(printf '%s\n' "$out" | awk '/^Uid:/ { print $2; exit }')" + cap_bnd="$(printf '%s\n' "$out" | awk '/^CapBnd:/ { print $2; exit }')" + cap_eff="$(printf '%s\n' "$out" | awk '/^CapEff:/ { print $2; exit }')" + no_new_privs="$(printf '%s\n' "$out" | awk '/^NoNewPrivs:/ { print $2; exit }')" + + if [ "${NEMOCLAW_E2E_EXPECT_NON_ROOT_ENTRYPOINT:-}" = "1" ]; then + if [ -n "$entry_uid" ] && [ "$entry_uid" != "0" ]; then + pass "Entrypoint PID 1 is non-root inside the sandbox (uid=${entry_uid})" + else + fail "Entrypoint PID 1 expected non-root uid, got '${entry_uid:-}'" + fi + elif [ -n "$entry_uid" ]; then + info "Entrypoint PID 1 uid=${entry_uid}" + fi + + if [ -z "$cap_bnd" ]; then + fail "Could not capture PID 1 CapBnd from sandbox ${sandbox_name}: ${out:0:300}" + return 0 + fi + + if [ "${NEMOCLAW_E2E_EXPECT_DROPPED_BOUNDS:-}" = "1" ]; then + security_posture_cap_absent "$cap_bnd" 21 CAP_SYS_ADMIN "Entrypoint PID 1" + security_posture_cap_absent "$cap_bnd" 19 CAP_SYS_PTRACE "Entrypoint PID 1" + security_posture_cap_absent "$cap_bnd" 13 CAP_NET_RAW "Entrypoint PID 1" + security_posture_cap_absent "$cap_bnd" 10 CAP_NET_BIND_SERVICE "Entrypoint PID 1" + security_posture_cap_absent "$cap_bnd" 1 CAP_DAC_OVERRIDE "Entrypoint PID 1" + else + present_caps="$(security_posture_dangerous_caps_present "$cap_bnd")" + if [ -n "$present_caps" ]; then + info "Entrypoint PID 1 residual CapBnd dangerous caps: ${present_caps}" + else + pass "Entrypoint PID 1 dangerous caps are absent from CapBnd" + fi + fi + + if [ -n "$cap_eff" ]; then + present_caps="$(security_posture_dangerous_caps_present "$cap_eff")" + if [ -n "$present_caps" ]; then + info "Entrypoint PID 1 residual CapEff dangerous caps: ${present_caps}" + else + pass "Entrypoint PID 1 dangerous caps are absent from CapEff" + fi + fi + + if [ "${NEMOCLAW_E2E_EXPECT_NO_NEW_PRIVS:-}" = "1" ]; then + if [ "$no_new_privs" = "1" ]; then + pass "Entrypoint PID 1 has NoNewPrivs=1" + else + fail "Entrypoint PID 1 expected NoNewPrivs=1, got '${no_new_privs:-}'" + fi + elif [ -n "$no_new_privs" ]; then + info "Entrypoint PID 1 NoNewPrivs=${no_new_privs}" + fi +} + +security_posture_assert_rc_files() { + local sandbox_name="$1" + local out rc + + rc=0 + # shellcheck disable=SC2016 # Remote shell snippet; expansion must happen inside the sandbox. + out="$(security_posture_sandbox_exec "$sandbox_name" 'bad=0; for f in /sandbox/.bashrc /sandbox/.profile; do if [ ! -f "$f" ]; then echo "MISSING $f"; bad=1; continue; fi; if [ -L "$f" ]; then echo "SYMLINK $f"; bad=1; fi; meta=$(stat -c "%a %U:%G" "$f" 2>/dev/null || true); echo "META $f $meta"; set -- $meta; mode="${1:-}"; owner="${2:-}"; if [ "$mode" != "444" ]; then echo "BAD_MODE $f $mode"; bad=1; fi; if [ "$owner" != "root:root" ]; then echo "BAD_OWNER $f $owner"; bad=1; fi; if grep -Eq "nemoclaw-configure-guard|^(openclaw|hermes)\(\)" "$f" 2>/dev/null; then echo "INLINE_GUARD $f"; bad=1; fi; done; exit "$bad"')" || rc=$? + info "rc-file metadata: ${out//$'\n'/; }" + if [ "$rc" -eq 0 ]; then + pass "Sandbox rc files are static root-owned 444 shims without inline configure guards" + else + fail "Sandbox rc files are not locked/static as expected: ${out:0:500}" + fi +} + +security_posture_assert_proxy_env() { + local sandbox_name="$1" + local agent_name="$2" + local function_name guard_arg out rc allow_non_root_owner + + case "$agent_name" in + hermes) + function_name="hermes" + guard_arg="setup" + ;; + *) + function_name="openclaw" + guard_arg="configure" + ;; + esac + + allow_non_root_owner=0 + if [ "${NEMOCLAW_E2E_EXPECT_NON_ROOT_HOST:-}" = "1" ]; then + # OpenShell's non-root host posture creates the runtime proxy-env file + # after dropping to the sandbox user. Keep root ownership required in + # normal lanes, but accept current-user ownership for that explicit lane. + allow_non_root_owner=1 + fi + + rc=0 + out="$(security_posture_sandbox_exec "$sandbox_name" "f=/tmp/nemoclaw-proxy-env.sh; allow_non_root_owner=${allow_non_root_owner}; bad=0; if [ ! -f \"\$f\" ]; then echo MISSING_PROXY_ENV; exit 1; fi; if [ -L \"\$f\" ]; then echo SYMLINK_PROXY_ENV; bad=1; fi; meta=\$(stat -c \"%a %U:%G\" \"\$f\" 2>/dev/null || true); echo \"META \$f \$meta\"; set -- \$meta; mode=\"\${1:-}\"; owner=\"\${2:-}\"; current_owner=\"\$(id -un):\$(id -gn)\"; if [ \"\$mode\" != \"444\" ]; then echo \"BAD_PROXY_ENV_MODE \$mode\"; bad=1; fi; case \"\$owner\" in root:root) ;; \"\$current_owner\") if [ \"\$allow_non_root_owner\" = \"1\" ]; then echo \"NON_ROOT_PROXY_ENV_OWNER \$owner\"; else echo \"BAD_PROXY_ENV_OWNER \$owner\"; bad=1; fi ;; *) echo \"BAD_PROXY_ENV_OWNER \$owner\"; bad=1 ;; esac; grep -Fq '# nemoclaw-configure-guard begin' \"\$f\" || { echo MISSING_GUARD_BEGIN; bad=1; }; grep -Fq '${function_name}() {' \"\$f\" || { echo MISSING_AGENT_GUARD_FUNCTION; bad=1; }; grep -Fq '# nemoclaw-configure-guard end' \"\$f\" || { echo MISSING_GUARD_END; bad=1; }; exit \"\$bad\"")" || rc=$? + info "runtime proxy-env metadata: ${out//$'\n'/; }" + if [ "$rc" -eq 0 ]; then + pass "Runtime proxy env is mode 444 with an accepted owner and carries the ${function_name} configure guard" + else + fail "Runtime proxy env is not locked or missing guard content: ${out:0:500}" + fi + + rc=0 + out="$(security_posture_sandbox_exec "$sandbox_name" ". /tmp/nemoclaw-proxy-env.sh || { echo SOURCE_FAILED; exit 1; }; if ${function_name} ${guard_arg} >/tmp/nemoclaw-security-guard-probe.out 2>&1; then echo GUARD_DID_NOT_BLOCK; cat /tmp/nemoclaw-security-guard-probe.out; exit 1; fi; cat /tmp/nemoclaw-security-guard-probe.out; grep -q 'cannot modify config inside the sandbox' /tmp/nemoclaw-security-guard-probe.out || { echo GUARD_MESSAGE_MISSING; exit 1; }")" || rc=$? + info "configure guard probe: ${out//$'\n'/; }" + if [ "$rc" -eq 0 ]; then + pass "${function_name} ${guard_arg} is blocked by the runtime guard after sourcing proxy-env" + else + fail "Runtime configure guard did not behave as expected: ${out:0:500}" + fi +} + +security_posture_assert_start_log() { + local sandbox_name="$1" + local agent_name="$2" + local out rc launch_pattern + + case "$agent_name" in + hermes) launch_pattern='hermes gateway launched' ;; + *) launch_pattern='openclaw gateway launched' ;; + esac + + rc=0 + out="$(security_posture_sandbox_exec "$sandbox_name" "log=/tmp/nemoclaw-start.log; bad=0; [ -f \"\$log\" ] || { echo MISSING_START_LOG; exit 1; }; if ! grep -qi '${launch_pattern}' \"\$log\"; then echo MISSING_GATEWAY_LAUNCH_MARKER; bad=1; fi; if grep -E 'mktemp:.*(/sandbox/\\.\\.(bashrc|profile)\\.tmp|/sandbox/\\.nemoclaw.*tmp)|Permission denied.*(/sandbox/\\.bashrc|/sandbox/\\.profile)' \"\$log\"; then echo START_LOG_HAS_RC_WRITE_FAILURE; bad=1; fi; tail -n 20 \"\$log\"; exit \"\$bad\"")" || rc=$? + info "start log probe: ${out//$'\n'/; }" + if [ "$rc" -eq 0 ]; then + pass "Startup log has no rc-file mktemp/permission failure" + else + fail "Startup log shows the rc-file write failure class: ${out:0:500}" + fi +} + +security_posture_assertions_run() { + local sandbox_name="$1" + local agent_name="${2:-openclaw}" + + section "Security posture regression checks" + security_posture_assert_host_user + security_posture_assert_entrypoint_process "$sandbox_name" + security_posture_assert_rc_files "$sandbox_name" + security_posture_assert_proxy_env "$sandbox_name" "$agent_name" + security_posture_assert_start_log "$sandbox_name" "$agent_name" +} diff --git a/test/e2e-vpn/lib/slack-api-proof.sh b/test/e2e-vpn/lib/slack-api-proof.sh new file mode 100755 index 00000000000..a6c01d5b9a8 --- /dev/null +++ b/test/e2e-vpn/lib/slack-api-proof.sh @@ -0,0 +1,744 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Shared hermetic Slack REST helpers for messaging E2E scripts. + +append_exit_trap_for_fake_slack_api() { + local command="$1" + local existing + existing="$(trap -p EXIT | sed "s/^trap -- '//;s/' EXIT$//")" + trap ''"${existing:+$existing; }$command"'' EXIT +} + +cleanup_fake_slack_api() { + if [ -n "${FAKE_SLACK_API_CONTAINER:-}" ]; then + docker rm -f "$FAKE_SLACK_API_CONTAINER" >/dev/null 2>&1 || true + fi + if [ -n "${FAKE_SLACK_API_PID:-}" ]; then + kill "$FAKE_SLACK_API_PID" 2>/dev/null || true + wait "$FAKE_SLACK_API_PID" 2>/dev/null || true + fi + if [ -n "${FAKE_SLACK_API_DIR:-}" ]; then + rm -rf "$FAKE_SLACK_API_DIR" 2>/dev/null || true + fi +} + +start_fake_slack_api() { + local bot_token="$1" + local app_token="$2" + mkdir -p "$REPO/.tmp" + FAKE_SLACK_API_DIR="$(mktemp -d "$REPO/.tmp/fake-slack.XXXXXX")" + FAKE_SLACK_API_PORT_FILE="$FAKE_SLACK_API_DIR/port" + FAKE_SLACK_API_CAPTURE_FILE="$FAKE_SLACK_API_DIR/capture.jsonl" + FAKE_SLACK_API_CONTAINER="nemoclaw-fake-slack-$$-$RANDOM" + FAKE_SLACK_API_HOST="host.docker.internal" + : >"$FAKE_SLACK_API_CAPTURE_FILE" + + if ! docker run -d --rm \ + --name "$FAKE_SLACK_API_CONTAINER" \ + -p 0:8080 \ + -e FAKE_SLACK_API_PORT=8080 \ + -e FAKE_SLACK_API_EXPECTED_BOT_TOKEN="$bot_token" \ + -e FAKE_SLACK_API_EXPECTED_APP_TOKEN="$app_token" \ + -e FAKE_SLACK_API_PORT_FILE=/tmp/fake-slack/port \ + -e FAKE_SLACK_API_CAPTURE_FILE=/tmp/fake-slack/capture.jsonl \ + -v "$FAKE_SLACK_API_DIR:/tmp/fake-slack" \ + -v "$REPO/test/e2e-vpn/lib:/opt/nemoclaw-e2e:ro" \ + node:22-bookworm-slim \ + node /opt/nemoclaw-e2e/fake-slack-api.cjs \ + >"$FAKE_SLACK_API_DIR/container.id" 2>"$FAKE_SLACK_API_DIR/server.log"; then + cat "$FAKE_SLACK_API_DIR/server.log" >&2 || true + return 1 + fi + append_exit_trap_for_fake_slack_api cleanup_fake_slack_api + + for _ in $(seq 1 50); do + if [ -s "$FAKE_SLACK_API_PORT_FILE" ]; then + local published_port + published_port="$(docker port "$FAKE_SLACK_API_CONTAINER" 8080/tcp 2>/dev/null | head -1 | sed 's/.*://')" + if [ -n "$published_port" ]; then + # Exported for callers that source this helper and apply policy/probes after startup. + export FAKE_SLACK_API_PORT + FAKE_SLACK_API_PORT="$published_port" + return 0 + fi + fi + if ! docker inspect "$FAKE_SLACK_API_CONTAINER" >/dev/null 2>&1; then + docker logs "$FAKE_SLACK_API_CONTAINER" >&2 || true + cat "$FAKE_SLACK_API_DIR/server.log" >&2 || true + return 1 + fi + sleep 0.1 + done + cat "$FAKE_SLACK_API_DIR/server.log" >&2 || true + return 1 +} + +fake_slack_api_allowed_ip_options() { + printf '%s' 'allowed-ip=10.0.0.0/8,allowed-ip=172.16.0.0/12,allowed-ip=192.168.0.0/16' +} + +apply_fake_slack_api_policy() { + local sandbox_name="$1" + local port="$2" + local host="${FAKE_SLACK_API_HOST:-host.openshell.internal}" + local allowed_ip_options + allowed_ip_options="$(fake_slack_api_allowed_ip_options)" + openshell policy update "$sandbox_name" \ + --add-endpoint "${host}:${port}:read-write:rest:enforce:request-body-credential-rewrite,${allowed_ip_options}" \ + --add-allow "${host}:${port}:GET:/**" \ + --add-allow "${host}:${port}:POST:/**" \ + --binary /usr/local/bin/node \ + --binary /usr/bin/node \ + --wait +} + +apply_fake_slack_socket_mode_policy() { + local sandbox_name="$1" + local port="$2" + local host="${FAKE_SLACK_API_HOST:-host.openshell.internal}" + local allowed_ip_options + allowed_ip_options="$(fake_slack_api_allowed_ip_options)" + openshell policy update "$sandbox_name" \ + --add-endpoint "${host}:${port}:read-write:websocket:enforce:websocket-credential-rewrite,${allowed_ip_options}" \ + --add-allow "${host}:${port}:GET:/**" \ + --add-allow "${host}:${port}:WEBSOCKET_TEXT:/**" \ + --binary /usr/local/bin/node \ + --binary /usr/bin/node \ + --wait +} + +run_fake_slack_api_node_request() { + local port="$1" + local path="$2" + local authorization="$3" + local host="${FAKE_SLACK_API_HOST:-host.openshell.internal}" + sandbox_exec_stdin "FAKE_SLACK_API_HOST='$host' FAKE_SLACK_API_PORT='$port' FAKE_SLACK_API_PATH='$path' FAKE_SLACK_API_AUTH='$authorization' node - 2>&1" <<'NODE' +const http = require("http"); + +const authorization = process.env.FAKE_SLACK_API_AUTH || ""; +const token = authorization.replace(/^Bearer\s+/, ""); +const data = `token=${encodeURIComponent(token)}`; +const options = { + hostname: process.env.FAKE_SLACK_API_HOST || "host.openshell.internal", + port: Number(process.env.FAKE_SLACK_API_PORT), + path: process.env.FAKE_SLACK_API_PATH, + method: "POST", + headers: { + Authorization: authorization, + "Content-Type": "application/x-www-form-urlencoded", + "Content-Length": data.length, + }, +}; + +const req = http.request(options, (res) => { + let body = ""; + res.on("data", (d) => { + body += d; + }); + res.on("end", () => { + console.log(`${res.statusCode} ${body.slice(0, 300)}`); + }); +}); + +req.on("error", (error) => { + console.log(`ERROR: ${error.message}`); +}); +req.setTimeout(30000, () => { + req.destroy(); + console.log("TIMEOUT"); +}); +req.write(data); +req.end(); +NODE +} + +run_fake_slack_channel_mention_proof() { + local port="$1" + local allowed_user="$2" + local denied_user="$3" + local host="${FAKE_SLACK_API_HOST:-host.openshell.internal}" + sandbox_exec_stdin "FAKE_SLACK_API_HOST='$host' FAKE_SLACK_API_PORT='$port' SLACK_ALLOWED_USER='$allowed_user' SLACK_DENIED_USER='$denied_user' node --preserve-symlinks --input-type=module - 2>&1" <<'NODE' +import { execFileSync } from "node:child_process"; +import fs from "node:fs"; +import http from "node:http"; +import { createRequire } from "node:module"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +function fail(message) { + console.error(message); + process.exit(1); +} + +function resolveOpenClawSlackApiLocation() { + const externalCandidates = []; + const coreCandidates = []; + const seen = new Set(); + const require = createRequire(import.meta.url); + const addExternalCandidate = (candidate) => { + if (!candidate) return; + const normalized = path.resolve(candidate); + if (!seen.has(normalized)) { + seen.add(normalized); + externalCandidates.push(normalized); + } + }; + const addCoreCandidate = (candidate) => { + if (!candidate) return; + const normalized = path.resolve(candidate); + if (!seen.has(normalized)) { + seen.add(normalized); + coreCandidates.push(normalized); + } + }; + const addPathWalk = (start) => { + if (!start) return; + let current = path.resolve(start); + for (let depth = 0; depth < 8; depth += 1) { + addExternalCandidate(path.join(current, "node_modules/@openclaw/slack")); + addCoreCandidate(current); + if (path.basename(current) === "openclaw") addCoreCandidate(current); + const parent = path.dirname(current); + if (parent === current) break; + current = parent; + } + }; + + if (process.env.OPENCLAW_SLACK_PACKAGE_ROOT) { + addExternalCandidate(process.env.OPENCLAW_SLACK_PACKAGE_ROOT); + } + addCoreCandidate(process.env.OPENCLAW_PACKAGE_ROOT); + for (const base of [process.cwd(), "/sandbox", "/usr/local/lib/node_modules", "/tmp/npm-global/lib/node_modules"]) { + try { + addExternalCandidate(path.dirname(require.resolve("@openclaw/slack/package.json", { paths: [base] }))); + } catch {} + try { + addCoreCandidate(path.dirname(require.resolve("openclaw/package.json", { paths: [base] }))); + } catch {} + } + try { + const globalRoot = execFileSync("npm", ["root", "-g"], { encoding: "utf8" }).trim(); + if (globalRoot) { + addExternalCandidate(path.join(globalRoot, "@openclaw/slack")); + addCoreCandidate(path.join(globalRoot, "openclaw")); + } + } catch {} + try { + addExternalCandidate(path.dirname(require.resolve("@openclaw/slack/package.json"))); + } catch {} + try { + addCoreCandidate(path.dirname(require.resolve("openclaw/package.json"))); + } catch {} + try { + const openclawBin = execFileSync("sh", ["-lc", "command -v openclaw || true"], { encoding: "utf8" }).trim(); + if (openclawBin) { + const realBin = execFileSync("readlink", ["-f", openclawBin], { encoding: "utf8" }).trim(); + addPathWalk(path.dirname(realBin)); + } + } catch {} + try { + const searchRoots = ["/usr/local", "/tmp/npm-global", "/sandbox"].filter((root) => fs.existsSync(root)); + const discovered = searchRoots.length + ? execFileSync("find", [ + ...searchRoots, + "(", + "-path", + "*/node_modules/@openclaw/slack/dist/test-api.js", + "-o", + "-path", + "*/node_modules/openclaw/dist/extensions/slack/test-api.js", + ")", + "-print", + "-quit", + ], { + encoding: "utf8", + }).trim() + : ""; + if (discovered.endsWith("/node_modules/@openclaw/slack/dist/test-api.js")) { + addExternalCandidate(path.resolve(discovered, "../..")); + } else if (discovered) { + addCoreCandidate(path.resolve(discovered, "../../../..")); + } + } catch {} + addExternalCandidate("/usr/local/lib/node_modules/@openclaw/slack"); + addExternalCandidate("/tmp/npm-global/lib/node_modules/@openclaw/slack"); + addCoreCandidate("/usr/local/lib/node_modules/openclaw"); + addCoreCandidate("/tmp/npm-global/lib/node_modules/openclaw"); + + const openclawRoot = coreCandidates.find((candidate) => + fs.existsSync(path.join(candidate, "package.json")) && + fs.existsSync(path.join(candidate, "dist/plugin-sdk/temp-path.js")) + ); + + for (const candidate of externalCandidates) { + const testApiPath = path.join(candidate, "dist/test-api.js"); + if (fs.existsSync(testApiPath)) { + console.error(`OpenClaw Slack external test API root: ${candidate}`); + if (openclawRoot) console.error(`OpenClaw Slack external peer OpenClaw root: ${openclawRoot}`); + return { kind: "external", root: candidate, testApiPath, openclawRoot }; + } + } + for (const candidate of coreCandidates) { + if (fs.existsSync(path.join(candidate, "dist/extensions/slack/test-api.js"))) { + console.error(`OpenClaw Slack core test API root: ${candidate}`); + return { kind: "core", root: candidate }; + } + } + return null; +} + +function createOpenClawSlackProofRoot(openclawRoot) { + const proofWorkspace = fs.mkdtempSync("/tmp/openclaw-slack-proof-"); + const proofRoot = path.join(proofWorkspace, "node_modules/openclaw"); + fs.mkdirSync(proofRoot, { recursive: true }); + fs.copyFileSync(path.join(openclawRoot, "package.json"), path.join(proofRoot, "package.json")); + fs.symlinkSync(path.join(openclawRoot, "dist"), path.join(proofRoot, "dist"), "dir"); + + const nodeModulesRoot = path.join(proofRoot, "node_modules"); + fs.mkdirSync(nodeModulesRoot, { recursive: true }); + + const linkNodeModules = (sourceNodeModules) => { + if (!fs.existsSync(sourceNodeModules)) return; + for (const entry of fs.readdirSync(sourceNodeModules)) { + if (entry === "openclaw") continue; + const sourceEntry = path.join(sourceNodeModules, entry); + const destEntry = path.join(nodeModulesRoot, entry); + if (entry.startsWith("@") && fs.statSync(sourceEntry).isDirectory()) { + fs.mkdirSync(destEntry, { recursive: true }); + for (const scopedEntry of fs.readdirSync(sourceEntry)) { + const sourceScopedEntry = path.join(sourceEntry, scopedEntry); + const destScopedEntry = path.join(destEntry, scopedEntry); + if (!fs.existsSync(destScopedEntry)) { + fs.symlinkSync(sourceScopedEntry, destScopedEntry, "dir"); + } + } + } else if (!fs.existsSync(destEntry)) { + fs.symlinkSync(sourceEntry, destEntry, "dir"); + } + } + }; + linkNodeModules(path.join(openclawRoot, "node_modules")); + linkNodeModules(path.dirname(openclawRoot)); + + const slackWebApiRoot = path.join(proofRoot, "node_modules/@slack/web-api"); + if (!fs.existsSync(slackWebApiRoot)) { + fs.mkdirSync(slackWebApiRoot, { recursive: true }); + fs.writeFileSync( + path.join(slackWebApiRoot, "package.json"), + JSON.stringify({ type: "module", exports: "./index.js" }), + ); + fs.writeFileSync( + path.join(slackWebApiRoot, "index.js"), + `export class WebClient { + constructor(token, options = {}) { + this.token = token; + this.options = options; + this.chat = { + postMessage: async () => { + throw new Error("stub @slack/web-api WebClient is not used by the NemoClaw E2E proof"); + }, + }; + } +} +`, + ); + } + + const proxyAgentRoot = path.join(proofRoot, "node_modules/https-proxy-agent"); + if (!fs.existsSync(proxyAgentRoot)) { + fs.mkdirSync(proxyAgentRoot, { recursive: true }); + fs.writeFileSync( + path.join(proxyAgentRoot, "package.json"), + JSON.stringify({ type: "module", exports: "./index.js" }), + ); + fs.writeFileSync( + path.join(proxyAgentRoot, "index.js"), + `export class HttpsProxyAgent { + constructor(url) { + this.url = url; + } +} +`, + ); + } + + return proofRoot; +} + +function linkNodeModulesEntries(nodeModulesRoot, sourceNodeModules, skip = new Set()) { + if (!fs.existsSync(sourceNodeModules)) return; + for (const entry of fs.readdirSync(sourceNodeModules)) { + const sourceEntry = path.join(sourceNodeModules, entry); + const destEntry = path.join(nodeModulesRoot, entry); + if (entry.startsWith("@") && fs.statSync(sourceEntry).isDirectory()) { + fs.mkdirSync(destEntry, { recursive: true }); + for (const scopedEntry of fs.readdirSync(sourceEntry)) { + const key = `${entry}/${scopedEntry}`; + if (skip.has(key)) continue; + const sourceScopedEntry = path.join(sourceEntry, scopedEntry); + const destScopedEntry = path.join(destEntry, scopedEntry); + if (!fs.existsSync(destScopedEntry)) { + fs.symlinkSync(sourceScopedEntry, destScopedEntry, "dir"); + } + } + } else if (!skip.has(entry) && !fs.existsSync(destEntry)) { + fs.symlinkSync(sourceEntry, destEntry, "dir"); + } + } +} + +function resolveSlackTestApiImport(testApiSource, exportName) { + const escapedExportName = exportName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const patterns = [ + new RegExp(`import\\s+\\{[^}]*\\bas\\s+${escapedExportName}\\b[^}]*\\}\\s+from\\s+["']([^"']+)["']`), + new RegExp(`import\\s+\\{[^}]*\\b${escapedExportName}\\b[^}]*\\}\\s+from\\s+["']([^"']+)["']`), + ]; + const match = patterns.map((pattern) => testApiSource.match(pattern)).find(Boolean); + if (!match) throw new Error(`OpenClaw Slack test API does not expose ${exportName}`); + return match[1]; +} + +function createExternalOpenClawSlackProofRoot(location) { + if (!location.openclawRoot) return location.root; + + const proofWorkspace = fs.mkdtempSync("/tmp/openclaw-slack-external-proof-"); + const nodeModulesRoot = path.join(proofWorkspace, "node_modules"); + const openclawScopeRoot = path.join(nodeModulesRoot, "@openclaw"); + fs.mkdirSync(openclawScopeRoot, { recursive: true }); + + const slackProofRoot = path.join(openclawScopeRoot, "slack"); + fs.symlinkSync(location.root, slackProofRoot, "dir"); + fs.symlinkSync(location.openclawRoot, path.join(nodeModulesRoot, "openclaw"), "dir"); + + linkNodeModulesEntries(nodeModulesRoot, path.resolve(location.root, "../.."), new Set(["openclaw", "@openclaw/slack"])); + linkNodeModulesEntries(nodeModulesRoot, path.join(location.root, "node_modules"), new Set(["openclaw", "@openclaw/slack"])); + linkNodeModulesEntries(nodeModulesRoot, path.dirname(location.openclawRoot), new Set(["openclaw", "@openclaw/slack"])); + linkNodeModulesEntries(nodeModulesRoot, path.join(location.openclawRoot, "node_modules"), new Set(["openclaw", "@openclaw/slack"])); + + return slackProofRoot; +} + +async function importSlackProofModulesFromDir(slackDir) { + const testApiSource = fs.readFileSync(path.join(slackDir, "test-api.js"), "utf8"); + const helperPath = resolveSlackTestApiImport(testApiSource, "createInboundSlackTestContext"); + const preparePath = resolveSlackTestApiImport(testApiSource, "prepareSlackMessage"); + const sendPath = resolveSlackTestApiImport(testApiSource, "sendMessageSlack"); + const [helperModule, prepareModule, sendModule] = await Promise.all([ + import(pathToFileURL(path.join(slackDir, helperPath)).href), + import(pathToFileURL(path.join(slackDir, preparePath)).href), + import(pathToFileURL(path.join(slackDir, sendPath)).href), + ]); + return { + createInboundSlackTestContext: helperModule.createInboundSlackTestContext ?? helperModule.t, + prepareSlackMessage: prepareModule.prepareSlackMessage ?? prepareModule.t, + sendMessageSlack: sendModule.sendMessageSlack ?? sendModule.t, + }; +} + +async function importOpenClawSlackProofApi(location) { + if (location.kind === "external") { + const proofRoot = createExternalOpenClawSlackProofRoot(location); + return importSlackProofModulesFromDir(path.join(proofRoot, "dist")); + } + + const proofRoot = createOpenClawSlackProofRoot(location.root); + return importSlackProofModulesFromDir(path.join(proofRoot, "dist/extensions/slack")); +} + +function postForm(pathname, fields, authorization) { + const body = new URLSearchParams(fields).toString(); + const options = { + hostname: process.env.FAKE_SLACK_API_HOST || "host.openshell.internal", + port: Number(process.env.FAKE_SLACK_API_PORT), + path: pathname, + method: "POST", + headers: { + Authorization: authorization, + "Content-Type": "application/x-www-form-urlencoded", + "Content-Length": Buffer.byteLength(body), + }, + }; + return new Promise((resolve, reject) => { + const req = http.request(options, (res) => { + let responseBody = ""; + res.on("data", (chunk) => { + responseBody += chunk; + }); + res.on("end", () => { + let parsed = {}; + try { + parsed = responseBody ? JSON.parse(responseBody) : {}; + } catch (error) { + reject(new Error(`invalid JSON from fake Slack: ${error.message}: ${responseBody}`)); + return; + } + resolve({ statusCode: res.statusCode, body: parsed }); + }); + }); + req.on("error", reject); + req.setTimeout(30000, () => { + req.destroy(new Error("fake Slack postMessage timed out")); + }); + req.write(body); + req.end(); + }); +} + +const cfg = JSON.parse(fs.readFileSync("/sandbox/.openclaw/openclaw.json", "utf8")); +const slackAccount = cfg.channels?.slack?.accounts?.default; +if (!slackAccount) fail("missing channels.slack.accounts.default"); +if (slackAccount.dmPolicy !== "allowlist") fail(`unexpected Slack dmPolicy: ${slackAccount.dmPolicy}`); +if (slackAccount.groupPolicy !== "allowlist") { + fail(`unexpected Slack groupPolicy: ${slackAccount.groupPolicy}`); +} +const wildcard = slackAccount.channels?.["*"]; +if (!wildcard?.enabled || wildcard.requireMention !== true) { + fail(`missing enabled requireMention wildcard Slack channel config: ${JSON.stringify(wildcard)}`); +} +const allowedUser = process.env.SLACK_ALLOWED_USER || "U0AR85ATALW"; +const deniedUser = process.env.SLACK_DENIED_USER || "U999DENIED"; +if (!Array.isArray(wildcard.users) || !wildcard.users.includes(allowedUser)) { + fail(`wildcard Slack channel users do not include ${allowedUser}: ${JSON.stringify(wildcard.users)}`); +} +if (wildcard.users.includes(deniedUser)) { + fail(`wildcard Slack channel users unexpectedly include denied user ${deniedUser}`); +} + +const channelId = "C0E2ESLACK"; +const baseMessage = { + channel: channelId, + channel_type: "channel", + team: "T1", + text: "<@B1> channel mention proof", +}; +const proofText = "NemoClaw Slack channel mention proof"; +const token = slackAccount.botToken; + +async function postChannelProofMessage() { + const response = await postForm( + "/api/chat.postMessage", + { + token, + channel: channelId, + text: proofText, + thread_ts: "1710000000.000100", + }, + `Bearer ${token}`, + ); + if (response.statusCode !== 200 || response.body?.ok !== true) { + throw new Error(`fake Slack chat.postMessage failed: ${response.statusCode} ${JSON.stringify(response.body)}`); + } + return response.body; +} + +async function runOpenClawPrivateProof(location) { + const slackApi = await importOpenClawSlackProofApi(location); + const { createInboundSlackTestContext, prepareSlackMessage, sendMessageSlack } = slackApi; + if ( + typeof createInboundSlackTestContext !== "function" || + typeof prepareSlackMessage !== "function" || + typeof sendMessageSlack !== "function" + ) { + fail("installed OpenClaw Slack test API does not expose the required proof helpers"); + } + // Records sender-facing feedback actions (chat.postEphemeral / chat.postMessage) + // so the proof can assert that a denied explicit @-mention still produces + // bounded feedback without preparing a command (NemoClaw #4752). + const senderFeedbackCalls = []; + const appClient = { + assistant: { + threads: { + setStatus: async () => ({ ok: true }), + }, + }, + conversations: { + info: async () => ({ + ok: true, + channel: { + id: channelId, + name: "nemoclaw-test", + is_channel: true, + }, + }), + open: async ({ users }) => ({ + ok: true, + channel: { id: `D${users}` }, + }), + }, + reactions: { + add: async () => ({ ok: true }), + remove: async () => ({ ok: true }), + }, + users: { + info: async ({ user }) => ({ + ok: true, + user: { + id: user, + name: user, + profile: { display_name: user, real_name: user }, + }, + }), + }, + chat: { + postEphemeral: async (payload) => { + senderFeedbackCalls.push({ + method: "chat.postEphemeral", + channel: payload.channel, + user: payload.user, + text: payload.text, + }); + return { ok: true, message_ts: "1710000000.000200" }; + }, + postMessage: async (payload) => { + senderFeedbackCalls.push({ + method: "chat.postMessage", + channel: payload.channel, + text: payload.text, + }); + return { ok: true, ts: "1710000000.000201" }; + }, + }, + }; + + const ctx = createInboundSlackTestContext({ + cfg, + appClient, + channelsConfig: slackAccount.channels, + defaultRequireMention: slackAccount.requireMention ?? true, + }); + ctx.botToken = slackAccount.botToken; + ctx.botUserId = "B1"; + ctx.botId = "B1"; + ctx.teamId = "T1"; + ctx.apiAppId = "A1"; + + const account = { + accountId: "default", + botToken: slackAccount.botToken, + appToken: slackAccount.appToken, + config: slackAccount, + }; + const allowedPrepared = await prepareSlackMessage({ + ctx, + account, + message: { ...baseMessage, user: allowedUser, ts: "1710000000.000100" }, + opts: { source: "app_mention", wasMentioned: true }, + }); + if (!allowedPrepared) fail("allowed Slack app_mention did not prepare"); + if (allowedPrepared.replyTarget !== `channel:${channelId}`) { + fail(`unexpected allowed replyTarget: ${allowedPrepared.replyTarget}`); + } + if (senderFeedbackCalls.length !== 0) { + fail(`allowed Slack app_mention unexpectedly produced sender feedback: ${JSON.stringify(senderFeedbackCalls)}`); + } + + senderFeedbackCalls.length = 0; + const deniedPrepared = await prepareSlackMessage({ + ctx, + account, + message: { ...baseMessage, user: deniedUser, ts: "1710000000.000101" }, + opts: { source: "app_mention", wasMentioned: true }, + }); + if (deniedPrepared !== null) fail("denied Slack app_mention unexpectedly prepared"); + + // NemoClaw #4752: a denied explicit @-mention must still prepare no command + // (asserted above) yet send exactly one bounded, sender-facing feedback + // action, addressed to the denied sender, without leaking the allowlist. + if (senderFeedbackCalls.length !== 1) { + fail( + `denied Slack app_mention expected exactly one sender feedback action, got ${senderFeedbackCalls.length}: ${JSON.stringify(senderFeedbackCalls)}`, + ); + } + const deniedFeedback = senderFeedbackCalls[0]; + if (deniedFeedback.method !== "chat.postEphemeral") { + fail(`denied Slack app_mention expected an ephemeral feedback action, got ${deniedFeedback.method}`); + } + if (deniedFeedback.channel !== channelId) { + fail(`denied Slack feedback targeted unexpected channel: ${deniedFeedback.channel}`); + } + if (deniedFeedback.user !== deniedUser) { + fail(`denied Slack feedback addressed unexpected user: ${deniedFeedback.user}`); + } + const deniedFeedbackText = deniedFeedback.text ?? ""; + if (!deniedFeedbackText) { + fail("denied Slack feedback message text was empty"); + } + if (deniedFeedbackText.includes(allowedUser) || /allow\s*list|allowlist|allowed users/i.test(deniedFeedbackText)) { + fail(`denied Slack feedback leaked allowlist details: ${deniedFeedbackText}`); + } + + const fakeClient = { + chat: { + postMessage: async (payload) => { + const response = await postForm( + "/api/chat.postMessage", + { + token, + channel: payload.channel || "", + text: payload.text || "", + ...(payload.thread_ts ? { thread_ts: payload.thread_ts } : {}), + ...(payload.blocks ? { blocks: JSON.stringify(payload.blocks) } : {}), + }, + `Bearer ${token}`, + ); + if (response.statusCode !== 200 || response.body?.ok !== true) { + throw new Error(`fake Slack chat.postMessage failed: ${response.statusCode} ${JSON.stringify(response.body)}`); + } + return response.body; + }, + }, + }; + + const sendResult = await sendMessageSlack(allowedPrepared.replyTarget, proofText, { + cfg, + token, + client: fakeClient, + accountId: "default", + }); + if (sendResult.channelId !== channelId) { + fail(`sendMessageSlack returned unexpected channelId: ${sendResult.channelId}`); + } + return { + proof: "openclaw-private-helper", + allowedReplyTarget: allowedPrepared.replyTarget, + deniedPrepared: deniedPrepared === null, + deniedFeedbackMethod: deniedFeedback.method, + deniedFeedbackCount: senderFeedbackCalls.length, + messageId: sendResult.messageId, + channelId: sendResult.channelId, + }; +} + +async function runHermeticSlackProof() { + const response = await postChannelProofMessage(); + return { + proof: "nemoclaw-hermetic", + allowedReplyTarget: `channel:${channelId}`, + deniedPrepared: true, + messageId: response.ts || response.message?.ts || null, + channelId, + }; +} + +const slackApiLocation = resolveOpenClawSlackApiLocation(); +let result; +if (slackApiLocation) { + try { + result = await runOpenClawPrivateProof(slackApiLocation); + } catch (error) { + fail(`[slack-proof] OpenClaw Slack helper failed: ${error.stack || error.message || String(error)}`); + } +} else { + fail("[slack-proof] could not find installed OpenClaw Slack proof helper"); +} + +console.log( + JSON.stringify({ + ok: true, + ...result, + }), +); +NODE +} diff --git a/test/e2e-vpn/lib/telegram-api-proof.sh b/test/e2e-vpn/lib/telegram-api-proof.sh new file mode 100755 index 00000000000..bfaa52fcb6d --- /dev/null +++ b/test/e2e-vpn/lib/telegram-api-proof.sh @@ -0,0 +1,311 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Shared hermetic Telegram Bot API helpers for OpenClaw messaging E2E checks. + +append_exit_trap_for_fake_telegram_api() { + local command="$1" + local existing + existing="$(trap -p EXIT | sed "s/^trap -- '//;s/' EXIT$//")" + trap ''"${existing:+$existing; }$command"'' EXIT +} + +cleanup_fake_telegram_api() { + if [ -n "${FAKE_TELEGRAM_API_CONTAINER:-}" ]; then + docker rm -f "$FAKE_TELEGRAM_API_CONTAINER" >/dev/null 2>&1 || true + fi + if [ -n "${FAKE_TELEGRAM_API_DIR:-}" ]; then + rm -rf "$FAKE_TELEGRAM_API_DIR" 2>/dev/null || true + fi +} + +start_fake_telegram_api() { + local token="$1" + mkdir -p "$REPO/.tmp" + FAKE_TELEGRAM_API_DIR="$(mktemp -d "$REPO/.tmp/fake-telegram.XXXXXX")" + FAKE_TELEGRAM_API_PORT_FILE="$FAKE_TELEGRAM_API_DIR/port" + FAKE_TELEGRAM_API_CAPTURE_FILE="$FAKE_TELEGRAM_API_DIR/capture.jsonl" + FAKE_TELEGRAM_API_CONTAINER="nemoclaw-fake-telegram-$$-$RANDOM" + FAKE_TELEGRAM_API_HOST="host.docker.internal" + : >"$FAKE_TELEGRAM_API_CAPTURE_FILE" + + if ! docker run -d --rm \ + --name "$FAKE_TELEGRAM_API_CONTAINER" \ + -p 0:8080 \ + -e FAKE_TELEGRAM_API_PORT=8080 \ + -e FAKE_TELEGRAM_API_EXPECTED_TOKEN="$token" \ + -e FAKE_TELEGRAM_API_PORT_FILE=/tmp/fake-telegram/port \ + -e FAKE_TELEGRAM_API_CAPTURE_FILE=/tmp/fake-telegram/capture.jsonl \ + -v "$FAKE_TELEGRAM_API_DIR:/tmp/fake-telegram" \ + -v "$REPO/test/e2e-vpn/lib:/opt/nemoclaw-e2e:ro" \ + node:22-bookworm-slim \ + node /opt/nemoclaw-e2e/fake-telegram-api.cjs \ + >"$FAKE_TELEGRAM_API_DIR/container.id" 2>"$FAKE_TELEGRAM_API_DIR/server.log"; then + cat "$FAKE_TELEGRAM_API_DIR/server.log" >&2 || true + return 1 + fi + append_exit_trap_for_fake_telegram_api cleanup_fake_telegram_api + + for _ in $(seq 1 50); do + if [ -s "$FAKE_TELEGRAM_API_PORT_FILE" ]; then + local published_port + published_port="$(docker port "$FAKE_TELEGRAM_API_CONTAINER" 8080/tcp 2>/dev/null | head -1 | sed 's/.*://')" + if [ -n "$published_port" ]; then + export FAKE_TELEGRAM_API_PORT + FAKE_TELEGRAM_API_PORT="$published_port" + return 0 + fi + fi + if ! docker inspect "$FAKE_TELEGRAM_API_CONTAINER" >/dev/null 2>&1; then + docker logs "$FAKE_TELEGRAM_API_CONTAINER" >&2 || true + cat "$FAKE_TELEGRAM_API_DIR/server.log" >&2 || true + return 1 + fi + sleep 0.1 + done + cat "$FAKE_TELEGRAM_API_DIR/server.log" >&2 || true + return 1 +} + +fake_telegram_api_allowed_ip_options() { + printf '%s' 'allowed-ip=10.0.0.0/8,allowed-ip=172.16.0.0/12,allowed-ip=192.168.0.0/16' +} + +apply_fake_telegram_api_policy() { + local sandbox_name="$1" + local port="$2" + local host="${FAKE_TELEGRAM_API_HOST:-host.openshell.internal}" + local allowed_ip_options + allowed_ip_options="$(fake_telegram_api_allowed_ip_options)" + openshell policy update "$sandbox_name" \ + --add-endpoint "${host}:${port}:read-write:rest:enforce:request-body-credential-rewrite,${allowed_ip_options}" \ + --add-allow "${host}:${port}:GET:/**" \ + --add-allow "${host}:${port}:POST:/**" \ + --binary /usr/local/bin/node \ + --binary /usr/bin/node \ + --wait +} + +run_openclaw_telegram_mock_send() { + local port="$1" + local target="$2" + local message="$3" + local host="${FAKE_TELEGRAM_API_HOST:-host.openshell.internal}" + local target_b64 message_b64 + target_b64=$(printf '%s' "$target" | base64 | tr -d '\n') + message_b64=$(printf '%s' "$message" | base64 | tr -d '\n') + + sandbox_exec_stdin "FAKE_TELEGRAM_API_HOST='$host' FAKE_TELEGRAM_API_PORT='$port' OPENCLAW_MESSAGE_TARGET_B64='$target_b64' OPENCLAW_MESSAGE_TEXT_B64='$message_b64' node --preserve-symlinks --input-type=module - 2>&1" <<'NODE' +import { execFileSync } from "node:child_process"; +import fs from "node:fs"; +import http from "node:http"; +import { createRequire } from "node:module"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +function decodeBase64(value) { + return Buffer.from(value || "", "base64").toString("utf8"); +} + +function addPathWalk(candidates, seen, start) { + if (!start) return; + let current = path.resolve(start); + for (let depth = 0; depth < 8; depth += 1) { + if (!seen.has(current)) { + seen.add(current); + candidates.push(path.join(current, "node_modules/openclaw/dist/extensions/telegram/test-api.js")); + candidates.push(path.join(current, "dist/extensions/telegram/test-api.js")); + } + const parent = path.dirname(current); + if (parent === current) break; + current = parent; + } +} + +function resolveTelegramTestApiPath() { + const require = createRequire(import.meta.url); + const candidates = []; + const seen = new Set(); + const add = (candidate) => { + if (candidate && !seen.has(candidate)) { + seen.add(candidate); + candidates.push(candidate); + } + }; + + for (const base of [process.cwd(), "/sandbox", "/usr/local/lib/node_modules", "/tmp/npm-global/lib/node_modules"]) { + try { + add(path.join(path.dirname(require.resolve("openclaw/package.json", { paths: [base] })), "dist/extensions/telegram/test-api.js")); + } catch {} + } + + try { + const globalRoot = execFileSync("npm", ["root", "-g"], { encoding: "utf8" }).trim(); + add(path.join(globalRoot, "openclaw/dist/extensions/telegram/test-api.js")); + } catch {} + + try { + const openclawBin = execFileSync("sh", ["-lc", "command -v openclaw || true"], { encoding: "utf8" }).trim(); + if (openclawBin) { + const realBin = execFileSync("readlink", ["-f", openclawBin], { encoding: "utf8" }).trim(); + addPathWalk(candidates, seen, path.dirname(realBin)); + } + } catch {} + + try { + const searchRoots = ["/usr/local", "/tmp/npm-global", "/sandbox"].filter((root) => fs.existsSync(root)); + if (searchRoots.length) { + const discovered = execFileSync("find", [ + ...searchRoots, + "-path", + "*/node_modules/openclaw/dist/extensions/telegram/test-api.js", + "-print", + "-quit", + ], { encoding: "utf8" }).trim(); + add(discovered); + } + } catch {} + + for (const candidate of candidates) { + if (candidate && fs.existsSync(candidate)) return candidate; + } + return null; +} + +function requestFakeTelegram(endpoint, fields, token) { + const payload = JSON.stringify(fields); + const options = { + hostname: process.env.FAKE_TELEGRAM_API_HOST || "host.openshell.internal", + port: Number(process.env.FAKE_TELEGRAM_API_PORT), + path: `/bot${token}/${endpoint}`, + method: "POST", + headers: { + "Content-Type": "application/json", + "Content-Length": Buffer.byteLength(payload), + "User-Agent": "nemoclaw-openclaw-telegram-plugin-e2e", + }, + }; + return new Promise((resolve, reject) => { + const req = http.request(options, (res) => { + let responseBody = ""; + res.on("data", (chunk) => { + responseBody += chunk; + }); + res.on("end", () => { + let parsed = {}; + try { + parsed = responseBody ? JSON.parse(responseBody) : {}; + } catch (error) { + reject(new Error(`invalid JSON from fake Telegram: ${error.message}: ${responseBody}`)); + return; + } + if (res.statusCode < 200 || res.statusCode >= 300 || parsed.ok !== true) { + reject(new Error(`fake Telegram ${endpoint} failed: HTTP ${res.statusCode} ${JSON.stringify(parsed)}`)); + return; + } + resolve(parsed.result); + }); + }); + req.on("error", reject); + req.setTimeout(30000, () => { + req.destroy(new Error("fake Telegram message API timed out")); + }); + req.write(payload); + req.end(); + }); +} + +async function main() { + const testApiPath = resolveTelegramTestApiPath(); + if (!testApiPath) throw new Error("could not find installed OpenClaw Telegram test-api.js"); + + const { sendMessageTelegram } = await import(pathToFileURL(testApiPath).href); + if (typeof sendMessageTelegram !== "function") { + throw new Error("installed Telegram test API does not export sendMessageTelegram"); + } + + const cfg = JSON.parse(fs.readFileSync("/sandbox/.openclaw/openclaw.json", "utf8")); + const account = cfg.channels?.telegram?.accounts?.default; + if (!account?.botToken) throw new Error("missing channels.telegram.accounts.default.botToken in openclaw.json"); + + const target = decodeBase64(process.env.OPENCLAW_MESSAGE_TARGET_B64); + const text = decodeBase64(process.env.OPENCLAW_MESSAGE_TEXT_B64); + const token = account.botToken; + const api = { + sendMessage: (chatId, body, params = {}) => requestFakeTelegram("sendMessage", { + chat_id: chatId, + text: body, + ...params, + }, token), + }; + + const result = await sendMessageTelegram(target, text, { + cfg, + token, + accountId: "default", + api, + }); + + console.log(JSON.stringify({ + ok: true, + proof: "openclaw-telegram-runtime-send", + chatId: result.chatId ?? target, + messageId: result.messageId ?? null, + })); +} + +main() + .then(() => { + console.log("__OPENCLAW_MESSAGE_SEND_EXIT__:0"); + }) + .catch((error) => { + console.error(error.stack || error.message || String(error)); + console.log("__OPENCLAW_MESSAGE_SEND_EXIT__:1"); + process.exit(1); + }); +NODE +} + +check_fake_telegram_capture_send() { + local expected_token="$1" + local expected_chat="$2" + local expected_text="$3" + node - "$FAKE_TELEGRAM_API_CAPTURE_FILE" "$expected_token" "$expected_chat" "$expected_text" <<'NODE' +const fs = require("fs"); +const [file, expectedToken, expectedChat, expectedText] = process.argv.slice(2); +const rows = fs + .readFileSync(file, "utf8") + .trim() + .split(/\n+/) + .filter(Boolean) + .map((line) => JSON.parse(line)) + .filter((row) => row.event === "request" && row.endpoint === "sendMessage"); +const last = rows.at(-1); +if (!last) { + console.log("NO_SEND_MESSAGE"); + process.exit(2); +} +if (last.tokenMatchesExpected !== true) { + console.log("BAD_TOKEN_REWRITE"); + process.exit(3); +} +if (last.tokenLooksPlaceholder) { + console.log("PLACEHOLDER_LEAK"); + process.exit(4); +} +if (String(last.chatId) !== String(expectedChat)) { + console.log(`BAD_CHAT ${last.chatId}`); + process.exit(5); +} +if (last.text !== expectedText) { + console.log(`BAD_TEXT ${last.text}`); + process.exit(6); +} +if (!expectedToken) { + console.log("MISSING_EXPECTED_TOKEN"); + process.exit(7); +} +console.log("OK"); +NODE +} diff --git a/test/e2e-vpn/test-agent-turn-latency-e2e.sh b/test/e2e-vpn/test-agent-turn-latency-e2e.sh new file mode 100755 index 00000000000..78e24966867 --- /dev/null +++ b/test/e2e-vpn/test-agent-turn-latency-e2e.sh @@ -0,0 +1,641 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Real agent turn latency E2E. +# +# Installs one OpenClaw sandbox and one Hermes sandbox against the configured +# hosted inference endpoint, verifies that both are configured for the requested +# model, and times one real model-backed turn through each runtime. +# +# Prerequisites: +# - Docker running +# - NVIDIA_API_KEY set for hosted inference +# - NEMOCLAW_NON_INTERACTIVE=1 +# - NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 +# +# Environment: +# NEMOCLAW_TURN_LATENCY_INSTALL_ATTEMPTS - install attempts for transient +# provider validation (default: 2) + +# Do not use errexit because this test records pass/fail counts and exits +# explicitly after critical failures or at the final summary. +set -uo pipefail + +: "${NEMOCLAW_E2E_DEFAULT_TIMEOUT:=7200}" +export NEMOCLAW_E2E_DEFAULT_TIMEOUT + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +# shellcheck source=test/e2e-vpn/e2e-timeout.sh +source "${SCRIPT_DIR}/e2e-timeout.sh" +# shellcheck source=test/e2e-vpn/lib/openclaw-json.sh +source "${SCRIPT_DIR}/lib/openclaw-json.sh" +# shellcheck source=test/e2e-vpn/lib/sandbox-teardown.sh +source "${SCRIPT_DIR}/lib/sandbox-teardown.sh" +# shellcheck source=test/e2e-vpn/lib/ci-compatible-inference.sh +. "${SCRIPT_DIR}/lib/ci-compatible-inference.sh" +# shellcheck source=test/e2e-vpn/lib/install-path-refresh.sh +source "${SCRIPT_DIR}/lib/install-path-refresh.sh" + +PASS=0 +FAIL=0 +SKIP=0 +TOTAL=0 + +pass() { + ((PASS++)) + ((TOTAL++)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} + +fail() { + ((FAIL++)) + ((TOTAL++)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} + +skip() { + ((SKIP++)) + ((TOTAL++)) + printf '\033[33m SKIP: %s\033[0m\n' "$1" +} + +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} + +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } + +is_positive_int() { + [[ "${1:-}" =~ ^[1-9][0-9]*$ ]] +} + +is_transient_provider_validation_log() { + local log_path="$1" + [ -f "$log_path" ] || return 1 + + grep -qiE 'endpoint validation failed|failed to verify inference endpoint|Chat Completions API validation' "$log_path" \ + && grep -qiE 'timed? out|timeout|curl failed \(exit (7|28|35|52|56)\)|ETIMEDOUT|ECONNRESET|EAI_AGAIN|ENOTFOUND|failed to connect|error sending request|HTTP (429|502|503|504)|returned HTTP (429|502|503|504)|temporar' "$log_path" +} + +monotonic_ms() { + python3 -c 'import time; print(time.monotonic_ns() // 1000000)' +} + +duration_s() { + python3 - "$1" <<'PY' +import sys + +ms = int(sys.argv[1]) +print(f"{ms / 1000:.3f}s") +PY +} + +strip_ansi() { + nemoclaw_e2e_strip_ansi +} + +parse_chat_content() { + python3 -c ' +import json +import sys + +try: + r = json.load(sys.stdin) + c = r["choices"][0]["message"] + content = c.get("content") or c.get("reasoning_content") or c.get("reasoning") or "" + print(content.strip()) +except Exception as exc: + print(f"PARSE_ERROR: {exc}", file=sys.stderr) + sys.exit(1) +' +} + +http_status_from_response() { + sed -n 's/^__NEMOCLAW_HTTP_STATUS__=//p' <<<"$1" | tail -1 +} + +http_body_from_response() { + sed '/^__NEMOCLAW_HTTP_STATUS__=/d' <<<"$1" +} + +get_route_output() { + local output + if output=$(openshell inference get -g nemoclaw 2>&1); then + printf '%s\n' "$output" + return 0 + fi + openshell inference get 2>&1 +} + +assert_route() { + local label="$1" + local output plain_output + if ! output=$(get_route_output); then + fail "${label}: openshell inference get failed: ${output:0:240}" + return + fi + plain_output=$(printf '%s' "$output" | strip_ansi) + + if nemoclaw_e2e_inference_output_matches "$plain_output" "$EXPECTED_ROUTE_PROVIDER" "$TURN_MODEL"; then + pass "${label}: OpenShell route is ${EXPECTED_ROUTE_PROVIDER} / ${TURN_MODEL}" + else + fail "${label}: route is not ${EXPECTED_ROUTE_PROVIDER} / ${TURN_MODEL}: ${plain_output:0:400}" + fi +} + +assert_openclaw_config() { + local sandbox="$1" + local config probe + config=$(openshell sandbox exec --name "$sandbox" -- cat /sandbox/.openclaw/openclaw.json 2>&1) || { + fail "OpenClaw config: could not read /sandbox/.openclaw/openclaw.json: ${config:0:240}" + return + } + + probe=$(EXPECTED_MODEL="$TURN_MODEL" python3 -c ' +import json +import os +import sys + +expected = os.environ["EXPECTED_MODEL"] +doc = json.load(sys.stdin) +errors = [] +primary = (((doc.get("agents") or {}).get("defaults") or {}).get("model") or {}).get("primary") +if primary != f"inference/{expected}": + errors.append(f"primary={primary!r}") + +provider = (((doc.get("models") or {}).get("providers") or {}).get("inference") or {}) +if provider.get("baseUrl") != "https://inference.local/v1": + errors.append("baseUrl={!r}".format(provider.get("baseUrl"))) +models = provider.get("models") or [] +if not models or models[0].get("id") != expected: + errors.append("model id={!r}".format(models[0].get("id") if models else None)) +if not models or models[0].get("name") != f"inference/{expected}": + errors.append("model name={!r}".format(models[0].get("name") if models else None)) + +if errors: + print("; ".join(errors)) + raise SystemExit(1) +print("OK") +' <<<"$config" 2>&1) || { + fail "OpenClaw config: expected Ultra model via inference.local: ${probe:0:400}" + return + } + pass "OpenClaw config uses inference/${TURN_MODEL}" +} + +assert_hermes_config() { + local sandbox="$1" + local config probe + config=$(openshell sandbox exec --name "$sandbox" -- cat /sandbox/.hermes/config.yaml 2>&1) || { + fail "Hermes config: could not read /sandbox/.hermes/config.yaml: ${config:0:240}" + return + } + + probe=$( + CONFIG_TEXT="$config" EXPECTED_MODEL="$TURN_MODEL" python3 - <<'PY' +import os +import re + +text = os.environ["CONFIG_TEXT"] +expected = os.environ["EXPECTED_MODEL"] +errors = [] + +model = {} +in_model = False +for line in text.splitlines(): + if re.match(r"^model:\s*$", line): + in_model = True + continue + if in_model and re.match(r"^[A-Za-z0-9_-]+:", line): + break + if in_model: + match = re.match(r"^\s+([A-Za-z0-9_-]+):\s*(.*?)\s*$", line) + if match: + value = match.group(2).strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'": + value = value[1:-1] + model[match.group(1)] = value + +if model.get("default") != expected: + errors.append(f"model.default={model.get('default')!r}") +if model.get("base_url") != "https://inference.local/v1": + errors.append(f"model.base_url={model.get('base_url')!r}") +if model.get("provider") != "custom": + errors.append(f"model.provider={model.get('provider')!r}") + +if re.search(r"(?ms)^models:\s*\n(?:[ \t].*\n)*?[ \t]+providers:", text): + errors.append("OpenClaw-style models.providers block present") + +if errors: + print("; ".join(errors)) + raise SystemExit(1) +print("OK") +PY + ) || { + fail "Hermes config: expected Ultra model via inference.local: ${probe:0:400}" + return + } + pass "Hermes config.yaml model block uses ${TURN_MODEL} via inference.local" +} + +assert_hermes_health() { + local sandbox="$1" + local health_response attempt + for attempt in 1 2 3 4 5 6 7 8 9 10; do + health_response=$(openshell sandbox exec --name "$sandbox" -- \ + curl -sf --max-time 10 http://localhost:8642/health 2>&1) || true + if grep -qi '"ok"' <<<"$health_response"; then + pass "Hermes health endpoint returns ok" + return + fi + [ "$attempt" -ge 10 ] || sleep 5 + done + fail "Hermes health endpoint did not return ok: ${health_response:0:240}" +} + +assert_latency_under_cap() { + local label="$1" + local elapsed_ms="$2" + local cap_ms=$((MAX_TURN_SECONDS * 1000)) + + if [ "$elapsed_ms" -le "$cap_ms" ]; then + pass "${label}: turn latency $(duration_s "$elapsed_ms") is within ${MAX_TURN_SECONDS}s cap" + else + fail "${label}: turn latency $(duration_s "$elapsed_ms") exceeded ${MAX_TURN_SECONDS}s cap" + fi +} + +destroy_sandbox() { + local sandbox="$1" + if command -v openshell >/dev/null 2>&1; then + openshell forward stop 8642 >/dev/null 2>&1 || true + openshell sandbox delete "$sandbox" >/dev/null 2>&1 || true + openshell gateway destroy -g nemoclaw >/dev/null 2>&1 || true + fi + if command -v nemoclaw >/dev/null 2>&1; then + nemoclaw "$sandbox" destroy --yes >/dev/null 2>&1 || true + NEMOCLAW_AGENT=hermes nemoclaw "$sandbox" destroy --yes >/dev/null 2>&1 || true + fi +} + +run_install() { + local label="$1" + local sandbox="$2" + local agent="$3" + local log_path="$4" + local install_pid tail_pid install_exit attempt + + section "Install ${label}" + info "Pre-cleaning sandbox ${sandbox} and the nemoclaw gateway..." + destroy_sandbox "$sandbox" + + cd "$REPO" || { + fail "${label}: could not cd to repo root: $REPO" + return 1 + } + + export NEMOCLAW_SANDBOX_NAME="$sandbox" + export NEMOCLAW_RECREATE_SANDBOX=1 + export NEMOCLAW_PROVIDER="$TURN_PROVIDER_KEY" + export NEMOCLAW_MODEL="$TURN_MODEL" + export NEMOCLAW_NON_INTERACTIVE="${NEMOCLAW_NON_INTERACTIVE:-1}" + export NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE="${NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE:-1}" + if [ "$agent" = "hermes" ]; then + export NEMOCLAW_AGENT=hermes + else + unset NEMOCLAW_AGENT + fi + + for ((attempt = 1; attempt <= TURN_INSTALL_ATTEMPTS; attempt++)); do + if [ "$attempt" -gt 1 ]; then + info "Retrying ${label} install after transient provider validation failure..." + destroy_sandbox "$sandbox" + fi + + info "Running install.sh for ${label} with ${TURN_PROVIDER_KEY} / ${TURN_MODEL} (attempt ${attempt}/${TURN_INSTALL_ATTEMPTS})..." + bash install.sh --non-interactive --yes-i-accept-third-party-software >"$log_path" 2>&1 & + install_pid=$! + tail -f "$log_path" --pid="$install_pid" 2>/dev/null & + tail_pid=$! + wait "$install_pid" + install_exit=$? + kill "$tail_pid" 2>/dev/null || true + wait "$tail_pid" 2>/dev/null || true + + if [ "$install_exit" -eq 0 ]; then + break + fi + + if is_transient_provider_validation_log "$log_path"; then + if [ "$attempt" -lt "$TURN_INSTALL_ATTEMPTS" ]; then + info "${label}: install attempt ${attempt}/${TURN_INSTALL_ATTEMPTS} hit transient provider validation; retrying..." + tail -40 "$log_path" || true + continue + fi + + skip "${label}: install skipped after ${TURN_INSTALL_ATTEMPTS} transient provider validation attempt(s)" + tail -80 "$log_path" || true + return 1 + fi + + break + done + + nemoclaw_refresh_install_env + export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" + # shellcheck source=/dev/null + [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" + nemoclaw_ensure_local_bin_on_path + + if [ "$install_exit" -ne 0 ]; then + fail "${label}: install.sh failed (exit ${install_exit})" + tail -80 "$log_path" || true + return 1 + fi + pass "${label}: install.sh completed" + + command -v nemoclaw >/dev/null 2>&1 || { + fail "${label}: nemoclaw not found on PATH" + return 1 + } + command -v openshell >/dev/null 2>&1 || { + fail "${label}: openshell not found on PATH" + return 1 + } + pass "${label}: nemoclaw and openshell are on PATH" +} + +run_openclaw_turn() { + local sandbox="$1" + local ssh_config stderr_file session_id start_ms end_ms raw rc reply stderr_text + + section "OpenClaw Turn Latency" + assert_route "OpenClaw" + assert_openclaw_config "$sandbox" + + ssh_config="$(mktemp)" + stderr_file="$(mktemp)" + if ! openshell sandbox ssh-config "$sandbox" >"$ssh_config" 2>/dev/null; then + rm -f "$ssh_config" "$stderr_file" + fail "OpenClaw: could not get SSH config for ${sandbox}" + return + fi + + session_id="e2e-openclaw-turn-latency-$(date +%s)-$$" + rc=0 + start_ms="$(monotonic_ms)" + raw=$(run_with_timeout "$COMMAND_TIMEOUT_SECONDS" ssh -F "$ssh_config" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o LogLevel=ERROR \ + "openshell-${sandbox}" \ + "openclaw agent --agent main --json --thinking off --session-id '${session_id}' -m 'What is 6 multiplied by 7? Reply with only the integer, no extra words.'" \ + 2>"$stderr_file") || rc=$? + end_ms="$(monotonic_ms)" + OPENCLAW_TURN_MS=$((end_ms - start_ms)) + stderr_text="$(<"$stderr_file")" + rm -f "$ssh_config" "$stderr_file" + + reply=$(printf '%s' "$raw" | parse_openclaw_agent_text 2>/dev/null) || reply="" + OPENCLAW_REPLY="$reply" + + if [ "$rc" -ne 0 ]; then + fail "OpenClaw: real agent turn failed (exit ${rc}); stdout='${raw:0:240}'; stderr='${stderr_text:0:240}'" + return + fi + + if printf '%s\n%s\n' "$raw" "$stderr_text" | grep -qiE 'SsrFBlockedError|transport error|ECONNREFUSED|EAI_AGAIN|gateway unavailable|network connection error'; then + fail "OpenClaw: real agent turn hit a provider or transport error" + return + fi + + if grep -qE '(^|[^0-9])42([^0-9]|$)' <<<"$reply"; then + pass "OpenClaw: real agent turn returned 42 in $(duration_s "$OPENCLAW_TURN_MS")" + assert_latency_under_cap "OpenClaw" "$OPENCLAW_TURN_MS" + else + fail "OpenClaw: expected 42 from real agent turn; reply='${reply:0:240}'; raw='${raw:0:240}'" + fi +} + +run_hermes_turn() { + local sandbox="$1" + local payload payload_arg remote start_ms end_ms response rc http_code body content + + section "Hermes Turn Latency" + assert_route "Hermes" + assert_hermes_config "$sandbox" + assert_hermes_health "$sandbox" + + payload=$(TURN_MODEL="$TURN_MODEL" python3 -c ' +import json +import os + +print(json.dumps({ + "model": os.environ["TURN_MODEL"], + "messages": [{"role": "user", "content": "What is 6 multiplied by 7? Reply with only the integer, no extra words."}], + "max_tokens": 64, +})) +') + payload_arg="$(printf '%q' "$payload")" + remote="set -a; [ ! -f /sandbox/.hermes/.env ] || . /sandbox/.hermes/.env; set +a; tmp=\$(mktemp); if [ -n \"\${API_SERVER_KEY:-}\" ]; then code=\$(curl -sS -o \"\$tmp\" -w '%{http_code}' --max-time ${COMMAND_TIMEOUT_SECONDS} http://localhost:8642/v1/chat/completions -H 'Content-Type: application/json' -H \"Authorization: Bearer \${API_SERVER_KEY}\" -d $payload_arg); else code=\$(curl -sS -o \"\$tmp\" -w '%{http_code}' --max-time ${COMMAND_TIMEOUT_SECONDS} http://localhost:8642/v1/chat/completions -H 'Content-Type: application/json' -d $payload_arg); fi; rc=\$?; cat \"\$tmp\"; rm -f \"\$tmp\"; printf '\n__NEMOCLAW_HTTP_STATUS__=%s\n' \"\${code:-000}\"; exit \"\$rc\"" + + rc=0 + start_ms="$(monotonic_ms)" + response=$(run_with_timeout "$COMMAND_TIMEOUT_SECONDS" openshell sandbox exec --name "$sandbox" -- sh -lc "$remote" 2>&1) || rc=$? + end_ms="$(monotonic_ms)" + HERMES_TURN_MS=$((end_ms - start_ms)) + + http_code=$(http_status_from_response "$response") + [ -n "$http_code" ] || http_code="000" + body=$(http_body_from_response "$response") + content=$(printf '%s' "$body" | parse_chat_content 2>/dev/null) || content="" + HERMES_REPLY="$content" + + if [ "$rc" -ne 0 ]; then + fail "Hermes: real daemon turn failed (exit ${rc}); HTTP ${http_code}: ${body:0:300}" + return + fi + if [ "$http_code" != "200" ]; then + fail "Hermes: real daemon turn returned HTTP ${http_code}: ${body:0:300}" + return + fi + + if grep -qE '(^|[^0-9])42([^0-9]|$)' <<<"$content"; then + pass "Hermes: real daemon turn returned 42 in $(duration_s "$HERMES_TURN_MS")" + assert_latency_under_cap "Hermes" "$HERMES_TURN_MS" + else + fail "Hermes: expected 42 from real daemon turn; content='${content:0:240}'; body='${body:0:240}'" + fi +} + +write_results_json() { + OPENCLAW_TURN_MS="${OPENCLAW_TURN_MS:-}" \ + HERMES_TURN_MS="${HERMES_TURN_MS:-}" \ + OPENCLAW_REPLY="${OPENCLAW_REPLY:-}" \ + HERMES_REPLY="${HERMES_REPLY:-}" \ + TURN_MODEL="$TURN_MODEL" \ + TURN_PROVIDER_KEY="$TURN_PROVIDER_KEY" \ + EXPECTED_ROUTE_PROVIDER="$EXPECTED_ROUTE_PROVIDER" \ + MAX_TURN_SECONDS="$MAX_TURN_SECONDS" \ + OPENCLAW_SANDBOX_NAME="$OPENCLAW_SANDBOX_NAME" \ + HERMES_SANDBOX_NAME="$HERMES_SANDBOX_NAME" \ + PASS="$PASS" FAIL="$FAIL" SKIP="$SKIP" TOTAL="$TOTAL" \ + python3 - <<'PY' >"$RESULTS_JSON" +import json +import os + +def maybe_int(name): + value = os.environ.get(name, "") + return int(value) if value.isdigit() else None + +doc = { + "model": os.environ["TURN_MODEL"], + "provider_key": os.environ["TURN_PROVIDER_KEY"], + "route_provider": os.environ["EXPECTED_ROUTE_PROVIDER"], + "max_turn_seconds": int(os.environ["MAX_TURN_SECONDS"]), + "sandboxes": { + "openclaw": os.environ["OPENCLAW_SANDBOX_NAME"], + "hermes": os.environ["HERMES_SANDBOX_NAME"], + }, + "turns": { + "openclaw": { + "elapsed_ms": maybe_int("OPENCLAW_TURN_MS"), + "reply_excerpt": os.environ.get("OPENCLAW_REPLY", "")[:200], + }, + "hermes": { + "elapsed_ms": maybe_int("HERMES_TURN_MS"), + "reply_excerpt": os.environ.get("HERMES_REPLY", "")[:200], + }, + }, + "summary": { + "pass": int(os.environ["PASS"]), + "fail": int(os.environ["FAIL"]), + "skip": int(os.environ["SKIP"]), + "total": int(os.environ["TOTAL"]), + }, +} +print(json.dumps(doc, indent=2, sort_keys=True)) +PY +} + +finish() { + section "Summary" + write_results_json + [ -n "${OPENCLAW_TURN_MS:-}" ] && info "OpenClaw turn: $(duration_s "$OPENCLAW_TURN_MS")" + [ -n "${HERMES_TURN_MS:-}" ] && info "Hermes turn: $(duration_s "$HERMES_TURN_MS")" + info "Results JSON: ${RESULTS_JSON}" + echo "" + echo "========================================" + echo "Real Agent Turn Latency E2E Summary" + echo "========================================" + echo "Total: $TOTAL" + echo "Passed: $PASS" + echo "Failed: $FAIL" + echo "Skipped: $SKIP" + echo "========================================" + + if [ "$FAIL" -gt 0 ]; then + exit 1 + fi + exit 0 +} + +if [ -d /workspace ] && [ -f /workspace/install.sh ]; then + REPO="/workspace" +elif [ -f "$(cd "${SCRIPT_DIR}/../.." && pwd)/install.sh" ]; then + REPO="$(cd "${SCRIPT_DIR}/../.." && pwd)" +else + echo "ERROR: Cannot find repo root." + exit 1 +fi + +OPENCLAW_SANDBOX_NAME="${NEMOCLAW_OPENCLAW_TURN_LATENCY_SANDBOX_NAME:-e2e-openclaw-turn-latency}" +HERMES_SANDBOX_NAME="${NEMOCLAW_HERMES_TURN_LATENCY_SANDBOX_NAME:-e2e-hermes-turn-latency}" +OPENCLAW_INSTALL_LOG="/tmp/nemoclaw-e2e-openclaw-turn-latency-install.log" +HERMES_INSTALL_LOG="/tmp/nemoclaw-e2e-hermes-turn-latency-install.log" +RESULTS_JSON="/tmp/nemoclaw-e2e-agent-turn-latency.json" +TURN_MODEL="" +TURN_PROVIDER_KEY="" +EXPECTED_ROUTE_PROVIDER="" + +MAX_TURN_SECONDS="${NEMOCLAW_TURN_LATENCY_MAX_SECONDS:-300}" +is_positive_int "$MAX_TURN_SECONDS" || MAX_TURN_SECONDS=300 +COMMAND_TIMEOUT_SECONDS="${NEMOCLAW_TURN_LATENCY_COMMAND_TIMEOUT_SECONDS:-$((MAX_TURN_SECONDS + 30))}" +is_positive_int "$COMMAND_TIMEOUT_SECONDS" || COMMAND_TIMEOUT_SECONDS=$((MAX_TURN_SECONDS + 30)) +TURN_INSTALL_ATTEMPTS="${NEMOCLAW_TURN_LATENCY_INSTALL_ATTEMPTS:-2}" +is_positive_int "$TURN_INSTALL_ATTEMPTS" || TURN_INSTALL_ATTEMPTS=2 + +OPENCLAW_TURN_MS="" +HERMES_TURN_MS="" +OPENCLAW_REPLY="" +HERMES_REPLY="" + +register_sandbox_for_teardown "$OPENCLAW_SANDBOX_NAME" +register_sandbox_for_teardown "$HERMES_SANDBOX_NAME" +nemoclaw_ensure_local_bin_on_path +nemoclaw_e2e_configure_compatible_inference || { + fail "Hosted CI inference could not be configured" + finish +} + +if nemoclaw_e2e_using_compatible_inference; then + TURN_MODEL="${NEMOCLAW_TURN_LATENCY_MODEL:-$(nemoclaw_e2e_hosted_inference_model)}" + TURN_PROVIDER_KEY="${NEMOCLAW_TURN_LATENCY_PROVIDER:-custom}" + EXPECTED_ROUTE_PROVIDER="${NEMOCLAW_TURN_LATENCY_ROUTE_PROVIDER:-$(nemoclaw_e2e_expected_route_provider)}" +else + TURN_MODEL="${NEMOCLAW_TURN_LATENCY_MODEL:-${NEMOCLAW_MODEL:-nvidia/nemotron-3-ultra-550b-a55b}}" + TURN_PROVIDER_KEY="${NEMOCLAW_TURN_LATENCY_PROVIDER:-custom}" + EXPECTED_ROUTE_PROVIDER="${NEMOCLAW_TURN_LATENCY_ROUTE_PROVIDER:-compatible-endpoint}" +fi + +section "Prerequisites" +if docker info >/dev/null 2>&1; then + pass "Docker is running" +else + fail "Docker is not running" + finish +fi + +if ! nemoclaw_e2e_require_hosted_inference_key; then + finish +fi + +if [ "${NEMOCLAW_NON_INTERACTIVE:-}" = "1" ] && [ "${NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE:-}" = "1" ]; then + pass "Non-interactive install flags are set" +else + fail "NEMOCLAW_NON_INTERACTIVE=1 and NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 are required" + finish +fi + +command -v python3 >/dev/null 2>&1 || { + fail "python3 not found on PATH" + finish +} +pass "python3 is available" + +info "Repo: ${REPO}" +info "Model: ${TURN_MODEL}" +info "Turn cap: ${MAX_TURN_SECONDS}s" + +run_install "OpenClaw" "$OPENCLAW_SANDBOX_NAME" "openclaw" "$OPENCLAW_INSTALL_LOG" || finish +run_openclaw_turn "$OPENCLAW_SANDBOX_NAME" + +if [ "${NEMOCLAW_E2E_KEEP_SANDBOX:-}" != "1" ]; then + section "Cleanup OpenClaw Sandbox" + destroy_sandbox "$OPENCLAW_SANDBOX_NAME" + pass "OpenClaw sandbox pre-Hermes cleanup completed" +fi + +run_install "Hermes" "$HERMES_SANDBOX_NAME" "hermes" "$HERMES_INSTALL_LOG" || finish +run_hermes_turn "$HERMES_SANDBOX_NAME" + +if [ "${NEMOCLAW_E2E_KEEP_SANDBOX:-}" != "1" ]; then + section "Cleanup Hermes Sandbox" + destroy_sandbox "$HERMES_SANDBOX_NAME" + pass "Hermes sandbox cleanup completed" +fi + +finish diff --git a/test/e2e-vpn/test-bedrock-runtime-compatible-anthropic.sh b/test/e2e-vpn/test-bedrock-runtime-compatible-anthropic.sh new file mode 100755 index 00000000000..3ffb706ddc1 --- /dev/null +++ b/test/e2e-vpn/test-bedrock-runtime-compatible-anthropic.sh @@ -0,0 +1,1020 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Bedrock Runtime compatible Anthropic endpoint E2E (#3767). +# +# Hermetic path: +# - starts a local HTTP/2 fake Bedrock Runtime endpoint +# - maps bedrock-runtime.us-east-1.amazonaws.com to localhost +# - onboards with NEMOCLAW_PROVIDER=anthropicCompatible and a fake pasted key +# - proves OpenShell owns the hidden Bedrock adapter token while the sandbox +# only sees https://inference.local/v1 +# - exercises OpenClaw and Hermes agent-specific runtime paths via the same +# nightly matrix script +# +# Environment: +# NEMOCLAW_AGENT openclaw or hermes +# NEMOCLAW_SANDBOX_NAME sandbox name +# NEMOCLAW_BEDROCK_RUNTIME_MOCK_PORT fake Bedrock endpoint port +# NEMOCLAW_E2E_KEEP_SANDBOX=1 keep sandbox for debugging + +set -uo pipefail + +export NEMOCLAW_E2E_DEFAULT_TIMEOUT=2700 +SCRIPT_DIR_TIMEOUT="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +# shellcheck source=test/e2e-vpn/e2e-timeout.sh +. "${SCRIPT_DIR_TIMEOUT}/e2e-timeout.sh" +# shellcheck source=test/e2e-vpn/lib/openclaw-json.sh +. "${SCRIPT_DIR_TIMEOUT}/lib/openclaw-json.sh" + +PASS=0 +FAIL=0 +SKIP=0 +TOTAL=0 + +pass() { + ((PASS++)) + ((TOTAL++)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} + +fail() { + ((FAIL++)) + ((TOTAL++)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} + +skip() { + ((SKIP++)) + ((TOTAL++)) + printf '\033[33m SKIP: %s\033[0m\n' "$1" +} + +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} + +info() { + printf '\033[1;34m [info]\033[0m %s\n' "$1" +} + +summary() { + echo "" + echo "============================================================" + echo " Bedrock Runtime Compatible Anthropic E2E Results" + echo "============================================================" + echo " Agent: $AGENT" + echo " PASS: $PASS" + echo " FAIL: $FAIL" + echo " SKIP: $SKIP" + echo " TOTAL: $TOTAL" + echo "============================================================" + if [ "$FAIL" -gt 0 ]; then + exit 1 + fi +} + +quote_for_remote_sh() { + local value="${1:-}" + printf "'%s'" "$(printf '%s' "$value" | sed "s/'/'\\\\''/g")" +} + +sandbox_exec_sh_script() { + local script="$1" + shift + local encoded remote_cmd arg + encoded="$(printf '%s' "$script" | base64 | tr -d '\n')" + remote_cmd="tmp=\$(mktemp); trap 'rm -f \"\$tmp\"' EXIT; printf %s $(quote_for_remote_sh "$encoded") | base64 -d > \"\$tmp\"; sh \"\$tmp\"" + for arg in "$@"; do + remote_cmd+=" $(quote_for_remote_sh "$arg")" + done + openshell sandbox exec --name "$SANDBOX_NAME" -- sh -lc "$remote_cmd" +} + +parse_chat_content() { + python3 -c ' +import json +import sys + +try: + response = json.load(sys.stdin) + message = response["choices"][0]["message"] + print((message.get("content") or message.get("reasoning_content") or "").strip()) +except Exception as exc: + print(f"PARSE_ERROR: {exc}", file=sys.stderr) + sys.exit(1) +' +} + +load_shell_path() { + local local_bin + if [ -f "$HOME/.bashrc" ]; then + # shellcheck source=/dev/null + source "$HOME/.bashrc" 2>/dev/null || true + fi + export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" + if [ -s "$NVM_DIR/nvm.sh" ]; then + # shellcheck source=/dev/null + . "$NVM_DIR/nvm.sh" + fi + local_bin="$HOME/.local/bin" + if [ -d "$local_bin" ]; then + PATH=":${PATH}:" + PATH="${PATH//:${local_bin}:/:}" + PATH="${PATH#:}" + PATH="${PATH%:}" + export PATH="$local_bin:$PATH" + fi +} + +cli_command_available_from_source() { + [ -f "$REPO/dist/nemoclaw.js" ] && command -v node >/dev/null 2>&1 && command -v openshell >/dev/null 2>&1 +} + +prepare_source_cli() { + local rc=0 + : >"$BUILD_LOG" + load_shell_path + + if ! command -v npm >/dev/null 2>&1; then + echo "npm is not available on PATH" >>"$BUILD_LOG" + return 127 + fi + if ! command -v node >/dev/null 2>&1; then + echo "node is not available on PATH" >>"$BUILD_LOG" + return 127 + fi + + info "Installing npm dependencies and building source CLI" + ( + cd "$REPO" \ + && npm ci --ignore-scripts \ + && npm run build:cli + ) >>"$BUILD_LOG" 2>&1 || rc=$? + if [ "$rc" -ne 0 ]; then + return "$rc" + fi + + if ! command -v openshell >/dev/null 2>&1; then + info "Installing OpenShell CLI" + bash "$REPO/scripts/install-openshell.sh" >>"$BUILD_LOG" 2>&1 || rc=$? + load_shell_path + if [ "$rc" -ne 0 ]; then + return "$rc" + fi + fi + + if ! command -v openshell >/dev/null 2>&1; then + echo "openshell is not available on PATH after installation" >>"$BUILD_LOG" + return 127 + fi +} + +stop_bedrock_mock() { + if [ -n "${BEDROCK_MOCK_PID:-}" ] && kill -0 "$BEDROCK_MOCK_PID" 2>/dev/null; then + kill "$BEDROCK_MOCK_PID" 2>/dev/null || true + wait "$BEDROCK_MOCK_PID" 2>/dev/null || true + fi + BEDROCK_MOCK_PID="" +} + +restore_hosts_file() { + if [ -n "${HOSTS_BACKUP:-}" ] && [ -f "$HOSTS_BACKUP" ]; then + sudo cp "$HOSTS_BACKUP" /etc/hosts 2>/dev/null || true + rm -f "$HOSTS_BACKUP" 2>/dev/null || true + HOSTS_BACKUP="" + fi +} + +stop_bedrock_adapter_best_effort() { + local state_file pid_file token_file pid endpoint + state_file="$HOME/.nemoclaw/bedrock-runtime-adapter.json" + pid_file="$HOME/.nemoclaw/bedrock-runtime-adapter.pid" + token_file="$HOME/.nemoclaw/bedrock-runtime-adapter-token" + if [ -f "$state_file" ]; then + endpoint=$( + python3 - "$state_file" <<'PY' 2>/dev/null || true +import json +import sys + +try: + print((json.load(open(sys.argv[1], encoding="utf-8")).get("endpointUrl") or "").strip()) +except Exception: + pass +PY + ) + if [ "$endpoint" != "$BEDROCK_ENDPOINT_URL" ]; then + return 0 + fi + fi + if [ -f "$pid_file" ]; then + pid="$(tr -d '\n' <"$pid_file" 2>/dev/null || true)" + if [ -n "$pid" ] && ps -p "$pid" -o args= 2>/dev/null | grep -q "bedrock-runtime-adapter.js"; then + kill "$pid" 2>/dev/null || true + fi + fi + rm -f "$pid_file" "$token_file" "$state_file" 2>/dev/null || true +} + +destroy_sandbox_best_effort() { + if [ "${NEMOCLAW_E2E_KEEP_SANDBOX:-}" = "1" ]; then + return 0 + fi + set +e + if cli_command_available_from_source; then + NEMOCLAW_AGENT="$AGENT" run_with_timeout 180 node "$REPO/bin/nemoclaw.js" "$SANDBOX_NAME" destroy --yes >/dev/null 2>&1 + elif command -v nemoclaw >/dev/null 2>&1; then + NEMOCLAW_AGENT="$AGENT" run_with_timeout 180 nemoclaw "$SANDBOX_NAME" destroy --yes >/dev/null 2>&1 + fi + if command -v openshell >/dev/null 2>&1; then + run_with_timeout 60 openshell sandbox delete "$SANDBOX_NAME" >/dev/null 2>&1 + run_with_timeout 60 openshell gateway destroy -g nemoclaw >/dev/null 2>&1 + fi + set -uo pipefail +} + +cleanup() { + stop_bedrock_mock + stop_bedrock_adapter_best_effort + restore_hosts_file + destroy_sandbox_best_effort +} + +map_bedrock_host_to_loopback() { + if ! command -v sudo >/dev/null 2>&1; then + fail "B0: sudo is required to edit /etc/hosts for Bedrock hostname mapping" + summary + fi + if ! sudo -n true >/dev/null 2>&1; then + fail "B0: passwordless sudo is required to edit /etc/hosts for Bedrock hostname mapping" + summary + fi + + HOSTS_BACKUP="$(mktemp)" + sudo cp /etc/hosts "$HOSTS_BACKUP" + printf '\n127.0.0.1 %s\n' "$BEDROCK_HOSTNAME" | sudo tee -a /etc/hosts >/dev/null + + if BEDROCK_HOSTNAME="$BEDROCK_HOSTNAME" python3 - <<'PY'; then +import os +import socket + +raise SystemExit(0 if socket.gethostbyname(os.environ["BEDROCK_HOSTNAME"]) == "127.0.0.1" else 1) +PY + pass "B0: Bedrock Runtime hostname maps to localhost" + else + fail "B0: Bedrock Runtime hostname did not resolve to localhost after hosts edit" + summary + fi +} + +start_bedrock_mock() { + : >"$BEDROCK_MOCK_LOG" + BEDROCK_FAKE_EXPECTED_BEARER="$COMPATIBLE_KEY" node - "$BEDROCK_MOCK_PORT" "$BEDROCK_MODEL" >"$BEDROCK_MOCK_LOG" 2>&1 <<'NODE' & +const http2 = require("node:http2"); +const { EventStreamCodec } = require("@smithy/core/event-streams"); +const { fromUtf8, toUtf8 } = require("@smithy/util-utf8"); + +const port = Number(process.argv[2]); +const expectedModel = process.argv[3]; +const expectedBearer = process.env.BEDROCK_FAKE_EXPECTED_BEARER || ""; +const codec = new EventStreamCodec(toUtf8, fromUtf8); + +function eventMessage(eventType, payload) { + return Buffer.from(codec.encode({ + headers: { + ":message-type": { type: "string", value: "event" }, + ":event-type": { type: "string", value: eventType }, + ":content-type": { type: "string", value: "application/json" }, + }, + body: fromUtf8(JSON.stringify(payload)), + })); +} + +function sendJson(stream, status, payload) { + stream.respond({ + [http2.constants.HTTP2_HEADER_STATUS]: status, + [http2.constants.HTTP2_HEADER_CONTENT_TYPE]: "application/json", + }); + stream.end(JSON.stringify(payload)); +} + +function conversePayload() { + return { + output: { + message: { + role: "assistant", + content: [{ text: "PONG" }], + }, + }, + stopReason: "end_turn", + usage: { + inputTokens: 1, + outputTokens: 1, + totalTokens: 2, + }, + metrics: { + latencyMs: 1, + }, + }; +} + +function sendConverseStream(stream) { + stream.respond({ + [http2.constants.HTTP2_HEADER_STATUS]: 200, + [http2.constants.HTTP2_HEADER_CONTENT_TYPE]: "application/vnd.amazon.eventstream", + }); + stream.write(eventMessage("messageStart", { role: "assistant" })); + stream.write(eventMessage("contentBlockDelta", { + contentBlockIndex: 0, + delta: { text: "PONG" }, + })); + stream.write(eventMessage("messageStop", { stopReason: "end_turn" })); + stream.write(eventMessage("metadata", { + usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 }, + metrics: { latencyMs: 1 }, + })); + stream.end(); +} + +function parseModelPath(pathname) { + const match = pathname.match(/^\/model\/(.+)\/(converse|converse-stream)$/); + if (!match) return null; + return { model: decodeURIComponent(match[1]), operation: match[2] }; +} + +const server = http2.createServer(); +server.on("stream", (stream, headers) => { + const method = headers[http2.constants.HTTP2_HEADER_METHOD] || ""; + const pathname = headers[http2.constants.HTTP2_HEADER_PATH] || ""; + const auth = headers[http2.constants.HTTP2_HEADER_AUTHORIZATION] || ""; + const chunks = []; + + stream.on("data", (chunk) => chunks.push(Buffer.from(chunk))); + stream.on("end", () => { + const parsed = parseModelPath(String(pathname)); + if (method !== "POST" || !parsed) { + sendJson(stream, 404, { message: "not found" }); + return; + } + + const opLabel = parsed.operation === "converse-stream" ? "converse-stream" : "converse"; + if (auth !== `Bearer ${expectedBearer}`) { + console.log(`POST /model/${opLabel} auth=missing`); + sendJson(stream, 401, { message: "missing bearer credential" }); + return; + } + + console.log(`POST /model/${opLabel} auth=ok`); + if (parsed.model !== expectedModel) { + sendJson(stream, 400, { message: "unexpected model id" }); + return; + } + + if (parsed.operation === "converse-stream") { + sendConverseStream(stream); + return; + } + sendJson(stream, 200, conversePayload()); + }); +}); + +server.on("sessionError", (err) => { + console.log(`session_error=${err && err.code ? err.code : "unknown"}`); +}); + +server.listen(port, "127.0.0.1", () => { + console.log("fake_bedrock_runtime_ready"); +}); +NODE + BEDROCK_MOCK_PID=$! + + for _ in $(seq 1 30); do + if node - "$BEDROCK_MOCK_PORT" <<'NODE' >/dev/null 2>&1; then +const net = require("node:net"); +const port = Number(process.argv[2]); +const socket = net.connect(port, "127.0.0.1"); +let done = false; +function finish(ok) { + if (done) return; + done = true; + socket.destroy(); + process.exit(ok ? 0 : 1); +} +socket.on("connect", () => finish(true)); +socket.on("error", () => finish(false)); +socket.setTimeout(500, () => finish(false)); +NODE + return 0 + fi + sleep 1 + done + return 1 +} + +run_bedrock_onboard() { + local onboard_exit=0 + export NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" + export NEMOCLAW_AGENT="$AGENT" + export NEMOCLAW_RECREATE_SANDBOX=1 + export NEMOCLAW_NON_INTERACTIVE=1 + export NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 + export NEMOCLAW_YES=1 + export NEMOCLAW_PROVIDER=anthropicCompatible + export NEMOCLAW_ENDPOINT_URL="$BEDROCK_ENDPOINT_URL" + export NEMOCLAW_MODEL="$BEDROCK_MODEL" + export NEMOCLAW_PREFERRED_API=openai-completions + export NEMOCLAW_POLICY_MODE=skip + export COMPATIBLE_ANTHROPIC_API_KEY="$COMPATIBLE_KEY" + + unset AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN AWS_PROFILE + unset AWS_WEB_IDENTITY_TOKEN_FILE AWS_CONTAINER_CREDENTIALS_RELATIVE_URI + unset AWS_CONTAINER_CREDENTIALS_FULL_URI AWS_BEARER_TOKEN_BEDROCK + unset AWS_REGION AWS_DEFAULT_REGION + unset NVIDIA_API_KEY OPENAI_API_KEY ANTHROPIC_API_KEY GEMINI_API_KEY COMPATIBLE_API_KEY + unset TELEGRAM_BOT_TOKEN DISCORD_BOT_TOKEN SLACK_BOT_TOKEN SLACK_APP_TOKEN + + destroy_sandbox_best_effort + info "Using source-built CLI at $REPO/bin/nemoclaw.js for agent=$AGENT" + run_with_timeout 1800 node "$REPO/bin/nemoclaw.js" onboard --fresh --non-interactive --yes-i-accept-third-party-software \ + >"$ONBOARD_LOG" 2>&1 || onboard_exit=$? + + if [ "$onboard_exit" -eq 0 ]; then + pass "B1: onboard completed for Bedrock Runtime compatible Anthropic endpoint" + else + fail "B1: onboard failed for Bedrock Runtime compatible Anthropic endpoint" + info "Last 120 lines of onboard log:" + tail -120 "$ONBOARD_LOG" 2>/dev/null || true + summary + fi +} + +assert_onboard_identity() { + local probe rc=0 + probe=$( + SANDBOX_NAME="$SANDBOX_NAME" AGENT="$AGENT" BEDROCK_MODEL="$BEDROCK_MODEL" python3 - <<'PY' +import json +import os +from pathlib import Path + +home = Path.home() +name = os.environ["SANDBOX_NAME"] +agent = os.environ["AGENT"] +model = os.environ["BEDROCK_MODEL"] +expected_provider = "compatible-anthropic-endpoint" +errors = [] + +session_path = home / ".nemoclaw" / "onboard-session.json" +registry_path = home / ".nemoclaw" / "sandboxes.json" + +try: + session = json.loads(session_path.read_text(encoding="utf-8")) +except Exception as exc: + session = None + errors.append(f"session read failed: {exc}") + +if isinstance(session, dict): + if session.get("sandboxName") != name: + errors.append(f"session sandboxName={session.get('sandboxName')!r}") + if session.get("agent") not in (None, agent): + errors.append(f"session agent={session.get('agent')!r}") + if session.get("provider") != expected_provider: + errors.append(f"session provider={session.get('provider')!r}") + if session.get("model") != model: + errors.append(f"session model={session.get('model')!r}") + +try: + registry = json.loads(registry_path.read_text(encoding="utf-8")) + sandbox = (registry.get("sandboxes") or {}).get(name) +except Exception as exc: + sandbox = None + errors.append(f"registry read failed: {exc}") + +if not isinstance(sandbox, dict): + errors.append(f"registry sandbox {name!r} missing") +else: + if sandbox.get("agent") not in (None, agent): + errors.append(f"registry agent={sandbox.get('agent')!r}") + if sandbox.get("provider") != expected_provider: + errors.append(f"registry provider={sandbox.get('provider')!r}") + if sandbox.get("model") != model: + errors.append(f"registry model={sandbox.get('model')!r}") + +if errors: + print("; ".join(errors)) + raise SystemExit(1) +print("OK") +PY + ) || rc=$? + if [ "$rc" -eq 0 ]; then + pass "B2: onboard state keeps provider identity as compatible-anthropic-endpoint" + else + fail "B2: onboard state did not preserve compatible-anthropic-endpoint identity: ${probe:0:500}" + fi +} + +assert_adapter_health() { + local health rc=0 + health=$(curl -sf --max-time 5 "http://127.0.0.1:${BEDROCK_ADAPTER_PORT}/health" 2>&1) || rc=$? + if [ "$rc" -ne 0 ]; then + fail "B3: Bedrock Runtime adapter health endpoint failed" + return + fi + + if HEALTH_JSON="$health" BEDROCK_ENDPOINT_URL="$BEDROCK_ENDPOINT_URL" python3 - <<'PY'; then +import json +import os + +health = json.loads(os.environ["HEALTH_JSON"]) +errors = [] +if health.get("ok") is not True: + errors.append(f"ok={health.get('ok')!r}") +if health.get("endpointUrl") != os.environ["BEDROCK_ENDPOINT_URL"]: + errors.append("endpointUrl mismatch") +if health.get("region") != "us-east-1": + errors.append(f"region={health.get('region')!r}") +if not health.get("tokenHash"): + errors.append("tokenHash missing") +if errors: + print("; ".join(errors)) + raise SystemExit(1) +PY + pass "B3: Bedrock Runtime adapter health reports fake endpoint and us-east-1" + else + fail "B3: Bedrock Runtime adapter health payload was not the expected fake endpoint" + fi +} + +assert_openshell_provider_route() { + local route provider_output plain_route + route=$(openshell inference get -g nemoclaw 2>&1 || openshell inference get 2>&1) || { + fail "B4: openshell inference get failed: ${route:0:300}" + return + } + plain_route=$(printf '%s' "$route" | python3 -c 'import re,sys; sys.stdout.write(re.sub(r"\x1b\[[0-9;]*m", "", sys.stdin.read()))') + if grep -Fq "Provider: compatible-anthropic-endpoint" <<<"$plain_route" \ + && grep -Fq "Model: ${BEDROCK_MODEL}" <<<"$plain_route"; then + pass "B4: OpenShell route points at compatible-anthropic-endpoint" + else + fail "B4: OpenShell route did not point at compatible-anthropic-endpoint: ${plain_route:0:400}" + fi + + provider_output=$(openshell provider get compatible-anthropic-endpoint 2>&1 || true) + if grep -Fq "compatible-anthropic-endpoint" <<<"$provider_output"; then + pass "B5: OpenShell provider registry contains compatible-anthropic-endpoint" + else + fail "B5: OpenShell provider registry did not expose compatible-anthropic-endpoint" + fi +} + +assert_openclaw_config() { + local output rc=0 script + script=$( + cat <<'SH' +python3 - "$1" <<'PY' +import json +import sys + +model = sys.argv[1] +cfg = json.load(open("/sandbox/.openclaw/openclaw.json", encoding="utf-8")) +errors = [] +providers = cfg.get("models", {}).get("providers", {}) +inference = providers.get("inference") if isinstance(providers, dict) else None +if sorted(providers.keys()) != ["inference"]: + errors.append("provider keys are %r" % sorted(providers.keys())) +if not isinstance(inference, dict): + errors.append("models.providers.inference is missing") +else: + if inference.get("baseUrl") != "https://inference.local/v1": + errors.append("inference baseUrl is %r" % inference.get("baseUrl")) + if inference.get("apiKey") != "unused": + errors.append("inference apiKey is not the non-secret placeholder") + if inference.get("api") != "openai-completions": + errors.append("inference api is %r" % inference.get("api")) +primary = cfg.get("agents", {}).get("defaults", {}).get("model", {}).get("primary") +if primary != "inference/" + model: + errors.append("primary model is %r" % primary) +print(json.dumps({ + "provider_keys": sorted(providers.keys()) if isinstance(providers, dict) else [], + "inference_base": inference.get("baseUrl") if isinstance(inference, dict) else None, + "inference_api_key": inference.get("apiKey") if isinstance(inference, dict) else None, + "primary": primary, + "errors": errors, +})) +sys.exit(1 if errors else 0) +PY +SH + ) + output=$(sandbox_exec_sh_script "$script" "$BEDROCK_MODEL" 2>&1) || rc=$? + info "OpenClaw config summary: ${output:0:500}" + if [ "$rc" -eq 0 ]; then + pass "B6: OpenClaw config uses only managed inference.local provider" + else + fail "B6: OpenClaw config did not use the expected inference.local provider shape" + fi +} + +assert_hermes_config() { + local config probe + config=$(openshell sandbox exec --name "$SANDBOX_NAME" -- cat /sandbox/.hermes/config.yaml 2>&1) || { + fail "B6: could not read Hermes config.yaml: ${config:0:240}" + return + } + + probe=$( + CONFIG_TEXT="$config" EXPECTED_MODEL="$BEDROCK_MODEL" python3 - <<'PY' +import os +import re + +text = os.environ["CONFIG_TEXT"] +expected = os.environ["EXPECTED_MODEL"] +errors = [] +model = {} +in_model = False +for line in text.splitlines(): + if re.match(r"^model:\s*$", line): + in_model = True + continue + if in_model and re.match(r"^[A-Za-z0-9_-]+:", line): + break + if in_model: + match = re.match(r"^\s+([A-Za-z0-9_-]+):\s*(.*?)\s*$", line) + if match: + value = match.group(2).strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'": + value = value[1:-1] + model[match.group(1)] = value + +if model.get("default") != expected: + errors.append(f"model.default={model.get('default')!r}") +if model.get("base_url") != "https://inference.local/v1": + errors.append(f"model.base_url={model.get('base_url')!r}") +api_key = model.get("api_key") +if not isinstance(api_key, str) or not api_key.startswith("sk-"): + errors.append(f"model.api_key={api_key!r}") +if re.search(r"(?ms)^models:\s*\n(?:[ \t].*\n)*?[ \t]+providers:", text): + errors.append("OpenClaw-style models.providers block present") +if "openshell:" in text: + errors.append("OpenShell provider placeholder present") + +if errors: + print("; ".join(errors)) + raise SystemExit(1) +print("OK") +PY + ) || { + fail "B6: Hermes config.yaml was not patched correctly: ${probe:0:400}" + return + } + pass "B6: Hermes config uses inference.local without OpenShell/OpenClaw provider blocks" +} + +check_sandbox_inference() { + local payload payload_arg response rc=0 content + payload=$(BEDROCK_MODEL="$BEDROCK_MODEL" python3 -c ' +import json +import os + +print(json.dumps({ + "model": os.environ["BEDROCK_MODEL"], + "messages": [{"role": "user", "content": "Reply with exactly one word: PONG"}], + "max_tokens": 32, +})) +') + payload_arg="$(printf '%q' "$payload")" + response=$(openshell sandbox exec --name "$SANDBOX_NAME" -- sh -lc "curl -sS --max-time 90 https://inference.local/v1/chat/completions -H 'Content-Type: application/json' -d $payload_arg" 2>&1) || rc=$? + content=$(printf '%s' "$response" | parse_chat_content 2>/dev/null) || true + if [ "$rc" -eq 0 ] && grep -qi "PONG" <<<"$content"; then + pass "B7: sandbox inference.local chat completion returned PONG" + else + fail "B7: sandbox inference.local chat completion failed: ${response:0:400}" + fi +} + +check_openclaw_agent_turn() { + local session_id remote_cmd raw reply rc=0 + session_id="bedrock-openclaw-e2e-$(date +%s)-$$" + remote_cmd="rm -f /sandbox/.openclaw/agents/main/sessions/${session_id}.jsonl.lock /sandbox/.openclaw/agents/main/sessions/${session_id}.trajectory.jsonl 2>/dev/null || true; nemoclaw-start openclaw agent --agent main --json --session-id $(quote_for_remote_sh "$session_id") -m 'Reply with only: PONG'" + raw=$(run_with_timeout 240 openshell sandbox exec --name "$SANDBOX_NAME" -- sh -lc "$remote_cmd" 2>&1) || rc=$? + + if printf '%s' "$raw" | grep -qiE "SsrFBlockedError|Blocked hostname|transport error|ECONNREFUSED|EAI_AGAIN|gateway unavailable|network connection error|bedrock_runtime_error"; then + fail "B8: OpenClaw agent turn hit a provider or transport error" + return + fi + + reply=$(printf '%s' "$raw" | parse_openclaw_agent_text 2>/dev/null) || true + + if [ "$rc" -eq 0 ] && grep -qi "PONG" <<<"$reply"; then + pass "B8: OpenClaw agent completed a Bedrock-backed turn through inference.local" + else + fail "B8: OpenClaw agent did not return PONG through Bedrock adapter" + fi +} + +check_hermes_api_chat() { + local payload payload_arg response rc=0 content remote + payload=$(BEDROCK_MODEL="$BEDROCK_MODEL" python3 -c ' +import json +import os + +print(json.dumps({ + "model": os.environ["BEDROCK_MODEL"], + "messages": [{"role": "user", "content": "Reply with exactly one word: PONG"}], + "max_tokens": 32, +})) +') + payload_arg="$(printf '%q' "$payload")" + remote="set -a; [ ! -f /sandbox/.hermes/.env ] || . /sandbox/.hermes/.env; set +a; if [ -n \"\${API_SERVER_KEY:-}\" ]; then curl -sS --max-time 120 http://localhost:8642/v1/chat/completions -H 'Content-Type: application/json' -H \"Authorization: Bearer \${API_SERVER_KEY}\" -d $payload_arg; else curl -sS --max-time 120 http://localhost:8642/v1/chat/completions -H 'Content-Type: application/json' -d $payload_arg; fi" + response=$(openshell sandbox exec --name "$SANDBOX_NAME" -- sh -lc "$remote" 2>&1) || rc=$? + content=$(printf '%s' "$response" | parse_chat_content 2>/dev/null) || true + if [ "$rc" -eq 0 ] && grep -qi "PONG" <<<"$content"; then + pass "B8: Hermes local chat API completed a Bedrock-backed turn through inference.local" + else + fail "B8: Hermes local chat API did not return PONG through Bedrock adapter: ${response:0:400}" + fi +} + +check_mock_observed_traffic() { + local converse_count stream_count + converse_count=$(grep -c "POST /model/converse auth=ok" "$BEDROCK_MOCK_LOG" 2>/dev/null || true) + stream_count=$(grep -c "POST /model/converse-stream auth=ok" "$BEDROCK_MOCK_LOG" 2>/dev/null || true) + if [ "$converse_count" -ge 1 ]; then + pass "B9: fake Bedrock Runtime endpoint observed authenticated Converse traffic" + else + fail "B9: fake Bedrock Runtime endpoint did not observe authenticated Converse traffic" + fi + if [ "$AGENT" = "openclaw" ]; then + if [ "$stream_count" -ge 1 ]; then + pass "B10: fake Bedrock Runtime endpoint observed authenticated ConverseStream traffic" + else + fail "B10: fake Bedrock Runtime endpoint did not observe OpenClaw streamed traffic" + fi + fi +} + +check_adapter_log_breadcrumbs() { + if [ ! -f "$ADAPTER_LOG" ]; then + fail "B11: Bedrock Runtime adapter host log was not written" + return + fi + if grep -Fq '"event":"request_completed"' "$ADAPTER_LOG" \ + && grep -Fq '"operation":"converse"' "$ADAPTER_LOG" \ + && grep -Fq "$BEDROCK_MODEL" "$ADAPTER_LOG"; then + if [ "$AGENT" = "openclaw" ]; then + if grep -Fq '"operation":"converse_stream"' "$ADAPTER_LOG"; then + pass "B11: Bedrock Runtime adapter host log records safe Converse and ConverseStream breadcrumbs" + else + fail "B11: Bedrock Runtime adapter host log did not record a ConverseStream breadcrumb" + fi + else + pass "B11: Bedrock Runtime adapter host log records safe Converse breadcrumbs" + fi + else + fail "B11: Bedrock Runtime adapter host log did not record expected request breadcrumbs" + fi +} + +collect_sandbox_snapshot() { + local script + script=$( + cat <<'SH' +set +e +emit_file() { + path="$1" + [ -r "$path" ] || return 0 + size=$(wc -c <"$path" 2>/dev/null || echo 0) + [ "$size" -le 1048576 ] || return 0 + printf '\n@@NEMOCLAW_E2E_FILE@@ %s\n' "$path" + tr '\000' '\n' <"$path" 2>/dev/null || true +} + +for root in /sandbox/.openclaw /sandbox/.hermes /etc/nemoclaw /tmp; do + [ -e "$root" ] || continue + find "$root" -maxdepth 4 -type f 2>/dev/null | while IFS= read -r file; do + case "$file" in + */node_modules/*|*/.git/*) continue ;; + esac + emit_file "$file" + done +done + +for proc_dir in /proc/[0-9]*; do + [ -d "$proc_dir" ] || continue + pid=$(basename "$proc_dir") + for name in environ cmdline; do + emit_file "$proc_dir/$name" + done +done +SH + ) + sandbox_exec_sh_script "$script" +} + +scan_file_for_leaks() { + local file_path="$1" + local label="$2" + PATTERN_FAKE_KEY="$COMPATIBLE_KEY" \ + PATTERN_ADAPTER_TOKEN="$ADAPTER_TOKEN" \ + PATTERN_AWS_ENV_NAME="AWS_BEARER_TOKEN_BEDROCK" \ + PATTERN_ADAPTER_ENV_NAME="NEMOCLAW_BEDROCK_RUNTIME_ADAPTER_TOKEN" \ + PATTERN_BEDROCK_HOST="$BEDROCK_HOSTNAME" \ + SCAN_FILE_PATH="$file_path" \ + SCAN_LABEL="$label" \ + python3 - <<'PY' +import os +from pathlib import Path + +path = Path(os.environ["SCAN_FILE_PATH"]) +label = os.environ["SCAN_LABEL"] +patterns = [ + ("fake user key", os.environ.get("PATTERN_FAKE_KEY", "")), + ("adapter token", os.environ.get("PATTERN_ADAPTER_TOKEN", "")), + ("AWS bearer env name", os.environ.get("PATTERN_AWS_ENV_NAME", "")), + ("adapter token env name", os.environ.get("PATTERN_ADAPTER_ENV_NAME", "")), + ("raw Bedrock hostname", os.environ.get("PATTERN_BEDROCK_HOST", "")), +] +current = label +locations = [] +for raw in path.read_text(encoding="utf-8", errors="replace").splitlines(): + if raw.startswith("@@NEMOCLAW_E2E_FILE@@ "): + current = raw.split(" ", 1)[1] + continue + for name, value in patterns: + if value and value in raw: + locations.append(f"{name}: {current}") + +if locations: + for item in sorted(set(locations)): + print(item) + raise SystemExit(1) +PY +} + +scan_for_leaks() { + local snapshot_file host_log_file scan_output rc=0 + ADAPTER_TOKEN="$(tr -d '\n' <"$HOME/.nemoclaw/bedrock-runtime-adapter-token" 2>/dev/null || true)" + if [ -z "$ADAPTER_TOKEN" ]; then + fail "B11: adapter token file was not created on the host" + return + fi + + snapshot_file="$(mktemp)" + host_log_file="$(mktemp)" + collect_sandbox_snapshot >"$snapshot_file" 2>/dev/null || true + { + printf '\n@@NEMOCLAW_E2E_FILE@@ %s\n' "$ONBOARD_LOG" + [ -f "$ONBOARD_LOG" ] && cat "$ONBOARD_LOG" + printf '\n@@NEMOCLAW_E2E_FILE@@ %s\n' "$ADAPTER_LOG" + [ -f "$ADAPTER_LOG" ] && cat "$ADAPTER_LOG" + printf '\n@@NEMOCLAW_E2E_FILE@@ %s\n' "$BEDROCK_MOCK_LOG" + [ -f "$BEDROCK_MOCK_LOG" ] && cat "$BEDROCK_MOCK_LOG" + } >"$host_log_file" + + scan_output=$(scan_file_for_leaks "$snapshot_file" "sandbox snapshot" 2>&1) || rc=$? + if [ "$rc" -eq 0 ]; then + scan_output=$(scan_file_for_leaks "$host_log_file" "host e2e logs" 2>&1) || rc=$? + fi + rm -f "$snapshot_file" "$host_log_file" 2>/dev/null || true + + if [ "$rc" -eq 0 ]; then + pass "B12: sandbox configs, env, proc, and logs contain no Bedrock token or hostname leaks" + else + fail "B12: leak scan found forbidden Bedrock token or hostname locations" + printf '%s\n' "$scan_output" | sed 's/^/ /' + fi +} + +# Repo root +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +if [ -d /workspace ] && [ -f /workspace/install.sh ]; then + REPO="/workspace" +elif [ -f "${SCRIPT_DIR}/../../install.sh" ]; then + REPO="$(cd "${SCRIPT_DIR}/../.." && pwd)" +elif [ -f "./install.sh" ]; then + REPO="$(pwd)" +else + echo "ERROR: Cannot find repo root." + exit 1 +fi + +AGENT="${NEMOCLAW_AGENT:-openclaw}" +case "$AGENT" in + openclaw | hermes) ;; + *) + echo "ERROR: NEMOCLAW_AGENT must be openclaw or hermes, got '$AGENT'" >&2 + exit 2 + ;; +esac + +BEDROCK_HOSTNAME="bedrock-runtime.us-east-1.amazonaws.com" +BEDROCK_MOCK_PORT="${NEMOCLAW_BEDROCK_RUNTIME_MOCK_PORT:-18147}" +BEDROCK_ADAPTER_PORT="${NEMOCLAW_BEDROCK_RUNTIME_ADAPTER_PORT:-11436}" +BEDROCK_ENDPOINT_URL="http://${BEDROCK_HOSTNAME}:${BEDROCK_MOCK_PORT}" +BEDROCK_MODEL="${NEMOCLAW_BEDROCK_RUNTIME_MODEL:-anthropic.claude-3-5-sonnet-20240620-v1:0}" +COMPATIBLE_KEY="${NEMOCLAW_BEDROCK_RUNTIME_FAKE_KEY:-fake-pasted-bedrock-runtime-key-e2e}" +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-bedrock-${AGENT}}" +ONBOARD_LOG="/tmp/nemoclaw-e2e-bedrock-runtime-${AGENT}-onboard.log" +BUILD_LOG="/tmp/nemoclaw-e2e-bedrock-runtime-${AGENT}-build.log" +BEDROCK_MOCK_LOG="/tmp/nemoclaw-e2e-bedrock-runtime-${AGENT}-mock.log" +ADAPTER_LOG="$HOME/.nemoclaw/bedrock-runtime-adapter.log" +BEDROCK_MOCK_PID="" +HOSTS_BACKUP="" +ADAPTER_TOKEN="" + +trap cleanup EXIT + +rm -f "$ADAPTER_LOG" 2>/dev/null || true + +echo "" +echo "============================================================" +echo " Bedrock Runtime Compatible Anthropic E2E (#3767)" +echo " $(date)" +echo "============================================================" +echo "" + +section "Phase 0: Prerequisites" +if docker info >/dev/null 2>&1; then + pass "Docker is running" +else + fail "Docker is not running" + summary +fi + +if command -v python3 >/dev/null 2>&1; then + pass "python3 is available" +else + fail "python3 not found" + summary +fi + +if [ "${NEMOCLAW_NON_INTERACTIVE:-}" = "1" ]; then + pass "NEMOCLAW_NON_INTERACTIVE=1" +else + fail "NEMOCLAW_NON_INTERACTIVE=1 is required" + summary +fi + +if [ "${NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE:-}" = "1" ]; then + pass "third-party software acceptance is set" +else + fail "NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 is required" + summary +fi + +load_shell_path +info "Repo: $REPO" +info "Agent: $AGENT" +info "Sandbox: $SANDBOX_NAME" +info "Model: $BEDROCK_MODEL" + +section "Phase 1: Source CLI and OpenShell" +if prepare_source_cli; then + pass "B0: source CLI and OpenShell are ready" +else + fail "B0: source CLI/OpenShell preparation failed" + info "Last 120 lines of build/setup log:" + tail -120 "$BUILD_LOG" 2>/dev/null || true + summary +fi + +section "Phase 2: Fake Bedrock Runtime endpoint" +map_bedrock_host_to_loopback +if start_bedrock_mock; then + pass "B0: fake Bedrock Runtime endpoint started" +else + fail "B0: fake Bedrock Runtime endpoint failed to start" + info "Mock log:" + sed 's/^/ /' "$BEDROCK_MOCK_LOG" 2>/dev/null || true + summary +fi + +section "Phase 3: Onboard" +run_bedrock_onboard + +section "Phase 4: Boundary assertions" +assert_onboard_identity +assert_adapter_health +assert_openshell_provider_route +if [ "$AGENT" = "hermes" ]; then + assert_hermes_config +else + assert_openclaw_config +fi + +section "Phase 5: Runtime requests" +check_sandbox_inference +if [ "$AGENT" = "hermes" ]; then + check_hermes_api_chat +else + check_openclaw_agent_turn +fi +check_mock_observed_traffic +check_adapter_log_breadcrumbs + +section "Phase 6: Leak scan" +scan_for_leaks + +trap - EXIT +cleanup +summary diff --git a/test/e2e-vpn/test-brave-search-e2e.sh b/test/e2e-vpn/test-brave-search-e2e.sh new file mode 100755 index 00000000000..102310304e7 --- /dev/null +++ b/test/e2e-vpn/test-brave-search-e2e.sh @@ -0,0 +1,438 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Brave Search E2E (Issue #2687) +# +# Verifies the issue's acceptance end-to-end: +# B0 BRAVE_API_KEY is present (skip-suite gate) +# B1 Non-interactive onboard with BRAVE_API_KEY succeeds +# B2a brave network policy preset is applied +# B2b openclaw web-search config selects brave (downstream of preset) +# B3a Real key never lands on disk in /sandbox/.openclaw/openclaw.json +# B3b Real key is not visible to sandbox-exec shells via printenv +# B4a Real Brave search via openclaw agent +# B4b Real Brave search via curl from inside the sandbox +# +# Required env (CI injects from secrets): +# BRAVE_API_KEY real Brave Search subscription token (skip-suite gate) +# NVIDIA_API_KEY drives the agent inference turn in B4a +# +# Secret hygiene: BRAVE_API_KEY is never echoed raw. All output that may +# contain it pipes through redact_stream; GitHub Actions auto-mask is the +# second line of defence. +# +# Usage: +# NEMOCLAW_NON_INTERACTIVE=1 NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ +# BRAVE_API_KEY=... NVIDIA_API_KEY=... \ +# bash test/e2e-vpn/test-brave-search-e2e.sh + +set -uo pipefail + +export NEMOCLAW_E2E_DEFAULT_TIMEOUT=1800 +SCRIPT_DIR_TIMEOUT="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +# shellcheck source=test/e2e-vpn/e2e-timeout.sh +. "${SCRIPT_DIR_TIMEOUT}/e2e-timeout.sh" +# shellcheck source=test/e2e-vpn/lib/openclaw-json.sh +. "${SCRIPT_DIR_TIMEOUT}/lib/openclaw-json.sh" + +PASS=0 +FAIL=0 +SKIP=0 +TOTAL=0 + +pass() { + ((PASS++)) + ((TOTAL++)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} +fail() { + ((FAIL++)) + ((TOTAL++)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} +skip() { + ((SKIP++)) + ((TOTAL++)) + printf '\033[33m SKIP: %s\033[0m\n' "$1" +} +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } + +summary() { + echo "" + echo "============================================================" + echo " Brave Search E2E Results" + echo "============================================================" + echo " PASS: $PASS" + echo " FAIL: $FAIL" + echo " SKIP: $SKIP" + echo " TOTAL: $TOTAL" + echo "============================================================" + if [ "$FAIL" -gt 0 ]; then exit 1; fi +} + +# Streaming line-by-line redactor. Replaces every literal occurrence of +# $1 with REDACTED. Defence in depth on top of GitHub Actions auto-mask. +redact_stream() { + local secret="${1:-}" + SECRET_TO_REDACT="$secret" python3 -u -c ' +import os, sys +secret = os.environ.get("SECRET_TO_REDACT", "") +for line in iter(sys.stdin.readline, ""): + sys.stdout.write(line.replace(secret, "REDACTED") if secret else line) + sys.stdout.flush() +' +} + +# ── Repo root ───────────────────────────────────────────────────── +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +if [ -d /workspace ] && [ -f /workspace/install.sh ]; then + REPO="/workspace" +elif [ -f "${SCRIPT_DIR}/../../install.sh" ]; then + REPO="$(cd "${SCRIPT_DIR}/../.." && pwd)" +elif [ -f "./install.sh" ]; then + REPO="$(pwd)" +else + echo "ERROR: Cannot find repo root." + exit 1 +fi + +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-brave-search}" +ONBOARD_LOG="/tmp/nemoclaw-e2e-brave-search-onboard.log" + +# Ship a shell script into the sandbox without quoting hell — base64 on +# the host, decode inside. Used by B2b's python heredoc. +quote_for_remote_sh() { + local value="${1:-}" + printf "'%s'" "$(printf '%s' "$value" | sed "s/'/'\\\\''/g")" +} + +sandbox_exec_sh_script() { + local script="$1" + shift + local encoded remote_cmd arg + encoded="$(printf '%s' "$script" | base64 | tr -d '\n')" + remote_cmd="tmp=\$(mktemp); trap 'rm -f \"\$tmp\"' EXIT; printf %s $(quote_for_remote_sh "$encoded") | base64 -d > \"\$tmp\"; sh \"\$tmp\"" + for arg in "$@"; do + remote_cmd+=" $(quote_for_remote_sh "$arg")" + done + openshell sandbox exec --name "$SANDBOX_NAME" -- sh -lc "$remote_cmd" +} + +load_shell_path() { + local local_bin + if [ -f "$HOME/.bashrc" ]; then + # shellcheck source=/dev/null + source "$HOME/.bashrc" 2>/dev/null || true + fi + export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" + if [ -s "$NVM_DIR/nvm.sh" ]; then + # shellcheck source=/dev/null + . "$NVM_DIR/nvm.sh" + fi + local_bin="$HOME/.local/bin" + if [ -d "$local_bin" ]; then + PATH=":${PATH}:" + PATH="${PATH//:${local_bin}:/:}" + PATH="${PATH#:}" + PATH="${PATH%:}" + export PATH="$local_bin:$PATH" + fi +} + +cli_command_available_from_source() { + [ -f "$REPO/dist/nemoclaw.js" ] && command -v node >/dev/null 2>&1 && command -v openshell >/dev/null 2>&1 +} + +destroy_sandbox_best_effort() { + if [ "${NEMOCLAW_E2E_KEEP_SANDBOX:-}" = "1" ]; then + return 0 + fi + if cli_command_available_from_source; then + run_with_timeout 120 node "$REPO/bin/nemoclaw.js" "$SANDBOX_NAME" destroy --yes >/dev/null 2>&1 || true + elif command -v nemoclaw >/dev/null 2>&1; then + run_with_timeout 120 nemoclaw "$SANDBOX_NAME" destroy --yes >/dev/null 2>&1 || true + fi + if command -v openshell >/dev/null 2>&1; then + run_with_timeout 60 openshell sandbox delete "$SANDBOX_NAME" >/dev/null 2>&1 || true + fi +} + +# B1 — non-interactive onboard with BRAVE_API_KEY. +# Output is mirrored to terminal AND captured to $ONBOARD_LOG, scrubbed +# by redact_stream as the first pipe stage. PIPESTATUS[0] captures the +# real onboard exit code (a plain $? would be tee's, which is always 0). +run_onboard_with_brave_key() { + local onboard_exit=0 onboard_cmd_desc + export NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" + export NEMOCLAW_RECREATE_SANDBOX=1 + export NEMOCLAW_NON_INTERACTIVE=1 + export NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 + + if cli_command_available_from_source; then + onboard_cmd_desc="source CLI onboard" + info "Using source-built CLI at $REPO/bin/nemoclaw.js" + destroy_sandbox_best_effort + run_with_timeout 1200 node "$REPO/bin/nemoclaw.js" onboard --fresh --non-interactive --yes-i-accept-third-party-software 2>&1 \ + | redact_stream "${BRAVE_API_KEY:-}" \ + | tee "$ONBOARD_LOG" + onboard_exit=${PIPESTATUS[0]} + else + onboard_cmd_desc="install.sh" + info "Source CLI is not built; running install.sh from this checkout." + bash "$REPO/install.sh" --non-interactive --yes-i-accept-third-party-software --fresh 2>&1 \ + | redact_stream "${BRAVE_API_KEY:-}" \ + | tee "$ONBOARD_LOG" + onboard_exit=${PIPESTATUS[0]} + load_shell_path + fi + + if [ "$onboard_exit" -eq 0 ]; then + pass "B1: ${onboard_cmd_desc} completed for Brave Search-enabled onboard" + else + fail "B1: ${onboard_cmd_desc} failed (exit $onboard_exit)" + summary + fi + + # Scrub the on-disk log in place before any failure-artifact upload. + if [ -n "${BRAVE_API_KEY:-}" ] && [ -f "$ONBOARD_LOG" ]; then + local redacted_log + redacted_log="$(mktemp)" + redact_stream "$BRAVE_API_KEY" <"$ONBOARD_LOG" >"$redacted_log" || true + mv "$redacted_log" "$ONBOARD_LOG" || rm -f "$redacted_log" + fi +} + +# B2 — brave preset is applied. +# B2a checks the gateway-level network policy; B2b checks openclaw's +# downstream web-search config (so a silent backend swap is also caught). +check_brave_preset_applied() { + local policy_output rc=0 config_check config_rc=0 config_script + + policy_output=$(openshell policy get --full "$SANDBOX_NAME" 2>&1) || rc=$? + if [ "$rc" -ne 0 ]; then + fail "B2a: openshell policy get failed (exit $rc)" + elif printf '%s' "$policy_output" | grep -q "api.search.brave.com"; then + pass "B2a: brave preset applied — api.search.brave.com is in the loaded gateway policy" + else + fail "B2a: brave preset NOT applied — api.search.brave.com is missing from the gateway policy" + fi + + config_script=$( + cat <<'SH' +python3 <<'PY' +import json +with open("/sandbox/.openclaw/openclaw.json") as f: + cfg = json.load(f) +s = cfg.get("tools", {}).get("web", {}).get("search", {}) +print(f"enabled={s.get('enabled')}") +print(f"provider={s.get('provider')}") +PY +SH + ) + config_check=$(sandbox_exec_sh_script "$config_script" 2>&1) || config_rc=$? + + if [ "$config_rc" -ne 0 ]; then + fail "B2b: could not read openclaw web-search config (exit $config_rc)" + elif printf '%s' "$config_check" | grep -q "^enabled=True$" \ + && printf '%s' "$config_check" | grep -q "^provider=brave$"; then + pass "B2b: brave preset wired through to openclaw — tools.web.search.provider=brave and enabled=true" + else + fail "B2b: openclaw web-search config does not select brave (got: $(printf '%s' "$config_check" | tr '\n' ' '))" + fi +} + +# B3 — real key must not leak into the sandbox. Matches NemoClaw's design +# intent (scripts/nemoclaw-start.sh:560-564). B3a checks the on-disk +# openclaw.json; B3b checks the env of a `sandbox exec` shell. +check_no_real_key_in_sandbox() { + local config_dump env_value + + config_dump=$(openshell sandbox exec --name "$SANDBOX_NAME" -- sh -lc \ + 'cat /sandbox/.openclaw/openclaw.json 2>/dev/null || true' 2>&1) || true + + # Accept both the canonical placeholder and any revision-scoped form + # OpenShell may emit (e.g. `openshell:resolve:env:v11_BRAVE_API_KEY`). + local placeholder_pattern='openshell:resolve:env:([A-Za-z0-9_]+_)?BRAVE_API_KEY' + + if [ -n "${BRAVE_API_KEY:-}" ] && printf '%s' "$config_dump" | grep -qF "$BRAVE_API_KEY"; then + fail "B3a: SECURITY — real BRAVE_API_KEY found verbatim in /sandbox/.openclaw/openclaw.json" + elif printf '%s' "$config_dump" | grep -qE "$placeholder_pattern"; then + pass "B3a: openclaw.json contains the placeholder, not the real key" + else + fail "B3a: openclaw.json has neither the real key nor the placeholder — web search not configured" + fi + + env_value=$(openshell sandbox exec --name "$SANDBOX_NAME" -- sh -lc \ + 'printenv BRAVE_API_KEY 2>/dev/null || true' 2>&1) || true + + if [ -n "${BRAVE_API_KEY:-}" ] && printf '%s' "$env_value" | grep -qF "$BRAVE_API_KEY"; then + fail "B3b: SECURITY — real BRAVE_API_KEY visible to sandbox shell via printenv" + elif [ -z "$env_value" ] || printf '%s' "$env_value" | grep -qE "$placeholder_pattern"; then + pass "B3b: sandbox shell env does not expose the real key (placeholder or empty)" + else + fail "B3b: unexpected non-empty BRAVE_API_KEY in sandbox env" + fi +} + +# B4a — real Brave search via openclaw agent. +# This is the realistic user path: SSH into sandbox, ask the agent to run +# its web-search tool, parse the JSON reply, assert NVIDIA-related text. +check_real_brave_search_via_agent() { + local session_id raw ssh_cfg reply rc=0 ssh_cmd + session_id="e2e-brave-agent-$(date +%s)-$$" + ssh_cfg="$(mktemp)" + + if ! openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_cfg" 2>/dev/null; then + rm -f "$ssh_cfg" + fail "B4a: agent web-search turn — could not get SSH config" + return + fi + + ssh_cmd="openclaw agent --agent main --json --session-id '${session_id}' -m 'Use the web search tool to find one result for the query: NVIDIA. Reply with only the title of the top result.'" + raw=$(run_with_timeout 120 ssh -F "$ssh_cfg" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" \ + "$ssh_cmd" \ + 2>/dev/null) || rc=$? + rm -f "$ssh_cfg" + + # Fail closed on explicit transport / proxy errors. Naked HTTP codes + # like 401/403 are NOT in this list — they appear in benign JSON content + # (URLs, timestamps) and would false-positive. + if printf '%s' "$raw" | grep -qiE "SsrFBlockedError|Blocked hostname|ECONNREFUSED|EAI_AGAIN|gateway unavailable|network connection error"; then + fail "B4a: agent web-search failed with provider/transport error (exit ${rc}): $(printf '%s' "${raw:0:300}" | redact_stream "${BRAVE_API_KEY:-}")" + return + fi + + reply=$(printf '%s' "$raw" | parse_openclaw_agent_text 2>/dev/null) || true + + # NVIDIA-related phrasing (nvidia, gpu, cuda, geforce) is overwhelmingly + # likely in any legitimate top-1 web result for the query "NVIDIA". + if [ "$rc" -eq 0 ] && printf '%s' "$reply" | grep -qiE "nvidia|geforce|cuda|gpu"; then + pass "B4a: openclaw agent web-search returned a real Brave result" + else + fail "B4a: agent web-search did not return a recognizable Brave result (exit ${rc}, reply='$(printf '%s' "${reply:0:200}" | redact_stream "${BRAVE_API_KEY:-}")')" + fi +} + +# B4b — real Brave search via curl from inside the sandbox. This proves the +# placeholder is rewritten to the real key at egress for Brave's custom +# X-Subscription-Token header. The placeholder is read from the running +# openclaw.json so we exercise whatever shape OpenShell wrote (canonical +# `openshell:resolve:env:BRAVE_API_KEY` or a revision-scoped variant). +check_real_brave_search_via_curl() { + local response status_code body rc=0 placeholder + placeholder=$(openshell sandbox exec --name "$SANDBOX_NAME" -- sh -lc \ + 'python3 -c " +import json, sys +try: + with open(\"/sandbox/.openclaw/openclaw.json\") as f: + cfg = json.load(f) + print(cfg.get(\"tools\", {}).get(\"web\", {}).get(\"search\", {}).get(\"apiKey\", \"\") or \"\") +except Exception: + sys.exit(1) +"' 2>/dev/null) || true + placeholder="${placeholder#"${placeholder%%[![:space:]]*}"}" + placeholder="${placeholder%"${placeholder##*[![:space:]]}"}" + if [ -z "$placeholder" ]; then + fail "B4b: could not read tools.web.search.apiKey placeholder from /sandbox/.openclaw/openclaw.json" + return + fi + + response=$(openshell sandbox exec --name "$SANDBOX_NAME" -- sh -lc \ + "curl -sS --max-time 20 -G 'https://api.search.brave.com/res/v1/web/search' \ + --data-urlencode 'q=NVIDIA' \ + --data-urlencode 'count=1' \ + -H 'X-Subscription-Token: ${placeholder}' \ + -w '\nHTTP_STATUS:%{http_code}\n'" \ + 2>&1) || rc=$? + + status_code=$(printf '%s' "$response" | grep -m1 -oE 'HTTP_STATUS:[0-9]+' | head -1 | cut -d: -f2) + body=$(printf '%s' "$response" | sed '/^HTTP_STATUS:/d') + + if [ "$status_code" = "200" ]; then + if printf '%s' "$body" | python3 -c ' +import json, sys +try: + doc = json.load(sys.stdin) +except Exception: + sys.exit(1) +results = (doc.get("web") or {}).get("results") or [] +sys.exit(0 if len(results) > 0 else 2) +' 2>/dev/null; then + pass "B4b: real Brave search via curl returned HTTP 200 with non-empty web.results[]" + else + fail "B4b: HTTP 200 but response had no web.results[] (body parsed empty)" + fi + elif [ "$status_code" = "401" ] || [ "$status_code" = "403" ]; then + fail "B4b: HTTP $status_code — proxy did not substitute the Brave placeholder at egress" + elif [ "$status_code" = "000" ] || [ -z "$status_code" ]; then + fail "B4b: curl never completed an HTTP transaction — check curl is in brave.yaml binaries allowlist. $(printf '%s' "${response:0:300}" | redact_stream "${BRAVE_API_KEY:-}")" + else + fail "B4b: unexpected HTTP status '${status_code:-}' from Brave (exit $rc)" + fi +} + +trap destroy_sandbox_best_effort EXIT + +echo "" +echo "============================================================" +echo " Brave Search E2E (#2687)" +echo " $(date)" +echo "============================================================" + +# B0 — skip-suite gate. Self-skips when BRAVE_API_KEY is not set so the +# script is safe to enable before the secret exists. +section "Phase 0: Brave Search secret gate" +if [ -z "${BRAVE_API_KEY:-}" ]; then + skip "B0: BRAVE_API_KEY is not set — skipping the entire Brave Search suite gracefully" + summary + # summary() only auto-exits on FAIL>0; a skip-only gate is a graceful + # success, so exit 0 explicitly so nothing else runs. + exit 0 +fi +pass "B0: BRAVE_API_KEY is available" + +section "Phase 0: Prerequisites" +if ! docker info >/dev/null 2>&1; then + fail "Docker is not running" + summary +fi +pass "Docker is running" + +if ! command -v python3 >/dev/null 2>&1; then + fail "python3 not found" + summary +fi +pass "python3 is available" + +load_shell_path +info "Repo: $REPO" +info "Sandbox: $SANDBOX_NAME" + +section "Phase 1: Non-interactive onboard with BRAVE_API_KEY" +run_onboard_with_brave_key + +section "Phase 2: Brave preset is applied to the sandbox" +check_brave_preset_applied + +section "Phase 3: Real key not leaked into the sandbox" +check_no_real_key_in_sandbox + +section "Phase 4a: Real Brave search via openclaw agent" +check_real_brave_search_via_agent + +section "Phase 4b: Real Brave search via curl from inside the sandbox" +check_real_brave_search_via_curl + +trap - EXIT +destroy_sandbox_best_effort +summary diff --git a/test/e2e-vpn/test-channels-add-remove.sh b/test/e2e-vpn/test-channels-add-remove.sh new file mode 100755 index 00000000000..92a98523534 --- /dev/null +++ b/test/e2e-vpn/test-channels-add-remove.sh @@ -0,0 +1,617 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Channel add/remove lifecycle E2E test. +# +# Covers Test 2 from issue #3462 ("onboard empty -> channels add -> channels remove"). +# Regression coverage for: +# - #3437 — `channels add ` + rebuild must apply the channel's matching +# network policy preset so the bridge boots with egress to its +# upstream API (the SSRF engine blocked all outbound traffic before +# the addSandboxChannel preset-apply fix). +# +# Telegram-only — Discord/Slack walk the same KNOWN_CHANNELS + preset lookup +# code path; telegram is the cheapest regression gate. +# +# Prerequisites: +# - Docker running +# - NVIDIA_API_KEY set (real key or fake OpenAI endpoint) +# - NEMOCLAW_NON_INTERACTIVE=1, NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 +# +# Usage: +# NEMOCLAW_NON_INTERACTIVE=1 NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ +# NVIDIA_API_KEY=nvapi-... bash test/e2e-vpn/test-channels-add-remove.sh + +set -uo pipefail + +export NEMOCLAW_E2E_DEFAULT_TIMEOUT=2400 +SCRIPT_DIR_TIMEOUT="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +# shellcheck source=test/e2e-vpn/e2e-timeout.sh +source "${SCRIPT_DIR_TIMEOUT}/e2e-timeout.sh" + +PASS=0 +FAIL=0 +SKIP=0 +TOTAL=0 + +pass() { + ((PASS++)) + ((TOTAL++)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} +fail() { + ((FAIL++)) + ((TOTAL++)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} +skip() { + ((SKIP++)) + ((TOTAL++)) + printf '\033[33m SKIP: %s\033[0m\n' "$1" +} +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } + +print_summary() { + section "Summary" + echo " Total: $TOTAL Pass: $PASS Fail: $FAIL Skip: $SKIP" + if [ "$FAIL" -gt 0 ]; then + echo "" + echo "FAILED" + exit 1 + fi + echo "" + if [ "$SKIP" -gt 0 ]; then + echo "PASSED (with $SKIP skipped)" + else + echo "ALL PASSED" + fi +} + +# Repo root resolution mirrors test-channels-stop-start.sh. +if [ -d /workspace ] && [ -f /workspace/install.sh ]; then + REPO="/workspace" +elif [ -f "$(cd "$(dirname "$0")/../.." && pwd)/install.sh" ]; then + REPO="$(cd "$(dirname "$0")/../.." && pwd)" +else + echo "ERROR: Cannot find repo root." + exit 1 +fi + +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-channels-add-remove}" +INSTALL_LOG="/tmp/nemoclaw-e2e-install.log" +REGISTRY="$HOME/.nemoclaw/sandboxes.json" +TELEGRAM_TOKEN="${TELEGRAM_BOT_TOKEN:-test-fake-telegram-token-add-remove-e2e}" +TELEGRAM_ALLOWED_IDS_VALUE="${TELEGRAM_ALLOWED_IDS:-123456789}" +TELEGRAM_REQUIRE_MENTION_VALUE="${TELEGRAM_REQUIRE_MENTION:-0}" + +is_fake_telegram_token() { + case "${1:-}" in + *fake*) return 0 ;; + *) return 1 ;; + esac +} + +maybe_skip_telegram_reachability_for_fake_token() { + if [ -z "${NEMOCLAW_SKIP_TELEGRAM_REACHABILITY:-}" ] && is_fake_telegram_token "$TELEGRAM_TOKEN"; then + # This E2E normally uses a fake token to exercise add/remove plumbing, not + # the live Telegram API. Remove once the test has a hermetic fake Telegram API. + export NEMOCLAW_SKIP_TELEGRAM_REACHABILITY=1 + info "Skipping Telegram reachability probe for fake-token E2E" + fi +} + +# shellcheck source=test/e2e-vpn/lib/sandbox-teardown.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib/sandbox-teardown.sh" +register_sandbox_for_teardown "$SANDBOX_NAME" + +# ── sandbox_exec: run a command inside the sandbox and capture output. ── +sandbox_exec() { + local cmd="$1" + local ssh_config + ssh_config="$(mktemp)" + openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config" 2>/dev/null + + local result + result=$(timeout 60 ssh -F "$ssh_config" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" \ + "$cmd" \ + 2>&1) || true + + rm -f "$ssh_config" + echo "$result" +} + +openclaw_has_telegram() { + # Read /sandbox/.openclaw/openclaw.json from inside the sandbox and check + # for `channels.telegram`. Exit 0 if present, 1 if absent, 2 if the file + # could not be read. + local out + out=$(sandbox_exec \ + "python3 -c 'import json,sys; d=json.load(open(\"/sandbox/.openclaw/openclaw.json\")); print(\"yes\" if \"telegram\" in d.get(\"channels\",{}) else \"no\")' 2>&1") || true + local verdict + verdict="$(printf '%s\n' "$out" | tail -n1 | tr -d '\r')" + case "$verdict" in + yes) return 0 ;; + no) return 1 ;; + *) return 2 ;; + esac +} + +# Print the policy-list snapshot so the test transcript shows gateway state +# alongside each pass/fail line. +print_policy_list() { + info "policy-list snapshot:" + nemoclaw "$SANDBOX_NAME" policy-list 2>&1 | sed 's/^/ /' || true +} + +# Check whether a named preset is currently applied. Matches only the +# applied marker (●); the inactive marker (○) is treated as "not applied". +policy_list_has_preset() { + local preset="$1" + nemoclaw "$SANDBOX_NAME" policy-list 2>/dev/null \ + | grep -E "^\s*●\s+${preset}\b" >/dev/null +} + +assert_host_telegram_config() { + local context="$1" + local output + if output="$(node -e ' +const fs = require("fs"); +const [registryPath, sandboxName, allowedIds, requireMention] = process.argv.slice(1); +const fail = (message) => { + console.error(message); + process.exit(1); +}; +if (!fs.existsSync(registryPath)) fail("registry file not found: " + registryPath); +const registry = JSON.parse(fs.readFileSync(registryPath, "utf8")); +const entry = registry.sandboxes?.[sandboxName]; +if (!entry) fail("sandbox " + sandboxName + " missing from registry"); +const plan = entry.messaging?.plan; +if (!plan || plan.schemaVersion !== 1) fail("messaging.plan missing or schemaVersion != 1"); +const channel = Array.isArray(plan.channels) + ? plan.channels.find((item) => item?.channelId === "telegram") + : null; +if (!channel) fail("telegram channel missing from messaging.plan.channels"); +const inputs = Array.isArray(channel.inputs) ? channel.inputs : []; +const inputValue = (id) => inputs.find((input) => input?.inputId === id)?.value; +if (inputValue("allowedIds") !== allowedIds) { + fail("allowedIds input expected " + allowedIds + ", got " + JSON.stringify(inputValue("allowedIds"))); +} +if (inputValue("requireMention") !== requireMention) { + fail("requireMention input expected " + requireMention + ", got " + JSON.stringify(inputValue("requireMention"))); +} +' "$REGISTRY" "$SANDBOX_NAME" "$TELEGRAM_ALLOWED_IDS_VALUE" "$TELEGRAM_REQUIRE_MENTION_VALUE" 2>&1)"; then + pass "host registry messaging.plan persists telegram config ${context}" + else + fail "host registry messaging.plan missing telegram config ${context}: ${output}" + fi +} + +assert_host_telegram_plan() { + local expected="$1" + local context="$2" + local output + if output="$(node -e ' +const fs = require("fs"); +const [registryPath, sandboxName, expected] = process.argv.slice(1); +const fail = (message) => { + console.error(message); + process.exit(1); +}; +if (!fs.existsSync(registryPath)) fail("registry file not found: " + registryPath); +const registry = JSON.parse(fs.readFileSync(registryPath, "utf8")); +const entry = registry.sandboxes?.[sandboxName]; +if (!entry) fail("sandbox " + sandboxName + " missing from registry"); +const state = entry.messaging; +if (!state || state.schemaVersion !== 1) fail("messaging state missing or schemaVersion != 1"); +const plan = state.plan; +if (!plan || plan.schemaVersion !== 1) fail("messaging.plan missing or schemaVersion != 1"); +if (plan.sandboxName !== sandboxName) { + fail("messaging.plan.sandboxName expected " + sandboxName + ", got " + JSON.stringify(plan.sandboxName)); +} +if (plan.agent !== "openclaw") fail("messaging.plan.agent expected openclaw, got " + JSON.stringify(plan.agent)); +const channels = Array.isArray(plan.channels) ? plan.channels : []; +const channel = channels.find((item) => item?.channelId === "telegram"); +const disabledChannels = Array.isArray(plan.disabledChannels) ? plan.disabledChannels : []; +const credentialBindings = Array.isArray(plan.credentialBindings) ? plan.credentialBindings : []; +const networkEntries = Array.isArray(plan.networkPolicy?.entries) ? plan.networkPolicy.entries : []; +const networkPresets = Array.isArray(plan.networkPolicy?.presets) ? plan.networkPolicy.presets : []; +if (Object.hasOwn(plan, "agentRender")) fail("messaging.plan.agentRender should not be persisted"); +if (channels.some((item) => item && Object.hasOwn(item, "hooks"))) fail("messaging.plan.channels hooks should not be persisted"); +if (expected === "active") { + if (!channel) fail("telegram channel missing from messaging.plan.channels"); + if (channel.active !== true) fail("telegram plan active expected true, got " + JSON.stringify(channel.active)); + if (channel.disabled === true) fail("telegram plan disabled unexpectedly true"); + if (!networkPresets.includes("telegram")) fail("telegram missing from messaging.plan.networkPolicy.presets"); + if (!networkEntries.some((entry) => entry?.channelId === "telegram")) { + fail("telegram missing from messaging.plan.networkPolicy.entries"); + } + if (!credentialBindings.some((entry) => entry?.channelId === "telegram" && entry?.providerEnvKey === "TELEGRAM_BOT_TOKEN")) { + fail("telegram TELEGRAM_BOT_TOKEN credential binding missing from messaging.plan"); + } + if (disabledChannels.includes("telegram")) fail("telegram unexpectedly listed in messaging.plan.disabledChannels"); +} else if (expected === "removed") { + if (channel) fail("telegram still present in messaging.plan.channels"); + if (disabledChannels.includes("telegram")) fail("telegram still present in messaging.plan.disabledChannels"); + if (networkPresets.includes("telegram")) fail("telegram still present in messaging.plan.networkPolicy.presets"); + if (networkEntries.some((entry) => entry?.channelId === "telegram")) { + fail("telegram still present in messaging.plan.networkPolicy.entries"); + } + if (credentialBindings.some((entry) => entry?.channelId === "telegram")) { + fail("telegram credential binding still present in messaging.plan"); + } +} else { + fail("unknown expected plan state: " + expected); +} +' "$REGISTRY" "$SANDBOX_NAME" "$expected" 2>&1)"; then + pass "host registry messaging.plan has telegram ${expected} ${context}" + else + fail "host registry messaging.plan expected telegram ${expected} ${context}: ${output}" + fi +} + +# Run rebuild with live tail of the rebuild log so the operator can see +# progress. Mirrors the install.sh tail pattern in Phase 1. +run_rebuild_with_live_log() { + local log_path="$1" + nemoclaw "$SANDBOX_NAME" rebuild --yes >"$log_path" 2>&1 & + local rebuild_pid=$! + tail -f "$log_path" --pid=$rebuild_pid 2>/dev/null & + local tail_pid=$! + wait $rebuild_pid + local rebuild_exit=$? + kill $tail_pid 2>/dev/null || true + wait $tail_pid 2>/dev/null || true + return $rebuild_exit +} + +# Egress probe through the L7 proxy from inside the sandbox. The telegram +# preset scopes egress to (binary IN [node]) AND (path /bot*/**), so probe +# with `node -e fetch` against a bot path. A 4xx from Telegram (e.g. 401 +# for the fake token) still counts as success — it proves the proxy let +# the CONNECT through. Proxy denial surfaces as a fetch error with no +# STATUS_ line. +telegram_egress_open() { + local body + body=$(sandbox_exec "node -e 'fetch(\"https://api.telegram.org/bot${TELEGRAM_TOKEN}/getMe\", {signal: AbortSignal.timeout(15000)}).then(r => console.log(\"STATUS_\" + r.status)).catch(e => console.log(\"ERROR_\" + (e.cause?.code || e.code || e.message)))' 2>&1" || true) + echo " [egress-probe] node fetch output:" + echo "$body" | head -20 | sed 's/^/ /' + # STATUS_2xx (valid token) or STATUS_4xx (e.g. 401 Unauthorized for the + # fake test token) — Telegram itself responded, meaning the proxy passed. + if echo "$body" | grep -qE "STATUS_[24][0-9][0-9]"; then + return 0 + fi + # Proxy denial signatures — fetch raises a network error before any HTTP + # status. The gateway L7 surfaces the rejection with one of these. + if echo "$body" | grep -qiE "policy_denied|engine:ssrf|forbidden by policy|CONNECT.*40[0-9]"; then + return 1 + fi + if echo "$body" | grep -qiE "fetch failed|ENOTFOUND|ECONNRESET|ETIMEDOUT"; then + return 2 + fi + return 2 +} + +# ══════════════════════════════════════════════════════════════════ +# Phase 0: Prerequisites +# ══════════════════════════════════════════════════════════════════ +section "Phase 0: Prerequisites" + +if [ -z "${NVIDIA_API_KEY:-}" ]; then + fail "C0: NVIDIA_API_KEY is required" + print_summary +fi +pass "C0: NVIDIA_API_KEY is set" + +if [ "${NEMOCLAW_NON_INTERACTIVE:-}" != "1" ]; then + fail "C0: NEMOCLAW_NON_INTERACTIVE=1 is required" + print_summary +fi +pass "C0: NEMOCLAW_NON_INTERACTIVE=1 is set" + +if [ "${NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE:-}" != "1" ]; then + fail "C0: NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 is required" + print_summary +fi +pass "C0: NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 is set" + +# ══════════════════════════════════════════════════════════════════ +# Phase 1: Install + onboard sandbox WITHOUT any messaging channel +# ══════════════════════════════════════════════════════════════════ +section "Phase 1: Install + onboard sandbox (no channel)" + +cd "$REPO" || exit 1 + +# Pre-cleanup: leftover sandboxes from prior runs. +info "Pre-cleanup..." +if command -v nemoclaw >/dev/null 2>&1; then + nemoclaw "$SANDBOX_NAME" destroy --yes 2>/dev/null || true +fi +if openshell --version >/dev/null 2>&1; then + openshell sandbox delete "$SANDBOX_NAME" 2>/dev/null || true + openshell gateway destroy -g nemoclaw 2>/dev/null || true +fi +pass "C1a: Pre-cleanup complete" + +# Intentionally do NOT export TELEGRAM_BOT_TOKEN here — onboard must see no +# messaging tokens and skip the messaging step entirely. This reproduces the +# exact entry condition of the #3437 bug (onboard empty -> later channels add). +unset TELEGRAM_BOT_TOKEN +unset TELEGRAM_ALLOWED_IDS +unset TELEGRAM_REQUIRE_MENTION + +export NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" +export NEMOCLAW_RECREATE_SANDBOX=1 +export NEMOCLAW_FRESH=1 + +info "Running install.sh --non-interactive (this takes 5-10 min on first run)..." +bash install.sh --non-interactive >"$INSTALL_LOG" 2>&1 & +install_pid=$! +tail -f "$INSTALL_LOG" --pid=$install_pid 2>/dev/null & +tail_pid=$! +wait $install_pid +install_exit=$? +kill $tail_pid 2>/dev/null || true +wait $tail_pid 2>/dev/null || true + +# Refresh PATH for nvm-managed installs. +if [ -f "$HOME/.bashrc" ]; then + # shellcheck source=/dev/null + source "$HOME/.bashrc" 2>/dev/null || true +fi +export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" +if [ -s "$NVM_DIR/nvm.sh" ]; then + # shellcheck source=/dev/null + . "$NVM_DIR/nvm.sh" +fi +if [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]]; then + export PATH="$HOME/.local/bin:$PATH" +fi + +if [ $install_exit -eq 0 ]; then + pass "C1b: install.sh + onboard completed (exit 0)" +else + fail "C1b: install.sh failed (exit $install_exit)" + tail -100 "$INSTALL_LOG" 2>/dev/null || true + print_summary +fi + +if ! openshell --version >/dev/null 2>&1; then + fail "C1c: openshell not on PATH after install" + print_summary +fi +pass "C1c: openshell installed" + +if ! command -v nemoclaw >/dev/null 2>&1; then + fail "C1d: nemoclaw not on PATH after install" + print_summary +fi +pass "C1d: nemoclaw installed" + +if openshell sandbox list 2>&1 | grep -q "${SANDBOX_NAME}.*Ready"; then + pass "C1e: Sandbox '${SANDBOX_NAME}' is Ready" +else + fail "C1e: Sandbox '${SANDBOX_NAME}' not Ready" + print_summary +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 2: Verify baseline state (no telegram anywhere) +# ══════════════════════════════════════════════════════════════════ +section "Phase 2: Verify baseline state (no channel)" + +if openshell provider get "${SANDBOX_NAME}-telegram-bridge" >/dev/null 2>&1; then + fail "C2a: Provider '${SANDBOX_NAME}-telegram-bridge' unexpectedly exists at baseline" +else + pass "C2a: No telegram-bridge provider at baseline" +fi + +if openclaw_has_telegram; then + fail "C2b: openclaw.json unexpectedly contains 'telegram' at baseline" +else + rc=$? + if [ "$rc" = "2" ]; then + fail "C2b: could not read openclaw.json inside sandbox at baseline" + else + pass "C2b: openclaw.json has no 'telegram' channel block at baseline" + fi +fi + +print_policy_list +if policy_list_has_preset telegram; then + fail "C2c: 'telegram' preset unexpectedly applied at baseline" +else + pass "C2c: 'telegram' preset not applied at baseline" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 3: channels add telegram + rebuild +# ══════════════════════════════════════════════════════════════════ +section "Phase 3: channels add telegram + rebuild" + +# Now provide the token — this mirrors the real user flow: after onboard, +# the operator decides to add a channel and exports the token first. +export TELEGRAM_BOT_TOKEN="$TELEGRAM_TOKEN" +export TELEGRAM_ALLOWED_IDS="$TELEGRAM_ALLOWED_IDS_VALUE" +export TELEGRAM_REQUIRE_MENTION="$TELEGRAM_REQUIRE_MENTION_VALUE" +maybe_skip_telegram_reachability_for_fake_token + +# Gateway-credential reuse gate. Before the fix, the rebuild preflight +# aborted with "provider credential not found" when NVIDIA_API_KEY was unset +# in the host env even though the inference provider was already registered +# in the OpenShell gateway. Drop the key from the env around `channels add` +# + rebuild so the post-add rebuild has to reuse the gateway-stored +# credential instead of demanding it back on the host. +NVIDIA_API_KEY_BACKUP="${NVIDIA_API_KEY:-}" +unset NVIDIA_API_KEY +info "NVIDIA_API_KEY unset for gateway-credential-reuse gate; gateway must hold the credential" + +if nemoclaw "$SANDBOX_NAME" channels add telegram >/tmp/nc-add.log 2>&1; then + add_rc=0 +else + add_rc=$? +fi +cat /tmp/nc-add.log +if [ "$add_rc" -eq 0 ] && grep -q "Registered telegram" /tmp/nc-add.log; then + pass "C3a: channels add telegram registered the bridge" +else + fail "C3a: channels add telegram did not register" + tail -20 /tmp/nc-add.log 2>/dev/null || true +fi +assert_host_telegram_config "after channels add" +assert_host_telegram_plan "active" "after channels add" + +info "Rebuilding sandbox to apply the add..." +if run_rebuild_with_live_log /tmp/nc-rebuild-add.log; then + pass "C3b: rebuild (post-add) completed" +else + fail "C3b: rebuild (post-add) failed" + tail -100 /tmp/nc-rebuild-add.log 2>/dev/null || true + # Restore env before bailing so later phases (and operators rerunning + # the script interactively) still see the original key. + if [ -n "$NVIDIA_API_KEY_BACKUP" ]; then + export NVIDIA_API_KEY="$NVIDIA_API_KEY_BACKUP" + fi + print_summary +fi + +# Gateway-credential reuse assertion: the rebuild must not have aborted with +# the "provider credential not found" error. +if grep -q "provider credential not found" /tmp/nc-rebuild-add.log; then + fail "C3c: REGRESSION — rebuild aborted on missing NVIDIA_API_KEY despite gateway-registered credential" +else + pass "C3c: rebuild reused gateway-stored credential without NVIDIA_API_KEY" +fi + +# Restore for the remaining phases — `channels remove` + rebuild should +# work in the normal env-present case too. +if [ -n "$NVIDIA_API_KEY_BACKUP" ]; then + export NVIDIA_API_KEY="$NVIDIA_API_KEY_BACKUP" +fi +unset NVIDIA_API_KEY_BACKUP + +# ══════════════════════════════════════════════════════════════════ +# Phase 4: Post-add assertions (Test 2 acceptance, regression #3437) +# ══════════════════════════════════════════════════════════════════ +section "Phase 4: Verify post-add state (regression #3437)" + +# C4a: regression gate for #3437. Pre-fix, `channels add` did not apply +# the matching policy preset, so the rebuilt sandbox lost egress to +# api.telegram.org. This assertion catches that regression. +print_policy_list +if policy_list_has_preset telegram; then + pass "C4a: 'telegram' preset present in policy list after add+rebuild (#3437 fixed)" +else + fail "C4a: REGRESSION — 'telegram' preset missing from policy list after add+rebuild (#3437)" +fi + +if openclaw_has_telegram; then + pass "C4b: openclaw.json contains 'telegram' channel block after add+rebuild" +else + rc=$? + if [ "$rc" = "2" ]; then + fail "C4b: could not read openclaw.json inside sandbox post-add" + else + fail "C4b: openclaw.json missing 'telegram' channel after add+rebuild" + fi +fi + +if openshell provider get "${SANDBOX_NAME}-telegram-bridge" >/dev/null 2>&1; then + pass "C4c: telegram-bridge provider exists in gateway after add+rebuild" +else + fail "C4c: telegram-bridge provider missing in gateway after add+rebuild" +fi + +assert_host_telegram_config "after add+rebuild" +assert_host_telegram_plan "active" "after add+rebuild" + +# C4d: network reachability. With the preset applied, the bridge-style +# probe (see telegram_egress_open) should reach Telegram and elicit a +# response; without it, the proxy denies the CONNECT. User-facing symptom +# of #3437 is the bot staying silent. +if telegram_egress_open; then + pass "C4d: egress to api.telegram.org reaches Telegram through L7 proxy" +else + rc=$? + if [ "$rc" = "2" ]; then + skip "C4d: egress probe inconclusive (network instability or unexpected proxy response)" + else + fail "C4d: egress to api.telegram.org blocked by proxy (preset not in effect)" + fi +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 5: channels remove telegram + rebuild +# ══════════════════════════════════════════════════════════════════ +section "Phase 5: channels remove telegram + rebuild" + +if nemoclaw "$SANDBOX_NAME" channels remove telegram >/tmp/nc-remove.log 2>&1; then + remove_rc=0 +else + remove_rc=$? +fi +cat /tmp/nc-remove.log +if [ "$remove_rc" -eq 0 ] && grep -q "Removed telegram" /tmp/nc-remove.log; then + pass "C5a: channels remove telegram unregistered the bridge" +else + fail "C5a: channels remove telegram did not unregister" + tail -20 /tmp/nc-remove.log 2>/dev/null || true +fi +assert_host_telegram_plan "removed" "after channels remove" + +info "Rebuilding sandbox to apply the remove..." +if run_rebuild_with_live_log /tmp/nc-rebuild-remove.log; then + pass "C5b: rebuild (post-remove) completed" +else + fail "C5b: rebuild (post-remove) failed" + tail -100 /tmp/nc-rebuild-remove.log 2>/dev/null || true + print_summary +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 6: Post-remove assertions (clean state restored) +# ══════════════════════════════════════════════════════════════════ +section "Phase 6: Verify post-remove state" + +if openclaw_has_telegram; then + fail "C6a: openclaw.json still contains 'telegram' after remove+rebuild" + info "openclaw.json channels after remove+rebuild:" + sandbox_exec "python3 -c 'import json; print(list(json.load(open(\"/sandbox/.openclaw/openclaw.json\")).get(\"channels\",{}).keys()))' 2>&1" | head -5 +else + rc=$? + if [ "$rc" = "2" ]; then + fail "C6a: could not read openclaw.json inside sandbox post-remove" + else + pass "C6a: openclaw.json excludes 'telegram' after remove+rebuild" + fi +fi + +if openshell provider get "${SANDBOX_NAME}-telegram-bridge" >/dev/null 2>&1; then + fail "C6b: telegram-bridge provider still exists in gateway after remove+rebuild" +else + pass "C6b: telegram-bridge provider removed from gateway after remove+rebuild" +fi + +# C6c: symmetric preset cleanup. `channels remove` should un-apply the +# channel's matching policy preset so the L7 proxy stops allow-listing the +# bridge's upstream API (defense-in-depth: bridge is gone, egress to +# api.telegram.org should follow). +print_policy_list +if policy_list_has_preset telegram; then + fail "C6c: REGRESSION — 'telegram' preset still applied after remove+rebuild (#3671)" +else + pass "C6c: 'telegram' preset removed from policy list after remove+rebuild" +fi + +assert_host_telegram_plan "removed" "after remove+rebuild" + +print_summary diff --git a/test/e2e-vpn/test-channels-stop-start.sh b/test/e2e-vpn/test-channels-stop-start.sh new file mode 100755 index 00000000000..fafb3612a0d --- /dev/null +++ b/test/e2e-vpn/test-channels-stop-start.sh @@ -0,0 +1,871 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Channel stop/start lifecycle E2E test. +# +# Covers Test 1 from issue #3462 ("onboard telegram -> channels stop -> channels start"). +# The regression surface is exercised for both supported agents (OpenClaw and +# Hermes) and every messaging channel (telegram, discord, wechat, slack, +# whatsapp). Set NEMOCLAW_CHANNELS_STOP_START_AGENT=openclaw or hermes to run a +# single-agent shard. +# +# Regression coverage: +# - #3453: `channels stop ` + rebuild must actually remove the channel +# from the baked agent config while preserving cached credentials. +# - #3381: `channels start ` + rebuild must reattach cached providers +# without re-prompting. +# +# Prerequisites: +# - Docker running +# - NVIDIA_API_KEY set +# - NEMOCLAW_NON_INTERACTIVE=1 +# - NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 +# +# Usage: +# NEMOCLAW_NON_INTERACTIVE=1 NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ +# NVIDIA_API_KEY=nvapi-... bash test/e2e-vpn/test-channels-stop-start.sh + +set -uo pipefail + +export NEMOCLAW_E2E_DEFAULT_TIMEOUT="${NEMOCLAW_E2E_DEFAULT_TIMEOUT:-7200}" +SCRIPT_DIR_TIMEOUT="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +# shellcheck source=test/e2e-vpn/e2e-timeout.sh +source "${SCRIPT_DIR_TIMEOUT}/e2e-timeout.sh" + +PASS=0 +FAIL=0 +SKIP=0 +TOTAL=0 + +pass() { + ((PASS++)) + ((TOTAL++)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} +fail() { + ((FAIL++)) + ((TOTAL++)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} +skip() { + ((SKIP++)) + ((TOTAL++)) + printf '\033[33m SKIP: %s\033[0m\n' "$1" +} +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } +pass_msg() { + ((PASS++)) + ((TOTAL++)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} +fail_msg() { + ((FAIL++)) + ((TOTAL++)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} + +print_summary() { + section "Summary" + echo " Total: $TOTAL Pass: $PASS Fail: $FAIL Skip: $SKIP" + if [ "$FAIL" -gt 0 ]; then + echo "" + echo "FAILED" + exit 1 + fi + echo "" + if [ "$SKIP" -gt 0 ]; then + echo "PASSED (with $SKIP skipped)" + else + echo "ALL PASSED" + fi +} + +if [ -d /workspace ] && [ -f /workspace/install.sh ]; then + REPO="/workspace" +elif [ -f "$(cd "$(dirname "$0")/../.." && pwd)/install.sh" ]; then + REPO="$(cd "$(dirname "$0")/../.." && pwd)" +else + echo "ERROR: Cannot find repo root." + exit 1 +fi + +BASE_SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-channels-stop-start}" +OPENCLAW_SANDBOX_NAME="${NEMOCLAW_CHANNELS_OPENCLAW_SANDBOX_NAME:-${BASE_SANDBOX_NAME}-openclaw}" +HERMES_SANDBOX_NAME="${NEMOCLAW_CHANNELS_HERMES_SANDBOX_NAME:-${BASE_SANDBOX_NAME}-hermes}" +REQUESTED_AGENT="${NEMOCLAW_CHANNELS_STOP_START_AGENT:-all}" +REGISTRY="$HOME/.nemoclaw/sandboxes.json" +OPENSHELL_BIN="${NEMOCLAW_OPENSHELL_BIN:-openshell}" +CHANNELS=(telegram discord wechat slack whatsapp) +TOKENLESS_CHANNELS=(whatsapp) +SELECTED_AGENT_SCENARIOS=() + +case "$REQUESTED_AGENT" in + all) + SELECTED_AGENT_SCENARIOS=("openclaw:${OPENCLAW_SANDBOX_NAME}" "hermes:${HERMES_SANDBOX_NAME}") + ;; + openclaw) + SELECTED_AGENT_SCENARIOS=("openclaw:${OPENCLAW_SANDBOX_NAME}") + ;; + hermes) + SELECTED_AGENT_SCENARIOS=("hermes:${HERMES_SANDBOX_NAME}") + ;; + *) + section "Phase 0: Prerequisites" + fail_msg "C0: NEMOCLAW_CHANNELS_STOP_START_AGENT must be all, openclaw, or hermes (got ${REQUESTED_AGENT})" + print_summary + ;; +esac + +ACTIVE_AGENT="" +ACTIVE_SANDBOX="" + +ORIG_TELEGRAM_BOT_TOKEN="${TELEGRAM_BOT_TOKEN:-}" +ORIG_TELEGRAM_ALLOWED_IDS="${TELEGRAM_ALLOWED_IDS:-}" +ORIG_TELEGRAM_REQUIRE_MENTION="${TELEGRAM_REQUIRE_MENTION:-}" +ORIG_DISCORD_BOT_TOKEN="${DISCORD_BOT_TOKEN:-}" +ORIG_DISCORD_SERVER_ID="${DISCORD_SERVER_ID:-}" +ORIG_DISCORD_SERVER_IDS="${DISCORD_SERVER_IDS:-}" +ORIG_DISCORD_USER_ID="${DISCORD_USER_ID:-}" +ORIG_DISCORD_ALLOWED_IDS="${DISCORD_ALLOWED_IDS:-}" +ORIG_DISCORD_REQUIRE_MENTION="${DISCORD_REQUIRE_MENTION:-}" +ORIG_SLACK_BOT_TOKEN="${SLACK_BOT_TOKEN:-}" +ORIG_SLACK_APP_TOKEN="${SLACK_APP_TOKEN:-}" +ORIG_SLACK_ALLOWED_USERS="${SLACK_ALLOWED_USERS:-}" +ORIG_WECHAT_BOT_TOKEN="${WECHAT_BOT_TOKEN:-}" +ORIG_WECHAT_ACCOUNT_ID="${WECHAT_ACCOUNT_ID:-}" +ORIG_WECHAT_BASE_URL="${WECHAT_BASE_URL:-}" +ORIG_WECHAT_USER_ID="${WECHAT_USER_ID:-}" +ORIG_WECHAT_ALLOWED_IDS="${WECHAT_ALLOWED_IDS:-}" + +openshell() { + if [ "$OPENSHELL_BIN" = "openshell" ]; then + command openshell "$@" + else + "$OPENSHELL_BIN" "$@" + fi +} + +# shellcheck source=test/e2e-vpn/lib/sandbox-teardown.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib/sandbox-teardown.sh" +for scenario in "${SELECTED_AGENT_SCENARIOS[@]}"; do + register_sandbox_for_teardown "${scenario#*:}" +done + +refresh_path() { + if [ -f "$HOME/.bashrc" ]; then + # shellcheck source=/dev/null + source "$HOME/.bashrc" 2>/dev/null || true + fi + export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" + if [ -s "$NVM_DIR/nvm.sh" ]; then + # shellcheck source=/dev/null + . "$NVM_DIR/nvm.sh" + fi + if [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]]; then + export PATH="$HOME/.local/bin:$PATH" + fi +} + +sandbox_exec() { + local cmd="$1" + local ssh_config + ssh_config="$(mktemp)" + openshell sandbox ssh-config "$ACTIVE_SANDBOX" >"$ssh_config" 2>/dev/null + + local result + result=$(run_with_timeout 60 ssh -F "$ssh_config" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o LogLevel=ERROR \ + "openshell-${ACTIVE_SANDBOX}" \ + "$cmd" \ + 2>&1) || true + + rm -f "$ssh_config" + echo "$result" +} + +registry_field() { + local field="$1" + if [ ! -f "$REGISTRY" ]; then + echo "null" + return + fi + if command -v jq >/dev/null 2>&1; then + jq -c --arg name "$ACTIVE_SANDBOX" --arg field "$field" \ + '.sandboxes[$name][$field]' "$REGISTRY" 2>/dev/null || echo "null" + else + node -e " +const r = JSON.parse(require('fs').readFileSync(process.argv[1], 'utf8')); +const v = (r.sandboxes || {})[process.argv[2]]?.[process.argv[3]]; +process.stdout.write(JSON.stringify(v ?? null)); +" "$REGISTRY" "$ACTIVE_SANDBOX" "$field" 2>/dev/null || echo "null" + fi +} + +registry_array_contains() { + local field="$1" + local item="$2" + local value + value="$(registry_field "$field")" + printf '%s' "$value" | grep -Fq "\"${item}\"" +} + +registry_plan_channel_contains() { + local item="$1" + node -e ' +const fs = require("fs"); +const [registryPath, sandboxName, channelId] = process.argv.slice(1); +if (!fs.existsSync(registryPath)) process.exit(1); +const registry = JSON.parse(fs.readFileSync(registryPath, "utf8")); +const plan = registry.sandboxes?.[sandboxName]?.messaging?.plan; +const channels = Array.isArray(plan?.channels) ? plan.channels : []; +process.exit(channels.some((channel) => channel?.channelId === channelId) ? 0 : 1); +' "$REGISTRY" "$ACTIVE_SANDBOX" "$item" +} + +registry_plan_disabled_contains() { + local item="$1" + node -e ' +const fs = require("fs"); +const [registryPath, sandboxName, channelId] = process.argv.slice(1); +if (!fs.existsSync(registryPath)) process.exit(1); +const registry = JSON.parse(fs.readFileSync(registryPath, "utf8")); +const disabled = registry.sandboxes?.[sandboxName]?.messaging?.plan?.disabledChannels; +process.exit(Array.isArray(disabled) && disabled.includes(channelId) ? 0 : 1); +' "$REGISTRY" "$ACTIVE_SANDBOX" "$item" +} + +provider_names_for_channel() { + local sandbox="$1" + local channel="$2" + case "$channel" in + telegram) printf '%s\n' "${sandbox}-telegram-bridge" ;; + discord) printf '%s\n' "${sandbox}-discord-bridge" ;; + wechat) printf '%s\n' "${sandbox}-wechat-bridge" ;; + slack) + printf '%s\n' "${sandbox}-slack-bridge" + printf '%s\n' "${sandbox}-slack-app" + ;; + esac +} + +channel_presence() { + local channel="$1" + local config_channel="$channel" + local out + if [ "$ACTIVE_AGENT" = "openclaw" ]; then + # NemoClaw's wechat channel maps to OpenClaw's upstream plugin key. + if [ "$channel" = "wechat" ]; then + config_channel="openclaw-weixin" + fi + out=$(sandbox_exec "python3 -c 'import json,sys; d=json.load(open(\"/sandbox/.openclaw/openclaw.json\")); print(\"yes\" if sys.argv[1] in d.get(\"channels\", {}) else \"no\")' '$config_channel'" | tail -1) || true + else + local probe + case "$channel" in + telegram) + probe='grep -Eq "^TELEGRAM_BOT_TOKEN=openshell:resolve:env:TELEGRAM_BOT_TOKEN$" /sandbox/.hermes/.env' + ;; + discord) + probe='grep -Eq "^DISCORD_BOT_TOKEN=openshell:resolve:env:DISCORD_BOT_TOKEN$" /sandbox/.hermes/.env' + ;; + wechat) + probe='grep -Eq "^WEIXIN_TOKEN=openshell:resolve:env:WECHAT_BOT_TOKEN$" /sandbox/.hermes/.env' + ;; + slack) + probe='grep -Eq "^SLACK_BOT_TOKEN=xoxb-OPENSHELL-RESOLVE-ENV-SLACK_BOT_TOKEN$" /sandbox/.hermes/.env && grep -Eq "^SLACK_APP_TOKEN=xapp-OPENSHELL-RESOLVE-ENV-SLACK_APP_TOKEN$" /sandbox/.hermes/.env' + ;; + whatsapp) + probe='grep -Eq "^WHATSAPP_ENABLED=true$" /sandbox/.hermes/.env && grep -Eq "^WHATSAPP_MODE=bot$" /sandbox/.hermes/.env' + ;; + esac + out=$(sandbox_exec "if [ -r /sandbox/.hermes/.env ]; then if ${probe}; then echo yes; else echo no; fi; else echo missing; fi" | tail -1) || true + fi + + case "$out" in + yes) echo "yes" ;; + no) echo "no" ;; + *) echo "error:${out}" ;; + esac +} + +dump_channel_state() { + info "registry.messaging.plan.channels: $(node -e 'const fs=require("fs"); const [p,n]=process.argv.slice(1); const r=fs.existsSync(p)?JSON.parse(fs.readFileSync(p,"utf8")):{}; const c=r.sandboxes?.[n]?.messaging?.plan?.channels; process.stdout.write(JSON.stringify(Array.isArray(c)?c.map((x)=>x?.channelId):null));' "$REGISTRY" "$ACTIVE_SANDBOX" 2>/dev/null || echo null)" + info "registry.messaging.plan.disabledChannels: $(node -e 'const fs=require("fs"); const [p,n]=process.argv.slice(1); const r=fs.existsSync(p)?JSON.parse(fs.readFileSync(p,"utf8")):{}; process.stdout.write(JSON.stringify(r.sandboxes?.[n]?.messaging?.plan?.disabledChannels ?? null));' "$REGISTRY" "$ACTIVE_SANDBOX" 2>/dev/null || echo null)" + if [ "$ACTIVE_AGENT" = "openclaw" ]; then + info "openclaw.json channels:" + sandbox_exec "python3 -c 'import json; print(list(json.load(open(\"/sandbox/.openclaw/openclaw.json\")).get(\"channels\", {}).keys()))' 2>&1" | head -10 || true + else + info ".hermes/.env messaging keys:" + sandbox_exec "grep -E '^(TELEGRAM_BOT_TOKEN|DISCORD_BOT_TOKEN|SLACK_BOT_TOKEN|SLACK_APP_TOKEN|WEIXIN_TOKEN|WHATSAPP_ENABLED|WHATSAPP_MODE|WHATSAPP_ALLOWED_USERS)=' /sandbox/.hermes/.env 2>/dev/null || true" | head -20 || true + fi +} + +assert_all_config_channels() { + local expected="$1" + local context="$2" + local channel status msg + for channel in "${CHANNELS[@]}"; do + status="$(channel_presence "$channel")" + if [ "$expected" = "present" ] && [ "$status" = "yes" ]; then + msg="${ACTIVE_AGENT}/${channel}: agent config contains channel ${context}" + pass_msg "$msg" + elif [ "$expected" = "absent" ] && [ "$status" = "no" ]; then + msg="${ACTIVE_AGENT}/${channel}: agent config excludes channel ${context}" + pass_msg "$msg" + else + msg="${ACTIVE_AGENT}/${channel}: expected channel ${expected} in agent config ${context}, got ${status}" + fail_msg "$msg" + dump_channel_state + fi + done +} + +assert_registry_channels() { + local expected="$1" + local context="$2" + local channel msg + for channel in "${CHANNELS[@]}"; do + if [ "$expected" = "present" ] && registry_plan_channel_contains "$channel"; then + msg="${ACTIVE_AGENT}/${channel}: registry.messaging.plan.channels contains channel ${context}" + pass_msg "$msg" + elif [ "$expected" = "absent" ] && ! registry_plan_channel_contains "$channel"; then + msg="${ACTIVE_AGENT}/${channel}: registry.messaging.plan.channels excludes channel ${context}" + pass_msg "$msg" + else + msg="${ACTIVE_AGENT}/${channel}: registry.messaging.plan.channels expected ${expected} ${context}" + fail_msg "$msg" + fi + done +} + +assert_disabled_channels() { + local expected="$1" + local context="$2" + local channel msg + for channel in "${CHANNELS[@]}"; do + if [ "$expected" = "present" ] && registry_plan_disabled_contains "$channel"; then + msg="${ACTIVE_AGENT}/${channel}: registry.messaging.plan.disabledChannels contains channel ${context}" + pass_msg "$msg" + elif [ "$expected" = "absent" ] && ! registry_plan_disabled_contains "$channel"; then + msg="${ACTIVE_AGENT}/${channel}: registry.messaging.plan.disabledChannels excludes channel ${context}" + pass_msg "$msg" + else + msg="${ACTIVE_AGENT}/${channel}: registry.messaging.plan.disabledChannels expected ${expected} ${context}" + fail_msg "$msg" + fi + done +} + +assert_host_messaging_config() { + local context="$1" + local output msg + if output="$(node -e ' +const fs = require("fs"); +const [registryPath, sandboxName, ...pairs] = process.argv.slice(1); +const fail = (message) => { + console.error(message); + process.exit(1); +}; +if (!fs.existsSync(registryPath)) fail("registry file not found: " + registryPath); +const registry = JSON.parse(fs.readFileSync(registryPath, "utf8")); +const entry = registry.sandboxes?.[sandboxName]; +if (!entry) fail("sandbox " + sandboxName + " missing from registry"); +const plan = entry.messaging?.plan; +if (!plan || plan.schemaVersion !== 1) fail("messaging.plan missing or schemaVersion != 1"); +const channels = Array.isArray(plan.channels) ? plan.channels : []; +const inputMap = { + TELEGRAM_ALLOWED_IDS: ["telegram", "allowedIds"], + TELEGRAM_REQUIRE_MENTION: ["telegram", "requireMention"], + DISCORD_SERVER_ID: ["discord", "serverId"], + DISCORD_USER_ID: ["discord", "userId"], + DISCORD_REQUIRE_MENTION: ["discord", "requireMention"], + SLACK_ALLOWED_USERS: ["slack", "allowedUsers"], + WECHAT_ALLOWED_IDS: ["wechat", "allowedIds"], +}; +for (let i = 0; i < pairs.length; i += 2) { + const key = pairs[i]; + const expected = pairs[i + 1]; + const mapping = inputMap[key]; + if (!mapping) fail("no plan input mapping for " + key); + const [channelId, inputId] = mapping; + const channel = channels.find((item) => item?.channelId === channelId); + if (!channel) fail(channelId + " missing from messaging.plan.channels"); + const inputs = Array.isArray(channel.inputs) ? channel.inputs : []; + const actual = inputs.find((input) => input?.inputId === inputId)?.value; + if (actual !== expected) { + fail(key + " expected " + expected + ", got " + JSON.stringify(actual)); + } +} +' "$REGISTRY" "$ACTIVE_SANDBOX" \ + TELEGRAM_ALLOWED_IDS "$TELEGRAM_ALLOWED_IDS" \ + TELEGRAM_REQUIRE_MENTION "$TELEGRAM_REQUIRE_MENTION" \ + DISCORD_SERVER_ID "$DISCORD_SERVER_ID" \ + DISCORD_USER_ID "$DISCORD_USER_ID" \ + DISCORD_REQUIRE_MENTION "$DISCORD_REQUIRE_MENTION" \ + SLACK_ALLOWED_USERS "$SLACK_ALLOWED_USERS" \ + WECHAT_ALLOWED_IDS "$WECHAT_ALLOWED_IDS" 2>&1)"; then + msg="${ACTIVE_AGENT}: host registry messaging.plan persists channel config ${context}" + pass_msg "$msg" + else + msg="${ACTIVE_AGENT}: host registry messaging.plan missing channel config ${context}: ${output}" + fail_msg "$msg" + fi +} + +assert_host_messaging_plan_state() { + local expected="$1" + local context="$2" + local channel output msg + for channel in "${CHANNELS[@]}"; do + if output="$(node -e ' +const fs = require("fs"); +const [registryPath, sandboxName, agent, channelId, expected] = process.argv.slice(1); +const fail = (message) => { + console.error(message); + process.exit(1); +}; +if (!fs.existsSync(registryPath)) fail("registry file not found: " + registryPath); +const registry = JSON.parse(fs.readFileSync(registryPath, "utf8")); +const entry = registry.sandboxes?.[sandboxName]; +if (!entry) fail("sandbox " + sandboxName + " missing from registry"); +const state = entry.messaging; +if (!state || state.schemaVersion !== 1) fail("messaging state missing or schemaVersion != 1"); +const plan = state.plan; +if (!plan || plan.schemaVersion !== 1) fail("messaging.plan missing or schemaVersion != 1"); +if (plan.sandboxName !== sandboxName) { + fail("messaging.plan.sandboxName expected " + sandboxName + ", got " + JSON.stringify(plan.sandboxName)); +} +if (plan.agent !== agent) fail("messaging.plan.agent expected " + agent + ", got " + JSON.stringify(plan.agent)); +const channels = Array.isArray(plan.channels) ? plan.channels : []; +const channel = channels.find((item) => item?.channelId === channelId); +if (!channel) fail(channelId + " missing from messaging.plan.channels"); +if (channel.configured !== true) { + fail(channelId + " messaging.plan configured expected true, got " + JSON.stringify(channel.configured)); +} +const disabledChannels = Array.isArray(plan.disabledChannels) ? plan.disabledChannels : []; +if (expected === "active") { + if (channel.active !== true) fail(channelId + " messaging.plan active expected true, got " + JSON.stringify(channel.active)); + if (channel.disabled === true) fail(channelId + " messaging.plan disabled unexpectedly true"); + if (disabledChannels.includes(channelId)) fail(channelId + " unexpectedly listed in messaging.plan.disabledChannels"); +} else if (expected === "disabled") { + if (channel.disabled !== true) fail(channelId + " messaging.plan disabled expected true, got " + JSON.stringify(channel.disabled)); + if (channel.active === true) fail(channelId + " messaging.plan active unexpectedly true"); + if (!disabledChannels.includes(channelId)) fail(channelId + " missing from messaging.plan.disabledChannels"); +} else { + fail("unknown expected plan state: " + expected); +} +const networkEntries = Array.isArray(plan.networkPolicy?.entries) ? plan.networkPolicy.entries : []; +const networkPresets = Array.isArray(plan.networkPolicy?.presets) ? plan.networkPolicy.presets : []; +if (!networkPresets.includes(channelId)) fail(channelId + " missing from messaging.plan.networkPolicy.presets"); +if (!networkEntries.some((entry) => entry?.channelId === channelId)) { + fail(channelId + " missing from messaging.plan.networkPolicy.entries"); +} +const credentialBindings = Array.isArray(plan.credentialBindings) ? plan.credentialBindings : []; +if (channelId !== "whatsapp" && !credentialBindings.some((entry) => entry?.channelId === channelId)) { + fail(channelId + " credential binding missing from messaging.plan"); +} +if (Object.hasOwn(plan, "agentRender")) fail("messaging.plan.agentRender should not be persisted"); +if (channels.some((item) => item && Object.hasOwn(item, "hooks"))) fail("messaging.plan.channels hooks should not be persisted"); +' "$REGISTRY" "$ACTIVE_SANDBOX" "$ACTIVE_AGENT" "$channel" "$expected" 2>&1)"; then + msg="${ACTIVE_AGENT}/${channel}: host registry messaging.plan has channel ${expected} ${context}" + pass_msg "$msg" + else + msg="${ACTIVE_AGENT}/${channel}: host registry messaging.plan expected ${expected} ${context}: ${output}" + fail_msg "$msg" + fi + done +} + +assert_provider_records_exist() { + local context="$1" + local channel provider msg + for channel in "${CHANNELS[@]}"; do + while IFS= read -r provider; do + if openshell provider get "$provider" >/dev/null 2>&1; then + msg="${ACTIVE_AGENT}/${provider}: provider record exists ${context}" + pass_msg "$msg" + else + msg="${ACTIVE_AGENT}/${provider}: provider record missing ${context}" + fail_msg "$msg" + fi + done < <(provider_names_for_channel "$ACTIVE_SANDBOX" "$channel") + done +} + +assert_policy_preset_active() { + local channel="$1" + local expected="$2" + local context="$3" + local log="/tmp/nc-channels-${ACTIVE_AGENT}-policy-list-${channel}.log" + local msg + if ! nemoclaw "$ACTIVE_SANDBOX" policy-list >"$log" 2>&1; then + msg="${ACTIVE_AGENT}/${channel}: policy-list failed ${context}" + fail_msg "$msg" + tail -30 "$log" 2>/dev/null || true + return + fi + + if [ "$expected" = "active" ]; then + if grep -q "● ${channel}" "$log"; then + msg="${ACTIVE_AGENT}/${channel}: channel policy preset active ${context}" + pass_msg "$msg" + else + msg="${ACTIVE_AGENT}/${channel}: channel policy preset not active ${context}" + fail_msg "$msg" + grep -F "$channel" "$log" | head -5 || true + fi + else + if grep -q "● ${channel}" "$log"; then + msg="${ACTIVE_AGENT}/${channel}: channel policy preset still active ${context}" + fail_msg "$msg" + grep -F "$channel" "$log" | head -5 || true + else + msg="${ACTIVE_AGENT}/${channel}: channel policy preset inactive ${context}" + pass_msg "$msg" + fi + fi +} + +is_fake_telegram_token() { + case "${1:-}" in + *fake*) return 0 ;; + *) return 1 ;; + esac +} +is_fake_slack_token() { + case "${1:-}" in + xoxb-fake-* | xoxb-test-* | xapp-fake-* | xapp-test-*) return 0 ;; + *) return 1 ;; + esac +} + +export_fake_channel_env() { + local suffix="$1" + export TELEGRAM_BOT_TOKEN="${ORIG_TELEGRAM_BOT_TOKEN:-test-fake-telegram-token-${suffix}}" + export TELEGRAM_ALLOWED_IDS="${ORIG_TELEGRAM_ALLOWED_IDS:-123456789,987654321}" + export TELEGRAM_REQUIRE_MENTION="${ORIG_TELEGRAM_REQUIRE_MENTION:-0}" + + export DISCORD_BOT_TOKEN="${ORIG_DISCORD_BOT_TOKEN:-test-fake-discord-token-${suffix}}" + export DISCORD_SERVER_ID="${ORIG_DISCORD_SERVER_ID:-1491590992753590594}" + export DISCORD_SERVER_IDS="${ORIG_DISCORD_SERVER_IDS:-${DISCORD_SERVER_ID}}" + export DISCORD_USER_ID="${ORIG_DISCORD_USER_ID:-1005536447329222676}" + export DISCORD_ALLOWED_IDS="${ORIG_DISCORD_ALLOWED_IDS:-${DISCORD_USER_ID}}" + export DISCORD_REQUIRE_MENTION="${ORIG_DISCORD_REQUIRE_MENTION:-0}" + + export SLACK_BOT_TOKEN="${ORIG_SLACK_BOT_TOKEN:-xoxb-fake-slack-token-${suffix}}" + export SLACK_APP_TOKEN="${ORIG_SLACK_APP_TOKEN:-xapp-fake-slack-app-token-${suffix}}" + export SLACK_ALLOWED_USERS="${ORIG_SLACK_ALLOWED_USERS:-U0123456789,U09ABCDEFGH}" + + export WECHAT_BOT_TOKEN="${ORIG_WECHAT_BOT_TOKEN:-test-fake-wechat-token-${suffix}}" + export WECHAT_ACCOUNT_ID="${ORIG_WECHAT_ACCOUNT_ID:-e2e-fake-account-${suffix}}" + export WECHAT_BASE_URL="${ORIG_WECHAT_BASE_URL:-https://ilinkai.wechat.com}" + export WECHAT_USER_ID="${ORIG_WECHAT_USER_ID:-wxid_${suffix}_operator}" + export WECHAT_ALLOWED_IDS="${ORIG_WECHAT_ALLOWED_IDS:-${WECHAT_USER_ID}}" +} + +pre_cleanup_sandbox() { + local sandbox="$1" + info "Pre-cleanup for ${sandbox}..." + if command -v nemoclaw >/dev/null 2>&1; then + nemoclaw "$sandbox" destroy --yes 2>/dev/null || true + fi + if openshell --version >/dev/null 2>&1; then + openshell sandbox delete "$sandbox" 2>/dev/null || true + local channel provider + for channel in "${CHANNELS[@]}"; do + while IFS= read -r provider; do + openshell provider delete "$provider" 2>/dev/null || true + done < <(provider_names_for_channel "$sandbox" "$channel") + done + openshell gateway destroy -g nemoclaw 2>/dev/null || true + fi +} + +install_for_active_agent() { + local log="/tmp/nemoclaw-e2e-channels-${ACTIVE_AGENT}-install.log" + export NEMOCLAW_SANDBOX_NAME="$ACTIVE_SANDBOX" + export NEMOCLAW_AGENT="$ACTIVE_AGENT" + export NEMOCLAW_POLICY_TIER="${NEMOCLAW_POLICY_TIER:-open}" + export NEMOCLAW_RECREATE_SANDBOX=1 + export NEMOCLAW_FRESH=1 + + if [ -z "${NEMOCLAW_SKIP_TELEGRAM_REACHABILITY:-}" ]; then + if is_fake_telegram_token "${TELEGRAM_BOT_TOKEN:-}"; then + export NEMOCLAW_SKIP_TELEGRAM_REACHABILITY=1 + info "Skipping onboarding Telegram reachability probe for fake-token E2E" + elif ! curl -fsS --max-time 10 https://api.telegram.org/ >/dev/null 2>&1; then + export NEMOCLAW_SKIP_TELEGRAM_REACHABILITY=1 + info "api.telegram.org unreachable from host; setting NEMOCLAW_SKIP_TELEGRAM_REACHABILITY=1" + fi + fi + if [ -z "${NEMOCLAW_SKIP_SLACK_AUTH_VALIDATION:-}" ] \ + && { is_fake_slack_token "$SLACK_BOT_TOKEN" || is_fake_slack_token "$SLACK_APP_TOKEN"; }; then + # This E2E normally uses fake Slack tokens to exercise channel lifecycle + # plumbing, not the live Slack API. + export NEMOCLAW_SKIP_SLACK_AUTH_VALIDATION=1 + info "Skipping onboarding Slack auth validation for fake-token E2E" + fi + + info "Running install.sh --non-interactive for ${ACTIVE_AGENT} (${ACTIVE_SANDBOX})..." + bash install.sh --non-interactive >"$log" 2>&1 & + local install_pid=$! + tail -f "$log" --pid=$install_pid 2>/dev/null & + local tail_pid=$! + wait $install_pid + local install_exit=$? + kill $tail_pid 2>/dev/null || true + wait $tail_pid 2>/dev/null || true + cp "$log" /tmp/nemoclaw-e2e-install.log 2>/dev/null || true + + refresh_path + + local msg + if [ "$install_exit" -eq 0 ]; then + msg="${ACTIVE_AGENT}: install.sh + onboard completed" + pass_msg "$msg" + else + msg="${ACTIVE_AGENT}: install.sh failed with exit ${install_exit}" + fail_msg "$msg" + tail -40 "$log" 2>/dev/null || true + print_summary + fi +} + +run_rebuild() { + local phase="$1" + local log="/tmp/nc-channels-${ACTIVE_AGENT}-rebuild-${phase}.log" + local msg + info "Rebuilding ${ACTIVE_SANDBOX} for ${phase}..." + if nemoclaw "$ACTIVE_SANDBOX" rebuild --yes >"$log" 2>&1; then + msg="${ACTIVE_AGENT}: rebuild completed after ${phase}" + pass_msg "$msg" + else + msg="${ACTIVE_AGENT}: rebuild failed after ${phase}" + fail_msg "$msg" + tail -40 "$log" 2>/dev/null || true + dump_channel_state + print_summary + fi +} + +ensure_tokenless_channels_enabled() { + local added=0 + local channel log rc msg + for channel in "${TOKENLESS_CHANNELS[@]}"; do + if registry_plan_channel_contains "$channel"; then + msg="${ACTIVE_AGENT}/${channel}: tokenless channel already registered" + pass_msg "$msg" + continue + fi + log="/tmp/nc-channels-${ACTIVE_AGENT}-add-${channel}.log" + if nemoclaw "$ACTIVE_SANDBOX" channels add "$channel" >"$log" 2>&1; then + rc=0 + else + rc=$? + fi + cat "$log" + if [ "$rc" -eq 0 ] && grep -q "Enabled ${channel} channel" "$log"; then + msg="${ACTIVE_AGENT}/${channel}: channels add registered tokenless QR channel" + pass_msg "$msg" + added=1 + else + msg="${ACTIVE_AGENT}/${channel}: channels add failed or did not register tokenless QR channel" + fail_msg "$msg" + tail -30 "$log" 2>/dev/null || true + fi + done + + if [ "$added" -eq 1 ]; then + run_rebuild "add-tokenless-channels" + fi +} + +stop_all_channels() { + local channel log rc msg + for channel in "${CHANNELS[@]}"; do + log="/tmp/nc-channels-${ACTIVE_AGENT}-stop-${channel}.log" + if nemoclaw "$ACTIVE_SANDBOX" channels stop "$channel" >"$log" 2>&1; then + rc=0 + else + rc=$? + fi + cat "$log" + if [ "$rc" -eq 0 ] && grep -q "Marked ${channel} disabled" "$log"; then + msg="${ACTIVE_AGENT}/${channel}: channels stop registered" + pass_msg "$msg" + else + msg="${ACTIVE_AGENT}/${channel}: channels stop failed or did not register" + fail_msg "$msg" + tail -20 "$log" 2>/dev/null || true + fi + done +} + +start_all_channels() { + local channel log rc msg + for channel in "${CHANNELS[@]}"; do + log="/tmp/nc-channels-${ACTIVE_AGENT}-start-${channel}.log" + if nemoclaw "$ACTIVE_SANDBOX" channels start "$channel" >"$log" 2>&1; then + rc=0 + else + rc=$? + fi + cat "$log" + if [ "$rc" -eq 0 ] && grep -q "Marked ${channel} enabled" "$log"; then + msg="${ACTIVE_AGENT}/${channel}: channels start registered" + pass_msg "$msg" + else + msg="${ACTIVE_AGENT}/${channel}: channels start failed or did not register" + fail_msg "$msg" + tail -20 "$log" 2>/dev/null || true + fi + done +} + +destroy_completed_sandbox() { + local sandbox="$1" + info "Destroying completed sandbox ${sandbox} before the next scenario..." + if command -v nemoclaw >/dev/null 2>&1; then + nemoclaw "$sandbox" destroy --yes >/dev/null 2>&1 || true + fi + if openshell --version >/dev/null 2>&1; then + openshell gateway destroy -g nemoclaw >/dev/null 2>&1 || true + fi +} + +run_agent_scenario() { + local agent="$1" + local sandbox="$2" + ACTIVE_AGENT="$agent" + ACTIVE_SANDBOX="$sandbox" + export NEMOCLAW_AGENT="$ACTIVE_AGENT" + + section "Scenario: ${agent} all messaging channels" + pre_cleanup_sandbox "$ACTIVE_SANDBOX" + export_fake_channel_env "${agent}" + + cd "$REPO" || exit 1 + install_for_active_agent + + local msg + if ! openshell --version >/dev/null 2>&1; then + msg="${ACTIVE_AGENT}: openshell not on PATH after install" + fail_msg "$msg" + print_summary + fi + msg="${ACTIVE_AGENT}: openshell installed" + pass_msg "$msg" + + if ! command -v nemoclaw >/dev/null 2>&1; then + msg="${ACTIVE_AGENT}: nemoclaw not on PATH after install" + fail_msg "$msg" + print_summary + fi + msg="${ACTIVE_AGENT}: nemoclaw installed" + pass_msg "$msg" + + if openshell sandbox list 2>&1 | grep -q "${ACTIVE_SANDBOX}.*Ready"; then + msg="${ACTIVE_AGENT}: sandbox ${ACTIVE_SANDBOX} is Ready" + pass_msg "$msg" + else + msg="${ACTIVE_AGENT}: sandbox ${ACTIVE_SANDBOX} is not Ready" + fail_msg "$msg" + openshell sandbox list 2>&1 || true + print_summary + fi + + ensure_tokenless_channels_enabled + + section "${agent}: baseline with all channels active" + assert_provider_records_exist "at baseline" + assert_all_config_channels "present" "at baseline" + assert_registry_channels "present" "at baseline" + assert_disabled_channels "absent" "at baseline" + assert_host_messaging_config "at baseline" + assert_host_messaging_plan_state "active" "at baseline" + for channel in "${CHANNELS[@]}"; do + assert_policy_preset_active "$channel" "active" "at baseline" + done + + section "${agent}: channels stop all + rebuild" + stop_all_channels + assert_host_messaging_config "after channels stop" + assert_host_messaging_plan_state "disabled" "after channels stop" + run_rebuild "stop-all" + + section "${agent}: verify stopped state" + assert_all_config_channels "absent" "after stop+rebuild" + assert_registry_channels "present" "after stop" + assert_disabled_channels "present" "after stop" + assert_provider_records_exist "after stop" + assert_host_messaging_config "after stop+rebuild" + assert_host_messaging_plan_state "disabled" "after stop+rebuild" + + section "${agent}: channels start all + rebuild" + start_all_channels + assert_host_messaging_config "after channels start" + assert_host_messaging_plan_state "active" "after channels start" + run_rebuild "start-all" + + section "${agent}: verify restarted state" + assert_all_config_channels "present" "after start+rebuild" + assert_registry_channels "present" "after start" + assert_disabled_channels "absent" "after start" + assert_provider_records_exist "after start" + assert_host_messaging_config "after start+rebuild" + assert_host_messaging_plan_state "active" "after start+rebuild" +} + +section "Phase 0: Prerequisites" + +if [ -z "${NVIDIA_API_KEY:-}" ]; then + msg="C0: NVIDIA_API_KEY is required" + fail_msg "$msg" + print_summary +fi +msg="C0: NVIDIA_API_KEY is set" +pass_msg "$msg" + +if [ "${NEMOCLAW_NON_INTERACTIVE:-}" != "1" ]; then + msg="C0: NEMOCLAW_NON_INTERACTIVE=1 is required" + fail_msg "$msg" + print_summary +fi +msg="C0: NEMOCLAW_NON_INTERACTIVE=1 is set" +pass_msg "$msg" + +if [ "${NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE:-}" != "1" ]; then + msg="C0: NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 is required" + fail_msg "$msg" + print_summary +fi +msg="C0: NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 is set" +pass_msg "$msg" + +if docker info >/dev/null 2>&1; then + msg="C0: Docker is running" + pass_msg "$msg" +else + msg="C0: Docker is not running" + fail_msg "$msg" + print_summary +fi + +refresh_path + +for index in "${!SELECTED_AGENT_SCENARIOS[@]}"; do + scenario="${SELECTED_AGENT_SCENARIOS[$index]}" + run_agent_scenario "${scenario%%:*}" "${scenario#*:}" + if [ "$index" -lt "$((${#SELECTED_AGENT_SCENARIOS[@]} - 1))" ]; then + destroy_completed_sandbox "${scenario#*:}" + fi +done + +print_summary diff --git a/test/e2e-vpn/test-cloud-inference-e2e.sh b/test/e2e-vpn/test-cloud-inference-e2e.sh new file mode 100755 index 00000000000..ea9ad9d66fd --- /dev/null +++ b/test/e2e-vpn/test-cloud-inference-e2e.sh @@ -0,0 +1,292 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Cloud Inference E2E — Live chat via inference.local + skill filesystem validation +# +# Tests end-to-end inference (sandbox → gateway → cloud API → response) +# and validates the OpenClaw skill filesystem layout inside the sandbox. +# +# Split from the cloud-experimental-e2e monolith (see #2644). +# Former phases: 5b (live chat), 5c (skill filesystem). +# +# Prerequisites: +# - Docker running +# - NVIDIA_API_KEY set (real key, starts with nvapi-) +# - NEMOCLAW_NON_INTERACTIVE=1, NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 +# +# Environment: +# NEMOCLAW_SANDBOX_NAME — sandbox name (default: e2e-cloud-inference) +# NEMOCLAW_RECREATE_SANDBOX=1 — recreate if exists +# E2E_PHASE_5B_MAX_ATTEMPTS — chat retries (default: 3) +# E2E_PHASE_5B_RETRY_SLEEP_SEC — seconds between retries (default: 5) +# NEMOCLAW_CLOUD_EXPERIMENTAL_MODEL — cloud model (default: nvidia/nemotron-3-super-120b-a12b) +# +# Usage: +# NEMOCLAW_NON_INTERACTIVE=1 NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ +# NVIDIA_API_KEY=nvapi-... bash test/e2e-vpn/test-cloud-inference-e2e.sh + +set -uo pipefail + +PASS=0 +FAIL=0 +SKIP=0 +TOTAL=0 + +pass() { + ((PASS++)) + ((TOTAL++)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} +fail() { + ((FAIL++)) + ((TOTAL++)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} +skip() { + ((SKIP++)) + ((TOTAL++)) + printf '\033[33m SKIP: %s\033[0m\n' "$1" +} +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } + +parse_chat_content() { + python3 -c " +import json, sys +try: + r = json.load(sys.stdin) + c = r['choices'][0]['message'] + content = c.get('content') or c.get('reasoning_content') or c.get('reasoning') or '' + print(content.strip()) +except Exception as e: + print(f'PARSE_ERROR: {e}', file=sys.stderr) + sys.exit(1) +" +} + +# ── Repo root ── +_script_dir="$(cd "$(dirname "$0")" && pwd)" +_candidate="$(cd "${_script_dir}/../.." && pwd)" +if [ -d /workspace ] && [ -f /workspace/package.json ] && [ -d /workspace/test/e2e ]; then + REPO="/workspace" +elif [ -f "${_candidate}/package.json" ] && [ -d "${_candidate}/test/e2e" ]; then + REPO="${_candidate}" +else + echo "ERROR: Cannot find repo root." + exit 1 +fi +unset _script_dir _candidate + +E2E_DIR="$(cd "$(dirname "$0")" && pwd)" +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-cloud-inference}" +CLOUD_MODEL="${NEMOCLAW_CLOUD_EXPERIMENTAL_MODEL:-nvidia/nemotron-3-super-120b-a12b}" + +# Source shared teardown helper +# shellcheck source=test/e2e-vpn/lib/sandbox-teardown.sh +. "${E2E_DIR}/lib/sandbox-teardown.sh" +# shellcheck source=test/e2e-vpn/lib/install-path-refresh.sh +. "${E2E_DIR}/lib/install-path-refresh.sh" +# shellcheck source=test/e2e-vpn/lib/ci-compatible-inference.sh +. "${E2E_DIR}/lib/ci-compatible-inference.sh" +nemoclaw_e2e_configure_compatible_inference +register_sandbox_for_teardown "$SANDBOX_NAME" + +# ══════════════════════════════════════════════════════════════════════ +# Phase 1: Install + Prerequisites +# ══════════════════════════════════════════════════════════════════════ +section "Phase 1: Install + Prerequisites" + +if ! docker info >/dev/null 2>&1; then + fail "Docker is not running" + exit 1 +fi +pass "Docker is running" + +if ! nemoclaw_e2e_require_hosted_inference_key; then + exit 1 +fi + +cd "$REPO" || { + fail "Could not cd to repo root" + exit 1 +} + +export NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" +export NEMOCLAW_RECREATE_SANDBOX="${NEMOCLAW_RECREATE_SANDBOX:-1}" + +info "Installing NemoClaw via install.sh --non-interactive..." +INSTALL_LOG="/tmp/nemoclaw-e2e-cloud-inference-install.log" +bash install.sh --non-interactive --yes-i-accept-third-party-software >"$INSTALL_LOG" 2>&1 & +install_pid=$! +tail -f "$INSTALL_LOG" --pid=$install_pid 2>/dev/null & +tail_pid=$! +wait "$install_pid" +install_exit=$? +kill "$tail_pid" 2>/dev/null || true +wait "$tail_pid" 2>/dev/null || true + +# Source shell profile +nemoclaw_refresh_install_env +export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" +# shellcheck source=/dev/null +[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" +nemoclaw_ensure_local_bin_on_path + +if [ "$install_exit" -ne 0 ]; then + fail "install.sh failed (exit $install_exit)" + tail -30 "$INSTALL_LOG" + exit 1 +fi +pass "NemoClaw installed" + +command -v nemoclaw >/dev/null 2>&1 || { + fail "nemoclaw not on PATH" + exit 1 +} +command -v openshell >/dev/null 2>&1 || { + fail "openshell not on PATH" + exit 1 +} +pass "CLIs on PATH" + +# ══════════════════════════════════════════════════════════════════════ +# Phase 2: Live chat via inference.local +# ══════════════════════════════════════════════════════════════════════ +section "Phase 2: Live chat (inference.local /v1/chat/completions)" + +command -v python3 >/dev/null 2>&1 || { + fail "python3 not on PATH" + exit 1 +} + +payload=$(CLOUD_MODEL="$CLOUD_MODEL" python3 -c " +import json, os +print(json.dumps({ + 'model': os.environ['CLOUD_MODEL'], + 'messages': [{'role': 'user', 'content': 'Reply with exactly one word: PONG'}], + 'max_tokens': 100, +})) +") || { + fail "Could not build chat payload" + exit 1 +} + +MAX_ATTEMPTS="${E2E_PHASE_5B_MAX_ATTEMPTS:-3}" +RETRY_SLEEP="${E2E_PHASE_5B_RETRY_SLEEP_SEC:-5}" +[[ "$MAX_ATTEMPTS" =~ ^[1-9][0-9]*$ ]] || MAX_ATTEMPTS=3 + +info "POST chat completion inside sandbox (model ${CLOUD_MODEL}, up to ${MAX_ATTEMPTS} attempts)..." + +TIMEOUT_CMD="" +command -v timeout >/dev/null 2>&1 && TIMEOUT_CMD="timeout 120" +command -v gtimeout >/dev/null 2>&1 && TIMEOUT_CMD="gtimeout 120" + +ssh_config="$(mktemp)" +if ! openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config" 2>/dev/null; then + rm -f "$ssh_config" + fail "openshell sandbox ssh-config failed for '${SANDBOX_NAME}'" + exit 1 +fi + +attempt=1 +chat_ok=0 +last_fail="" +while [ "$attempt" -le "$MAX_ATTEMPTS" ]; do + set +e + chat_out=$( + $TIMEOUT_CMD ssh -F "$ssh_config" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" \ + "curl -sS --max-time 90 https://inference.local/v1/chat/completions -H 'Content-Type: application/json' -d $(printf '%q' "$payload")" \ + 2>&1 + ) + chat_rc=$? + set -uo pipefail + + if [ "$chat_rc" -ne 0 ]; then + last_fail="ssh/curl failed (exit ${chat_rc}): ${chat_out:0:400}" + elif [ -z "$chat_out" ]; then + last_fail="empty response from inference.local" + else + chat_text=$(printf '%s' "$chat_out" | parse_chat_content 2>/dev/null) || chat_text="" + if echo "$chat_text" | grep -qi "PONG"; then + pass "Chat completion returned PONG (attempt ${attempt}/${MAX_ATTEMPTS})" + chat_ok=1 + break + fi + last_fail="expected PONG, got: ${chat_text:0:300}" + fi + + if [ "$attempt" -ge "$MAX_ATTEMPTS" ]; then break; fi + info "Attempt ${attempt}/${MAX_ATTEMPTS} failed — ${last_fail}" + info "Sleeping ${RETRY_SLEEP}s..." + sleep "$RETRY_SLEEP" + attempt=$((attempt + 1)) +done + +rm -f "$ssh_config" + +if [ "$chat_ok" -ne 1 ]; then + fail "Live chat: $last_fail" + exit 1 +fi + +# ══════════════════════════════════════════════════════════════════════ +# Phase 3: Skill filesystem validation +# ══════════════════════════════════════════════════════════════════════ +section "Phase 3: Skill filesystem validation" + +info "Validating repo .agents/skills (SKILL.md frontmatter + body)..." +if ! bash "$E2E_DIR/e2e-cloud-experimental/features/skill/lib/validate_repo_skills.sh" --repo "$REPO"; then + fail "Repo skill validation failed" + exit 1 +fi +pass "Repo agent skills (SKILL.md) valid" + +info "Checking /sandbox/.openclaw inside sandbox..." +set +e +sb_out=$(SANDBOX_NAME="$SANDBOX_NAME" bash "$E2E_DIR/e2e-cloud-experimental/features/skill/lib/validate_sandbox_openclaw_skills.sh" 2>/dev/null) +sb_rc=$? +set -uo pipefail + +if [ "$sb_rc" -ne 0 ]; then + fail "Sandbox OpenClaw layout check failed (exit ${sb_rc}): ${sb_out:0:240}" + exit 1 +fi +pass "Sandbox /sandbox/.openclaw + openclaw.json OK" + +if echo "$sb_out" | grep -q "SKILLS_SUBDIR=present"; then + pass "Sandbox /sandbox/.openclaw/skills present" +elif echo "$sb_out" | grep -q "SKILLS_SUBDIR=absent"; then + skip "/sandbox/.openclaw/skills absent (migration snapshot had no skills dir)" +else + fail "Unexpected sandbox check output: ${sb_out:0:240}" + exit 1 +fi + +# ══════════════════════════════════════════════════════════════════════ +# Summary +# ══════════════════════════════════════════════════════════════════════ +echo "" +echo "========================================" +echo " Cloud Inference E2E Results:" +echo " Passed: $PASS" +echo " Failed: $FAIL" +echo " Skipped: $SKIP" +echo " Total: $TOTAL" +echo "========================================" + +if [ "$FAIL" -eq 0 ]; then + printf '\033[1;32m\n Cloud Inference E2E PASSED.\033[0m\n' + exit 0 +else + printf '\033[1;31m\n %d test(s) failed.\033[0m\n' "$FAIL" + exit 1 +fi diff --git a/test/e2e-vpn/test-cloud-onboard-e2e.sh b/test/e2e-vpn/test-cloud-onboard-e2e.sh new file mode 100755 index 00000000000..b88ee7c2ed3 --- /dev/null +++ b/test/e2e-vpn/test-cloud-onboard-e2e.sh @@ -0,0 +1,348 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Cloud Onboard E2E — Install via public URL + sandbox health + security +# +# Tests the public installer flow (curl nvidia.com/nemoclaw.sh | bash), +# verifies the sandbox is healthy, checks Landlock read-only enforcement, +# API key leak detection, and inference.local HTTPS. +# +# Split from the cloud-experimental-e2e monolith (see #2644). +# Former phases: 0 (pre-cleanup), 1 (prereqs), 3 (install), 5 (checks/*.sh), 6 (cleanup). +# +# Prerequisites: +# - Docker running +# - NVIDIA_API_KEY set (real key, starts with nvapi-) +# - Network access to inference.nvidia.com +# +# Environment: +# NEMOCLAW_NON_INTERACTIVE=1 — required for non-interactive install +# NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 — required for non-interactive install +# NEMOCLAW_SANDBOX_NAME — sandbox name (default: e2e-cloud-onboard) +# NEMOCLAW_RECREATE_SANDBOX=1 — recreate if exists +# NEMOCLAW_POLICY_MODE=custom — custom policy mode +# NEMOCLAW_POLICY_PRESETS=npm,pypi — policy presets +# RUN_E2E_CLOUD_ONBOARD_INTERACTIVE_INSTALL=0 — set 0 for non-interactive (default), 1 for expect +# NEMOCLAW_INSTALL_SCRIPT_URL — override public installer URL +# NEMOCLAW_PUBLIC_INSTALL_REF — Git ref used for the public install script and clone +# NEMOCLAW_INSTALL_REF — Git ref cloned by public installer +# NEMOCLAW_PUBLIC_INSTALL_CWD — override temp cwd for public install +# E2E_CLOUD_ONBOARD_INSTALL_LOG — install log path +# +# Usage: +# NEMOCLAW_NON_INTERACTIVE=1 NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ +# NVIDIA_API_KEY=nvapi-... bash test/e2e-vpn/test-cloud-onboard-e2e.sh + +set -uo pipefail + +PASS=0 +FAIL=0 +SKIP=0 +TOTAL=0 + +pass() { + ((PASS++)) + ((TOTAL++)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} +fail() { + ((FAIL++)) + ((TOTAL++)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} +skip() { + ((SKIP++)) + ((TOTAL++)) + printf '\033[33m SKIP: %s\033[0m\n' "$1" +} +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } + +# ── Repo root ── +_script_dir="$(cd "$(dirname "$0")" && pwd)" +_candidate="$(cd "${_script_dir}/../.." && pwd)" +if [ -d /workspace ] && [ -f /workspace/package.json ] && [ -d /workspace/test/e2e ]; then + REPO="/workspace" +elif [ -f "${_candidate}/package.json" ] && [ -d "${_candidate}/test/e2e" ]; then + REPO="${_candidate}" +else + echo "ERROR: Cannot find repo root (expected package.json and test/e2e at checkout root)." + exit 1 +fi +unset _script_dir _candidate + +E2E_DIR="$(cd "$(dirname "$0")" && pwd)" +E2E_CHECKS_DIR="${E2E_DIR}/e2e-cloud-experimental/checks" +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-cloud-onboard}" +CLOUD_MODEL="${NEMOCLAW_CLOUD_EXPERIMENTAL_MODEL:-nvidia/nemotron-3-super-120b-a12b}" +INSTALL_LOG="${E2E_CLOUD_ONBOARD_INSTALL_LOG:-/tmp/nemoclaw-e2e-cloud-onboard-install.log}" +INTERACTIVE_INSTALL="${RUN_E2E_CLOUD_ONBOARD_INTERACTIVE_INSTALL:-0}" +PUBLIC_INSTALL_CWD="${NEMOCLAW_PUBLIC_INSTALL_CWD:-}" + +# Source shared teardown helper +# shellcheck source=test/e2e-vpn/lib/sandbox-teardown.sh +. "${E2E_DIR}/lib/sandbox-teardown.sh" +# shellcheck source=test/e2e-vpn/lib/install-path-refresh.sh +. "${E2E_DIR}/lib/install-path-refresh.sh" +# shellcheck source=test/e2e-vpn/lib/ci-compatible-inference.sh +. "${E2E_DIR}/lib/ci-compatible-inference.sh" +nemoclaw_e2e_configure_compatible_inference +if nemoclaw_e2e_using_compatible_inference; then + CLOUD_MODEL="$(nemoclaw_e2e_hosted_inference_model)" +fi +register_sandbox_for_teardown "$SANDBOX_NAME" + +# ══════════════════════════════════════════════════════════════════════ +# Phase 1: Pre-cleanup +# ══════════════════════════════════════════════════════════════════════ +section "Phase 1: Pre-cleanup" + +info "Destroying leftover sandbox, forwards, and gateway for '${SANDBOX_NAME}'..." +SANDBOX_NAME="$SANDBOX_NAME" bash "${E2E_DIR}/e2e-cloud-experimental/cleanup.sh" 2>/dev/null || true +pass "Pre-cleanup complete" + +# ══════════════════════════════════════════════════════════════════════ +# Phase 2: Prerequisites +# ══════════════════════════════════════════════════════════════════════ +section "Phase 2: Prerequisites" + +if docker info >/dev/null 2>&1; then + pass "Docker is running" +else + fail "Docker is not running — cannot continue" + exit 1 +fi + +if ! nemoclaw_e2e_require_hosted_inference_key; then + exit 1 +fi + +HOSTED_INFERENCE_BASE_URL="$(nemoclaw_e2e_hosted_inference_base_url)" + +if nemoclaw_e2e_probe_hosted_inference; then + pass "Network access to ${HOSTED_INFERENCE_BASE_URL}" +else + fail "Cannot reach ${HOSTED_INFERENCE_BASE_URL}" + exit 1 +fi + +if [ "$INTERACTIVE_INSTALL" != "1" ]; then + if [ "${NEMOCLAW_NON_INTERACTIVE:-}" != "1" ]; then + fail "NEMOCLAW_NON_INTERACTIVE=1 is required for non-interactive install" + exit 1 + fi + if [ "${NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE:-}" != "1" ]; then + fail "NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 is required for non-interactive install" + exit 1 + fi + pass "Non-interactive mode configured" +else + skip "Interactive install mode not supported in split tests (use non-interactive)" +fi + +if [[ "$(uname -s)" == "Linux" ]]; then + pass "Host OS is Linux" +else + skip "Host is not Linux — test nominally targets Ubuntu (continuing)" +fi + +# ══════════════════════════════════════════════════════════════════════ +# Phase 3: Install via public URL +# ══════════════════════════════════════════════════════════════════════ +section "Phase 3: Install via public URL" + +export NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" +export NEMOCLAW_EXPERIMENTAL=1 +export NEMOCLAW_PROVIDER="${NEMOCLAW_PROVIDER:-cloud}" +export NEMOCLAW_MODEL="${NEMOCLAW_MODEL:-$CLOUD_MODEL}" +export NEMOCLAW_POLICY_MODE="${NEMOCLAW_POLICY_MODE:-custom}" +export NEMOCLAW_POLICY_PRESETS="${NEMOCLAW_POLICY_PRESETS:-npm,pypi}" + +PUBLIC_INSTALL_REF="${NEMOCLAW_PUBLIC_INSTALL_REF:-${GITHUB_SHA:-}}" +if [ -n "$PUBLIC_INSTALL_REF" ]; then + export NEMOCLAW_INSTALL_REF="$PUBLIC_INSTALL_REF" + export NEMOCLAW_INSTALL_TAG="$PUBLIC_INSTALL_REF" +fi +if [ -z "${NEMOCLAW_INSTALL_SCRIPT_URL:-}" ] && [ -n "$PUBLIC_INSTALL_REF" ]; then + NEMOCLAW_INSTALL_SCRIPT_URL="https://raw.githubusercontent.com/NVIDIA/NemoClaw/${PUBLIC_INSTALL_REF}/install.sh" +else + NEMOCLAW_INSTALL_SCRIPT_URL="${NEMOCLAW_INSTALL_SCRIPT_URL:-https://www.nvidia.com/nemoclaw.sh}" +fi +export NEMOCLAW_INSTALL_SCRIPT_URL + +info "Model: ${CLOUD_MODEL}, Policy: ${NEMOCLAW_POLICY_MODE} ${NEMOCLAW_POLICY_PRESETS}" +if [ -n "${NEMOCLAW_INSTALL_REF:-}" ]; then + info "Public installer will clone NemoClaw ref: ${NEMOCLAW_INSTALL_REF}" +else + info "Public installer will clone NemoClaw ref: latest" +fi + +if [ "$INTERACTIVE_INSTALL" = "1" ]; then + # Interactive install via expect is not currently supported in the split + # tests. The original monolith inlined the expect heredoc; the standalone + # wrapper (expect-interactive-install.sh) was never self-contained. + # TODO(#2644): re-implement interactive install if needed. + fail "Interactive install (RUN_E2E_CLOUD_ONBOARD_INTERACTIVE_INSTALL=1) is not yet supported — use non-interactive mode" + exit 1 +else + if [ -z "$PUBLIC_INSTALL_CWD" ]; then + PUBLIC_INSTALL_CWD="$(mktemp -d "${TMPDIR:-/tmp}/nemoclaw-public-install.XXXXXX")" + else + mkdir -p "$PUBLIC_INSTALL_CWD" + fi + info "Installing (non-interactive): curl -fsSL ${NEMOCLAW_INSTALL_SCRIPT_URL} | bash" + info "Public install cwd: ${PUBLIC_INSTALL_CWD}" + ( + cd "$PUBLIC_INSTALL_CWD" || exit 1 + curl -fsSL "$NEMOCLAW_INSTALL_SCRIPT_URL" | bash + ) >"$INSTALL_LOG" 2>&1 & + install_pid=$! + tail -f "$INSTALL_LOG" --pid=$install_pid 2>/dev/null & + tail_pid=$! + wait "$install_pid" + install_exit=$? + kill "$tail_pid" 2>/dev/null || true + wait "$tail_pid" 2>/dev/null || true +fi + +# Source shell profile to pick up nvm/PATH changes +nemoclaw_refresh_install_env +export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" +# shellcheck source=/dev/null +[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" +nemoclaw_ensure_local_bin_on_path + +if [ "$install_exit" -eq 0 ]; then + pass "Public install completed (exit 0)" +else + fail "Public install failed (exit $install_exit)" + info "Last 30 lines of install log:" + tail -30 "$INSTALL_LOG" + exit 1 +fi + +if grep -q "NemoClaw package.json found in the selected source checkout" "$INSTALL_LOG"; then + fail "Public install unexpectedly used the local source checkout" + info "Last 30 lines of install log:" + tail -30 "$INSTALL_LOG" + exit 1 +fi + +if grep -q "Installing NemoClaw from GitHub" "$INSTALL_LOG" \ + && grep -q "Resolved install ref:" "$INSTALL_LOG" \ + && grep -q "Cloning NemoClaw source" "$INSTALL_LOG"; then + pass "Public install used the GitHub clone path" +else + fail "Public install did not show the GitHub clone path" + info "Last 40 lines of install log:" + tail -40 "$INSTALL_LOG" + exit 1 +fi + +if [ -n "$PUBLIC_INSTALL_REF" ]; then + if grep -q "Resolved install ref: ${PUBLIC_INSTALL_REF}" "$INSTALL_LOG"; then + pass "Public install used requested ref ${PUBLIC_INSTALL_REF}" + else + fail "Public install did not use requested ref ${PUBLIC_INSTALL_REF}" + info "Last 40 lines of install log:" + tail -40 "$INSTALL_LOG" + exit 1 + fi +fi + +if command -v nemoclaw >/dev/null 2>&1; then + pass "nemoclaw on PATH ($(command -v nemoclaw))" +else + fail "nemoclaw not found on PATH after install" + exit 1 +fi + +if command -v openshell >/dev/null 2>&1; then + pass "openshell on PATH ($(openshell --version 2>&1 || echo unknown))" +else + fail "openshell not found on PATH after install" + exit 1 +fi + +if nemoclaw --help >/dev/null 2>&1; then + pass "nemoclaw --help exits 0" +else + fail "nemoclaw --help failed" + exit 1 +fi + +# ══════════════════════════════════════════════════════════════════════ +# Phase 4: Sandbox checks suite +# ══════════════════════════════════════════════════════════════════════ +section "Phase 4: Sandbox checks (Landlock, security, inference.local)" + +if nemoclaw_e2e_using_compatible_inference; then + export NEMOCLAW_E2E_CLOUD_API_KEY_ENV=COMPATIBLE_API_KEY +else + export NEMOCLAW_E2E_CLOUD_API_KEY_ENV=NVIDIA_API_KEY +fi +export SANDBOX_NAME CLOUD_EXPERIMENTAL_MODEL="$CLOUD_MODEL" REPO NVIDIA_API_KEY COMPATIBLE_API_KEY +export PATH="/usr/local/bin:${HOME}/.local/bin:${PATH}" + +shopt -s nullglob +case_scripts=("$E2E_CHECKS_DIR"/*.sh) +shopt -u nullglob + +if [ "${#case_scripts[@]}" -eq 0 ]; then + skip "No checks scripts in ${E2E_CHECKS_DIR}" +else + info "Running ${#case_scripts[@]} check script(s) from ${E2E_CHECKS_DIR}" + for case_script in "${case_scripts[@]}"; do + info "Running $(basename "$case_script")..." + set +e + bash "$case_script" + c_rc=$? + set -uo pipefail + if [ "$c_rc" -eq 0 ]; then + pass "$(basename "$case_script" .sh)" + else + fail "$(basename "$case_script" .sh) exited ${c_rc}" + exit 1 + fi + done +fi + +# ══════════════════════════════════════════════════════════════════════ +# Phase 5: Cleanup +# ══════════════════════════════════════════════════════════════════════ +section "Phase 5: Cleanup" + +if [ "${NEMOCLAW_E2E_KEEP_SANDBOX:-}" = "1" ]; then + skip "Cleanup skipped (NEMOCLAW_E2E_KEEP_SANDBOX=1)" +else + info "Destroying sandbox '${SANDBOX_NAME}'..." + if ! SANDBOX_NAME="$SANDBOX_NAME" bash "${E2E_DIR}/e2e-cloud-experimental/cleanup.sh" --verify; then + fail "Cleanup or verification failed" + exit 1 + fi + pass "Cleanup complete" +fi + +# ══════════════════════════════════════════════════════════════════════ +# Summary +# ══════════════════════════════════════════════════════════════════════ +echo "" +echo "========================================" +echo " Cloud Onboard E2E Results:" +echo " Passed: $PASS" +echo " Failed: $FAIL" +echo " Skipped: $SKIP" +echo " Total: $TOTAL" +echo "========================================" + +if [ "$FAIL" -eq 0 ]; then + printf '\033[1;32m\n Cloud Onboard E2E PASSED.\033[0m\n' + exit 0 +else + printf '\033[1;31m\n %d test(s) failed.\033[0m\n' "$FAIL" + exit 1 +fi diff --git a/test/e2e-vpn/test-common-egress-agent-e2e.sh b/test/e2e-vpn/test-common-egress-agent-e2e.sh new file mode 100755 index 00000000000..e0061da1b12 --- /dev/null +++ b/test/e2e-vpn/test-common-egress-agent-e2e.sh @@ -0,0 +1,485 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Common Egress Agent E2E +# +# Proves the safe common-egress defaults through real agent turns: +# C1 OpenClaw balanced includes weather and the agent fetches Open-Meteo. +# C2 OpenClaw open includes public-reference and the agent fetches Wikidata. +# C3 Hermes open includes public-reference plus all Hermes Nous policy presets, +# and the Hermes agent fetches Wikidata through its API-server agent path. +# +# Required env: +# NVIDIA_API_KEY hosted inference credential +# NEMOCLAW_NON_INTERACTIVE=1 required +# NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 required +# +# Optional env: +# NEMOCLAW_COMMON_EGRESS_SKIP_OPENCLAW=1 skip OpenClaw phases +# NEMOCLAW_COMMON_EGRESS_SKIP_HERMES=1 skip Hermes phase +# NEMOCLAW_COMMON_EGRESS_KEEP_SANDBOX=1 preserve created sandboxes + +set -uo pipefail + +export NEMOCLAW_E2E_DEFAULT_TIMEOUT=3600 +SCRIPT_DIR_TIMEOUT="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +# shellcheck source=test/e2e-vpn/e2e-timeout.sh +. "${SCRIPT_DIR_TIMEOUT}/e2e-timeout.sh" +# shellcheck source=test/e2e-vpn/lib/openclaw-json.sh +. "${SCRIPT_DIR_TIMEOUT}/lib/openclaw-json.sh" + +PASS=0 +FAIL=0 +SKIP=0 +TOTAL=0 +SANDBOXES_TO_CLEAN="" + +pass() { + ((PASS++)) + ((TOTAL++)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} +fail() { + ((FAIL++)) + ((TOTAL++)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} +skip() { + ((SKIP++)) + ((TOTAL++)) + printf '\033[33m SKIP: %s\033[0m\n' "$1" +} +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } + +summary() { + echo "" + echo "============================================================" + echo " Common Egress Agent E2E Results" + echo "============================================================" + echo " PASS: $PASS" + echo " FAIL: $FAIL" + echo " SKIP: $SKIP" + echo " TOTAL: $TOTAL" + echo "============================================================" + if [ "$FAIL" -gt 0 ]; then exit 1; fi +} + +quote_for_remote_sh() { + local value="${1:-}" + printf "'%s'" "$(printf '%s' "$value" | sed "s/'/'\\\\''/g")" +} + +load_shell_path() { + if [ -f "$HOME/.bashrc" ]; then + # shellcheck source=/dev/null + source "$HOME/.bashrc" 2>/dev/null || true + fi + export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" + if [ -s "$NVM_DIR/nvm.sh" ]; then + # shellcheck source=/dev/null + . "$NVM_DIR/nvm.sh" + fi + if [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]]; then + export PATH="$HOME/.local/bin:$PATH" + fi +} + +parse_chat_content() { + python3 -c ' +import json +import sys +try: + doc = json.load(sys.stdin) + message = doc["choices"][0]["message"] + content = message.get("content") or message.get("reasoning_content") or "" + print(content.strip()) +except Exception as exc: + print(f"PARSE_ERROR: {exc}", file=sys.stderr) + sys.exit(1) +' +} + +http_status_from_response() { + sed -n 's/^__NEMOCLAW_HTTP_STATUS__=//p' | tail -1 +} + +http_body_from_response() { + sed '/^__NEMOCLAW_HTTP_STATUS__=/d' +} + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +if [ -d /workspace ] && [ -f /workspace/install.sh ]; then + REPO="/workspace" +elif [ -f "${SCRIPT_DIR}/../../install.sh" ]; then + REPO="$(cd "${SCRIPT_DIR}/../.." && pwd)" +elif [ -f "./install.sh" ]; then + REPO="$(pwd)" +else + echo "ERROR: Cannot find repo root." + exit 1 +fi + +cli_command_available_from_source() { + [ -f "$REPO/dist/nemoclaw.js" ] && command -v node >/dev/null 2>&1 && command -v openshell >/dev/null 2>&1 +} + +run_nemoclaw_cli_with_timeout() { + local seconds="$1" + shift + if cli_command_available_from_source; then + run_with_timeout "$seconds" node "$REPO/bin/nemoclaw.js" "$@" + elif command -v nemoclaw >/dev/null 2>&1; then + run_with_timeout "$seconds" nemoclaw "$@" + else + return 127 + fi +} + +destroy_sandbox_best_effort() { + local sandbox="$1" + if [ "${NEMOCLAW_E2E_KEEP_SANDBOX:-}" = "1" ] || [ "${NEMOCLAW_COMMON_EGRESS_KEEP_SANDBOX:-}" = "1" ]; then + return 0 + fi + run_nemoclaw_cli_with_timeout 120 "$sandbox" destroy --yes >/dev/null 2>&1 || true + if command -v openshell >/dev/null 2>&1; then + run_with_timeout 60 openshell sandbox delete "$sandbox" >/dev/null 2>&1 || true + fi +} + +cleanup_all() { + local sandbox + [ -z "$SANDBOXES_TO_CLEAN" ] && return 0 + while IFS= read -r sandbox; do + [ -z "$sandbox" ] && continue + destroy_sandbox_best_effort "$sandbox" + done <"$log" 2>&1 || rc=$? + else + info "Onboarding ${sandbox} via install.sh (agent=${agent}, tier=${tier})" + run_with_timeout 1800 bash "$REPO/install.sh" --non-interactive --yes-i-accept-third-party-software --fresh >"$log" 2>&1 || rc=$? + load_shell_path + fi + + if [ "$rc" -eq 0 ]; then + pass "onboard completed for ${sandbox} (${agent}, ${tier})" + else + fail "onboard failed for ${sandbox} (${agent}, ${tier}); tail of ${log}:" + tail -80 "$log" 2>/dev/null || true + summary + fi +} + +assert_policy_contains() { + local sandbox="$1" + shift + local label="$1" + shift + local policy_output rc=0 missing=() + policy_output=$(openshell policy get --full "$sandbox" 2>&1) || rc=$? + if [ "$rc" -ne 0 ]; then + fail "${label}: openshell policy get failed for ${sandbox} (exit ${rc})" + return + fi + local needle + for needle in "$@"; do + if ! grep -Fq "$needle" <<<"$policy_output"; then + missing+=("$needle") + fi + done + if [ "${#missing[@]}" -eq 0 ]; then + pass "${label}: expected policy endpoints are present" + else + fail "${label}: missing policy entries for ${sandbox}: ${missing[*]}" + fi +} + +assert_policy_absent() { + local sandbox="$1" + local label="$2" + local needle="$3" + local policy_output rc=0 + policy_output=$(openshell policy get --full "$sandbox" 2>&1) || rc=$? + if [ "$rc" -ne 0 ]; then + fail "${label}: openshell policy get failed for ${sandbox} (exit ${rc})" + return + fi + if grep -Fq "$needle" <<<"$policy_output"; then + fail "${label}: unexpected policy entry '${needle}' found in ${sandbox}" + else + pass "${label}: '${needle}' is not present" + fi +} + +run_openclaw_agent_assertion() { + local sandbox="$1" + local label="$2" + local prompt="$3" + local expected="$4" + local ssh_cfg raw reply rc=0 session_id remote_cmd stderr_file stderr_text combined log_file attempt last_fail recover_rc + log_file="/tmp/nemoclaw-e2e-common-egress-${sandbox}-agent.log" + + ssh_cfg="$(mktemp)" + if ! openshell sandbox ssh-config "$sandbox" >"$ssh_cfg" 2>/dev/null; then + rm -f "$ssh_cfg" + fail "${label}: could not get SSH config" + return + fi + + last_fail="" + for attempt in 1 2 3; do + rc=0 + stderr_file="$(mktemp)" + session_id="e2e-common-egress-$(date +%s)-$$-${attempt}" + remote_cmd="rm -f /sandbox/.openclaw/agents/main/sessions/$(quote_for_remote_sh "${session_id}.jsonl.lock") /sandbox/.openclaw/agents/main/sessions/$(quote_for_remote_sh "${session_id}.trajectory.jsonl") 2>/dev/null || true; openclaw agent --agent main --json --thinking off --session-id $(quote_for_remote_sh "$session_id") -m $(quote_for_remote_sh "$prompt")" + raw=$(run_with_timeout 180 ssh -F "$ssh_cfg" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o LogLevel=ERROR \ + "openshell-${sandbox}" \ + "$remote_cmd" \ + 2>"$stderr_file") || rc=$? + stderr_text="$(cat "$stderr_file" 2>/dev/null || true)" + combined="${raw} +${stderr_text}" + { + printf '=== %s attempt=%s rc=%s ===\n' "$label" "$attempt" "$rc" + printf '%s\n' '--- stdout ---' + printf '%s\n' "$raw" + printf '%s\n' '--- stderr ---' + printf '%s\n' "$stderr_text" + } >>"$log_file" + rm -f "$stderr_file" + + if printf '%s' "$combined" | grep -qiE "SsrFBlockedError|Blocked hostname"; then + rm -f "$ssh_cfg" + fail "${label}: agent hit policy block (exit ${rc}): ${combined:0:300}" + return + fi + + reply=$(printf '%s' "$raw" | parse_openclaw_agent_text 2>/dev/null) || true + if [ "$rc" -eq 0 ] && grep -Fq "$expected" <<<"$reply"; then + rm -f "$ssh_cfg" + pass "${label}: OpenClaw agent returned ${expected}" + return + fi + last_fail="reply='${reply:0:240}' (exit ${rc}, raw='${raw:0:240}', stderr='${stderr_text:0:240}')" + + if [ "$attempt" -lt 3 ] && printf '%s' "$combined" | grep -qiE "scope upgrade pending approval|pairing required: device is asking for more scopes"; then + info "${label}: pending OpenClaw scope upgrade detected; running recover before retry" + recover_rc=0 + { + printf '=== %s recover after attempt=%s ===\n' "$label" "$attempt" + } >>"$log_file" + run_nemoclaw_cli_with_timeout 120 "$sandbox" recover >>"$log_file" 2>&1 || recover_rc=$? + if [ "$recover_rc" -ne 0 ]; then + info "${label}: recover exited ${recover_rc}; retrying agent turn" + fi + sleep $((attempt * 15)) + continue + fi + + if [ "$attempt" -lt 3 ] && printf '%s' "$combined" | grep -qiE "ECONNREFUSED|EAI_AGAIN|ECONNRESET|ETIMEDOUT|gateway unavailable|network connection error|DNS error|fetch failed|LLM request timed out|FailoverError|inference service unavailable|rawError=503"; then + info "${label}: transient agent/inference error detected; retrying after backoff" + sleep $((attempt * 15)) + continue + fi + + [ "$attempt" -ge 3 ] || sleep 5 + done + + rm -f "$ssh_cfg" + fail "${label}: expected ${expected}, got ${last_fail}" +} + +run_hermes_agent_assertion() { + local sandbox="$1" + local label="$2" + local prompt="$3" + local expected="$4" + local payload response reply rc=0 model remote attempt last_fail http_code body log_file + model="${NEMOCLAW_MODEL:-nvidia/nemotron-3-super-120b-a12b}" + log_file="/tmp/nemoclaw-e2e-common-egress-${sandbox}-agent.log" + payload=$( + MODEL="$model" PROMPT="$prompt" python3 - <<'PY' +import json +import os + +print(json.dumps({ + "model": os.environ["MODEL"], + "messages": [{"role": "user", "content": os.environ["PROMPT"]}], + "max_tokens": 300, +})) +PY + ) + remote="set -a; [ ! -f /sandbox/.hermes/.env ] || . /sandbox/.hermes/.env; set +a; tmp=\$(mktemp); if [ -n \"\${API_SERVER_KEY:-}\" ]; then code=\$(curl -sS -o \"\$tmp\" -w '%{http_code}' --max-time 120 http://localhost:8642/v1/chat/completions -H 'Content-Type: application/json' -H \"Authorization: Bearer \${API_SERVER_KEY}\" -d $(quote_for_remote_sh "$payload")); else code=\$(curl -sS -o \"\$tmp\" -w '%{http_code}' --max-time 120 http://localhost:8642/v1/chat/completions -H 'Content-Type: application/json' -d $(quote_for_remote_sh "$payload")); fi; rc=\$?; cat \"\$tmp\"; rm -f \"\$tmp\"; printf '\n__NEMOCLAW_HTTP_STATUS__=%s\n' \"\${code:-000}\"; exit \"\$rc\"" + last_fail="" + + for attempt in 1 2 3; do + rc=0 + response=$(run_with_timeout 150 openshell sandbox exec --name "$sandbox" -- sh -lc "$remote" 2>&1) || rc=$? + http_code=$(printf '%s' "$response" | http_status_from_response) + [ -n "$http_code" ] || http_code="000" + body=$(printf '%s' "$response" | http_body_from_response) + reply=$(printf '%s' "$body" | parse_chat_content 2>/dev/null) || true + { + printf '=== %s attempt=%s rc=%s http=%s ===\n' "$label" "$attempt" "$rc" "$http_code" + printf '%s\n' "$response" + } >>"$log_file" + + if [ "$rc" -eq 0 ] && [ "$http_code" = "200" ] && grep -Fq "$expected" <<<"$reply"; then + pass "${label}: Hermes agent returned ${expected}" + return + fi + last_fail="exit ${rc}, HTTP ${http_code}, reply='${reply:0:240}', raw='${body:0:240}'" + [ "$attempt" -ge 3 ] || sleep 5 + done + + fail "${label}: expected ${expected}, got ${last_fail}" +} + +trap cleanup_all EXIT + +echo "" +echo "============================================================" +echo " Common Egress Agent E2E" +echo " $(date)" +echo "============================================================" + +# shellcheck source=test/e2e-vpn/lib/ci-compatible-inference.sh +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/lib/ci-compatible-inference.sh" +nemoclaw_e2e_configure_compatible_inference || summary + +section "Phase 0: Prerequisites" +load_shell_path +info "Repo: $REPO" + +if ! docker info >/dev/null 2>&1; then + fail "Docker is not running" + summary +fi +pass "Docker is running" + +if ! nemoclaw_e2e_require_hosted_inference_key; then + summary +fi + +if [ "${NEMOCLAW_NON_INTERACTIVE:-}" != "1" ]; then + fail "NEMOCLAW_NON_INTERACTIVE=1 is required" + summary +fi +pass "NEMOCLAW_NON_INTERACTIVE=1" + +if [ "${NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE:-}" != "1" ]; then + fail "NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 is required" + summary +fi +pass "Third-party software acceptance env is set" + +if [ "${NEMOCLAW_COMMON_EGRESS_SKIP_OPENCLAW:-}" != "1" ]; then + section "Phase 1: OpenClaw balanced weather" + OPENCLAW_BALANCED_SANDBOX="${NEMOCLAW_COMMON_EGRESS_OPENCLAW_BALANCED_SANDBOX:-e2e-common-egress-openclaw-balanced}" + run_onboard "$OPENCLAW_BALANCED_SANDBOX" "openclaw" "balanced" + assert_policy_contains "$OPENCLAW_BALANCED_SANDBOX" "C1 policy" "api.open-meteo.com" "geocoding-api.open-meteo.com" + assert_policy_absent "$OPENCLAW_BALANCED_SANDBOX" "C1 balanced scope" "restcountries.com" + WEATHER_AGENT_PROMPT=$( + cat <<'PROMPT' +Use the web_fetch tool to fetch exactly this URL: +https://api.open-meteo.com/v1/forecast?latitude=47.4979&longitude=19.0402¤t=temperature_2m +After web_fetch returns, reply exactly WEATHER_AGENT_OK if the fetched response contains temperature_2m. Do not fetch any other URL. +PROMPT + ) + run_openclaw_agent_assertion "$OPENCLAW_BALANCED_SANDBOX" "C1 agent weather" "$WEATHER_AGENT_PROMPT" "WEATHER_AGENT_OK" + + section "Phase 2: OpenClaw open public reference" + OPENCLAW_OPEN_SANDBOX="${NEMOCLAW_COMMON_EGRESS_OPENCLAW_OPEN_SANDBOX:-e2e-common-egress-openclaw-open}" + run_onboard "$OPENCLAW_OPEN_SANDBOX" "openclaw" "open" + assert_policy_contains "$OPENCLAW_OPEN_SANDBOX" "C2 policy" "www.wikidata.org" "nominatim.openstreetmap.org" "query.wikidata.org" + REFERENCE_AGENT_PROMPT=$( + cat <<'PROMPT' +Use the web_fetch tool to fetch exactly this URL: +https://www.wikidata.org/w/api.php?action=wbgetentities&ids=Q30&props=labels&languages=en&format=json +After web_fetch returns, reply exactly REFERENCE_AGENT_OK if the fetched response says entity Q30 has the English label United States. Do not fetch any other URL. +PROMPT + ) + run_openclaw_agent_assertion "$OPENCLAW_OPEN_SANDBOX" "C2 agent reference" "$REFERENCE_AGENT_PROMPT" "REFERENCE_AGENT_OK" +else + skip "OpenClaw common-egress phases skipped by NEMOCLAW_COMMON_EGRESS_SKIP_OPENCLAW=1" +fi + +if [ "${NEMOCLAW_COMMON_EGRESS_SKIP_HERMES:-}" != "1" ]; then + section "Phase 3: Hermes open public reference" + HERMES_SANDBOX="${NEMOCLAW_COMMON_EGRESS_HERMES_SANDBOX:-e2e-common-egress-hermes-open}" + run_onboard "$HERMES_SANDBOX" "hermes" "open" + assert_policy_contains "$HERMES_SANDBOX" "C3 common policy" "www.wikidata.org" "api.open-meteo.com" + assert_policy_contains "$HERMES_SANDBOX" "C3 Hermes Nous policy" "/firecrawl" "/fal-queue" "/openai-audio" "/browser-use" "/modal" + HERMES_REFERENCE_AGENT_PROMPT=$( + cat <<'PROMPT' +Use your terminal tool to run this Python check exactly once: +python3 - <<'PY' +import json +import urllib.request + +url = "https://www.wikidata.org/w/api.php?action=wbgetentities&ids=Q30&props=labels&languages=en&format=json" +with urllib.request.urlopen(url, timeout=20) as response: + doc = json.load(response) +ok = doc.get("success") == 1 and doc.get("entities", {}).get("Q30", {}).get("labels", {}).get("en", {}).get("value") == "United States" +print("HERMES_REFERENCE_AGENT_OK" if ok else "HERMES_REFERENCE_AGENT_BAD") +PY +After the command completes, reply exactly HERMES_REFERENCE_AGENT_OK if that exact token appeared. Do not fetch any other URL. +PROMPT + ) + run_hermes_agent_assertion "$HERMES_SANDBOX" "C3 agent reference" "$HERMES_REFERENCE_AGENT_PROMPT" "HERMES_REFERENCE_AGENT_OK" +else + skip "Hermes common-egress phase skipped by NEMOCLAW_COMMON_EGRESS_SKIP_HERMES=1" +fi + +trap - EXIT +cleanup_all +summary diff --git a/test/e2e-vpn/test-concurrent-gateway-ports.sh b/test/e2e-vpn/test-concurrent-gateway-ports.sh new file mode 100755 index 00000000000..e1386b7140f --- /dev/null +++ b/test/e2e-vpn/test-concurrent-gateway-ports.sh @@ -0,0 +1,307 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Concurrent gateway ports — exercises multiple NemoClaw-managed sandboxes on a +# single host with fully segregated gateways, dashboards, and registries. A +# second onboard with NEMOCLAW_GATEWAY_PORT set to a non-default port must not +# touch the first sandbox's gateway process, dashboard SSH forward, or sandbox +# container. +# +# Scenario shape: +# 1. Onboard sandbox A on the default gateway port (8080) + default dashboard +# port (18789). +# 2. Onboard sandbox B with NEMOCLAW_GATEWAY_PORT set to a non-default port +# that drives the per-port binding path. The dashboard port should +# auto-allocate from the 18789-18799 range without colliding with A. +# 3. Verify both sandboxes coexist: distinct gateways, distinct dashboards, +# distinct sandbox containers, no SIGKILL of A during B's onboard, and +# nemoclaw list reports two entries with two distinct dashboard URLs. +# 4. Destroy B and verify A remains healthy. +# +# This script intentionally uses a local fake OpenAI-compatible endpoint so it +# does not depend on real NVIDIA endpoints, matching the pattern in +# test-double-onboard.sh. + +# ShellCheck cannot see EXIT trap invocations of cleanup helpers in this E2E script. +# shellcheck disable=SC2317 +set -uo pipefail + +export NEMOCLAW_E2E_DEFAULT_TIMEOUT=4800 +SCRIPT_DIR_TIMEOUT="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +# shellcheck source=test/e2e-vpn/e2e-timeout.sh +source "${SCRIPT_DIR_TIMEOUT}/e2e-timeout.sh" + +PASS=0 +FAIL=0 +TOTAL=0 + +pass() { + ((PASS++)) + ((TOTAL++)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} +fail() { + ((FAIL++)) + ((TOTAL++)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } + +PHASE_TIMEOUT="${NEMOCLAW_E2E_PHASE_TIMEOUT:-1200}" + +SANDBOX_A="e2e-cgp-a" +SANDBOX_B="e2e-cgp-b" +GATEWAY_PORT_A=8080 +GATEWAY_PORT_B="${NEMOCLAW_E2E_GATEWAY_PORT_B:-18080}" +DASHBOARD_PORT_A=18789 + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +# shellcheck source=test/e2e-vpn/lib/openai-compatible-api-proof.sh +source "${SCRIPT_DIR}/lib/openai-compatible-api-proof.sh" +FAKE_OPENAI_HOST="127.0.0.1" +FAKE_OPENAI_PORT="${NEMOCLAW_E2E_FAKE_PORT:-18180}" +FAKE_OPENAI_LOG="$(mktemp)" +FAKE_BASE_URL="http://${FAKE_OPENAI_HOST}:${FAKE_OPENAI_PORT}/v1" + +if command -v node >/dev/null 2>&1 && [ -f "$REPO_ROOT/bin/nemoclaw.js" ]; then + NEMOCLAW_CMD=(node "$REPO_ROOT/bin/nemoclaw.js") +else + NEMOCLAW_CMD=(nemoclaw) +fi + +# shellcheck disable=SC2329 +cleanup() { + stop_fake_openai_compatible_api + rm -f "$FAKE_OPENAI_LOG" +} +trap cleanup EXIT + +start_fake_openai() { + if start_fake_openai_compatible_api; then + FAKE_BASE_URL="$FAKE_OPENAI_BASE_URL" + info "Fake OpenAI server up on ${FAKE_BASE_URL} (pid ${FAKE_OPENAI_PID})" + return 0 + fi + fail "Fake OpenAI server did not become ready on ${FAKE_BASE_URL}; see ${FAKE_OPENAI_LOG}" + cat "$FAKE_OPENAI_LOG" + exit 1 +} + +dashboard_port_from_list() { + local sandbox="$1" + "${NEMOCLAW_CMD[@]}" list 2>/dev/null \ + | awk -v want="${sandbox}" ' + /^[[:space:]]+[A-Za-z0-9_-]+( \*)?[[:space:]]*$/ { + name=$1 + inblock=(name == want) ? 1 : 0 + next + } + inblock && /dashboard:[[:space:]]*http:\/\/[0-9.]+:[0-9]+/ { + match($0, /:[0-9]+/) + print substr($0, RSTART+1, RLENGTH-1) + exit + } + ' +} + +dump_diagnostics() { + local label="${1:-unknown}" + info "=== Diagnostics for ${label} ===" + info "nemoclaw list:" + "${NEMOCLAW_CMD[@]}" list 2>&1 | sed 's/^/ /' || true + info "openshell sandbox list:" + openshell sandbox list 2>&1 | sed 's/^/ /' || true + info "openshell forward list:" + openshell forward list 2>&1 | sed 's/^/ /' || true + info "docker ps -a:" + docker ps -a --format 'table {{.Names}}\t{{.Status}}' 2>&1 | sed 's/^/ /' || true + info "ss -ltn (gateway/dashboard ports):" + ss -ltn 2>&1 | grep -E ":(${GATEWAY_PORT_A}|${GATEWAY_PORT_B}|1878[0-9]|1879[0-9])" | sed 's/^/ /' || true +} + +onboard_sandbox() { + local name="$1" + local gateway_port="$2" + local label="onboard-${name}" + local start_time + start_time="$(date +%s)" + info "Starting onboard of '${name}' with NEMOCLAW_GATEWAY_PORT=${gateway_port}" + if COMPATIBLE_API_KEY=dummy \ + NEMOCLAW_NON_INTERACTIVE=1 \ + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ + NEMOCLAW_PROVIDER=custom \ + NEMOCLAW_ENDPOINT_URL="${FAKE_BASE_URL}" \ + NEMOCLAW_MODEL=test-model \ + NEMOCLAW_POLICY_MODE=skip \ + NEMOCLAW_DASHBOARD_PORT='' \ + CHAT_UI_URL='' \ + NEMOCLAW_GATEWAY_PORT="${gateway_port}" \ + NEMOCLAW_SANDBOX_NAME="${name}" \ + timeout "${PHASE_TIMEOUT}" "${NEMOCLAW_CMD[@]}" onboard --non-interactive \ + >"/tmp/${name}-onboard.log" 2>&1; then + local elapsed + elapsed=$(($(date +%s) - start_time)) + pass "${label} completed in ${elapsed}s" + return 0 + fi + fail "${label} did not complete within ${PHASE_TIMEOUT}s" + dump_diagnostics "${label}" + tail -200 "/tmp/${name}-onboard.log" | sed 's/^/ /' + return 1 +} + +destroy_default_install_sandbox() { + local default_name + default_name="$("${NEMOCLAW_CMD[@]}" list 2>/dev/null \ + | grep -E '^[[:space:]]+[a-zA-Z0-9_-]+ \*' \ + | awk '{print $1}' \ + | head -1 || true)" + if [ -z "${default_name}" ]; then + info "no pre-existing default sandbox to destroy" + return 0 + fi + if [ "${default_name}" = "${SANDBOX_A}" ] || [ "${default_name}" = "${SANDBOX_B}" ]; then + info "default sandbox is one under test (${default_name}); skipping pre-destroy" + return 0 + fi + info "destroying pre-existing default sandbox '${default_name}' (created by install.sh)" + if NEMOCLAW_NON_INTERACTIVE=1 timeout 300 "${NEMOCLAW_CMD[@]}" "${default_name}" destroy --yes \ + >"/tmp/${default_name}-predestroy.log" 2>&1; then + pass "pre-existing default sandbox '${default_name}' destroyed" + else + fail "could not destroy pre-existing default sandbox '${default_name}'" + tail -100 "/tmp/${default_name}-predestroy.log" | sed 's/^/ /' + return 1 + fi +} + +gateway_name_for_port() { + local port="$1" + if [ "${port}" = "8080" ]; then + echo "nemoclaw" + else + echo "nemoclaw-${port}" + fi +} + +sandbox_phase() { + local name="$1" + local gateway="${2:-}" + local args=("sandbox" "list") + if [ -n "${gateway}" ]; then + args+=("-g" "${gateway}") + fi + openshell "${args[@]}" 2>/dev/null \ + | sed 's/\x1b\[[0-9;]*m//g' \ + | awk -v want="${name}" '$1 == want { print $NF; exit }' +} + +verify_sandbox_alive() { + local name="$1" + local label="${2:-${name} alive}" + local gateway="${3:-}" + local retries="${4:-12}" + local phase="" + for _ in $(seq 1 "${retries}"); do + phase="$(sandbox_phase "${name}" "${gateway}")" + case "${phase}" in + Ready | Running) + pass "${label} (phase=${phase})" + return 0 + ;; + Error | Failed | CrashLoopBackOff) + fail "${label} terminal (phase='${phase}')" + return 1 + ;; + esac + sleep 5 + done + fail "${label} did not reach Ready/Running within ${retries} polls (last phase='${phase:-missing}')" + return 1 +} + +# === Scenario === + +section "Stage 0: prepare fake inference endpoint" +start_fake_openai + +section "Stage 0.5: destroy default sandbox created by install.sh (if any)" +destroy_default_install_sandbox || exit 1 + +section "Stage 1: onboard sandbox A on default gateway port (${GATEWAY_PORT_A})" +GATEWAY_A_NAME="$(gateway_name_for_port "${GATEWAY_PORT_A}")" +GATEWAY_B_NAME="$(gateway_name_for_port "${GATEWAY_PORT_B}")" +onboard_sandbox "${SANDBOX_A}" "${GATEWAY_PORT_A}" || exit 1 +verify_sandbox_alive "${SANDBOX_A}" "Sandbox A reaches Ready/Running on default port" "${GATEWAY_A_NAME}" + +DASHBOARD_A="$(dashboard_port_from_list "${SANDBOX_A}")" +if [ -n "${DASHBOARD_A}" ] && [ "${DASHBOARD_A}" = "${DASHBOARD_PORT_A}" ]; then + pass "Sandbox A holds default dashboard port ${DASHBOARD_PORT_A}" +else + fail "Sandbox A dashboard port is '${DASHBOARD_A:-missing}', expected ${DASHBOARD_PORT_A}" +fi + +section "Stage 2: onboard sandbox B with NEMOCLAW_GATEWAY_PORT=${GATEWAY_PORT_B}" +onboard_sandbox "${SANDBOX_B}" "${GATEWAY_PORT_B}" || { + info "B onboard failed; capturing pre-fail state of A for diagnostics" + dump_diagnostics "stage-2-onboard-B" + exit 1 +} + +section "Stage 3: assert both sandboxes coexist" +verify_sandbox_alive "${SANDBOX_A}" "Sandbox A still alive after B's onboard" "${GATEWAY_A_NAME}" +verify_sandbox_alive "${SANDBOX_B}" "Sandbox B reaches Ready/Running on per-port gateway" "${GATEWAY_B_NAME}" + +DASHBOARD_B="$(dashboard_port_from_list "${SANDBOX_B}")" +if [ -n "${DASHBOARD_B}" ] && [ "${DASHBOARD_B}" != "${DASHBOARD_A:-${DASHBOARD_PORT_A}}" ]; then + pass "Sandbox B got a distinct dashboard port (A=${DASHBOARD_A:-missing} B=${DASHBOARD_B})" +else + fail "Sandbox B dashboard port collides with A: A=${DASHBOARD_A:-missing} B=${DASHBOARD_B:-missing}" + dump_diagnostics "dashboard-port-collision" +fi + +if ss -ltn 2>/dev/null | grep -qE ":${GATEWAY_PORT_A}\\b"; then + pass "Sandbox A gateway port ${GATEWAY_PORT_A} still listening" +else + fail "Sandbox A gateway port ${GATEWAY_PORT_A} no longer listening — recreate destroyed first gateway" + dump_diagnostics "gateway-port-A-missing" +fi + +if ss -ltn 2>/dev/null | grep -qE ":${GATEWAY_PORT_B}\\b"; then + pass "Sandbox B gateway port ${GATEWAY_PORT_B} listening" +else + fail "Sandbox B gateway port ${GATEWAY_PORT_B} not listening" + dump_diagnostics "gateway-port-B-missing" +fi + +LIST_OUTPUT="$("${NEMOCLAW_CMD[@]}" list 2>&1 || true)" +if echo "${LIST_OUTPUT}" | grep -qE "^[[:space:]]+${SANDBOX_A}( \*)?[[:space:]]*$" \ + && echo "${LIST_OUTPUT}" | grep -qE "^[[:space:]]+${SANDBOX_B}( \*)?[[:space:]]*$"; then + pass "nemoclaw list shows both sandbox A and B" +else + fail "nemoclaw list missing one of A/B" + # shellcheck disable=SC2001 + echo "${LIST_OUTPUT}" | sed 's/^/ /' +fi + +section "Stage 4: destroy sandbox B; assert sandbox A still healthy" +if NEMOCLAW_NON_INTERACTIVE=1 timeout 300 "${NEMOCLAW_CMD[@]}" "${SANDBOX_B}" destroy --yes \ + >"/tmp/${SANDBOX_B}-destroy.log" 2>&1; then + pass "Sandbox B destroyed" +else + fail "Sandbox B destroy timed out or failed" + tail -100 "/tmp/${SANDBOX_B}-destroy.log" | sed 's/^/ /' +fi +verify_sandbox_alive "${SANDBOX_A}" "Sandbox A still alive after B's destroy" "${GATEWAY_A_NAME}" + +section "Summary: PASS=${PASS} FAIL=${FAIL} TOTAL=${TOTAL}" +if [ "${FAIL}" -gt 0 ]; then + exit 1 +fi +exit 0 diff --git a/test/e2e-vpn/test-credential-migration.sh b/test/e2e-vpn/test-credential-migration.sh new file mode 100755 index 00000000000..62ec1c4e88e --- /dev/null +++ b/test/e2e-vpn/test-credential-migration.sh @@ -0,0 +1,302 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Credential Migration E2E +# +# Validates the host-side credential storage hardening: +# +# 1. A pre-existing plaintext ~/.nemoclaw/credentials.json from an earlier +# release is staged into process.env at onboard time and the value is +# registered with the OpenShell gateway. The legacy file is then +# securely removed (zero-filled, then unlinked) — only after a +# successful onboard, so an interrupted run can be retried without +# losing the user's only copy. +# +# 2. The migration loop is gated on KNOWN_CREDENTIAL_ENV_KEYS so a stale +# or tampered credentials.json cannot inject unrelated variables (PATH, +# NODE_OPTIONS, OPENSHELL_GATEWAY) into the onboard process. +# +# 3. After a normal env-var-driven onboard, no plaintext credentials.json +# exists under ~/.nemoclaw/. +# +# 4. `nemoclaw credentials list` reports providers from the OpenShell +# gateway, not from disk. +# +# 5. If ~/.nemoclaw/credentials.json exists as a symlink to an unrelated +# file, the secure-unlink path removes the symlink without touching +# the target. +# +# This test deliberately lays down legacy state under the runner's HOME, so +# it should run on an ephemeral CI runner. Local dev runs are destructive +# to ~/.nemoclaw/ — set NEMOCLAW_E2E_KEEP_SANDBOX=1 to skip the teardown +# and inspect post-mortem. +# +# Prerequisites: +# - Docker running +# - openshell + nemoclaw on PATH +# - NVIDIA_API_KEY set (used as the migrated value) +# +# Usage: +# NEMOCLAW_NON_INTERACTIVE=1 NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ +# NVIDIA_API_KEY=nvapi-... bash test/e2e-vpn/test-credential-migration.sh + +set -uo pipefail + +export NEMOCLAW_E2E_DEFAULT_TIMEOUT=2400 +SCRIPT_DIR_TIMEOUT="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +# shellcheck source=test/e2e-vpn/e2e-timeout.sh +source "${SCRIPT_DIR_TIMEOUT}/e2e-timeout.sh" + +PASS=0 +FAIL=0 +TOTAL=0 + +pass() { + ((PASS++)) + ((TOTAL++)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} +fail() { + ((FAIL++)) + ((TOTAL++)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } +indent() { awk '{print " " $0}'; } + +# Resolve repo root the same way the other E2E scripts do. +if [ -d /workspace ] && [ -f /workspace/install.sh ]; then + REPO="/workspace" +elif [ -f "$(cd "$(dirname "$0")/../.." && pwd)/install.sh" ]; then + REPO="$(cd "$(dirname "$0")/../.." && pwd)" +else + echo "ERROR: Cannot find repo root." + exit 1 +fi + +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-cred-migration}" + +# shellcheck source=test/e2e-vpn/lib/sandbox-teardown.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib/sandbox-teardown.sh" +register_sandbox_for_teardown "$SANDBOX_NAME" + +# shellcheck source=test/e2e-vpn/lib/install-path-refresh.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib/install-path-refresh.sh" + +# ══════════════════════════════════════════════════════════════════ +# Phase 0: Prerequisites +# ══════════════════════════════════════════════════════════════════ +section "Phase 0: Prerequisites" + +if [ -z "${NVIDIA_API_KEY:-}" ]; then + fail "NVIDIA_API_KEY not set" + exit 1 +fi +pass "NVIDIA_API_KEY is set" + +if ! command -v openshell >/dev/null 2>&1 || ! command -v nemoclaw >/dev/null 2>&1; then + info "openshell or nemoclaw not found; running install" + bash "$REPO/install.sh" --yes-i-accept-third-party-software \ + >/tmp/nemoclaw-e2e-install.log 2>&1 || { + fail "install.sh failed; see /tmp/nemoclaw-e2e-install.log" + exit 1 + } + # Refresh PATH so install.sh-managed binaries are visible + nemoclaw_refresh_install_env +fi + +command -v openshell >/dev/null 2>&1 || { + fail "openshell still missing after install" + exit 1 +} +command -v nemoclaw >/dev/null 2>&1 || { + fail "nemoclaw still missing after install" + exit 1 +} +pass "openshell + nemoclaw on PATH" + +REAL_API_KEY="$NVIDIA_API_KEY" +NEMOCLAW_DIR="$HOME/.nemoclaw" +LEGACY_FILE="$NEMOCLAW_DIR/credentials.json" + +# ══════════════════════════════════════════════════════════════════ +# Phase 1: Pre-seed a legacy credentials.json and verify migration +# ══════════════════════════════════════════════════════════════════ +section "Phase 1: Legacy credentials.json migration" + +# Start from a clean ~/.nemoclaw to avoid interference from prior runs. +rm -rf "$NEMOCLAW_DIR" +mkdir -p "$NEMOCLAW_DIR" +chmod 700 "$NEMOCLAW_DIR" + +# Tampered fixture: includes an unrelated key the migrator must ignore. +cat >"$LEGACY_FILE" </dev/null || stat -f '%i' "$LEGACY_FILE" 2>/dev/null || echo "") +[ -n "$LEGACY_INODE_BEFORE" ] && info "Legacy file inode before onboard: $LEGACY_INODE_BEFORE" + +# Run onboard WITHOUT NVIDIA_API_KEY in the env. The only place the value +# can come from is the legacy credentials.json — exactly the migration +# path we want to exercise. +ONBOARD_LOG="$(mktemp)" +( + unset NVIDIA_API_KEY + NEMOCLAW_NON_INTERACTIVE=1 \ + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ + NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" \ + NEMOCLAW_RECREATE_SANDBOX=1 \ + nemoclaw onboard --non-interactive >"$ONBOARD_LOG" 2>&1 +) & +ONBOARD_PID=$! +wait "$ONBOARD_PID" +ONBOARD_EXIT=$? + +if [ "$ONBOARD_EXIT" -eq 0 ]; then + pass "nemoclaw onboard succeeded with only the legacy file as the credential source" +else + fail "nemoclaw onboard failed (exit $ONBOARD_EXIT); see log below" + tail -50 "$ONBOARD_LOG" || true + rm -f "$ONBOARD_LOG" + exit 1 +fi + +if grep -q "Staged .* legacy credential" "$ONBOARD_LOG"; then + pass "Migration notice was emitted to stderr" +else + fail "Expected migration notice on stderr; not found in onboard log" + tail -30 "$ONBOARD_LOG" || true +fi +rm -f "$ONBOARD_LOG" + +# After a successful onboard, the legacy file must be gone. +if [ -e "$LEGACY_FILE" ]; then + fail "Legacy credentials.json still exists after successful onboard" +else + pass "Legacy credentials.json was removed after onboard" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 2: Verify the value reached the OpenShell gateway +# ══════════════════════════════════════════════════════════════════ +section "Phase 2: Gateway provider registration" + +if ! PROVIDERS_OUT=$(openshell -g nemoclaw provider list --names 2>&1); then + fail "openshell -g nemoclaw provider list --names failed" + printf '%s\n' "$PROVIDERS_OUT" | indent + exit 1 +fi +info "Providers in nemoclaw gateway:" +printf '%s\n' "$PROVIDERS_OUT" | indent + +# The legacy NVIDIA_API_KEY should have been registered as one of the +# inference providers (compatible-endpoint, nvidia-nim, etc. — the exact name +# depends on what onboarding chose). Just assert that at least one +# provider was registered. +PROVIDER_COUNT=$(echo "$PROVIDERS_OUT" | grep -E -c '^[a-zA-Z][a-zA-Z0-9_-]*$' || true) +if [ "$PROVIDER_COUNT" -ge 1 ]; then + pass "At least one provider is registered with the gateway ($PROVIDER_COUNT total)" +else + fail "No providers registered with the gateway after migration" +fi + +# Negative assertion: the unrelated keys from the tampered file must not +# have leaked anywhere observable. The strongest check available without +# spawning another nemoclaw process is to verify they are NOT registered +# as gateway provider names — since `openshell provider create +# --credential KEY` would have failed for non-allowlisted keys, but a bug +# could conceivably push them through. +if echo "$PROVIDERS_OUT" | grep -q "OPENSHELL_GATEWAY\|NODE_OPTIONS"; then + fail "A non-allowlisted key from the tampered file appears as a gateway provider" +else + pass "Non-allowlisted keys from the tampered file did not become providers" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 3: nemoclaw credentials list reads from the gateway, not disk +# ══════════════════════════════════════════════════════════════════ +section "Phase 3: nemoclaw credentials list" + +if ! CREDS_LIST_OUT=$(nemoclaw credentials list 2>&1); then + fail "nemoclaw credentials list failed" + printf '%s\n' "$CREDS_LIST_OUT" | indent + exit 1 +fi +info "Output:" +printf '%s\n' "$CREDS_LIST_OUT" | indent + +if echo "$CREDS_LIST_OUT" | grep -q "Providers registered with the OpenShell gateway"; then + pass "credentials list surfaces gateway-registered providers" +else + fail "credentials list did not produce the expected gateway header" +fi + +# The disk should still have NO plaintext credentials.json regardless of +# what the gateway holds. +if [ -e "$LEGACY_FILE" ]; then + fail "credentials.json reappeared on disk after credentials list" +else + pass "No plaintext credentials.json on disk after credentials list" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 4: Symlink-safe secure unlink +# ══════════════════════════════════════════════════════════════════ +section "Phase 4: Symlink-safe secure unlink" + +# Plant a symlink at the credentials path pointing at an unrelated victim +# file. A naive secureUnlink would zero-fill and unlink the target; the +# hardened path must remove the symlink itself and leave the target +# intact. +VICTIM_FILE="$(mktemp)" +VICTIM_PAYLOAD="important data the attacker should not touch" +printf '%s' "$VICTIM_PAYLOAD" >"$VICTIM_FILE" +ln -s "$VICTIM_FILE" "$LEGACY_FILE" + +# Drive removeLegacyCredentialsFile() directly via a tiny node one-liner. +# Using the compiled module from dist/ matches what the CLI imports. +node -e " +const { removeLegacyCredentialsFile } = require('${REPO}/dist/lib/credentials/store.js'); +removeLegacyCredentialsFile(); +" >/dev/null 2>&1 || { + fail "node invocation of removeLegacyCredentialsFile failed" +} + +if [ -L "$LEGACY_FILE" ] || [ -e "$LEGACY_FILE" ]; then + fail "Symlink at credentials path was not removed" +else + pass "Symlink at credentials path was removed" +fi + +if [ ! -e "$VICTIM_FILE" ]; then + fail "Victim file was deleted; secureUnlink followed the symlink" +elif [ "$(cat "$VICTIM_FILE")" != "$VICTIM_PAYLOAD" ]; then + fail "Victim file contents were modified; secureUnlink wrote through the symlink" +else + pass "Victim file is untouched (link removed without following the target)" +fi +rm -f "$VICTIM_FILE" + +# ══════════════════════════════════════════════════════════════════ +# Summary +# ══════════════════════════════════════════════════════════════════ +section "Summary" +echo " Total: $TOTAL" +echo " Passed: $PASS" +echo " Failed: $FAIL" + +if [ "$FAIL" -gt 0 ]; then + exit 1 +fi +exit 0 diff --git a/test/e2e-vpn/test-credential-sanitization.sh b/test/e2e-vpn/test-credential-sanitization.sh new file mode 100755 index 00000000000..3346206a044 --- /dev/null +++ b/test/e2e-vpn/test-credential-sanitization.sh @@ -0,0 +1,816 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Credential Sanitization & Blueprint Digest E2E Tests +# +# Validates that PR #156's fix correctly strips credentials from migration +# bundles and that empty blueprint digests are no longer silently accepted. +# +# Attack surface: +# Before the fix, createSnapshotBundle() copied the entire ~/.openclaw +# directory into the sandbox, including auth-profiles.json with live API +# keys, GitHub PATs, and npm tokens. A compromised agent could read these +# and exfiltrate them. Additionally, blueprint.yaml shipped with digest: "" +# which caused the integrity check to silently pass (JS falsy). +# +# Prerequisites: +# - Docker running +# - NemoClaw installed and sandbox running (test-full-e2e.sh Phase 0-3) +# - NVIDIA_API_KEY set +# - openshell on PATH +# +# Environment variables: +# NEMOCLAW_SANDBOX_NAME — sandbox name (default: e2e-test) +# NVIDIA_API_KEY — required +# +# Usage: +# NEMOCLAW_NON_INTERACTIVE=1 NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 NVIDIA_API_KEY=nvapi-... bash test/e2e-vpn/test-credential-sanitization.sh +# +# See: https://github.com/NVIDIA/NemoClaw/pull/156 + +set -uo pipefail + +PASS=0 +FAIL=0 +SKIP=0 +TOTAL=0 + +pass() { + ((PASS++)) + ((TOTAL++)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} +fail() { + ((FAIL++)) + ((TOTAL++)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} +skip() { + ((SKIP++)) + ((TOTAL++)) + printf '\033[33m SKIP: %s\033[0m\n' "$1" +} +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } + +# Determine repo root +if [ -d /workspace ] && [ -f /workspace/install.sh ]; then + REPO="/workspace" +elif [ -f "$(cd "$(dirname "$0")/../.." && pwd)/install.sh" ]; then + REPO="$(cd "$(dirname "$0")/../.." && pwd)" +else + echo "ERROR: Cannot find repo root." + exit 1 +fi + +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-test}" + +# shellcheck source=test/e2e-vpn/lib/sandbox-teardown.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib/sandbox-teardown.sh" +register_sandbox_for_teardown "$SANDBOX_NAME" + +# Run a command inside the sandbox and capture output. +# Returns __PROBE_FAILED__ and exit 1 if SSH setup or execution fails, +# so callers can distinguish "no output" from "probe never ran". +sandbox_exec() { + local cmd="$1" + local ssh_config + ssh_config="$(mktemp)" + if ! openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config" 2>/dev/null; then + rm -f "$ssh_config" + echo "__PROBE_FAILED__" + return 1 + fi + + local result + local rc=0 + result=$(timeout 60 ssh -F "$ssh_config" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" \ + "$cmd" \ + 2>&1) || rc=$? + + rm -f "$ssh_config" + if [ "$rc" -ne 0 ] && [ -z "$result" ]; then + echo "__PROBE_FAILED__" + return 1 + fi + echo "$result" +} + +# ══════════════════════════════════════════════════════════════════ +# Phase 0: Prerequisites +# ══════════════════════════════════════════════════════════════════ +section "Phase 0: Prerequisites" + +if [ -z "${NVIDIA_API_KEY:-}" ]; then + fail "NVIDIA_API_KEY not set" + exit 1 +fi +pass "NVIDIA_API_KEY is set" + +if ! command -v openshell >/dev/null 2>&1; then + fail "openshell not found on PATH" + exit 1 +fi +pass "openshell found" + +if ! command -v nemoclaw >/dev/null 2>&1; then + fail "nemoclaw not found on PATH" + exit 1 +fi +pass "nemoclaw found" + +if ! command -v node >/dev/null 2>&1; then + fail "node not found on PATH" + exit 1 +fi +pass "node found" + +# Verify sandbox is running +# shellcheck disable=SC2034 # status_output captures stderr for diagnostics on failure +if status_output=$(nemoclaw "$SANDBOX_NAME" status 2>&1); then + pass "Sandbox '${SANDBOX_NAME}' is running" +else + fail "Sandbox '${SANDBOX_NAME}' not running — run test-full-e2e.sh first" + exit 1 +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 1: Credential Stripping from Migration Bundles +# +# We create a mock ~/.openclaw directory with known fake credentials, +# then run the sanitization functions and verify the output. +# ══════════════════════════════════════════════════════════════════ +section "Phase 1: Credential Stripping (Unit-Level on Real Stack)" + +# Deliberately non-matching fake tokens that will NOT trigger secret scanners. +FAKE_NVIDIA_KEY="test-fake-nvidia-key-0000000000000000" +FAKE_GITHUB_TOKEN="test-fake-github-token-1111111111111111" +FAKE_NPM_TOKEN="test-fake-npm-token-2222222222222222" +FAKE_GATEWAY_TOKEN="test-fake-gateway-token-333333333333" + +# Create a temp directory simulating the state that would be migrated +MOCK_DIR=$(mktemp -d /tmp/nemoclaw-cred-test-XXXXXX) +MOCK_STATE="$MOCK_DIR/.openclaw" +mkdir -p "$MOCK_STATE" + +# Create openclaw.json with credential fields +cat >"$MOCK_STATE/openclaw.json" <"$AUTH_DIR/auth-profiles.json" <"$MOCK_STATE/workspace/project.md" + +# Copy to simulate bundle +BUNDLE_DIR="$MOCK_DIR/bundle/openclaw" +mkdir -p "$BUNDLE_DIR" +cp -r "$MOCK_STATE"/* "$BUNDLE_DIR/" 2>/dev/null || true +cp -r "$MOCK_STATE"/.[!.]* "$BUNDLE_DIR/" 2>/dev/null || true +# Actually copy the directory contents properly +rm -rf "$BUNDLE_DIR" +cp -r "$MOCK_STATE" "$BUNDLE_DIR" + +# Run the sanitization logic via node (mirrors production sanitizeCredentialsInBundle) +info "C1-C5: Running credential sanitization on mock bundle..." +sanitize_result=$(cd "$REPO" && node -e " +const fs = require('fs'); +const path = require('path'); + +// --- Credential field detection (mirrors migration-state.ts) --- +const CREDENTIAL_FIELDS = new Set([ + 'apiKey', 'api_key', 'token', 'secret', 'password', 'resolvedKey', +]); +const CREDENTIAL_FIELD_PATTERN = + /(?:access|refresh|client|bearer|auth|api|private|public|signing|session)(?:Token|Key|Secret|Password)$/; + +function isCredentialField(key) { + return CREDENTIAL_FIELDS.has(key) || CREDENTIAL_FIELD_PATTERN.test(key); +} + +function stripCredentials(obj) { + if (obj === null || obj === undefined) return obj; + if (typeof obj !== 'object') return obj; + if (Array.isArray(obj)) return obj.map(stripCredentials); + const result = {}; + for (const [key, value] of Object.entries(obj)) { + if (isCredentialField(key)) { + result[key] = '[STRIPPED_BY_MIGRATION]'; + } else { + result[key] = stripCredentials(value); + } + } + return result; +} + +function walkAndRemoveFile(dirPath, targetName) { + let entries; + try { entries = fs.readdirSync(dirPath); } catch { return; } + for (const entry of entries) { + const fullPath = path.join(dirPath, entry); + try { + const stat = fs.lstatSync(fullPath); + if (stat.isSymbolicLink()) continue; + if (stat.isDirectory()) { + walkAndRemoveFile(fullPath, targetName); + } else if (entry === targetName) { + fs.rmSync(fullPath, { force: true }); + } + } catch {} + } +} + +const bundleDir = '$BUNDLE_DIR'; + +// 1. Remove auth-profiles.json +const agentsDir = path.join(bundleDir, 'agents'); +if (fs.existsSync(agentsDir)) { + walkAndRemoveFile(agentsDir, 'auth-profiles.json'); +} + +// 2. Strip credential fields from openclaw.json +const configPath = path.join(bundleDir, 'openclaw.json'); +if (fs.existsSync(configPath)) { + const config = JSON.parse(fs.readFileSync(configPath, 'utf-8')); + const sanitized = stripCredentials(config); + fs.writeFileSync(configPath, JSON.stringify(sanitized, null, 2)); +} + +console.log('SANITIZED'); +" 2>&1) + +if echo "$sanitize_result" | grep -q "SANITIZED"; then + pass "Sanitization ran successfully" +else + fail "Sanitization script failed: ${sanitize_result:0:200}" +fi + +# C1: No nvapi- strings in the entire bundle +info "C1: Checking for API key leaks in bundle..." +nvapi_hits=$(grep -r "test-fake-nvidia-key" "$BUNDLE_DIR" 2>/dev/null || true) +if [ -z "$nvapi_hits" ]; then + pass "C1: No fake NVIDIA key found in bundle" +else + fail "C1: Fake NVIDIA key found in bundle: ${nvapi_hits:0:200}" +fi + +# Also check for the other fake tokens +github_hits=$(grep -r "test-fake-github-token" "$BUNDLE_DIR" 2>/dev/null || true) +npm_hits=$(grep -r "test-fake-npm-token" "$BUNDLE_DIR" 2>/dev/null || true) +gateway_hits=$(grep -r "test-fake-gateway-token" "$BUNDLE_DIR" 2>/dev/null || true) + +if [ -z "$github_hits" ] && [ -z "$npm_hits" ] && [ -z "$gateway_hits" ]; then + pass "C1b: No fake GitHub/npm/gateway tokens found in bundle" +else + fail "C1b: Fake tokens found — github: ${github_hits:0:80}, npm: ${npm_hits:0:80}, gateway: ${gateway_hits:0:80}" +fi + +# C2: auth-profiles.json must not exist anywhere in the bundle +info "C2: Checking for auth-profiles.json..." +auth_files=$(find "$BUNDLE_DIR" -name "auth-profiles.json" 2>/dev/null || true) +if [ -z "$auth_files" ]; then + pass "C2: auth-profiles.json deleted from bundle" +else + fail "C2: auth-profiles.json still exists: $auth_files" +fi + +# C3: openclaw.json credential fields must be [STRIPPED_BY_MIGRATION] +info "C3: Checking credential field sanitization in openclaw.json..." +config_content=$(cat "$BUNDLE_DIR/openclaw.json" 2>/dev/null || echo "{}") + +nvidia_apikey=$(echo "$config_content" | python3 -c " +import json, sys +config = json.load(sys.stdin) +print(config.get('nvidia', {}).get('apiKey', 'MISSING')) +" 2>/dev/null || echo "PARSE_ERROR") + +gateway_token=$(echo "$config_content" | python3 -c " +import json, sys +config = json.load(sys.stdin) +print(config.get('gateway', {}).get('auth', {}).get('token', 'MISSING')) +" 2>/dev/null || echo "PARSE_ERROR") + +if [ "$nvidia_apikey" = "[STRIPPED_BY_MIGRATION]" ]; then + pass "C3a: nvidia.apiKey replaced with sentinel" +else + fail "C3a: nvidia.apiKey not sanitized (got: $nvidia_apikey)" +fi + +if [ "$gateway_token" = "[STRIPPED_BY_MIGRATION]" ]; then + pass "C3b: gateway.auth.token replaced with sentinel" +else + fail "C3b: gateway.auth.token not sanitized (got: $gateway_token)" +fi + +# C4: Non-credential fields must be preserved +info "C4: Checking non-credential field preservation..." +model_primary=$(echo "$config_content" | python3 -c " +import json, sys +config = json.load(sys.stdin) +print(config.get('agents', {}).get('defaults', {}).get('model', {}).get('primary', 'MISSING')) +" 2>/dev/null || echo "PARSE_ERROR") + +gateway_mode=$(echo "$config_content" | python3 -c " +import json, sys +config = json.load(sys.stdin) +print(config.get('gateway', {}).get('mode', 'MISSING')) +" 2>/dev/null || echo "PARSE_ERROR") + +if [ "$model_primary" = "nvidia/nemotron-3-super-120b-a12b" ]; then + pass "C4a: agents.defaults.model.primary preserved" +else + fail "C4a: agents.defaults.model.primary corrupted (got: $model_primary)" +fi + +if [ "$gateway_mode" = "local" ]; then + pass "C4b: gateway.mode preserved" +else + fail "C4b: gateway.mode corrupted (got: $gateway_mode)" +fi + +# C5: Workspace files must be intact +info "C5: Checking workspace file integrity..." +if [ -f "$BUNDLE_DIR/workspace/project.md" ]; then + project_content=$(cat "$BUNDLE_DIR/workspace/project.md") + if [ "$project_content" = "# My Project" ]; then + pass "C5: workspace/project.md intact" + else + fail "C5: workspace/project.md content changed" + fi +else + fail "C5: workspace/project.md missing from bundle" +fi + +# Cleanup mock directory +rm -rf "$MOCK_DIR" + +# ══════════════════════════════════════════════════════════════════ +# Phase 2: Runtime Sandbox Credential Check +# +# Verify that credentials are NOT accessible from inside the running +# sandbox. This tests the end-to-end flow: migrate → sandbox start → +# agent cannot read credentials from filesystem. +# ══════════════════════════════════════════════════════════════════ +section "Phase 2: Runtime Sandbox Credential Check" + +# C6: auth-profiles.json must not exist inside the sandbox +info "C6: Checking for auth-profiles.json inside sandbox..." +c6_result=$(sandbox_exec "find /sandbox -name 'auth-profiles.json' 2>/dev/null | head -5") + +if [ "$c6_result" = "__PROBE_FAILED__" ]; then + fail "C6: Sandbox probe failed — SSH did not execute; cannot verify auth-profiles.json absence" +elif [ -z "$c6_result" ]; then + pass "C6: No auth-profiles.json found inside sandbox" +else + fail "C6: auth-profiles.json found inside sandbox: $c6_result" +fi + +# C7: No real secret patterns in sandbox config files +info "C7: Checking for secret patterns in sandbox config..." + +# Search for real API key patterns (not our test fakes). +# Exclude policy preset files and installed extension code/dependencies; package +# sources can contain detector strings like nvapi-, ghp_, or npm_ without storing +# user secrets. +c7_scan_pattern() { + local pattern="$1" + sandbox_exec "grep -r '$pattern' /sandbox/.openclaw/ /sandbox/.nemoclaw/ 2>/dev/null | grep -v 'STRIPPED' | grep -v '/policies/' | grep -v '/plugin-runtime-deps/' | grep -Ev '/extensions/[^/]+/(dist|node_modules)/' | head -5" || true +} + +c7_nvapi=$(c7_scan_pattern "nvapi-") +c7_ghp=$(c7_scan_pattern "ghp_") +c7_npm=$(c7_scan_pattern "npm_") + +if [ "$c7_nvapi" = "__PROBE_FAILED__" ] || [ "$c7_ghp" = "__PROBE_FAILED__" ] || [ "$c7_npm" = "__PROBE_FAILED__" ]; then + fail "C7: Sandbox probe failed — SSH did not execute; cannot verify secret absence" +elif [ -z "$c7_nvapi" ] && [ -z "$c7_ghp" ] && [ -z "$c7_npm" ]; then + pass "C7: No secret patterns (nvapi-, ghp_, npm_) found in sandbox config" +else + fail "C7: Secret patterns found in sandbox — nvapi: ${c7_nvapi:0:100}, ghp: ${c7_ghp:0:100}, npm: ${c7_npm:0:100}" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 3: Symlink Safety +# ══════════════════════════════════════════════════════════════════ +section "Phase 3: Symlink Safety" + +# C8: Symlinked auth-profiles.json must NOT delete the target file +info "C8: Testing symlink traversal protection..." + +SYMLINK_DIR=$(mktemp -d /tmp/nemoclaw-symlink-test-XXXXXX) +OUTSIDE_DIR="$SYMLINK_DIR/outside" +BUNDLE_SYM_DIR="$SYMLINK_DIR/bundle/agents" +mkdir -p "$OUTSIDE_DIR" "$BUNDLE_SYM_DIR" + +# Create a real file outside the bundle +echo '{"shouldNotBeDeleted": true}' >"$OUTSIDE_DIR/auth-profiles.json" + +# Create a symlink inside the bundle pointing to the outside file +ln -s "$OUTSIDE_DIR/auth-profiles.json" "$BUNDLE_SYM_DIR/auth-profiles.json" + +# Run walkAndRemoveFile — it should skip symlinks +c8_result=$(cd "$REPO" && node -e " +const fs = require('fs'); +const path = require('path'); + +function walkAndRemoveFile(dirPath, targetName) { + let entries; + try { entries = fs.readdirSync(dirPath); } catch { return; } + for (const entry of entries) { + const fullPath = path.join(dirPath, entry); + try { + const stat = fs.lstatSync(fullPath); + if (stat.isSymbolicLink()) continue; // SKIP SYMLINKS + if (stat.isDirectory()) { + walkAndRemoveFile(fullPath, targetName); + } else if (entry === targetName) { + fs.rmSync(fullPath, { force: true }); + } + } catch {} + } +} + +walkAndRemoveFile('$BUNDLE_SYM_DIR', 'auth-profiles.json'); + +// Check if the outside file still exists +if (fs.existsSync('$OUTSIDE_DIR/auth-profiles.json')) { + console.log('SAFE'); +} else { + console.log('EXPLOITED'); +} +" 2>&1) + +if echo "$c8_result" | grep -q "SAFE"; then + pass "C8: Symlink traversal blocked — outside file preserved" +else + fail "C8: Symlink traversal — outside file was DELETED through symlink!" +fi + +rm -rf "$SYMLINK_DIR" + +# ══════════════════════════════════════════════════════════════════ +# Phase 4: Blueprint Digest Verification +# ══════════════════════════════════════════════════════════════════ +section "Phase 4: Blueprint Digest Verification" + +# C9: Empty digest string must be treated as a FAILURE +info "C9: Testing empty digest rejection..." + +c9_result=$(cd "$REPO" && node -e " +// Simulate the FIXED verifyBlueprintDigest behavior: +// Empty/missing digest must be a hard failure, not a silent pass. + +function verifyBlueprintDigest_FIXED(manifest) { + if (!manifest.digest || manifest.digest.trim() === '') { + return { valid: false, reason: 'Blueprint has no digest — verification required' }; + } + // In real code, this would compute and compare the hash + return { valid: true }; +} + +// The bug: digest: '' is falsy in JS, so the OLD code did: +// if (manifest.digest && ...) — which skipped verification entirely +function verifyBlueprintDigest_VULNERABLE(manifest) { + if (manifest.digest && manifest.digest !== 'WRONG') { + return { valid: true }; + } + if (!manifest.digest) { + // This is the bug: empty string silently passes + return { valid: true, reason: 'no digest to verify' }; + } + return { valid: false, reason: 'digest mismatch' }; +} + +// Test the FIXED version +const result = verifyBlueprintDigest_FIXED({ digest: '' }); +if (!result.valid) { + console.log('REJECTED_EMPTY'); +} else { + console.log('ACCEPTED_EMPTY'); +} + +// Also test with undefined/null +const result2 = verifyBlueprintDigest_FIXED({ digest: undefined }); +if (!result2.valid) { + console.log('REJECTED_UNDEFINED'); +} else { + console.log('ACCEPTED_UNDEFINED'); +} +" 2>&1) + +if echo "$c9_result" | grep -q "REJECTED_EMPTY"; then + pass "C9a: Empty digest string correctly rejected" +else + fail "C9a: Empty digest string was ACCEPTED — bypass still possible!" +fi + +if echo "$c9_result" | grep -q "REJECTED_UNDEFINED"; then + pass "C9b: Undefined digest correctly rejected" +else + fail "C9b: Undefined digest was ACCEPTED — bypass still possible!" +fi + +# C10: Wrong digest must fail verification +info "C10: Testing wrong digest rejection..." + +c10_result=$(cd "$REPO" && node -e " +const crypto = require('crypto'); + +function verifyDigest(manifest, blueprintContent) { + if (!manifest.digest || manifest.digest.trim() === '') { + return { valid: false, reason: 'no digest' }; + } + const computed = crypto.createHash('sha256').update(blueprintContent).digest('hex'); + if (manifest.digest !== computed) { + return { valid: false, reason: 'digest mismatch: expected ' + manifest.digest + ', got ' + computed }; + } + return { valid: true }; +} + +const content = 'blueprint content here'; +const wrongDigest = 'deadbeef0000000000000000000000000000000000000000000000000000dead'; +const result = verifyDigest({ digest: wrongDigest }, content); +console.log(result.valid ? 'ACCEPTED_WRONG' : 'REJECTED_WRONG'); +" 2>&1) + +if echo "$c10_result" | grep -q "REJECTED_WRONG"; then + pass "C10: Wrong digest correctly rejected" +else + fail "C10: Wrong digest was ACCEPTED — verification broken!" +fi + +# C11: Correct digest must pass +info "C11: Testing correct digest acceptance..." + +c11_result=$(cd "$REPO" && node -e " +const crypto = require('crypto'); + +function verifyDigest(manifest, blueprintContent) { + if (!manifest.digest || manifest.digest.trim() === '') { + return { valid: false, reason: 'no digest' }; + } + const computed = crypto.createHash('sha256').update(blueprintContent).digest('hex'); + if (manifest.digest !== computed) { + return { valid: false, reason: 'digest mismatch' }; + } + return { valid: true }; +} + +const content = 'blueprint content here'; +const correctDigest = crypto.createHash('sha256').update(content).digest('hex'); +const result = verifyDigest({ digest: correctDigest }, content); +console.log(result.valid ? 'ACCEPTED_CORRECT' : 'REJECTED_CORRECT'); +" 2>&1) + +if echo "$c11_result" | grep -q "ACCEPTED_CORRECT"; then + pass "C11: Correct digest correctly accepted" +else + fail "C11: Correct digest was REJECTED — false negative!" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 5: Pattern-Based Credential Field Detection +# ══════════════════════════════════════════════════════════════════ +section "Phase 5: Pattern-Based Credential Detection" + +# C12: Pattern-matched credential fields must be stripped +info "C12: Testing pattern-based credential field stripping..." + +c12_result=$(cd "$REPO" && node -e " +const CREDENTIAL_FIELDS = new Set([ + 'apiKey', 'api_key', 'token', 'secret', 'password', 'resolvedKey', +]); +const CREDENTIAL_FIELD_PATTERN = + /(?:access|refresh|client|bearer|auth|api|private|public|signing|session)(?:Token|Key|Secret|Password)$/; + +function isCredentialField(key) { + return CREDENTIAL_FIELDS.has(key) || CREDENTIAL_FIELD_PATTERN.test(key); +} + +function stripCredentials(obj) { + if (obj === null || obj === undefined) return obj; + if (typeof obj !== 'object') return obj; + if (Array.isArray(obj)) return obj.map(stripCredentials); + const result = {}; + for (const [key, value] of Object.entries(obj)) { + if (isCredentialField(key)) { + result[key] = '[STRIPPED_BY_MIGRATION]'; + } else { + result[key] = stripCredentials(value); + } + } + return result; +} + +const config = { + provider: { + accessToken: 'test-access-token-value', + refreshToken: 'test-refresh-token-value', + privateKey: 'test-private-key-value', + clientSecret: 'test-client-secret-value', + signingKey: 'test-signing-key-value', + bearerToken: 'test-bearer-token-value', + sessionToken: 'test-session-token-value', + authKey: 'test-auth-key-value', + } +}; + +const sanitized = stripCredentials(config); +const allStripped = Object.values(sanitized.provider).every(v => v === '[STRIPPED_BY_MIGRATION]'); +console.log(allStripped ? 'ALL_STRIPPED' : 'SOME_LEAKED'); + +// Print any that weren't stripped for debugging +for (const [k, v] of Object.entries(sanitized.provider)) { + if (v !== '[STRIPPED_BY_MIGRATION]') { + console.log('LEAKED: ' + k + ' = ' + v); + } +} +" 2>&1) + +if echo "$c12_result" | grep -q "ALL_STRIPPED"; then + pass "C12: All pattern-matched credential fields stripped" +else + fail "C12: Some credential fields NOT stripped: ${c12_result}" +fi + +# C13: Non-credential fields with partial keyword overlap must be preserved +info "C13: Testing non-credential field preservation..." + +c13_result=$(cd "$REPO" && node -e " +const CREDENTIAL_FIELDS = new Set([ + 'apiKey', 'api_key', 'token', 'secret', 'password', 'resolvedKey', +]); +const CREDENTIAL_FIELD_PATTERN = + /(?:access|refresh|client|bearer|auth|api|private|public|signing|session)(?:Token|Key|Secret|Password)$/; + +function isCredentialField(key) { + return CREDENTIAL_FIELDS.has(key) || CREDENTIAL_FIELD_PATTERN.test(key); +} + +function stripCredentials(obj) { + if (obj === null || obj === undefined) return obj; + if (typeof obj !== 'object') return obj; + if (Array.isArray(obj)) return obj.map(stripCredentials); + const result = {}; + for (const [key, value] of Object.entries(obj)) { + if (isCredentialField(key)) { + result[key] = '[STRIPPED_BY_MIGRATION]'; + } else { + result[key] = stripCredentials(value); + } + } + return result; +} + +const config = { + displayName: 'should-be-preserved', + sortKey: 'should-also-be-preserved', + modelName: 'nvidia/nemotron-3-super-120b-a12b', + keyRef: { source: 'env', id: 'NVIDIA_API_KEY' }, + description: 'A secret garden (but not a real secret)', + tokenizer: 'sentencepiece', + endpoint: 'https://api.nvidia.com/v1', + sessionId: 'abc-123', + accessLevel: 'admin', + publicUrl: 'https://example.com', +}; + +const sanitized = stripCredentials(config); +const results = []; + +// These should ALL be preserved (not stripped) +const expected = { + displayName: 'should-be-preserved', + sortKey: 'should-also-be-preserved', + modelName: 'nvidia/nemotron-3-super-120b-a12b', + description: 'A secret garden (but not a real secret)', + tokenizer: 'sentencepiece', + endpoint: 'https://api.nvidia.com/v1', + sessionId: 'abc-123', + accessLevel: 'admin', + publicUrl: 'https://example.com', +}; + +let allPreserved = true; +for (const [key, expectedVal] of Object.entries(expected)) { + if (sanitized[key] !== expectedVal) { + console.log('CORRUPTED: ' + key + ' = ' + JSON.stringify(sanitized[key]) + ' (expected: ' + expectedVal + ')'); + allPreserved = false; + } +} + +// keyRef is an object — check it's preserved structurally +if (JSON.stringify(sanitized.keyRef) !== JSON.stringify({ source: 'env', id: 'NVIDIA_API_KEY' })) { + console.log('CORRUPTED: keyRef'); + allPreserved = false; +} + +console.log(allPreserved ? 'ALL_PRESERVED' : 'SOME_CORRUPTED'); +" 2>&1) + +if echo "$c13_result" | grep -q "ALL_PRESERVED"; then + pass "C13: All non-credential fields preserved correctly" +else + fail "C13: Some non-credential fields were corrupted: ${c13_result}" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 6: Shipped Blueprint Digest Check +# ══════════════════════════════════════════════════════════════════ +section "Phase 6: Shipped Blueprint Check" + +# Verify the shipped blueprint.yaml has the known empty digest issue +info "Checking shipped blueprint.yaml digest field..." +BLUEPRINT_FILE="$REPO/nemoclaw-blueprint/blueprint.yaml" +if [ -f "$BLUEPRINT_FILE" ]; then + digest_line=$(grep "^digest:" "$BLUEPRINT_FILE" || true) + if echo "$digest_line" | grep -qE 'digest:\s*""'; then + info "Shipped blueprint has digest: \"\" (empty) — this is the known vulnerability" + info "After PR #156, empty digest will cause a hard verification failure" + pass "Blueprint digest field found and identified" + elif echo "$digest_line" | grep -qE 'digest:\s*$'; then + info "Shipped blueprint has empty digest field" + pass "Blueprint digest field found (empty)" + elif [ -n "$digest_line" ]; then + info "Blueprint digest: $digest_line" + pass "Blueprint has a digest value set" + else + skip "No digest field found in blueprint.yaml" + fi +else + skip "blueprint.yaml not found at $BLUEPRINT_FILE" +fi + +# ══════════════════════════════════════════════════════════════════ +# Summary +# ══════════════════════════════════════════════════════════════════ +echo "" +echo "========================================" +echo " Credential Sanitization Test Results:" +echo " Passed: $PASS" +echo " Failed: $FAIL" +echo " Skipped: $SKIP" +echo " Total: $TOTAL" +echo "========================================" + +if [ "$FAIL" -eq 0 ]; then + printf '\n\033[1;32m Credential sanitization tests PASSED — no credential leaks found.\033[0m\n' + exit 0 +else + printf '\n\033[1;31m %d test(s) failed — CREDENTIAL LEAKS OR BYPASS DETECTED.\033[0m\n' "$FAIL" + exit 1 +fi diff --git a/test/e2e-vpn/test-cron-preflight-inference-local-e2e.sh b/test/e2e-vpn/test-cron-preflight-inference-local-e2e.sh new file mode 100755 index 00000000000..91bc54fe93d --- /dev/null +++ b/test/e2e-vpn/test-cron-preflight-inference-local-e2e.sh @@ -0,0 +1,380 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Cron preflight inference.local E2E. +# +# Onboards a fresh sandbox against the configured hosted inference provider +# (whose base URL resolves through `inference.local`), then loads OpenClaw's +# cron isolated-agent preflight runtime directly from the in-sandbox dist and +# invokes `preflightCronModelProvider` against the onboarded provider/model. +# Asserts the call returns `status: "available"` and never reports `EAI_AGAIN` +# or the "local provider endpoint is not reachable" message. +# +# This probes the exact runtime path Patch 6 modifies — the cron CLI surfaces +# (`openclaw cron add` / `openclaw cron run`) need `operator.admin` scope, which +# the in-sandbox auto-pair approval sweep deliberately omits from its allowlist, +# so the scheduler boundary is intentionally bypassed in favour of a direct +# runtime probe. +# +# Prerequisites: +# - Docker running +# - NVIDIA_API_KEY set for hosted inference +# - NEMOCLAW_NON_INTERACTIVE=1, NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 +# +# Environment: +# NEMOCLAW_SANDBOX_NAME — sandbox name (default: e2e-cron-preflight) +# NEMOCLAW_RECREATE_SANDBOX=1 — destroy + recreate if exists +# NEMOCLAW_CRON_PREFLIGHT_MODEL — model for non-hosted provider runs +# NEMOCLAW_CRON_PREFLIGHT_KEEP=1 — keep the sandbox after the test for inspection +# +# Usage: +# NEMOCLAW_NON_INTERACTIVE=1 NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ +# NVIDIA_API_KEY=... bash test/e2e-vpn/test-cron-preflight-inference-local-e2e.sh + +set -uo pipefail + +PASS=0 +FAIL=0 +SKIP=0 +TOTAL=0 + +pass() { + PASS=$((PASS + 1)) + TOTAL=$((TOTAL + 1)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} +fail() { + FAIL=$((FAIL + 1)) + TOTAL=$((TOTAL + 1)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} +skip() { + SKIP=$((SKIP + 1)) + TOTAL=$((TOTAL + 1)) + printf '\033[33m SKIP: %s\033[0m\n' "$1" +} +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } + +# ── Repo root ── +_script_dir="$(cd "$(dirname "$0")" && pwd)" +_candidate="$(cd "${_script_dir}/../.." && pwd)" +if [ -d /workspace ] && [ -f /workspace/package.json ] && [ -d /workspace/test/e2e ]; then + REPO="/workspace" +elif [ -f "${_candidate}/package.json" ] && [ -d "${_candidate}/test/e2e" ]; then + REPO="${_candidate}" +else + echo "ERROR: Cannot find repo root." + exit 1 +fi +unset _script_dir _candidate +cd "$REPO" || { + echo "ERROR: Cannot cd into repo root '$REPO'." + exit 1 +} + +E2E_DIR="${REPO}/test/e2e" +SANDBOX="${NEMOCLAW_SANDBOX_NAME:-e2e-cron-preflight}" +MODEL="${NEMOCLAW_CRON_PREFLIGHT_MODEL:-nvidia/nemotron-3-super-120b-a12b}" +INSTALL_LOG="/tmp/nemoclaw-e2e-cron-preflight-install.log" + +# shellcheck source=test/e2e-vpn/lib/sandbox-teardown.sh +. "${E2E_DIR}/lib/sandbox-teardown.sh" +# shellcheck source=test/e2e-vpn/lib/install-path-refresh.sh +. "${E2E_DIR}/lib/install-path-refresh.sh" +# shellcheck source=test/e2e-vpn/lib/ci-compatible-inference.sh +. "${E2E_DIR}/lib/ci-compatible-inference.sh" + +# ── Prereqs ── +section "Prerequisites" +if ! command -v docker >/dev/null 2>&1; then + skip "docker not installed" + echo " Total: $TOTAL Pass: $PASS Fail: $FAIL Skip: $SKIP" + exit 0 +fi +if ! command -v jq >/dev/null 2>&1; then + skip "jq not installed" + echo " Total: $TOTAL Pass: $PASS Fail: $FAIL Skip: $SKIP" + exit 0 +fi +if ! nemoclaw_e2e_configure_compatible_inference; then + fail "hosted CI inference could not be configured" + echo " Total: $TOTAL Pass: $PASS Fail: $FAIL Skip: $SKIP" + exit 1 +fi +if nemoclaw_e2e_using_compatible_inference; then + if ! nemoclaw_e2e_require_hosted_inference_key; then + echo " Total: $TOTAL Pass: $PASS Fail: $FAIL Skip: $SKIP" + exit 1 + fi +else + if [ -z "${NVIDIA_API_KEY:-}" ]; then + skip "NVIDIA_API_KEY not set" + echo " Total: $TOTAL Pass: $PASS Fail: $FAIL Skip: $SKIP" + exit 0 + fi + if [ "${NVIDIA_API_KEY:0:6}" != "nvapi-" ]; then + skip "NVIDIA_API_KEY does not start with nvapi-" + echo " Total: $TOTAL Pass: $PASS Fail: $FAIL Skip: $SKIP" + exit 0 + fi +fi +if [ "${NEMOCLAW_NON_INTERACTIVE:-}" != "1" ]; then + skip "NEMOCLAW_NON_INTERACTIVE must be 1; refusing to risk an interactive onboard prompt" + echo " Total: $TOTAL Pass: $PASS Fail: $FAIL Skip: $SKIP" + exit 0 +fi +if [ "${NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE:-}" != "1" ]; then + skip "NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE must be 1; refusing to risk an interactive onboard prompt" + echo " Total: $TOTAL Pass: $PASS Fail: $FAIL Skip: $SKIP" + exit 0 +fi +pass "prerequisites satisfied" + +# ── Install NemoClaw + onboard sandbox ── +section "Install NemoClaw + onboard sandbox '$SANDBOX'" +export NEMOCLAW_SANDBOX_NAME="$SANDBOX" +export NEMOCLAW_RECREATE_SANDBOX="${NEMOCLAW_RECREATE_SANDBOX:-1}" +export NEMOCLAW_PROVIDER="" +export NEMOCLAW_MODEL="${NEMOCLAW_MODEL:-$MODEL}" + +info "Installing NemoClaw via install.sh --non-interactive..." +bash install.sh --non-interactive --yes-i-accept-third-party-software >"$INSTALL_LOG" 2>&1 & +install_pid=$! +tail -f "$INSTALL_LOG" --pid=$install_pid 2>/dev/null & +tail_pid=$! +wait "$install_pid" +install_exit=$? +kill "$tail_pid" 2>/dev/null || true +wait "$tail_pid" 2>/dev/null || true + +nemoclaw_refresh_install_env +export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" +# shellcheck source=/dev/null +[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" +nemoclaw_ensure_local_bin_on_path + +if [ "$install_exit" -ne 0 ]; then + fail "install.sh failed (exit $install_exit)" + tail -30 "$INSTALL_LOG" + exit 1 +fi +pass "NemoClaw installed + sandbox onboarded" + +command -v nemoclaw >/dev/null 2>&1 || { + fail "nemoclaw not on PATH after install" + exit 1 +} + +# Wire the documented `NEMOCLAW_CRON_PREFLIGHT_KEEP` flag through to the shared +# teardown helper (which honours only `NEMOCLAW_E2E_KEEP_SANDBOX`) so the +# documented escape hatch actually preserves the sandbox for inspection. +export NEMOCLAW_E2E_KEEP_SANDBOX="${NEMOCLAW_E2E_KEEP_SANDBOX:-${NEMOCLAW_CRON_PREFLIGHT_KEEP:-}}" +register_sandbox_for_teardown "$SANDBOX" + +# ── Probe the cron preflight directly ── +# +# The cron CLI surfaces (`openclaw cron add` / `openclaw cron run`) require +# `operator.admin` scope, which the in-sandbox auto-pair approval sweep +# deliberately excludes from its allowlist. There is no declarative way for +# an external CLI to call those RPCs without an interactive scope-upgrade +# approval, which is plumbing noise for what this test is actually checking. +# +# Patch 6 only changes the `fetchWithSsrFGuard` call inside +# `probeLocalProviderEndpoint`. Invoke that function directly via a node +# script loaded from the in-sandbox OpenClaw dist instead: the probe asserts +# the same behaviour (managed inference base URL reachable from cron +# preflight) without any gateway, scheduler, or device pairing involvement. +section "Probe cron preflight against managed inference base URL" + +PROBE_SRC=$( + cat <<'PROBE_JS' +const fs = require("node:fs"); +const path = require("node:path"); +const url = require("node:url"); + +const AUDIT_CONTEXT = "cron-model-provider-preflight"; +const EXPORT_NAME = "preflightCronModelProvider"; +const EXPECTED_HOSTNAME = "inference.local"; +const DIST_ROOTS = [ + "/usr/local/lib/node_modules/openclaw/dist", + "/usr/lib/node_modules/openclaw/dist", +]; + +function isExpectedManagedProvider(provider) { + if (!provider || typeof provider.baseUrl !== "string") return false; + try { + return new URL(provider.baseUrl).hostname.toLowerCase() === EXPECTED_HOSTNAME; + } catch { + return false; + } +} + +function findPreflightModule(root) { + const stack = [root]; + while (stack.length > 0) { + const dir = stack.pop(); + let entries; + try { + entries = fs.readdirSync(dir, { withFileTypes: true }); + } catch { + continue; + } + for (const entry of entries) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) { + stack.push(full); + continue; + } + if (!entry.isFile()) continue; + if (!(full.endsWith(".js") || full.endsWith(".mjs") || full.endsWith(".cjs"))) continue; + let body; + try { + body = fs.readFileSync(full, "utf8"); + } catch { + continue; + } + if (body.includes(AUDIT_CONTEXT) && body.includes(EXPORT_NAME)) { + return full; + } + } + } + return null; +} + +(async () => { + let target = null; + const scanned = []; + for (const root of DIST_ROOTS) { + if (!fs.existsSync(root)) continue; + scanned.push(root); + target = findPreflightModule(root); + if (target) break; + } + if (!target) { + console.error(JSON.stringify({ error: "preflight-source-not-found", scanned })); + process.exit(3); + } + + let mod; + try { + mod = await import(url.pathToFileURL(target).href); + } catch (err) { + console.error( + JSON.stringify({ + error: "preflight-import-threw", + target, + message: String(err && err.stack ? err.stack : err), + }), + ); + process.exit(3); + } + const preflightCronModelProvider = mod[EXPORT_NAME]; + if (typeof preflightCronModelProvider !== "function") { + console.error( + JSON.stringify({ + error: "preflight-export-missing", + target, + exports: Object.keys(mod), + }), + ); + process.exit(3); + } + + const configPath = process.env.OPENCLAW_CONFIG_PATH || "/sandbox/.openclaw/openclaw.json"; + let cfg; + try { + cfg = JSON.parse(fs.readFileSync(configPath, "utf8")); + } catch (err) { + console.error( + JSON.stringify({ error: "config-read-failed", configPath, message: String(err) }), + ); + process.exit(3); + } + + const providers = (cfg.models && cfg.models.providers) || {}; + const providerKey = Object.keys(providers).find((key) => + isExpectedManagedProvider(providers[key]), + ); + if (!providerKey) { + console.error( + JSON.stringify({ + error: "no-managed-inference-local-provider", + expectedHost: EXPECTED_HOSTNAME, + providers: Object.entries(providers).map(([key, value]) => ({ + key, + baseUrl: value && typeof value.baseUrl === "string" ? value.baseUrl : null, + })), + }), + ); + process.exit(3); + } + const providerCfg = providers[providerKey]; + const modelKey = + providerCfg.defaultModel || + (Array.isArray(providerCfg.models) ? providerCfg.models[0] : undefined) || + "ping"; + + try { + const result = await preflightCronModelProvider({ + cfg, + provider: providerKey, + model: modelKey, + }); + console.log( + JSON.stringify({ providerKey, modelKey, baseUrl: providerCfg.baseUrl, target, result }), + ); + process.exit(result && result.status === "available" ? 0 : 1); + } catch (err) { + console.error( + JSON.stringify({ + error: "preflight-threw", + message: String(err && err.stack ? err.stack : err), + }), + ); + process.exit(2); + } +})(); +PROBE_JS +) +PROBE_B64="$(printf '%s' "$PROBE_SRC" | base64 -w 0)" + +# openshell sandbox exec rejects any command argument that contains a newline +# or carriage return ("command argument N contains newline or carriage return +# characters"), so the inner `sh -c` payload must be a single physical line. +# Chain with `&&` for success-only steps and `;` for the cleanup tail so the +# probe exit code is preserved end-to-end. +PROBE_SHELL=". /tmp/nemoclaw-proxy-env.sh && __probe=\"\$(mktemp /tmp/nemoclaw-preflight-probe.XXXXXX.cjs)\" && printf %s '$PROBE_B64' | base64 -d > \"\$__probe\" && node \"\$__probe\"; __rc=\$?; rm -f \"\$__probe\"; exit \"\$__rc\"" +PROBE_OUT="$(nemoclaw "$SANDBOX" exec -- sh -c "$PROBE_SHELL" 2>&1)" +PROBE_RC=$? +info "preflight probe output (rc=$PROBE_RC):" +printf '%s\n' "$PROBE_OUT" | sed 's/^/ /' + +# Probe stdout/stderr are interleaved (captured via 2>&1). Pick the structured +# JSON result line (the only line that starts with `{"providerKey"`) before +# parsing, so undici experimental-feature warnings on stderr do not break jq. +PROBE_JSON="$(printf '%s\n' "$PROBE_OUT" | grep -E '^\s*\{"providerKey"' | tail -n 1)" +STATUS="$(printf '%s' "$PROBE_JSON" | jq -r '.result.status // empty' 2>/dev/null || true)" +REASON="$(printf '%s' "$PROBE_JSON" | jq -r '.result.reason // ""' 2>/dev/null || true)" + +section "Assertions" +if [ "$PROBE_RC" -ge 2 ]; then + fail "probe harness failed (rc=$PROBE_RC); preflight did not run" +elif printf '%s' "$REASON" | grep -qi "EAI_AGAIN"; then + fail "preflight raised EAI_AGAIN; reason='$REASON'" +elif printf '%s' "$REASON" | grep -qi "local provider endpoint is not reachable"; then + fail "preflight reported endpoint unreachable; reason='$REASON'" +elif [ "$STATUS" = "available" ]; then + pass "preflight status=available" +else + fail "unexpected probe status='$STATUS' rc=$PROBE_RC reason='$REASON'" +fi + +section "Summary" +echo " Total: $TOTAL Pass: $PASS Fail: $FAIL Skip: $SKIP" +[ "$FAIL" -eq 0 ] || exit 1 +exit 0 diff --git a/test/e2e-vpn/test-dashboard-remote-bind.sh b/test/e2e-vpn/test-dashboard-remote-bind.sh new file mode 100755 index 00000000000..9fa259f8c8d --- /dev/null +++ b/test/e2e-vpn/test-dashboard-remote-bind.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -uo pipefail + +section() { printf '\n=== %s ===\n' "$1"; } +pass() { echo "PASS: $1"; } +fail() { + echo "FAIL: $1" + exit 1 +} +info() { echo "INFO: $1"; } + +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-test}" +DASHBOARD_PORT="${NEMOCLAW_DASHBOARD_PORT:-18789}" +REMOTE_HOST="${NEMOCLAW_E2E_REMOTE_HOST:-$(hostname -I 2>/dev/null | awk '{print $1}')}" +if [ -z "$REMOTE_HOST" ]; then + REMOTE_HOST="$(hostname -f 2>/dev/null || hostname)" +fi + +section "Preconditions" +info "Sandbox: ${SANDBOX_NAME}" +info "Dashboard port: ${DASHBOARD_PORT}" +info "Remote host candidate: ${REMOTE_HOST}" + +if ! command -v nemoclaw >/dev/null 2>&1; then + fail "nemoclaw CLI is not on PATH" +fi +if ! command -v openshell >/dev/null 2>&1; then + fail "openshell CLI is not on PATH" +fi +pass "Required CLIs are available" + +section "Restart dashboard forward with explicit all-interface bind" +# The coverage guard mirrors issue #3259: remote SSH-deployed hosts need an +# explicit operator-controlled way to bind the dashboard forward on all +# interfaces. On main, NEMOCLAW_DASHBOARD_BIND is ignored and the forward stays +# localhost-only; the fix should make this opt-in produce 0.0.0.0:. +openshell forward stop "${DASHBOARD_PORT}" >/dev/null 2>&1 || true +CONNECT_LOG="$(mktemp -t nemoclaw-dashboard-remote-bind.XXXXXX.log)" +trap 'rm -f "${CONNECT_LOG}"' EXIT +if NEMOCLAW_DASHBOARD_BIND=0.0.0.0 nemoclaw "${SANDBOX_NAME}" connect >"${CONNECT_LOG}" 2>&1; then + pass "nemoclaw connect completed with NEMOCLAW_DASHBOARD_BIND=0.0.0.0" +else + cat "${CONNECT_LOG}" + fail "nemoclaw connect failed with NEMOCLAW_DASHBOARD_BIND=0.0.0.0" +fi + +section "Verify OpenShell forward bind" +FORWARD_LIST="$(openshell forward list 2>/dev/null || true)" +printf '%s\n' "${FORWARD_LIST}" +FORWARD_LINE="$(printf '%s\n' "${FORWARD_LIST}" | awk -v sandbox="${SANDBOX_NAME}" -v port="${DASHBOARD_PORT}" '$0 ~ sandbox && $0 ~ port {print; exit}')" +if [ -z "${FORWARD_LINE}" ]; then + fail "No OpenShell forward found for ${SANDBOX_NAME} on ${DASHBOARD_PORT}" +fi +info "Matched forward: ${FORWARD_LINE}" + +case "${FORWARD_LINE}" in + *"0.0.0.0:${DASHBOARD_PORT}"* | *"*:""${DASHBOARD_PORT}"* | *"0.0.0.0 "*" ${DASHBOARD_PORT} "*) + pass "Dashboard forward binds all interfaces for remote origin (${DASHBOARD_PORT})" + ;; + *"127.0.0.1:${DASHBOARD_PORT}"* | *"localhost:${DASHBOARD_PORT}"* | *"127.0.0.1 "*" ${DASHBOARD_PORT} "*) + fail "Dashboard forward is still localhost-only; expected 0.0.0.0:${DASHBOARD_PORT}" + ;; + *) + fail "Could not prove dashboard forward uses 0.0.0.0:${DASHBOARD_PORT} from: ${FORWARD_LINE}" + ;; +esac + +section "Summary" +pass "Remote dashboard bind guard completed" diff --git a/test/e2e-vpn/test-device-auth-health.sh b/test/e2e-vpn/test-device-auth-health.sh new file mode 100755 index 00000000000..a544e3b77af --- /dev/null +++ b/test/e2e-vpn/test-device-auth-health.sh @@ -0,0 +1,375 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# ============================================================================= +# test-device-auth-health.sh +# Device Auth Health Probe E2E — Regression test for #2342 +# +# Validates that gateway health probes work correctly when device auth is +# enabled (the default). Previously, `curl -sf` treated HTTP 401 as failure, +# causing false "Health Offline" readings in the dashboard and unnecessary +# process recovery attempts. +# +# What this proves: +# 1. Onboard succeeds with device auth ON (verifyDeployment doesn't block) +# 2. /health endpoint returns 200 from inside sandbox (auth-free) +# 3. / endpoint returns 401 from inside sandbox (device auth active) +# 4. `nemoclaw status` reports gateway Running (not Offline) +# 5. isSandboxGatewayRunning() correctly treats 401 as alive +# 6. After gateway restart, status still reports Running (not Offline) +# +# Prerequisites: +# - Docker running +# - NVIDIA_API_KEY set (real key, starts with nvapi-) +# - Network access to inference.nvidia.com +# +# Environment variables: +# NEMOCLAW_NON_INTERACTIVE=1 — required +# NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 — required +# NVIDIA_API_KEY — required +# NEMOCLAW_SANDBOX_NAME — sandbox name (default: e2e-health-auth) +# NEMOCLAW_E2E_TIMEOUT_SECONDS — overall timeout (default: 600) +# NEMOCLAW_DASHBOARD_PORT — dashboard port (default: 18789) +# +# Usage: +# NEMOCLAW_NON_INTERACTIVE=1 \ +# NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ +# NVIDIA_API_KEY=nvapi-... \ +# bash test/e2e-vpn/test-device-auth-health.sh +# ============================================================================= + +# ShellCheck cannot see EXIT trap invocations of cleanup helpers in this E2E script. +# shellcheck disable=SC2317 +set -uo pipefail + +# ── Overall timeout ────────────────────────────────────────────────────────── +export NEMOCLAW_E2E_DEFAULT_TIMEOUT=1200 +SCRIPT_DIR_TIMEOUT="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +# shellcheck source=test/e2e-vpn/e2e-timeout.sh +source "${SCRIPT_DIR_TIMEOUT}/e2e-timeout.sh" + +# ── Config ─────────────────────────────────────────────────────────────────── +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-health-auth}" +DASHBOARD_PORT="${NEMOCLAW_DASHBOARD_PORT:-18789}" + +# ── Counters ───────────────────────────────────────────────────────────────── +PASS=0 +FAIL=0 +SKIP=0 +TOTAL=0 + +# ── Helpers ────────────────────────────────────────────────────────────────── +pass() { + ((PASS++)) + ((TOTAL++)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} +fail() { + ((FAIL++)) + ((TOTAL++)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} +skip() { + ((SKIP++)) + ((TOTAL++)) + printf '\033[33m SKIP: %s\033[0m\n' "$1" +} +section() { + echo "" + printf '\033[1;36m══════ %s ══════\033[0m\n' "$1" +} +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } + +# shellcheck source=test/e2e-vpn/lib/sandbox-teardown.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib/sandbox-teardown.sh" +register_sandbox_for_teardown "$SANDBOX_NAME" + +# shellcheck disable=SC2329 +cleanup_ssh() { [[ -n "${SSH_CONFIG:-}" ]] && rm -f "$SSH_CONFIG"; } +trap 'cleanup_ssh' EXIT + +# Execute a command inside the sandbox via SSH (the established E2E pattern). +SSH_CONFIG="" +setup_ssh() { + SSH_CONFIG="$(mktemp)" + local attempt + for attempt in $(seq 1 5); do + if openshell sandbox ssh-config "$SANDBOX_NAME" >"$SSH_CONFIG" 2>/dev/null; then + if [[ -s "$SSH_CONFIG" ]]; then + return 0 + fi + fi + sleep 3 + done + info "Failed to get SSH config for '$SANDBOX_NAME' after 5 attempts" + return 1 +} +sandbox_exec() { + local cmd="$1" + if [[ -z "$SSH_CONFIG" ]] || [[ ! -s "$SSH_CONFIG" ]]; then + setup_ssh || return 1 + fi + ssh -F "$SSH_CONFIG" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" "$cmd" 2>/dev/null +} + +# ══════════════════════════════════════════════════════════════════════════════ +# Phase 0: Preflight +# ══════════════════════════════════════════════════════════════════════════════ +section "Phase 0: Preflight" + +if [[ -z "${NVIDIA_API_KEY:-}" ]]; then + echo "ERROR: NVIDIA_API_KEY not set" >&2 + exit 1 +fi + +if ! docker info >/dev/null 2>&1; then + echo "ERROR: Docker not running" >&2 + exit 1 +fi + +info "Sandbox name: ${SANDBOX_NAME}" +info "Dashboard port: ${DASHBOARD_PORT}" +info "Device auth: ENABLED (default — no NEMOCLAW_DISABLE_DEVICE_AUTH)" +pass "Preflight checks passed" + +# ══════════════════════════════════════════════════════════════════════════════ +# Phase 1: Install & Onboard (device auth ON) +# ══════════════════════════════════════════════════════════════════════════════ +section "Phase 1: Install & Onboard" + +# Clean up any previous sandbox with the same name +rm -f "$HOME/.nemoclaw/onboard.lock" 2>/dev/null || true + +INSTALL_LOG="/tmp/nemoclaw-e2e-health-install.log" + +info "Installing NemoClaw (install.sh runs onboard in non-interactive mode)..." +INSTALL_EXIT=0 +NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" \ + NEMOCLAW_NON_INTERACTIVE=1 \ + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ + NEMOCLAW_RECREATE_SANDBOX=1 \ + GITHUB_TOKEN="${GITHUB_TOKEN:-}" \ + bash scripts/install.sh --non-interactive 2>&1 | tee "$INSTALL_LOG" || INSTALL_EXIT=$? + +# Source shell profile to pick up PATH changes from install.sh +# shellcheck disable=SC1091 +source "$HOME/.bashrc" 2>/dev/null || true +if [[ -d "$HOME/.local/bin" ]] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]]; then + export PATH="$HOME/.local/bin:$PATH" +fi +export PATH="/usr/local/bin:$PATH" +hash -r + +if [[ $INSTALL_EXIT -ne 0 ]]; then + fail "Install failed with exit code $INSTALL_EXIT" + info "See $INSTALL_LOG for details" + exit 1 +fi + +if ! command -v nemoclaw >/dev/null 2>&1; then + fail "nemoclaw not found on PATH after install" + info "PATH=$PATH" + exit 1 +fi + +# Detect actual dashboard port (may differ from default if port was taken) +ACTUAL_PORT=$(openshell forward list 2>/dev/null | grep "$SANDBOX_NAME" | awk '{print $3}' | head -1) +if [[ -n "$ACTUAL_PORT" ]]; then + DASHBOARD_PORT="$ACTUAL_PORT" + info "Detected actual dashboard port: ${DASHBOARD_PORT}" +fi + +# Verify sandbox exists +if nemoclaw list 2>/dev/null | grep -q "$SANDBOX_NAME"; then + pass "Onboard succeeded — sandbox '${SANDBOX_NAME}' registered" +else + fail "Sandbox '${SANDBOX_NAME}' not found in nemoclaw list after onboard" + exit 1 +fi + +# ══════════════════════════════════════════════════════════════════════════════ +# Phase 2: Health Endpoint Probes (inside sandbox) +# ══════════════════════════════════════════════════════════════════════════════ +section "Phase 2: Health Endpoint Probes" + +# Ensure SSH is ready before probing +info "Setting up SSH to sandbox..." +if ! setup_ssh; then + info "SSH setup failed — falling back to host-side probes only" +fi + +# 2a: /health should return 200 (unaffected by device auth) +info "Probing /health endpoint inside sandbox..." +HEALTH_CODE="" +for attempt in $(seq 1 10); do + HEALTH_CODE=$( + sandbox_exec \ + "curl -so /dev/null -w '%{http_code}' --max-time 3 http://localhost:${DASHBOARD_PORT}/health" + ) || true + if [[ "$HEALTH_CODE" == "200" ]]; then + break + fi + info " Attempt ${attempt}/10: /health returned ${HEALTH_CODE:-empty}, retrying..." + sleep 3 +done + +if [[ "$HEALTH_CODE" == "200" ]]; then + pass "/health returns 200 (auth-free health endpoint via sandbox exec)" +elif [[ -z "$HEALTH_CODE" ]]; then + # SSH exec not working — fall back to host probe (Phase 4 covers this) + skip "/health via sandbox exec returned empty (SSH may not be available; host probe in Phase 4)" +else + fail "/health returned ${HEALTH_CODE} — expected 200" +fi + +# 2b: / should return 401 (proves device auth is active) +info "Probing / endpoint inside sandbox (expect 401 = device auth active)..." +ROOT_CODE=$( + sandbox_exec \ + "curl -so /dev/null -w '%{http_code}' --max-time 3 http://localhost:${DASHBOARD_PORT}/" +) || true + +if [[ "$ROOT_CODE" == "401" ]]; then + pass "/ returns 401 (device auth is active — confirms test premise)" +elif [[ "$ROOT_CODE" == "200" ]]; then + skip "/ returns 200 — device auth not active on this image (test still valid for /health)" +elif [[ -z "$ROOT_CODE" ]]; then + skip "/ via sandbox exec returned empty (SSH may not be available; host probe in Phase 4)" +else + fail "/ returned ${ROOT_CODE:-empty} — expected 401 (device auth) or 200 (no auth)" +fi + +# ══════════════════════════════════════════════════════════════════════════════ +# Phase 3: Status Command (isSandboxGatewayRunning regression) +# ══════════════════════════════════════════════════════════════════════════════ +section "Phase 3: Status Command" + +# The key regression: `nemoclaw status` must NOT report "Offline" +# when device auth returns 401 on the probe endpoint. +info "Running nemoclaw ${SANDBOX_NAME} status..." +STATUS_OUTPUT=$(nemoclaw "$SANDBOX_NAME" status 2>&1) || true + +# Check for the "Health Offline" false negative +if echo "$STATUS_OUTPUT" | grep -qi "offline"; then + fail "Status reports 'Offline' — #2342 REGRESSION: 401 treated as dead" + info "Status output: $(echo "$STATUS_OUTPUT" | head -10)" +else + pass "Status does NOT report 'Offline' (gateway correctly detected as alive)" +fi + +# Check it shows positive running indicators +if echo "$STATUS_OUTPUT" | grep -qiE "running|online|healthy|OpenClaw|Ready"; then + pass "Status shows positive health indicator (Running/Online/Healthy)" +else + info "Status output (no positive indicator found): $(echo "$STATUS_OUTPUT" | head -10)" + skip "Could not confirm positive health indicator (output format may vary)" +fi + +# ══════════════════════════════════════════════════════════════════════════════ +# Phase 4: Host-Side Port Forward Probe +# ══════════════════════════════════════════════════════════════════════════════ +section "Phase 4: Host-Side Port Forward Probe" + +# The port forward from host should also work. verifyDeployment() probes this. +info "Probing dashboard from host via port forward..." +HOST_HEALTH_CODE=$(curl -so /dev/null -w '%{http_code}' --max-time 5 \ + "http://127.0.0.1:${DASHBOARD_PORT}/health" 2>/dev/null) || true + +if [[ "$HOST_HEALTH_CODE" == "200" ]] || [[ "$HOST_HEALTH_CODE" == "401" ]]; then + pass "Host port forward to dashboard is live (HTTP ${HOST_HEALTH_CODE})" +else + # Port forward may not be active in all E2E environments + if [[ "$HOST_HEALTH_CODE" == "000" ]] || [[ -z "$HOST_HEALTH_CODE" ]]; then + skip "Port forward not reachable from host (may not be configured in this environment)" + else + fail "Host health probe returned ${HOST_HEALTH_CODE} — expected 200 or 401" + fi +fi + +# ══════════════════════════════════════════════════════════════════════════════ +# Phase 5: Gateway Restart + Health Re-check +# ══════════════════════════════════════════════════════════════════════════════ +section "Phase 5: Gateway Restart + Health Re-check" + +# Kill the gateway process inside the sandbox to simulate a restart scenario. +# This tests that isSandboxGatewayRunning() + process recovery work correctly +# with the new HTTP status code pattern. +# +# NOTE: Gateway auto-restart depends on the process supervisor inside the +# sandbox. If recovery doesn't work, we still validate that status doesn't +# falsely report Offline on the attempt. +info "Killing gateway process inside sandbox..." +sandbox_exec "pkill -f 'openclaw.*gateway' 2>/dev/null || true" +sleep 3 + +# Run status — this triggers process recovery which uses the fixed health probe +info "Running nemoclaw ${SANDBOX_NAME} status (triggers recovery)..." +RECOVERY_STATUS=$(nemoclaw "$SANDBOX_NAME" status 2>&1) || true + +# The key assertion: even during recovery, status must NOT report Offline +# due to 401 being misinterpreted. It may say "recovering" or show the +# gateway as temporarily down, but NOT "Health Offline" from #2342. +if echo "$RECOVERY_STATUS" | grep -qi "offline"; then + fail "Status reports 'Offline' during recovery — #2342 regression" +else + pass "Status does not report 'Offline' during recovery attempt" +fi + +# Wait for recovery to complete and gateway to become healthy again +info "Waiting for gateway to recover..." +RECOVERED=false +for attempt in $(seq 1 30); do + RECOVER_HEALTH=$( + sandbox_exec \ + "curl -so /dev/null -w '%{http_code}' --max-time 3 http://localhost:${DASHBOARD_PORT}/health" + ) || true + if [[ "$RECOVER_HEALTH" == "200" ]] || [[ "$RECOVER_HEALTH" == "401" ]]; then + RECOVERED=true + break + fi + sleep 5 +done + +if $RECOVERED; then + pass "Gateway recovered after restart (HTTP ${RECOVER_HEALTH} on /health)" +else + # Recovery may not be supported in all environments — skip rather than fail + skip "Gateway did not recover within 150s (process supervisor may not be active)" +fi + +# ══════════════════════════════════════════════════════════════════════════════ +# Phase 6: Verify verifyDeployment() Output in Onboard Log +# ══════════════════════════════════════════════════════════════════════════════ +section "Phase 6: Verify Deployment Diagnostics" + +# Check that the onboard log includes verification output (not a crash/skip) +if grep -qi "verification\|✓.*Gateway\|✓.*Dashboard\|verif" "$INSTALL_LOG" 2>/dev/null; then + pass "Onboard log contains deployment verification output" +elif grep -qi "Dashboard is live" "$INSTALL_LOG" 2>/dev/null; then + pass "Onboard log confirms dashboard readiness check passed" +else + skip "Could not confirm verification output in onboard log (format may vary)" +fi + +# ══════════════════════════════════════════════════════════════════════════════ +# Summary +# ══════════════════════════════════════════════════════════════════════════════ +section "Summary" +echo "" +printf ' Total: %d | \033[32mPass: %d\033[0m | \033[31mFail: %d\033[0m | \033[33mSkip: %d\033[0m\n' \ + "$TOTAL" "$PASS" "$FAIL" "$SKIP" +echo "" + +if [[ $FAIL -gt 0 ]]; then + echo "RESULT: FAILED — $FAIL test(s) failed" + exit 1 +fi + +echo "RESULT: PASSED — all health probes work correctly with device auth enabled" +exit 0 diff --git a/test/e2e-vpn/test-diagnostics.sh b/test/e2e-vpn/test-diagnostics.sh new file mode 100755 index 00000000000..9c6b40d2ebb --- /dev/null +++ b/test/e2e-vpn/test-diagnostics.sh @@ -0,0 +1,513 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# ============================================================================= +# test-diagnostics.sh +# NemoClaw Diagnostics & Credential E2E Tests +# +# Covers: +# TC-DIAG-04: nemoclaw --version (semver output, exit 0) +# TC-DIAG-02: nemoclaw debug --quick (fast, non-empty archive) +# TC-DIAG-01: nemoclaw debug --output (tarball, no credentials in archive) +# TC-DIAG-06: nemoclaw debug --sandbox rejected; registered name accepted +# TC-DIAG-05: /nemoclaw status inside sandbox (model + provider) +# TC-DIAG-03: credentials list (no values) + credentials reset +# +# Prerequisites: +# - Docker running +# - NVIDIA_API_KEY set +# ============================================================================= + +set -euo pipefail + +# ── Overall timeout ────────────────────────────────────────────────────────── +if [ -z "${NEMOCLAW_E2E_NO_TIMEOUT:-}" ]; then + export NEMOCLAW_E2E_NO_TIMEOUT=1 + TIMEOUT_SECONDS="${NEMOCLAW_E2E_TIMEOUT_SECONDS:-3600}" + if command -v timeout >/dev/null 2>&1; then + exec timeout -s TERM "$TIMEOUT_SECONDS" bash "$0" "$@" + elif command -v gtimeout >/dev/null 2>&1; then + exec gtimeout -s TERM "$TIMEOUT_SECONDS" bash "$0" "$@" + fi +fi + +# ── Config ─────────────────────────────────────────────────────────────────── +SANDBOX_NAME="e2e-diag" +LOG_FILE="test-diagnostics-$(date +%Y%m%d-%H%M%S).log" +touch "$LOG_FILE" + +if command -v gtimeout >/dev/null 2>&1; then + TIMEOUT_CMD="gtimeout" +elif command -v timeout >/dev/null 2>&1; then + TIMEOUT_CMD="timeout" +else + TIMEOUT_CMD="" +fi + +# ── Colors ─────────────────────────────────────────────────────────────────── +GREEN='\033[0;32m' +RED='\033[0;31m' +YELLOW='\033[1;33m' +CYAN='\033[0;36m' +NC='\033[0m' + +PASS=0 +FAIL=0 +SKIP=0 +TOTAL=0 + +# Log a timestamped message. +log() { echo -e "${CYAN}[$(date +%H:%M:%S)]${NC} $*" | tee -a "$LOG_FILE"; } +# Record a passing assertion. +pass() { + ((PASS += 1)) + ((TOTAL += 1)) + echo -e "${GREEN} PASS${NC} $1" | tee -a "$LOG_FILE" +} +# Record a failing assertion. +fail() { + ((FAIL += 1)) + ((TOTAL += 1)) + echo -e "${RED} FAIL${NC} $1 — $2" | tee -a "$LOG_FILE" +} +# Record a skipped test. +skip() { + ((SKIP += 1)) + ((TOTAL += 1)) + echo -e "${YELLOW} SKIP${NC} $1 — $2" | tee -a "$LOG_FILE" +} + +# ── Resolve repo root ──────────────────────────────────────────────────────── +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" + +# shellcheck source=test/e2e-vpn/lib/install-path-refresh.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib/install-path-refresh.sh" + +# ── Install NemoClaw if not present ────────────────────────────────────────── +install_nemoclaw() { + export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" + if [ -s "$NVM_DIR/nvm.sh" ]; then + # shellcheck source=/dev/null + . "$NVM_DIR/nvm.sh" + fi + nemoclaw_ensure_local_bin_on_path + + if command -v nemoclaw >/dev/null 2>&1; then + log "nemoclaw already installed: $(nemoclaw --version 2>/dev/null || echo unknown)" + return + fi + log "=== Installing NemoClaw via install.sh ===" + NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" \ + NVIDIA_API_KEY="${NVIDIA_API_KEY:-nvapi-DUMMY-FOR-INSTALL}" \ + NEMOCLAW_NON_INTERACTIVE=1 \ + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ + bash "$REPO_ROOT/install.sh" --non-interactive --yes-i-accept-third-party-software \ + 2>&1 | tee -a "$LOG_FILE" + nemoclaw_refresh_install_env + if ! command -v nemoclaw >/dev/null 2>&1; then + log "ERROR: install.sh failed — nemoclaw not found" + exit 1 + fi +} + +# ── Pre-flight ─────────────────────────────────────────────────────────────── +preflight() { + log "=== Pre-flight checks ===" + if ! docker info >/dev/null 2>&1; then + log "ERROR: Docker is not running." + exit 1 + fi + log "Docker is running" + + local api_key="${NVIDIA_API_KEY:-}" + if [[ -z "$api_key" ]]; then + log "ERROR: NVIDIA_API_KEY not set" + exit 1 + fi + + install_nemoclaw + log "nemoclaw: $(nemoclaw --version 2>/dev/null || echo unknown)" + log "Pre-flight complete" +} + +# Execute a command inside the sandbox via SSH. +sandbox_exec() { + local cmd="$1" + local ssh_cfg + ssh_cfg="$(mktemp)" + if ! openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_cfg" 2>/dev/null; then + rm -f "$ssh_cfg" + echo "" + return 1 + fi + local result ssh_exit=0 + result=$(${TIMEOUT_CMD:+$TIMEOUT_CMD 120} ssh -F "$ssh_cfg" \ + -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" "$cmd" 2>&1) || ssh_exit=$? + rm -f "$ssh_cfg" + echo "$result" + return $ssh_exit +} + +# Onboard a sandbox with default settings. +onboard_sandbox() { + local name="$1" + log " Onboarding sandbox '$name'..." + rm -f "$HOME/.nemoclaw/onboard.lock" 2>/dev/null || true + NEMOCLAW_SANDBOX_NAME="$name" \ + NEMOCLAW_NON_INTERACTIVE=1 \ + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ + NEMOCLAW_POLICY_TIER="open" \ + ${TIMEOUT_CMD:+$TIMEOUT_CMD 600} nemoclaw onboard --non-interactive --yes-i-accept-third-party-software \ + 2>&1 | tee -a "$LOG_FILE" || { + log "FATAL: Onboard failed for '$name'" + return 1 + } + log " Sandbox '$name' onboarded" +} + +# ============================================================================= +# TC-DIAG-04: nemoclaw --version +# ============================================================================= +test_diag_04_version() { + log "=== TC-DIAG-04: nemoclaw --version ===" + + local version_output version_rc=0 + version_output=$(nemoclaw --version 2>&1) || version_rc=$? + + log " Output: $version_output (exit $version_rc)" + + if [[ $version_rc -ne 0 ]]; then + fail "TC-DIAG-04: Exit code" "nemoclaw --version exited with $version_rc" + return + fi + + if echo "$version_output" | grep -qE '[0-9]+\.[0-9]+\.[0-9]+'; then + pass "TC-DIAG-04: Version output matches semver ($version_output)" + else + fail "TC-DIAG-04: Format" "Output does not match semver pattern: $version_output" + fi +} + +# ============================================================================= +# TC-DIAG-02: nemoclaw debug --quick +# ============================================================================= +test_diag_02_debug_quick() { + log "=== TC-DIAG-02: nemoclaw debug --quick ===" + + local debug_dir + debug_dir=$(mktemp -d) + local output_file="${debug_dir}/quick-debug.tar.gz" + + local start_time + start_time=$(date +%s) + + local debug_output debug_rc=0 + debug_output=$(${TIMEOUT_CMD:+$TIMEOUT_CMD 30} nemoclaw debug --quick --output "$output_file" 2>&1) || debug_rc=$? + + local end_time + end_time=$(date +%s) + local elapsed=$((end_time - start_time)) + + log " Completed in ${elapsed}s (exit $debug_rc)" + log " Output: ${debug_output:0:300}" + + if [[ $debug_rc -ne 0 ]]; then + fail "TC-DIAG-02: Exit code" "debug --quick exited with $debug_rc" + rm -rf "$debug_dir" + return + fi + + if [[ -f "$output_file" ]] && [[ -s "$output_file" ]]; then + pass "TC-DIAG-02: debug --quick produced non-empty archive (${elapsed}s)" + else + fail "TC-DIAG-02: Output" "No archive produced or archive is empty" + fi + + if [[ $elapsed -le 30 ]]; then + pass "TC-DIAG-02: Completed within time limit (${elapsed}s)" + else + fail "TC-DIAG-02: Timing" "Took ${elapsed}s (expected ≤30s)" + fi + + rm -rf "$debug_dir" +} + +# ============================================================================= +# TC-DIAG-01: nemoclaw debug --output (full tarball + credential sanitization) +# ============================================================================= +test_diag_01_debug_tarball() { + log "=== TC-DIAG-01: Full Debug Tarball + Credential Sanitization ===" + + local debug_dir + debug_dir=$(mktemp -d) + local output_file="${debug_dir}/debug-full.tar.gz" + local extract_dir="${debug_dir}/extracted" + + local debug_output debug_rc=0 + debug_output=$(nemoclaw debug --output "$output_file" 2>&1) || debug_rc=$? + log " Debug output (exit $debug_rc): ${debug_output:0:300}" + + if [[ $debug_rc -ne 0 ]] || [[ ! -f "$output_file" ]]; then + fail "TC-DIAG-01: Setup" "debug --output failed or no file produced" + rm -rf "$debug_dir" + return + fi + + pass "TC-DIAG-01: Debug tarball created" + + mkdir -p "$extract_dir" + if ! tar xzf "$output_file" -C "$extract_dir" 2>/dev/null; then + fail "TC-DIAG-01: Extract" "Could not extract tarball" + rm -rf "$debug_dir" + return + fi + + local real_key="${NVIDIA_API_KEY:-}" + if [[ -z "$real_key" ]]; then + skip "TC-DIAG-01: Credential check" "NVIDIA_API_KEY not set" + rm -rf "$debug_dir" + return + fi + + log " Scanning extracted files for credential leaks..." + local leaks + leaks=$(grep -rl "$real_key" "$extract_dir" 2>/dev/null || true) + + if [[ -z "$leaks" ]]; then + pass "TC-DIAG-01: No API key found in debug tarball" + else + fail "TC-DIAG-01: Credential leak" "API key found in: $leaks" + fi + + local pattern_leaks + pattern_leaks=$(grep -rlE "nvapi-[A-Za-z0-9_-]{10,}" "$extract_dir" 2>/dev/null || true) + if [[ -z "$pattern_leaks" ]]; then + pass "TC-DIAG-01: No nvapi- pattern credentials in tarball" + else + fail "TC-DIAG-01: Pattern leak" "nvapi- pattern found in: $pattern_leaks" + fi + + rm -rf "$debug_dir" +} + +# ============================================================================= +# TC-DIAG-06: debug --sandbox NAME validation +# Registered names succeed; unknown names exit non-zero, name the sandbox, +# and leave no partial tarball. +# ============================================================================= +test_diag_06_debug_sandbox_validation() { + log "=== TC-DIAG-06: debug --sandbox NAME validation ===" + + local debug_dir + debug_dir=$(mktemp -d) + + local good_output="${debug_dir}/known.tar.gz" + local good_rc=0 good_log="" + good_log=$(${TIMEOUT_CMD:+$TIMEOUT_CMD 30} nemoclaw debug --quick --sandbox "$SANDBOX_NAME" --output "$good_output" 2>&1) || good_rc=$? + log " Registered name exit=$good_rc" + if [[ $good_rc -eq 0 ]] && [[ -s "$good_output" ]]; then + pass "TC-DIAG-06: Registered --sandbox produced non-empty archive" + else + fail "TC-DIAG-06: Registered name" "exit=$good_rc, output=${good_log:0:300}" + fi + + # Unique per-run name avoids collisions when another e2e job leaves a + # sandbox with a shared "does-not-exist" placeholder behind. + local bad_name + bad_name="nemoclaw-e2e-missing-$$-$(date +%s)-${RANDOM}" + local bad_output="${debug_dir}/unknown.tar.gz" + local bad_rc=0 bad_log="" + bad_log=$(${TIMEOUT_CMD:+$TIMEOUT_CMD 30} nemoclaw debug --quick --sandbox "$bad_name" --output "$bad_output" 2>&1) || bad_rc=$? + log " Unknown name exit=$bad_rc" + + if [[ $bad_rc -ne 0 ]]; then + pass "TC-DIAG-06: Unknown --sandbox exits non-zero" + else + fail "TC-DIAG-06: Unknown name exit code" "expected non-zero, got 0" + fi + + if echo "$bad_log" | grep -q "$bad_name"; then + pass "TC-DIAG-06: Error message names the unknown sandbox" + else + fail "TC-DIAG-06: Error message" "did not mention '$bad_name'" + fi + + if echo "$bad_log" | grep -qi "not registered"; then + pass "TC-DIAG-06: Error message reports 'not registered'" + else + fail "TC-DIAG-06: Error message" "missing 'not registered' guidance" + fi + + if [[ ! -e "$bad_output" ]]; then + pass "TC-DIAG-06: No partial tarball written for unknown sandbox" + else + local size + size=$(stat -c '%s' "$bad_output" 2>/dev/null || stat -f '%z' "$bad_output" 2>/dev/null || echo "?") + fail "TC-DIAG-06: Tarball cleanup" "partial tarball persisted at $bad_output (${size} bytes)" + fi + + rm -rf "$debug_dir" +} + +# ============================================================================= +# TC-DIAG-05: Sandbox inference config visible inside sandbox +# ============================================================================= +test_diag_05_sandbox_config() { + log "=== TC-DIAG-05: Sandbox Inference Config ===" + + log " Checking openclaw.json config inside sandbox..." + local config_output + config_output=$(sandbox_exec "cat /sandbox/.openclaw/openclaw.json 2>/dev/null" 2>&1) || true + + if [[ -z "$config_output" ]]; then + fail "TC-DIAG-05: Config" "Could not read openclaw.json inside sandbox" + return + fi + + pass "TC-DIAG-05: openclaw.json readable inside sandbox" + + log " Checking nemoclaw status from host..." + local status_output + status_output=$(nemoclaw "$SANDBOX_NAME" status 2>&1) || true + if echo "$status_output" | grep -qiE "Model.*nemotron\|Model.*nvidia\|Model.*llama"; then + pass "TC-DIAG-05: nemoclaw status shows model info" + elif echo "$status_output" | grep -qi "Model"; then + pass "TC-DIAG-05: nemoclaw status shows Model field" + else + fail "TC-DIAG-05: Status" "No model info in nemoclaw status output" + fi +} + +# ============================================================================= +# TC-DIAG-03: credentials list + credentials reset +# ============================================================================= +test_diag_03_credentials() { + log "=== TC-DIAG-03: Credentials List and Reset ===" + + local real_key="${NVIDIA_API_KEY:-}" + + log " Step 1: Running credentials list..." + local list_output list_rc=0 + list_output=$(nemoclaw credentials list 2>&1) || list_rc=$? + log " List output (exit $list_rc): ${list_output:0:400}" + + if [[ $list_rc -ne 0 ]]; then + fail "TC-DIAG-03: List" "credentials list exited with $list_rc" + return + fi + + if echo "$list_output" | grep -qi "No stored credentials"; then + pass "TC-DIAG-03: credentials list works (store empty — API key passed via env on CI)" + + log " Step 2: Verifying credentials list does not leak env var..." + if [[ -n "$real_key" ]] && echo "$list_output" | grep -qF "$real_key"; then + fail "TC-DIAG-03: Value leak" "Real API key visible in credentials list output" + else + pass "TC-DIAG-03: credentials list does not expose env key values" + fi + return + fi + + if echo "$list_output" | grep -qiE "NVIDIA_API_KEY\|nvidia.api"; then + pass "TC-DIAG-03: credentials list shows key name" + else + skip "TC-DIAG-03: Key name" "Expected credential key not found in list" + return + fi + + if [[ -n "$real_key" ]] && echo "$list_output" | grep -qF "$real_key"; then + fail "TC-DIAG-03: Value leak" "Real API key value visible in credentials list" + else + pass "TC-DIAG-03: credentials list does not expose key values" + fi + + log " Step 2: Running credentials reset NVIDIA_API_KEY..." + local reset_output reset_rc=0 + reset_output=$(nemoclaw credentials reset NVIDIA_API_KEY --yes 2>&1) || reset_rc=$? + log " Reset output (exit $reset_rc): ${reset_output:0:300}" + + if [[ $reset_rc -eq 0 ]]; then + pass "TC-DIAG-03: credentials reset completed" + else + fail "TC-DIAG-03: Reset" "credentials reset failed (exit $reset_rc)" + return + fi + + log " Step 3: Verifying key removed from list..." + local post_list + post_list=$(nemoclaw credentials list 2>&1) || true + if echo "$post_list" | grep -qiE "NVIDIA_API_KEY"; then + fail "TC-DIAG-03: Post-reset" "NVIDIA_API_KEY still in list after reset" + else + pass "TC-DIAG-03: NVIDIA_API_KEY removed after reset" + fi +} + +# Clean up sandbox and services on exit. +teardown() { + # Do not unlink ~/.nemoclaw/onboard.lock: see rationale in + # test/e2e-vpn/lib/sandbox-teardown.sh — the lock is PID-ownership-aware + # and onboard cleans up stale locks itself. + set +e + nemoclaw "$SANDBOX_NAME" destroy --yes 2>/dev/null || true + set -e +} + +# Print final PASS/FAIL/SKIP counts and exit. +summary() { + echo "" + echo "============================================================" + echo " Diagnostics E2E Results" + echo "============================================================" + echo -e " ${GREEN}PASS: $PASS${NC}" + echo -e " ${RED}FAIL: $FAIL${NC}" + echo -e " ${YELLOW}SKIP: $SKIP${NC}" + echo " TOTAL: $TOTAL" + echo "============================================================" + echo " Log: $LOG_FILE" + echo "============================================================" + echo "" + + if [[ $FAIL -gt 0 ]]; then + exit 1 + fi + exit 0 +} + +# Entry point: preflight → tests → summary. +main() { + echo "" + echo "============================================================" + echo " NemoClaw Diagnostics E2E Tests" + echo " $(date)" + echo "============================================================" + echo "" + + preflight + + # No sandbox needed + test_diag_04_version + test_diag_02_debug_quick + + # Onboard sandbox for remaining tests + log "=== Onboarding sandbox ===" + if ! onboard_sandbox "$SANDBOX_NAME"; then + log "FATAL: Could not onboard sandbox" + exit 1 + fi + + test_diag_01_debug_tarball + test_diag_06_debug_sandbox_validation + test_diag_05_sandbox_config + test_diag_03_credentials # modifies state — runs last + + teardown + trap - EXIT + summary +} + +trap teardown EXIT +main "$@" diff --git a/test/e2e-vpn/test-docs-validation.sh b/test/e2e-vpn/test-docs-validation.sh new file mode 100755 index 00000000000..39697f2b824 --- /dev/null +++ b/test/e2e-vpn/test-docs-validation.sh @@ -0,0 +1,163 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Docs Validation E2E — CLI/docs parity + markdown link validation +# +# Runs check-docs.sh to verify nemoclaw --help matches commands.mdx +# and that markdown links resolve. No sandbox needed — just needs +# nemoclaw installed. +# +# Split from the cloud-experimental-e2e monolith (see #2644). +# Former phase: 5f (documentation checks). +# +# Prerequisites: +# - nemoclaw installed and on PATH +# - Node.js on PATH (for CLI help output) +# +# Environment: +# CHECK_DOC_LINKS_REMOTE=1 — curl http(s) links (default: 1; set 0 to skip) +# CHECK_DOC_LINKS_VERBOSE=1 — log each URL while curling +# +# Usage: +# bash test/e2e-vpn/test-docs-validation.sh +# CHECK_DOC_LINKS_REMOTE=0 bash test/e2e-vpn/test-docs-validation.sh + +# ShellCheck cannot see EXIT trap invocations of cleanup helpers in this E2E script. +# shellcheck disable=SC2317 +set -uo pipefail + +PASS=0 +FAIL=0 +SKIP=0 +TOTAL=0 + +pass() { + ((PASS++)) + ((TOTAL++)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} +fail() { + ((FAIL++)) + ((TOTAL++)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} +# shellcheck disable=SC2329 +skip() { + ((SKIP++)) + ((TOTAL++)) + printf '\033[33m SKIP: %s\033[0m\n' "$1" +} +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } + +# ── Repo root ── +_script_dir="$(cd "$(dirname "$0")" && pwd)" +_candidate="$(cd "${_script_dir}/../.." && pwd)" +if [ -d /workspace ] && [ -f /workspace/package.json ] && [ -d /workspace/test/e2e ]; then + REPO="/workspace" +elif [ -f "${_candidate}/package.json" ] && [ -d "${_candidate}/test/e2e" ]; then + REPO="${_candidate}" # exported for child scripts +else + echo "ERROR: Cannot find repo root." + exit 1 +fi +unset _script_dir _candidate +export REPO + +E2E_DIR="$(cd "$(dirname "$0")" && pwd)" + +# ══════════════════════════════════════════════════════════════════════ +# Phase 1: Prerequisites +# ══════════════════════════════════════════════════════════════════════ +section "Phase 1: Prerequisites" + +# check-docs.sh needs nemoclaw on PATH for CLI parity check. +# In nightly CI the install step runs before this job. +if command -v nemoclaw >/dev/null 2>&1; then + pass "nemoclaw on PATH" +else + # Try sourcing nvm in case it wasn't inherited + export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" + # shellcheck source=/dev/null + [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" + [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]] && export PATH="$HOME/.local/bin:$PATH" + + if command -v nemoclaw >/dev/null 2>&1; then + pass "nemoclaw on PATH (after sourcing nvm)" + else + fail "nemoclaw not on PATH — install NemoClaw first" + exit 1 + fi +fi + +# ══════════════════════════════════════════════════════════════════════ +# Phase 2: CLI / docs parity (check-docs.sh --only-cli) +# ══════════════════════════════════════════════════════════════════════ +section "Phase 2: CLI / docs parity" + +info "Running check-docs.sh --only-cli (nemoclaw --help vs commands.mdx)..." +set +e +bash "${E2E_DIR}/e2e-cloud-experimental/check-docs.sh" --only-cli +cli_rc=$? +set -uo pipefail + +if [ "$cli_rc" -eq 0 ]; then + pass "CLI / docs parity check passed" +else + fail "CLI / docs parity check failed (exit ${cli_rc})" + exit 1 +fi + +# ══════════════════════════════════════════════════════════════════════ +# Phase 3: Markdown link validation (check-docs.sh --only-links) +# ══════════════════════════════════════════════════════════════════════ +section "Phase 3: Markdown link validation" + +if [ "${CHECK_DOC_LINKS_REMOTE:-1}" = "0" ]; then + info "Running check-docs.sh --only-links --local-only (no remote probes)..." + set +e + bash "${E2E_DIR}/e2e-cloud-experimental/check-docs.sh" --only-links --local-only + links_rc=$? + set -uo pipefail +else + info "Running check-docs.sh --only-links (includes remote http(s) probes)..." + set +e + bash "${E2E_DIR}/e2e-cloud-experimental/check-docs.sh" --only-links + links_rc=$? + set -uo pipefail +fi + +if [ "$links_rc" -eq 0 ]; then + pass "Markdown link validation passed" +else + # Remote link probes can fail due to rate limiting (429) — warn but don't block + if [ "${CHECK_DOC_LINKS_REMOTE:-1}" != "0" ]; then + info "Link validation failed — may be due to remote rate limiting. Re-run with CHECK_DOC_LINKS_REMOTE=0 to check local links only." + fi + fail "Markdown link validation failed (exit ${links_rc})" + exit 1 +fi + +# ══════════════════════════════════════════════════════════════════════ +# Summary +# ══════════════════════════════════════════════════════════════════════ +echo "" +echo "========================================" +echo " Docs Validation E2E Results:" +echo " Passed: $PASS" +echo " Failed: $FAIL" +echo " Skipped: $SKIP" +echo " Total: $TOTAL" +echo "========================================" + +if [ "$FAIL" -eq 0 ]; then + printf '\033[1;32m\n Docs Validation E2E PASSED.\033[0m\n' + exit 0 +else + printf '\033[1;31m\n %d test(s) failed.\033[0m\n' "$FAIL" + exit 1 +fi diff --git a/test/e2e-vpn/test-double-onboard.sh b/test/e2e-vpn/test-double-onboard.sh new file mode 100755 index 00000000000..bb78ce6e802 --- /dev/null +++ b/test/e2e-vpn/test-double-onboard.sh @@ -0,0 +1,910 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Double onboard / lifecycle recovery: +# - prove repeat onboard reuses the healthy shared NemoClaw gateway +# - prove onboarding a second sandbox does not destroy the first sandbox +# - prove stale registry entries are reconciled against live OpenShell state +# - prove gateway rebuilds surface the expected lifecycle guidance +# +# This script intentionally uses a local fake OpenAI-compatible endpoint so it +# matches the current onboarding flow. Older versions of this test relied on a +# missing/invalid NVIDIA_API_KEY causing a late failure after sandbox creation; +# that no longer reflects current non-interactive onboarding behavior. + +# ShellCheck cannot see EXIT trap invocations of cleanup helpers in this E2E script. +# shellcheck disable=SC2317 +set -uo pipefail + +# Three sequential sandbox creations (~5-7 min each) plus cleanup phases need +# well over the default 900s. 80 min leaves a 10 min buffer under the 90-min +# CI job timeout. +export NEMOCLAW_E2E_DEFAULT_TIMEOUT=4800 +SCRIPT_DIR_TIMEOUT="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +# shellcheck source=test/e2e-vpn/e2e-timeout.sh +source "${SCRIPT_DIR_TIMEOUT}/e2e-timeout.sh" + +PASS=0 +FAIL=0 +TOTAL=0 + +pass() { + ((PASS++)) + ((TOTAL++)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} +fail() { + ((FAIL++)) + ((TOTAL++)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } + +# TODO(#2562): replace shell timeout with structured timeout once unified abstraction lands + +# Per-phase timeout in seconds (20 min per onboard phase, generous for CI) +PHASE_TIMEOUT="${NEMOCLAW_E2E_PHASE_TIMEOUT:-1200}" + +# Elapsed-time helpers +phase_start_time() { date +%s; } +phase_elapsed() { + local start="$1" + local now + now="$(date +%s)" + echo $((now - start)) +} + +# Diagnostic dump — called on phase timeout or failure to aid debugging +dump_diagnostics() { + local phase_label="${1:-unknown}" + info "=== Diagnostics for ${phase_label} ===" + if [ -n "${RUN_ONBOARD_OUTPUT:-}" ]; then + info "Captured nemoclaw onboard stdout/stderr (exit=${RUN_ONBOARD_EXIT:-?}):" + printf '%s\n' "$RUN_ONBOARD_OUTPUT" | sed 's/^/ /' + fi + info "openshell status:" + openshell status 2>&1 | sed 's/^/ /' || true + info "openshell sandbox list:" + openshell sandbox list 2>&1 | sed 's/^/ /' || true + info "openshell forward list:" + openshell forward list 2>&1 | sed 's/^/ /' || true + for sandbox_name in "${SANDBOX_A:-}" "${SANDBOX_B:-}"; do + [ -n "$sandbox_name" ] || continue + info "${sandbox_name} /etc/resolv.conf:" + openshell sandbox exec --name "$sandbox_name" -- cat /etc/resolv.conf 2>&1 | sed 's/^/ /' || true + info "${sandbox_name} inference.local /v1/models probe:" + openshell sandbox exec --name "$sandbox_name" -- sh -c 'curl -sk -o /tmp/nemoclaw-e2e-models.out -w "%{http_code}" --connect-timeout 3 --max-time 8 https://inference.local/v1/models; printf "\\n"; head -c 300 /tmp/nemoclaw-e2e-models.out 2>/dev/null; printf "\\n"' 2>&1 | sed 's/^/ /' || true + done + info "docker ps:" + docker ps 2>&1 | sed 's/^/ /' || true + info "Docker DNS proxy/gateway logs:" + docker ps --format '{{.Names}}' 2>/dev/null | grep -Ei 'dns|proxy|gateway|nemoclaw' | while read -r container_name; do + [ -n "$container_name" ] || continue + info "docker logs ${container_name}:" + docker logs --tail 80 "$container_name" 2>&1 | sed 's/^/ /' || true + done + info "OpenShell inference route:" + openshell inference get 2>&1 | sed 's/^/ /' || true + info "=== End diagnostics ===" +} + +registry_has() { + local sandbox_name="$1" + [ -f "$REGISTRY" ] && grep -q "$sandbox_name" "$REGISTRY" +} + +wait_openshell_sandbox_absent() { + local sandbox_name="$1" + local timeout="${2:-60}" + local deadline=$((SECONDS + timeout)) + local output status + + while [ "$SECONDS" -le "$deadline" ]; do + output="$(openshell sandbox get "$sandbox_name" 2>&1)" + status=$? + if [ "$status" -ne 0 ] && grep -qiE 'NotFound|Not Found|sandbox not found' <<<"$output"; then + return 0 + fi + sleep 1 + done + + info "OpenShell still reports sandbox '$sandbox_name' after ${timeout}s:" + printf '%s\n' "$output" | sed 's/^/ /' + return 1 +} + +docker_driver_gateway_pid_file() { + printf '%s/.local/state/nemoclaw/openshell-docker-gateway/openshell-gateway.pid\n' "$HOME" +} + +gateway_runtime_id() { + local pid_file pid cid + pid_file="$(docker_driver_gateway_pid_file)" + if [ -f "$pid_file" ]; then + pid="$(tr -d '[:space:]' <"$pid_file" 2>/dev/null || true)" + if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then + printf 'pid:%s\n' "$pid" + return 0 + fi + fi + + cid="$(docker ps -qf "name=openshell-cluster-nemoclaw" 2>/dev/null | head -1)" + if [ -n "$cid" ]; then + printf 'container:%s\n' "$cid" + return 0 + fi + + return 1 +} + +gateway_alias_endpoint() { + local scheme="https" + if [ "$(uname -s)" = "Linux" ]; then + scheme="http" + fi + printf '%s://127.0.0.1:%s\n' "$scheme" "${NEMOCLAW_GATEWAY_PORT:-8080}" +} + +stop_gateway_runtime() { + local pid_file pid cid + openshell forward stop 18789 2>/dev/null || true + openshell gateway stop -g nemoclaw 2>/dev/null || true + + pid_file="$(docker_driver_gateway_pid_file)" + if [ -f "$pid_file" ]; then + pid="$(tr -d '[:space:]' <"$pid_file" 2>/dev/null || true)" + if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then + kill "$pid" 2>/dev/null || true + for _ in $(seq 1 10); do + kill -0 "$pid" 2>/dev/null || break + sleep 1 + done + if kill -0 "$pid" 2>/dev/null; then + kill -9 "$pid" 2>/dev/null || true + fi + fi + fi + + cid="$(docker ps -qf "name=openshell-cluster-nemoclaw" 2>/dev/null | head -1)" + if [ -n "$cid" ]; then + docker stop "$cid" >/dev/null 2>&1 || true + fi +} + +SANDBOX_A="e2e-double-a" +SANDBOX_B="e2e-double-b" +INSTALL_SANDBOX_NAME="${NEMOCLAW_E2E_INSTALL_SANDBOX_NAME:-}" +ALT_GATEWAY_NAME="e2e-double-alt" +REGISTRY="$HOME/.nemoclaw/sandboxes.json" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +# shellcheck source=test/e2e-vpn/lib/openai-compatible-api-proof.sh +source "${SCRIPT_DIR}/lib/openai-compatible-api-proof.sh" +FAKE_OPENAI_HOST="127.0.0.1" +FAKE_OPENAI_PORT="${NEMOCLAW_FAKE_PORT:-18080}" +FAKE_OPENAI_LOG="$(mktemp)" +FAKE_BASE_URL="http://${FAKE_OPENAI_HOST}:${FAKE_OPENAI_PORT}/v1" + +if command -v node >/dev/null 2>&1 && [ -f "$REPO_ROOT/bin/nemoclaw.js" ]; then + NEMOCLAW_CMD=(node "$REPO_ROOT/bin/nemoclaw.js") +else + NEMOCLAW_CMD=(nemoclaw) +fi + +# shellcheck disable=SC2329 +cleanup() { + stop_fake_openai_compatible_api + rm -f "$FAKE_OPENAI_LOG" +} +trap cleanup EXIT + +start_fake_openai() { + start_fake_openai_compatible_api || return 1 + FAKE_BASE_URL="$FAKE_OPENAI_BASE_URL" +} + +# TODO(#2562): replace shell timeout with structured timeout once unified abstraction lands +run_onboard() { + local sandbox_name="$1" + local recreate="${2:-0}" + local log_file + log_file="$(mktemp)" + + local -a env_args=( + "COMPATIBLE_API_KEY=dummy" + "NEMOCLAW_NON_INTERACTIVE=1" + "NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1" + "NEMOCLAW_PROVIDER=custom" + "NEMOCLAW_ENDPOINT_URL=${FAKE_BASE_URL}" + "NEMOCLAW_MODEL=test-model" + "NEMOCLAW_SANDBOX_NAME=${sandbox_name}" + "NEMOCLAW_POLICY_MODE=skip" + "NEMOCLAW_DASHBOARD_PORT=" + "CHAT_UI_URL=" + ) + if [ "$recreate" = "1" ]; then + env_args+=("NEMOCLAW_RECREATE_SANDBOX=1") + fi + + run_with_timeout "$PHASE_TIMEOUT" env "${env_args[@]}" "${NEMOCLAW_CMD[@]}" onboard --non-interactive >"$log_file" 2>&1 + RUN_ONBOARD_EXIT=$? + RUN_ONBOARD_OUTPUT="$(cat "$log_file")" + rm -f "$log_file" +} + +run_nemoclaw() { + "${NEMOCLAW_CMD[@]}" "$@" +} + +stop_forward_if_set() { + local port="${1:-}" + if [ -n "$port" ]; then + openshell forward stop "$port" 2>/dev/null || true + fi +} + +dashboard_port_from_list() { + local sandbox_name="$1" + + LIST_OUTPUT="$list_output" python3 - "$sandbox_name" <<'PY' +import os +import re +import sys + +target = sys.argv[1] +current = None + +for line in os.environ.get("LIST_OUTPUT", "").splitlines(): + if line.startswith(" ") and not line.startswith(" "): + stripped = line.strip() + current = stripped.split()[0] if stripped else None + continue + + if current == target: + match = re.search(r"dashboard:\s+http://127\.0\.0\.1:(\d+)/?", line) + if match: + print(match.group(1)) + sys.exit(0) + +sys.exit(1) +PY +} + +gateway_name_from_output() { + local output="$1" + + GATEWAY_OUTPUT="$output" python3 <<'PY' +import os +import re +import sys + +clean = re.sub(r"\x1b\[[0-9;]*m", "", os.environ.get("GATEWAY_OUTPUT", "")) +match = re.search(r"^\s*Gateway:\s+([^\s]+)", clean, re.MULTILINE) +if match: + print(match.group(1)) + sys.exit(0) +sys.exit(1) +PY +} + +forward_owner_for_port() { + local port="$1" + + FORWARD_OUTPUT="$forward_output" python3 - "$port" <<'PY' +import os +import re +import sys + +target = sys.argv[1] +clean = re.sub(r"\x1b\[[0-9;]*m", "", os.environ.get("FORWARD_OUTPUT", "")) + +for line in clean.splitlines(): + parts = line.strip().split() + if len(parts) < 5 or parts[0].lower() == "sandbox": + continue + status = " ".join(parts[4:]).lower() + if parts[2] == target and "running" in status: + print(parts[0]) + sys.exit(0) + +sys.exit(1) +PY +} + +# ══════════════════════════════════════════════════════════════════ +# Phase 0: Pre-cleanup +# ══════════════════════════════════════════════════════════════════ +section "Phase 0: Pre-cleanup" +info "Destroying any leftover test sandboxes/gateway from previous runs..." +if [ -x "$REPO_ROOT/bin/nemoclaw.js" ] || command -v nemoclaw >/dev/null 2>&1; then + if [ -n "$INSTALL_SANDBOX_NAME" ]; then + run_nemoclaw "$INSTALL_SANDBOX_NAME" destroy --yes 2>/dev/null || true + fi + run_nemoclaw "$SANDBOX_A" destroy --yes 2>/dev/null || true + run_nemoclaw "$SANDBOX_B" destroy --yes 2>/dev/null || true +fi +if [ -n "$INSTALL_SANDBOX_NAME" ]; then + openshell sandbox delete "$INSTALL_SANDBOX_NAME" 2>/dev/null || true +fi +openshell sandbox delete "$SANDBOX_A" 2>/dev/null || true +openshell sandbox delete "$SANDBOX_B" 2>/dev/null || true +stop_gateway_runtime +openshell gateway destroy -g nemoclaw 2>/dev/null || true +openshell gateway destroy -g "$ALT_GATEWAY_NAME" 2>/dev/null || true +pass "Pre-cleanup complete" + +# ══════════════════════════════════════════════════════════════════ +# Phase 1: Prerequisites + fake endpoint +# ══════════════════════════════════════════════════════════════════ +section "Phase 1: Prerequisites" + +if docker info >/dev/null 2>&1; then + pass "Docker is running" +else + fail "Docker is not running — cannot continue" + exit 1 +fi + +if command -v openshell >/dev/null 2>&1; then + pass "openshell CLI installed" +else + fail "openshell CLI not found — cannot continue" + exit 1 +fi + +if [ -x "$REPO_ROOT/bin/nemoclaw.js" ] || command -v nemoclaw >/dev/null 2>&1; then + pass "nemoclaw CLI available" +else + fail "nemoclaw CLI not found — cannot continue" + exit 1 +fi + +if command -v python3 >/dev/null 2>&1; then + pass "python3 installed" +else + fail "python3 not found — cannot continue" + exit 1 +fi + +if start_fake_openai; then + pass "Fake OpenAI-compatible endpoint started at ${FAKE_BASE_URL}" +else + fail "Failed to start fake OpenAI-compatible endpoint" + info "Fake server log:" + sed 's/^/ /' "$FAKE_OPENAI_LOG" + exit 1 +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 2: First onboard (e2e-double-a) +# ══════════════════════════════════════════════════════════════════ +section "Phase 2: First onboard ($SANDBOX_A)" +info "Running successful non-interactive onboard against local compatible endpoint..." + +PHASE2_START="$(phase_start_time)" +run_onboard "$SANDBOX_A" +output1="$RUN_ONBOARD_OUTPUT" +exit1="$RUN_ONBOARD_EXIT" +info "Phase 2 elapsed: $(phase_elapsed "$PHASE2_START")s" + +if [ "$exit1" -eq 0 ]; then + pass "First onboard completed successfully" +elif [ "$exit1" -eq 124 ]; then + fail "First onboard timed out after ${PHASE_TIMEOUT}s (exit 124)" + dump_diagnostics "Phase 2" +else + fail "First onboard exited $exit1 (expected 0)" + dump_diagnostics "Phase 2" +fi + +if grep -q "Sandbox '${SANDBOX_A}' created" <<<"$output1"; then + pass "Sandbox '$SANDBOX_A' created" +else + fail "Sandbox '$SANDBOX_A' creation not confirmed in output" +fi + +if openshell gateway info -g nemoclaw 2>/dev/null | grep -q "nemoclaw"; then + pass "Gateway is running after first onboard" +else + fail "Gateway is not running after first onboard" +fi + +if openshell sandbox get "$SANDBOX_A" >/dev/null 2>&1; then + pass "Sandbox '$SANDBOX_A' exists in openshell" +else + fail "Sandbox '$SANDBOX_A' not found in openshell" +fi + +if registry_has "$SANDBOX_A"; then + pass "Registry contains '$SANDBOX_A'" +else + fail "Registry does not contain '$SANDBOX_A'" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 3: Second onboard — SAME name (recreate) +# ══════════════════════════════════════════════════════════════════ +section "Phase 3: Second onboard ($SANDBOX_A — same name, recreate)" +info "Running nemoclaw onboard with NEMOCLAW_RECREATE_SANDBOX=1..." + +GATEWAY_ID_BEFORE=$(gateway_runtime_id || true) +PHASE3_START="$(phase_start_time)" +run_onboard "$SANDBOX_A" "1" +output2="$RUN_ONBOARD_OUTPUT" +exit2="$RUN_ONBOARD_EXIT" +info "Phase 3 elapsed: $(phase_elapsed "$PHASE3_START")s" + +if [ "$exit2" -eq 0 ]; then + pass "Second onboard completed successfully" +elif [ "$exit2" -eq 124 ]; then + fail "Second onboard timed out after ${PHASE_TIMEOUT}s (exit 124)" + dump_diagnostics "Phase 3" +else + fail "Second onboard exited $exit2 (expected 0)" + dump_diagnostics "Phase 3" +fi + +GATEWAY_ID_AFTER=$(gateway_runtime_id || true) +if [ -n "$GATEWAY_ID_BEFORE" ] && [ "$GATEWAY_ID_BEFORE" = "$GATEWAY_ID_AFTER" ]; then + pass "Healthy gateway runtime reused on second onboard ($GATEWAY_ID_BEFORE)" +else + fail "Gateway runtime changed on second onboard (before=$GATEWAY_ID_BEFORE after=$GATEWAY_ID_AFTER)" +fi + +if grep -q "Port 8080 is not available" <<<"$output2"; then + fail "Port 8080 conflict detected (regression)" +else + pass "No port 8080 conflict on second onboard" +fi + +if grep -q "Port 18789 is not available" <<<"$output2"; then + fail "Port 18789 conflict detected on second onboard" +else + pass "No port 18789 conflict on second onboard" +fi + +if openshell sandbox get "$SANDBOX_A" >/dev/null 2>&1; then + pass "Sandbox '$SANDBOX_A' still exists after recreate" +else + fail "Sandbox '$SANDBOX_A' missing after recreate" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 4: Third onboard — DIFFERENT name +# ══════════════════════════════════════════════════════════════════ +section "Phase 4: Third onboard ($SANDBOX_B — different name)" +info "Running nemoclaw onboard with new sandbox name..." + +ALT_GATEWAY_ENDPOINT="$(gateway_alias_endpoint)" +alt_gateway_add_output="$(openshell gateway add --local --name "$ALT_GATEWAY_NAME" "$ALT_GATEWAY_ENDPOINT" 2>&1 || true)" +if openshell gateway select "$ALT_GATEWAY_NAME" >/dev/null 2>&1; then + selected_gateway_output="$( + openshell status 2>&1 || true + openshell gateway info 2>&1 || true + )" + selected_gateway="$(gateway_name_from_output "$selected_gateway_output" 2>/dev/null || true)" + if [ "$selected_gateway" = "$ALT_GATEWAY_NAME" ]; then + pass "Alternate gateway alias selected before third onboard" + else + fail "Alternate gateway alias was not selected before third onboard (selected=${selected_gateway:-unknown})" + fi +else + fail "Could not select alternate gateway alias before third onboard (add output=${alt_gateway_add_output:-empty})" +fi + +GATEWAY_ID_BEFORE3=$(gateway_runtime_id || true) +PHASE4_START="$(phase_start_time)" +run_onboard "$SANDBOX_B" +output3="$RUN_ONBOARD_OUTPUT" +exit3="$RUN_ONBOARD_EXIT" +info "Phase 4 elapsed: $(phase_elapsed "$PHASE4_START")s" + +if [ "$exit3" -eq 0 ]; then + pass "Third onboard completed successfully" +elif [ "$exit3" -eq 124 ]; then + fail "Third onboard timed out after ${PHASE_TIMEOUT}s (exit 124)" + dump_diagnostics "Phase 4" +else + fail "Third onboard exited $exit3 (expected 0)" + dump_diagnostics "Phase 4" +fi + +GATEWAY_ID_AFTER3=$(gateway_runtime_id || true) +if [ -n "$GATEWAY_ID_BEFORE3" ] && [ "$GATEWAY_ID_BEFORE3" = "$GATEWAY_ID_AFTER3" ]; then + pass "Healthy gateway runtime reused on third onboard ($GATEWAY_ID_BEFORE3)" +else + fail "Gateway runtime changed on third onboard (before=$GATEWAY_ID_BEFORE3 after=$GATEWAY_ID_AFTER3)" +fi + +if grep -q "Port 8080 is not available" <<<"$output3"; then + fail "Port 8080 conflict on third onboard" +else + pass "No port 8080 conflict on third onboard" +fi + +if grep -q "Port 18789 is not available" <<<"$output3"; then + fail "Port 18789 conflict on third onboard" +else + pass "No port 18789 conflict on third onboard" +fi + +selected_gateway_output="$( + openshell status 2>&1 || true + openshell gateway info 2>&1 || true +)" +selected_gateway="$(gateway_name_from_output "$selected_gateway_output" 2>/dev/null || true)" +if [ "$selected_gateway" = "nemoclaw" ]; then + pass "Named gateway reselected during third onboard" +else + fail "Named gateway was not reselected during third onboard (selected=${selected_gateway:-unknown})" +fi + +if openshell sandbox get "$SANDBOX_B" >/dev/null 2>&1; then + pass "Sandbox '$SANDBOX_B' created" +else + fail "Sandbox '$SANDBOX_B' was not created" +fi + +if openshell sandbox get "$SANDBOX_A" >/dev/null 2>&1; then + pass "First sandbox '$SANDBOX_A' still exists after creating '$SANDBOX_B'" +else + fail "First sandbox '$SANDBOX_A' disappeared after creating '$SANDBOX_B' (regression: #849)" +fi + +# #2174 regression: B must auto-allocate to a different dashboard port, +# surface it in nemoclaw list, and not collide with A's dashboard. +if grep -q "is taken. Using port" <<<"$output3"; then + info "Second-sandbox onboard logged port auto-allocation (#2174)" +else + info "Second-sandbox onboard did not emit the optional auto-allocation warning; verifying assigned ports directly." +fi + +LIST_LOG="$(mktemp)" +run_nemoclaw list >"$LIST_LOG" 2>&1 || true +list_output="$(cat "$LIST_LOG")" +rm -f "$LIST_LOG" + +port_a="$(dashboard_port_from_list "$SANDBOX_A" 2>/dev/null || true)" +port_b="$(dashboard_port_from_list "$SANDBOX_B" 2>/dev/null || true)" + +if [ -n "$port_a" ] && [ -n "$port_b" ]; then + pass "nemoclaw list shows dashboard ports for both test sandboxes (#2174)" +else + fail "nemoclaw list did not show dashboard ports for both test sandboxes (a=${port_a:-missing} b=${port_b:-missing})" + info "Observed nemoclaw list output:" + printf '%s\n' "$list_output" | sed 's/^/ /' +fi + +if [ -n "$port_a" ] && [ -n "$port_b" ] && [ "$port_a" != "$port_b" ]; then + pass "nemoclaw list shows distinct dashboard ports for test sandboxes (#2174)" +else + fail "test sandboxes did not have distinct dashboard ports (#2174): ${SANDBOX_A}=${port_a:-missing} ${SANDBOX_B}=${port_b:-missing}" +fi + +if [ -n "$port_a" ] && [ -n "$port_b" ] && [ "$port_a" != "$port_b" ]; then + info "Stopping '$SANDBOX_B' dashboard forward to verify stored-port recovery..." + openshell forward stop "$port_b" 2>/dev/null || true + + PROBE_LOG="$(mktemp)" + PROBE_ATTEMPTS="${NEMOCLAW_E2E_PROBE_ATTEMPTS:-3}" + PROBE_DELAY_SECONDS="${NEMOCLAW_E2E_PROBE_DELAY_SECONDS:-3}" + PROBE_TIMEOUT_SECONDS="${NEMOCLAW_E2E_PROBE_TIMEOUT_SECONDS:-30}" + probe_exit=1 + probe_output="" + for attempt in $(seq 1 "$PROBE_ATTEMPTS"); do + info "Probe-only connect attempt ${attempt}/${PROBE_ATTEMPTS} for '$SANDBOX_B'..." + run_with_timeout "$PROBE_TIMEOUT_SECONDS" "${NEMOCLAW_CMD[@]}" "$SANDBOX_B" connect --probe-only >"$PROBE_LOG" 2>&1 + probe_exit=$? + probe_output="$(cat "$PROBE_LOG")" + [ "$probe_exit" -eq 0 ] && break + [ "$attempt" -lt "$PROBE_ATTEMPTS" ] && sleep "$PROBE_DELAY_SECONDS" + done + rm -f "$PROBE_LOG" + + if [ "$probe_exit" -eq 0 ]; then + pass "Probe-only connect recovered '$SANDBOX_B' dashboard forward" + else + fail "Probe-only connect exited $probe_exit after stopping '$SANDBOX_B' dashboard forward" + info "Observed probe output:" + printf '%s\n' "$probe_output" | sed 's/^/ /' + dump_diagnostics "probe-only dashboard forward recovery" + fi + + forward_output="$(openshell forward list 2>&1 || true)" + owner_a="$(forward_owner_for_port "$port_a" 2>/dev/null || true)" + owner_b="$(forward_owner_for_port "$port_b" 2>/dev/null || true)" + + if [ "$owner_b" = "$SANDBOX_B" ]; then + pass "Second sandbox dashboard forward restored on its recorded port" + else + fail "Second sandbox dashboard forward owner mismatch on port $port_b (owner=${owner_b:-missing})" + info "Observed forward list:" + printf '%s\n' "$forward_output" | sed 's/^/ /' + fi + + if [ "$owner_a" = "$SANDBOX_A" ]; then + pass "First sandbox dashboard forward kept its recorded port" + else + fail "First sandbox dashboard forward owner mismatch on port $port_a (owner=${owner_a:-missing})" + info "Observed forward list:" + printf '%s\n' "$forward_output" | sed 's/^/ /' + fi +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 5: Stale registry reconciliation +# ══════════════════════════════════════════════════════════════════ +section "Phase 5: Stale registry reconciliation" +info "Deleting '$SANDBOX_A' directly in OpenShell to leave a stale NemoClaw registry entry..." + +openshell sandbox delete "$SANDBOX_A" 2>/dev/null || true +if wait_openshell_sandbox_absent "$SANDBOX_A" 60; then + pass "OpenShell reports '$SANDBOX_A' absent after direct deletion" +else + fail "OpenShell still reports '$SANDBOX_A' after direct deletion" +fi + +if registry_has "$SANDBOX_A"; then + pass "Registry still contains stale '$SANDBOX_A' entry" +else + fail "Registry was unexpectedly cleaned before status reconciliation" +fi + +STATUS_LOG="$(mktemp)" +run_nemoclaw "$SANDBOX_A" status >"$STATUS_LOG" 2>&1 +status_exit=$? +status_output="$(cat "$STATUS_LOG")" +rm -f "$STATUS_LOG" + +if [ "$status_exit" -eq 1 ]; then + pass "Stale sandbox status exited 1" +else + fail "Stale sandbox status exited $status_exit (expected 1)" +fi + +if grep -q "No local registry entry was removed" <<<"$status_output"; then + pass "Stale sandbox status emitted non-destructive guidance (#4578)" +else + fail "Stale sandbox status did not emit non-destructive guidance (#4578)" +fi + +# #4497: neither status nor connect may delete the stale local entry — the +# metadata is what `rebuild` / `onboard --recreate-sandbox` need to recover. +if grep -q "Removed stale local registry entry" <<<"$status_output"; then + fail "status removed the local registry entry (must be preserved, #4497)" +else + pass "status preserved the stale registry entry" +fi + +if registry_has "$SANDBOX_A"; then + pass "Registry still contains '$SANDBOX_A' after status" +else + fail "Registry entry for '$SANDBOX_A' was removed by status (must be preserved, #4497)" +fi + +# Bound every Phase 5 recovery probe so a reintroduced prompt or hang fails the +# job fast instead of stalling to the phase timeout. Mirrors the probe-only +# connect in Phase 4. +RECOVERY_PROBE_TIMEOUT_SECONDS="${NEMOCLAW_E2E_RECOVERY_PROBE_TIMEOUT_SECONDS:-180}" + +# A routine `connect` against the same stale entry must also preserve it. +CONNECT_LOG="$(mktemp)" +run_with_timeout "$RECOVERY_PROBE_TIMEOUT_SECONDS" \ + env NEMOCLAW_NON_INTERACTIVE=1 "${NEMOCLAW_CMD[@]}" "$SANDBOX_A" connect >"$CONNECT_LOG" 2>&1 +connect_exit=$? +connect_output="$(cat "$CONNECT_LOG")" +rm -f "$CONNECT_LOG" + +if [ "$connect_exit" -eq 1 ]; then + pass "Stale sandbox connect exited 1" +else + fail "Stale sandbox connect exited $connect_exit (expected 1)" +fi + +if grep -q "Removed stale local registry entry" <<<"$connect_output"; then + fail "connect removed the local registry entry (must be preserved, #4497)" +else + pass "connect preserved the stale registry entry" +fi + +if registry_has "$SANDBOX_A"; then + pass "Registry still contains '$SANDBOX_A' after connect (#4497)" +else + fail "connect removed '$SANDBOX_A' from the registry (must be preserved, #4497)" +fi + +# #4497 (reopened) acceptance gate — the EXACT reporter workflow: +# status recommends `rebuild --yes` → connect preserves the registry → +# `rebuild --yes` must actually RECOVER the sandbox, not dead-end. +# +# The first fix (PR #4647) only stopped connect from deleting the entry; rebuild +# still aborted at its backup step with "Cannot back up state" whenever the live +# sandbox was absent — precisely this stale state. A probe that only checks for +# "does not exist" would pass against that bug, so this drives the full recovery +# rebuild and asserts it (a) never prints the dead-end errors, (b) reports the +# stale state and skips the impossible backup, and (c) recreates a live sandbox. +# +# The recreate runs `onboard --resume` in-process, so it needs the same provider +# env the original onboard used. Allow a full phase timeout for the rebuild. +REBUILD_LOG="$(mktemp)" +rebuild_exit=0 +run_with_timeout "$PHASE_TIMEOUT" \ + env \ + COMPATIBLE_API_KEY=dummy \ + NEMOCLAW_NON_INTERACTIVE=1 \ + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ + NEMOCLAW_PROVIDER=custom \ + "NEMOCLAW_ENDPOINT_URL=${FAKE_BASE_URL}" \ + NEMOCLAW_MODEL=test-model \ + "NEMOCLAW_SANDBOX_NAME=${SANDBOX_A}" \ + NEMOCLAW_POLICY_MODE=skip \ + NEMOCLAW_DASHBOARD_PORT= \ + CHAT_UI_URL= \ + "${NEMOCLAW_CMD[@]}" "$SANDBOX_A" rebuild --yes >"$REBUILD_LOG" 2>&1 || rebuild_exit=$? +rebuild_output="$(cat "$REBUILD_LOG")" +rm -f "$REBUILD_LOG" + +# A timeout (124 from `timeout`/`gtimeout`) must fail, not silently pass. +if [ "$rebuild_exit" -eq 124 ]; then + dump_diagnostics "stale rebuild recovery (#4497)" + fail "rebuild recovery timed out after ${PHASE_TIMEOUT}s (#4497)" +fi + +# (a) The pre-fix dead-ends must never appear. +if grep -q "Cannot back up state" <<<"$rebuild_output"; then + fail "rebuild dead-ended at 'Cannot back up state' on a stale sandbox (#4497)" +elif grep -q "does not exist" <<<"$rebuild_output"; then + fail "rebuild could not locate the preserved sandbox '$SANDBOX_A' (#4497)" +else + pass "rebuild did not dead-end on the stale sandbox (#4497)" +fi + +# (b) It must recognize the stale state and skip the impossible backup. +if grep -q "absent from the live OpenShell gateway" <<<"$rebuild_output" \ + && grep -q "No live workspace state to back up" <<<"$rebuild_output"; then + pass "rebuild reported the stale state and skipped backup (#4497)" +else + dump_diagnostics "stale rebuild recovery markers (#4497)" + fail "rebuild did not report the stale-recovery path (#4497)" +fi +if grep -q "Creating new sandbox with current image" <<<"$rebuild_output"; then + pass "rebuild proceeded to recreate from preserved metadata (#4497)" +else + fail "rebuild did not proceed to recreate the sandbox (#4497)" +fi + +# (c) The recovery must succeed end-to-end: a live sandbox is back and the +# registry entry survived the whole workflow. +if [ "$rebuild_exit" -eq 0 ]; then + pass "rebuild recovery exited 0 (#4497)" +else + dump_diagnostics "stale rebuild recovery exit=$rebuild_exit (#4497)" + fail "rebuild recovery exited $rebuild_exit (expected 0, #4497)" +fi +if openshell sandbox get "$SANDBOX_A" >/dev/null 2>&1; then + pass "OpenShell reports '$SANDBOX_A' live again after recovery rebuild (#4497)" +else + dump_diagnostics "stale rebuild recovery liveness (#4497)" + fail "'$SANDBOX_A' is still absent from OpenShell after recovery rebuild (#4497)" +fi +if registry_has "$SANDBOX_A"; then + pass "Registry still contains '$SANDBOX_A' after recovery rebuild (#4497)" +else + fail "Recovery rebuild lost the '$SANDBOX_A' registry entry (#4497)" +fi + +# Teardown the now-live sandbox while the gateway is healthy so it does not leak +# into Phase 7 cleanup (which runs against a stopped gateway). +run_with_timeout "$RECOVERY_PROBE_TIMEOUT_SECONDS" \ + env NEMOCLAW_NON_INTERACTIVE=1 "${NEMOCLAW_CMD[@]}" "$SANDBOX_A" destroy --yes 2>/dev/null || true +openshell sandbox delete "$SANDBOX_A" 2>/dev/null || true +if registry_has "$SANDBOX_A"; then + fail "destroy did not purge the recovered '$SANDBOX_A' registry entry (#4497)" +else + pass "destroy purged the recovered '$SANDBOX_A' registry entry (#4497)" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 6: Gateway lifecycle response +# ══════════════════════════════════════════════════════════════════ +section "Phase 6: Gateway lifecycle response" +info "Stopping the NemoClaw gateway runtime to verify current lifecycle behavior..." + +openshell forward stop 18789 2>/dev/null || true +stop_gateway_runtime + +GATEWAY_LOG="$(mktemp)" +run_nemoclaw "$SANDBOX_B" status >"$GATEWAY_LOG" 2>&1 +gateway_status_exit=$? +gateway_status_output="$(cat "$GATEWAY_LOG")" +rm -f "$GATEWAY_LOG" + +if [ "$gateway_status_exit" -eq 0 ] || [ "$gateway_status_exit" -eq 1 ]; then + pass "Post-stop status exited $gateway_status_exit" +else + fail "Post-stop status exited $gateway_status_exit (expected 0 or 1)" +fi + +if grep -qE \ + "Recovered NemoClaw gateway runtime|gateway is no longer configured after restart/rebuild|gateway is still refusing connections after restart|gateway trust material rotated after restart" \ + <<<"$gateway_status_output"; then + pass "Gateway lifecycle response was explicit after gateway stop" +else + fail "Gateway lifecycle response was not explicit after gateway stop" + info "Observed status output:" + printf '%s\n' "$gateway_status_output" | sed 's/^/ /' +fi + +if registry_has "$SANDBOX_B"; then + pass "Registry still contains '$SANDBOX_B' after gateway stop" +else + fail "Registry is missing '$SANDBOX_B' after gateway stop" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 7: Final cleanup +# ══════════════════════════════════════════════════════════════════ +section "Phase 7: Final cleanup" + +run_nemoclaw "$SANDBOX_A" destroy --yes 2>/dev/null || true +run_nemoclaw "$SANDBOX_B" destroy --yes 2>/dev/null || true +if [ -n "$INSTALL_SANDBOX_NAME" ]; then + run_nemoclaw "$INSTALL_SANDBOX_NAME" destroy --yes 2>/dev/null || true +fi +openshell sandbox delete "$SANDBOX_A" 2>/dev/null || true +openshell sandbox delete "$SANDBOX_B" 2>/dev/null || true +if [ -n "$INSTALL_SANDBOX_NAME" ]; then + openshell sandbox delete "$INSTALL_SANDBOX_NAME" 2>/dev/null || true +fi +stop_forward_if_set "${port_a:-}" +stop_forward_if_set "${port_b:-}" +openshell forward stop 18789 2>/dev/null || true +stop_gateway_runtime +openshell gateway destroy -g nemoclaw 2>/dev/null || true +openshell gateway destroy -g "$ALT_GATEWAY_NAME" 2>/dev/null || true + +# `status` and `connect` intentionally preserve stale registry entries (#4497), +# so final cleanup relies on the explicit `destroy --yes` calls above. Do not +# run a post-destroy status probe here: it can restart the gateway without +# removing registry state. + +if openshell sandbox get "$SANDBOX_A" >/dev/null 2>&1; then + fail "Sandbox '$SANDBOX_A' still exists after cleanup" +else + pass "Sandbox '$SANDBOX_A' cleaned up" +fi + +if openshell sandbox get "$SANDBOX_B" >/dev/null 2>&1; then + fail "Sandbox '$SANDBOX_B' still exists after cleanup" +else + pass "Sandbox '$SANDBOX_B' cleaned up" +fi + +if [ -f "$REGISTRY" ] && grep -q "$SANDBOX_A\|$SANDBOX_B" "$REGISTRY"; then + fail "Registry still contains test sandbox entries" +else + pass "Registry cleaned up" +fi + +pass "Final cleanup complete" + +echo "" +echo "========================================" +echo " Double Onboard E2E Results:" +echo " Passed: $PASS" +echo " Failed: $FAIL" +echo " Total: $TOTAL" +echo "========================================" + +if [ "$FAIL" -eq 0 ]; then + printf '\n\033[1;32m Double onboard and lifecycle recovery PASSED.\033[0m\n' + exit 0 +else + printf '\n\033[1;31m %d test(s) failed.\033[0m\n' "$FAIL" + exit 1 +fi diff --git a/test/e2e-vpn/test-full-e2e.sh b/test/e2e-vpn/test-full-e2e.sh new file mode 100755 index 00000000000..d8d362e7564 --- /dev/null +++ b/test/e2e-vpn/test-full-e2e.sh @@ -0,0 +1,521 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Full E2E: install → onboard → verify inference (REAL services, no mocks) +# +# Proves the COMPLETE user journey including real inference against +# VPN NVIDIA inference. Runs install.sh --non-interactive which handles +# Node.js, openshell, NemoClaw, and onboard setup automatically. +# +# Prerequisites: +# - Docker running +# - NVIDIA_API_KEY set (real key, starts with nvapi-) +# - Network access to inference.nvidia.com +# +# Environment variables: +# NEMOCLAW_NON_INTERACTIVE=1 — required (enables non-interactive install + onboard) +# NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 — required for non-interactive install/onboard +# NEMOCLAW_SANDBOX_NAME — sandbox name (default: e2e-nightly) +# NEMOCLAW_RECREATE_SANDBOX=1 — recreate sandbox if it exists from a previous run +# NVIDIA_API_KEY — required for VPN NVIDIA inference inference +# +# Usage: +# NEMOCLAW_NON_INTERACTIVE=1 NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 NVIDIA_API_KEY=nvapi-... bash test/e2e-vpn/test-full-e2e.sh +# +# See: https://github.com/NVIDIA/NemoClaw/issues/71 + +set -uo pipefail + +PASS=0 +FAIL=0 +SKIP=0 +TOTAL=0 + +pass() { + ((PASS++)) + ((TOTAL++)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} +fail() { + ((FAIL++)) + ((TOTAL++)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} +skip() { + ((SKIP++)) + ((TOTAL++)) + printf '\033[33m SKIP: %s\033[0m\n' "$1" +} +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } + +# Parse chat completion response — handles both content and reasoning_content +# (nemotron-3-super is a reasoning model that may put output in reasoning_content) +parse_chat_content() { + python3 -c " +import json, sys +try: + r = json.load(sys.stdin) + c = r['choices'][0]['message'] + content = c.get('content') or c.get('reasoning_content') or '' + print(content.strip()) +except Exception as e: + print(f'PARSE_ERROR: {e}', file=sys.stderr) + sys.exit(1) +" +} + +# shellcheck source=test/e2e-vpn/lib/openclaw-json.sh +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/lib/openclaw-json.sh" +# shellcheck source=test/e2e-vpn/lib/ci-compatible-inference.sh +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/lib/ci-compatible-inference.sh" + +# Determine repo root +if [ -d /workspace ] && [ -f /workspace/install.sh ]; then + REPO="/workspace" +elif [ -f "$(cd "$(dirname "$0")/../.." && pwd)/install.sh" ]; then + REPO="$(cd "$(dirname "$0")/../.." && pwd)" +else + echo "ERROR: Cannot find repo root." + exit 1 +fi + +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-nightly}" +nemoclaw_e2e_configure_compatible_inference + +# shellcheck source=test/e2e-vpn/lib/sandbox-teardown.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib/sandbox-teardown.sh" +register_sandbox_for_teardown "$SANDBOX_NAME" + +# ══════════════════════════════════════════════════════════════════ +# Phase 0: Pre-cleanup +# ══════════════════════════════════════════════════════════════════ +section "Phase 0: Pre-cleanup" +info "Destroying any leftover sandbox/gateway from previous runs..." +if command -v nemoclaw >/dev/null 2>&1; then + nemoclaw "$SANDBOX_NAME" destroy --yes 2>/dev/null || true +fi +if command -v openshell >/dev/null 2>&1; then + openshell sandbox delete "$SANDBOX_NAME" 2>/dev/null || true + openshell gateway destroy -g nemoclaw 2>/dev/null || true +fi +pass "Pre-cleanup complete" + +# ══════════════════════════════════════════════════════════════════ +# Phase 1: Prerequisites +# ══════════════════════════════════════════════════════════════════ +section "Phase 1: Prerequisites" + +if docker info >/dev/null 2>&1; then + pass "Docker is running" +else + fail "Docker is not running — cannot continue" + exit 1 +fi + +if ! nemoclaw_e2e_require_hosted_inference_key; then + exit 1 +fi + +HOSTED_INFERENCE_BASE_URL="$(nemoclaw_e2e_hosted_inference_base_url)" +HOSTED_INFERENCE_MODEL="$(nemoclaw_e2e_hosted_inference_model)" +HOSTED_INFERENCE_KEY="$(nemoclaw_e2e_hosted_inference_key)" + +if nemoclaw_e2e_probe_hosted_inference; then + pass "Network access to ${HOSTED_INFERENCE_BASE_URL}" +else + fail "Cannot reach ${HOSTED_INFERENCE_BASE_URL}" + exit 1 +fi + +if [ "${NEMOCLAW_NON_INTERACTIVE:-}" != "1" ]; then + fail "NEMOCLAW_NON_INTERACTIVE=1 is required" + exit 1 +fi + +if [ "${NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE:-}" != "1" ]; then + fail "NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 is required for non-interactive install" + exit 1 +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 2: Install nemoclaw (non-interactive mode) +# ══════════════════════════════════════════════════════════════════ +section "Phase 2: Install nemoclaw (non-interactive mode)" + +cd "$REPO" || { + fail "Could not cd to repo root: $REPO" + exit 1 +} + +info "Running install.sh --non-interactive..." +info "This installs Node.js, openshell, NemoClaw, and runs onboard." +info "Expected duration: 5-10 minutes on first run." + +INSTALL_LOG="/tmp/nemoclaw-e2e-install.log" +# Write to a file instead of piping through tee. openshell's background +# port-forward inherits pipe file descriptors, which prevents tee from exiting. +# Use tail -f in the background for real-time output in CI logs. +bash install.sh --non-interactive >"$INSTALL_LOG" 2>&1 & +install_pid=$! +tail -f "$INSTALL_LOG" --pid=$install_pid 2>/dev/null & +tail_pid=$! +wait $install_pid +install_exit=$? +kill $tail_pid 2>/dev/null || true +wait $tail_pid 2>/dev/null || true + +# Source shell profile to pick up nvm/PATH changes from install.sh +if [ -f "$HOME/.bashrc" ]; then + # shellcheck source=/dev/null + source "$HOME/.bashrc" 2>/dev/null || true +fi +# Ensure nvm is loaded in current shell +export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" +if [ -s "$NVM_DIR/nvm.sh" ]; then + # shellcheck source=/dev/null + . "$NVM_DIR/nvm.sh" +fi +# Ensure ~/.local/bin is on PATH (openshell may be installed there in non-interactive mode) +if [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]]; then + export PATH="$HOME/.local/bin:$PATH" +fi + +if [ $install_exit -eq 0 ]; then + pass "install.sh completed (exit 0)" +else + fail "install.sh failed (exit $install_exit)" + exit 1 +fi + +# Verify nemoclaw is on PATH +if command -v nemoclaw >/dev/null 2>&1; then + pass "nemoclaw installed at $(command -v nemoclaw)" +else + fail "nemoclaw not found on PATH after install" + exit 1 +fi + +# Verify openshell was installed +if command -v openshell >/dev/null 2>&1; then + pass "openshell installed ($(openshell --version 2>&1 || echo unknown))" +else + fail "openshell not found on PATH after install" + exit 1 +fi + +if nemoclaw --help >/dev/null 2>&1; then + pass "nemoclaw --help exits 0" +else + fail "nemoclaw --help failed" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 3: Sandbox verification +# ══════════════════════════════════════════════════════════════════ +section "Phase 3: Sandbox verification" + +# 3a: nemoclaw list +if list_output=$(nemoclaw list 2>&1); then + if grep -Fq -- "$SANDBOX_NAME" <<<"$list_output"; then + pass "nemoclaw list contains '${SANDBOX_NAME}'" + else + fail "nemoclaw list does not contain '${SANDBOX_NAME}'" + fi +else + fail "nemoclaw list failed: ${list_output:0:200}" +fi + +# 3b: nemoclaw status +if status_output=$(nemoclaw "$SANDBOX_NAME" status 2>&1); then + pass "nemoclaw ${SANDBOX_NAME} status exits 0" +else + fail "nemoclaw ${SANDBOX_NAME} status failed: ${status_output:0:200}" +fi + +# 3c: Inference must be configured by onboard (no fallback — if onboard +# failed to configure it, that's a bug we want to catch) +if inf_check=$(openshell inference get 2>&1); then + inf_check_plain="$(sed -E $'s/\x1B\\[[0-9;]*[A-Za-z]//g' <<<"$inf_check")" + if nemoclaw_e2e_using_compatible_inference; then + if grep -Eqi "Provider:[[:space:]]*(custom|compatible-endpoint)" <<<"$inf_check_plain" && grep -Fq "$HOSTED_INFERENCE_MODEL" <<<"$inf_check_plain"; then + pass "Inference configured via onboard (CI-compatible endpoint)" + else + fail "Inference not configured — onboard did not set up CI-compatible provider: ${inf_check_plain:0:200}" + fi + elif grep -qi "compatible-endpoint" <<<"$inf_check_plain"; then + pass "Inference configured via onboard" + else + fail "Inference not configured — onboard did not set up compatible-endpoint provider" + fi +else + fail "openshell inference get failed: ${inf_check:0:200}" +fi + +# 3d: Policy presets applied +if policy_output=$(openshell policy get --full "$SANDBOX_NAME" 2>&1); then + if grep -qi "network_policies" <<<"$policy_output"; then + pass "Policy applied to sandbox" + else + fail "No network policy found on sandbox" + fi + + # Check that at least npm or pypi preset endpoints are present (onboard auto-suggests these) + if grep -qi "registry.npmjs.org\|pypi.org" <<<"$policy_output"; then + pass "Policy presets (npm/pypi) detected in sandbox policy" + else + skip "Could not confirm npm/pypi presets in policy (may vary by environment)" + fi +else + fail "openshell policy get failed: ${policy_output:0:200}" +fi + +# 3e: NemoClaw plugin remains registered after gateway policy initialization. +# Regression coverage for #2021: OpenClaw's policy-changed registry rebuild can +# drop path/npm-origin plugins from plugins[], which removes the /nemoclaw TUI +# command surface. The startup refresh should restore the registry before users +# interact with the sandbox. This non-interactive E2E cannot drive OpenClaw's +# terminal autocomplete directly; the interactive TUI/chat surface is owned by +# the openclaw-tui-chat-correlation-e2e scenario. Here we validate the runtime +# slash alias that the TUI consumes. The direct command help path is also +# probed, but only a NemoClaw-specific missing-command failure is fatal because +# OpenClaw can exit non-zero for unrelated plugin config warnings. +info "[PLUGIN] verifying NemoClaw plugin registry entry, slash alias, and command help..." +ssh_config="$(mktemp)" +plugin_check_output="" +plugin_check_timeout_cmd=() +command -v timeout >/dev/null 2>&1 && plugin_check_timeout_cmd=(timeout 90) +command -v gtimeout >/dev/null 2>&1 && plugin_check_timeout_cmd=(gtimeout 90) +if openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config" 2>/dev/null; then + for plugin_attempt in 1 2 3 4 5; do + plugin_check_output=$("${plugin_check_timeout_cmd[@]}" ssh -F "$ssh_config" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" \ + "inspect_log=/tmp/nemoclaw-e2e-plugin-inspect.log; help_log=/tmp/nemoclaw-e2e-plugin-help.log; manifest=/sandbox/.openclaw/extensions/nemoclaw/openclaw.plugin.json; if ! HOME=/sandbox openclaw plugins inspect nemoclaw >\"\$inspect_log\" 2>&1; then printf 'inspect failed: '; head -c 600 \"\$inspect_log\"; exit 1; fi; if ! HOME=/sandbox openclaw nemoclaw --help >\"\$help_log\" 2>&1 && grep -Eiq '(nemoclaw|/nemoclaw).*(not found|not installed)|not found.*(nemoclaw|/nemoclaw)' \"\$help_log\"; then printf 'help missing nemoclaw: '; head -c 600 \"\$help_log\"; exit 1; fi; if ! grep -Eq '\"name\"[[:space:]]*:[[:space:]]*\"nemoclaw\"' \"\$manifest\"; then printf 'manifest missing nemoclaw name'; exit 1; fi; if ! grep -Eq '\"kind\"[[:space:]]*:[[:space:]]*\"runtime-slash\"' \"\$manifest\"; then printf 'manifest missing runtime-slash alias'; exit 1; fi; printf 'plugin-ok'" \ + 2>&1) || true + grep -Fq "plugin-ok" <<<"$plugin_check_output" && break + [ "$plugin_attempt" -lt 5 ] && sleep 3 + done +fi +rm -f "$ssh_config" +if grep -Fq "plugin-ok" <<<"$plugin_check_output"; then + pass "NemoClaw OpenClaw plugin is registered with runtime slash alias" +else + fail "NemoClaw OpenClaw plugin registry/slash-alias/help check failed: ${plugin_check_output:0:300}" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 4: Live inference — the real proof +# ══════════════════════════════════════════════════════════════════ +section "Phase 4: Live inference" + +# ── Test 4a: Direct VPN NVIDIA inference ── +info "[LIVE] Direct API test → ${HOSTED_INFERENCE_BASE_URL}..." +api_response=$(curl -s --max-time 30 \ + -X POST "${HOSTED_INFERENCE_BASE_URL}/chat/completions" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $HOSTED_INFERENCE_KEY" \ + -d '{ + "model": "'"${HOSTED_INFERENCE_MODEL}"'", + "messages": [{"role": "user", "content": "Reply with exactly one word: PONG"}], + "max_tokens": 100 + }' 2>/dev/null) || true + +if [ -n "$api_response" ]; then + api_content=$(echo "$api_response" | parse_chat_content 2>/dev/null) || true + if grep -qi "PONG" <<<"$api_content"; then + pass "[LIVE] Direct API: model responded with PONG" + else + fail "[LIVE] Direct API: expected PONG, got: ${api_content:0:200}" + fi +else + fail "[LIVE] Direct API: empty response from curl" +fi + +# ── Test 4b: OpenShell DNS+proxy can route inference.local from the sandbox ── +# This is a routing-layer check, not an openclaw check. The HTTP request is +# made by `curl` from inside the sandbox; nothing in this path exercises +# openclaw's HTTP client or its SSRF guard. See Phase 4c for the openclaw- +# mediated assertion. (NemoClaw #2490 / openclaw 2026.4.9 SSRF regression +# was invisible to this step because curl bypasses openclaw entirely.) +info "[ROUTING] inference.local DNS + OpenShell proxy reachable from sandbox..." +ssh_config="$(mktemp)" +sandbox_response="" + +if openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config" 2>/dev/null; then + # Use timeout if available (Linux, Homebrew), fall back to plain ssh + TIMEOUT_CMD="" + command -v timeout >/dev/null 2>&1 && TIMEOUT_CMD="timeout 90" + command -v gtimeout >/dev/null 2>&1 && TIMEOUT_CMD="gtimeout 90" + sandbox_response=$($TIMEOUT_CMD ssh -F "$ssh_config" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" \ + "curl -s --max-time 60 https://inference.local/v1/chat/completions \ + -H 'Content-Type: application/json' \ + -d '{\"model\":\"nvidia/nemotron-3-super-120b-a12b\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with exactly one word: PONG\"}],\"max_tokens\":100}'" \ + 2>&1) || true +fi +rm -f "$ssh_config" + +# Retry sandbox inference up to 3 times — live models are not deterministic +# and the gateway proxy can return unexpected responses on first attempt. (#1969) +TIMEOUT_CMD="${TIMEOUT_CMD:-}" +sandbox_content="" +pong_ok=false +for pong_attempt in 1 2 3; do + if [ -n "$sandbox_response" ]; then + sandbox_content=$(echo "$sandbox_response" | parse_chat_content 2>/dev/null) || true + if grep -qi "PONG" <<<"$sandbox_content"; then + pong_ok=true + break + fi + info "Sandbox inference attempt ${pong_attempt}/3: got '${sandbox_content:0:80}', retrying in 5s..." + else + info "Sandbox inference attempt ${pong_attempt}/3: empty response, retrying in 5s..." + fi + [ "$pong_attempt" -lt 3 ] || break + sleep 5 + # Re-fetch with verbose curl on retry to diagnose proxy issues (#1969) + ssh_config="$(mktemp)" + sandbox_response="" + if openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config" 2>/dev/null; then + info "Retry $((pong_attempt + 1)): using curl -v to capture proxy request/response headers" + sandbox_response=$($TIMEOUT_CMD ssh -F "$ssh_config" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" \ + "curl -v --max-time 60 https://inference.local/v1/chat/completions \ + -H 'Content-Type: application/json' \ + -d '{\"model\":\"nvidia/nemotron-3-super-120b-a12b\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with exactly one word: PONG\"}],\"max_tokens\":100}'" \ + 2>&1) || true + info "Verbose response (first 500 chars): ${sandbox_response:0:500}" + fi + rm -f "$ssh_config" +done +if $pong_ok; then + pass "[ROUTING] inference.local: OpenShell routed curl to VPN NVIDIA inference and returned PONG" + info "Routing path proven: sandbox curl → DNS forwarder → gateway proxy → VPN NVIDIA inference (does not exercise openclaw HTTP client; see Phase 4c)" +else + fail "[ROUTING] inference.local: expected PONG after 3 attempts, got: ${sandbox_content:0:200}" +fi + +# ── Test 4c: openclaw-mediated turn against inference.local ── +# This is the only assertion in this file that proves openclaw can complete +# a turn against inference.local. Prior to this step, every "[LIVE] inference" +# label in the suite was actually a [ROUTING] check via curl (see 4b above). +# +# Properties of this assertion that prevent the false-positive class that +# masked the openclaw 2026.4.9 SSRF regression: +# * Uses `openclaw agent --json`. With --json the CLI calls +# routeLogsToStderr() (openclaw/src/commands/agent-via-gateway.ts:57), +# so stdout is a clean JSON envelope; prompt-echo on stderr cannot +# pollute the assertion. +# * Asserts on parsed model reply text from the JSON envelope, not on +# the merged stdout/stderr or a single brittle envelope shape. +# * The expected token (the integer 42) is not a literal substring of the +# prompt, so an error path that quoted the prompt back cannot satisfy +# the grep. +info "[LIVE] openclaw agent → openclaw HTTP client → inference.local..." +ssh_config="$(mktemp)" +agent_response="" + +if openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config" 2>/dev/null; then + agent_session_id="e2e-live-$(date +%s)-$$" + # 2>/dev/null discards stderr (progress + log lines) so stdout is JSON-only. + agent_response=$($TIMEOUT_CMD ssh -F "$ssh_config" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" \ + "openclaw agent --agent main --json --session-id '${agent_session_id}' -m 'What is 6 multiplied by 7? Reply with only the integer, no extra words.'" \ + 2>/dev/null) || true +fi +rm -f "$ssh_config" + +agent_reply=$(printf '%s' "$agent_response" | parse_openclaw_agent_text 2>/dev/null) || true + +if grep -qE "(^|[^0-9])42([^0-9]|$)" <<<"$agent_reply"; then + pass "[LIVE] openclaw agent: model answered 6×7=42 through openclaw → inference.local" +else + fail "[LIVE] openclaw agent: expected '42' in agent reply, got: ${agent_reply:0:200}" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 5: NemoClaw CLI operations +# ══════════════════════════════════════════════════════════════════ +section "Phase 5: NemoClaw CLI operations" + +# Note: Policy enforcement (proxy blocking, L4/L7 rules, SSRF protection) +# and sandbox command execution are tested extensively in OpenShell's own +# E2E suite (e2e/python/test_sandbox_policy.py, test_sandbox_api.py). +# NemoClaw tests only that its onboard correctly *configured* the policies +# (Phase 3d above), not that OpenShell *enforces* them. + +# ── Test 5a: nemoclaw logs ── +info "Testing sandbox log retrieval..." +logs_output=$(nemoclaw "$SANDBOX_NAME" logs 2>&1) || true +if [ -n "$logs_output" ]; then + pass "nemoclaw logs: produced output ($(echo "$logs_output" | wc -l | tr -d ' ') lines)" +else + fail "nemoclaw logs: no output" +fi + +# ══════════════════════════════════════════════════════════════════ +# Optional Phase 5b: Security posture regression checks +# ══════════════════════════════════════════════════════════════════ +if [ "${NEMOCLAW_E2E_SECURITY_POSTURE:-}" = "1" ]; then + # shellcheck source=test/e2e-vpn/lib/security-posture-assertions.sh + . "$(dirname "${BASH_SOURCE[0]}")/lib/security-posture-assertions.sh" + security_posture_assertions_run "$SANDBOX_NAME" "openclaw" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 6: Cleanup +# ══════════════════════════════════════════════════════════════════ +section "Phase 6: Cleanup" + +[[ "${NEMOCLAW_E2E_KEEP_SANDBOX:-}" = "1" ]] || nemoclaw "$SANDBOX_NAME" destroy --yes 2>&1 | tail -3 || true +openshell gateway destroy -g nemoclaw 2>/dev/null || true + +# Verify against the registry file directly. `nemoclaw list` triggers +# gateway recovery which can restart a destroyed gateway and re-import stale +# sandbox entries — that's a separate issue (#TBD), so avoid it here. +registry_file="${HOME}/.nemoclaw/sandboxes.json" +if [ -f "$registry_file" ] && grep -Fq "\"${SANDBOX_NAME}\"" "$registry_file"; then + fail "Sandbox ${SANDBOX_NAME} still in registry after destroy" +else + pass "Sandbox ${SANDBOX_NAME} removed" +fi + +# ══════════════════════════════════════════════════════════════════ +# Summary +# ══════════════════════════════════════════════════════════════════ +echo "" +echo "========================================" +echo " Full E2E Results:" +echo " Passed: $PASS" +echo " Failed: $FAIL" +echo " Skipped: $SKIP" +echo " Total: $TOTAL" +echo "========================================" + +if [ "$FAIL" -eq 0 ]; then + printf '\n\033[1;32m Full E2E PASSED — real inference verified end-to-end.\033[0m\n' + exit 0 +else + printf '\n\033[1;31m %d test(s) failed.\033[0m\n' "$FAIL" + exit 1 +fi diff --git a/test/e2e-vpn/test-gateway-drift-preflight.sh b/test/e2e-vpn/test-gateway-drift-preflight.sh new file mode 100755 index 00000000000..88edec5942a --- /dev/null +++ b/test/e2e-vpn/test-gateway-drift-preflight.sh @@ -0,0 +1,423 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -uo pipefail + +section() { printf '\n=== %s ===\n' "$1"; } +pass() { echo "PASS: $1"; } +info() { echo "INFO: $1"; } +fail() { + echo "FAIL: $1" >&2 + if [ -n "${CASE_DIR:-}" ] && [ -d "$CASE_DIR" ]; then + echo "--- fake openshell calls ---" >&2 + cat "$CASE_DIR/openshell-calls.log" 2>/dev/null >&2 || true + echo "--- fake docker calls ---" >&2 + cat "$CASE_DIR/docker-calls.log" 2>/dev/null >&2 || true + echo "--- command output ---" >&2 + cat "$CASE_DIR/command.out" 2>/dev/null >&2 || true + fi + exit 1 +} + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +WORK_ROOT="$(mktemp -d -t nemoclaw-gateway-drift-preflight.XXXXXX)" +export NEMOCLAW_DISABLE_GATEWAY_DRIFT_PREFLIGHT=0 +LIVE_GATEWAY_PID="" + +cleanup() { + if [ -n "$LIVE_GATEWAY_PID" ]; then + kill "$LIVE_GATEWAY_PID" 2>/dev/null || true + fi + rm -rf "$WORK_ROOT" +} +trap cleanup EXIT + +load_shell_path() { + if [ -f "$HOME/.bashrc" ]; then + # shellcheck source=/dev/null + source "$HOME/.bashrc" 2>/dev/null || true + fi + export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" + if [ -s "$NVM_DIR/nvm.sh" ]; then + # shellcheck source=/dev/null + . "$NVM_DIR/nvm.sh" + fi +} + +write_registry() { + local home="$1" + mkdir -p "$home/.nemoclaw" + cat >"$home/.nemoclaw/sandboxes.json" <<'JSON' +{ + "sandboxes": { + "alpha": { + "name": "alpha", + "model": "test-model", + "provider": "compatible-endpoint", + "gpuEnabled": false, + "policies": [], + "agent": "openclaw", + "agentVersion": "test-version" + } + }, + "defaultSandbox": "alpha" +} +JSON + chmod 600 "$home/.nemoclaw/sandboxes.json" +} + +write_fake_openshell() { + local bin_dir="$1" + cat >"$bin_dir/openshell" <<'SH' +#!/usr/bin/env bash +set -uo pipefail +: "${NEMOCLAW_FAKE_CASE_DIR:?}" +printf '%s\n' "$*" >> "$NEMOCLAW_FAKE_CASE_DIR/openshell-calls.log" +case "${1:-}" in + --version|-V) + printf 'openshell 0.0.37\n' + exit 0 + ;; + status) + printf 'Server Status\n\n Gateway: nemoclaw\n Gateway endpoint: http://127.0.0.1:8080\n Status: Connected\n' + exit 0 + ;; + gateway) + if [ "${2:-}" = "info" ]; then + printf 'Gateway Info\n\n Gateway: nemoclaw\n Gateway endpoint: http://127.0.0.1:8080\n' + exit 0 + fi + ;; + sandbox) + if [ "${2:-}" = "list" ]; then + printf '%s\n' 'Error: status: Internal, message: "failed to decode Protobuf message: Sandbox.metadata: SandboxResponse.sandbox: invalid wire type value: 6"' >&2 + exit "${NEMOCLAW_FAKE_SANDBOX_LIST_EXIT:-1}" + fi + ;; +esac +printf 'unexpected openshell args: %s\n' "$*" >&2 +exit 9 +SH + chmod +x "$bin_dir/openshell" +} + +write_fake_docker() { + local bin_dir="$1" + local gateway_running="${NEMOCLAW_FAKE_GATEWAY_RUNNING:-true}" + local gateway_ports="${NEMOCLAW_FAKE_GATEWAY_PORTS:-}" + if [ -z "$gateway_ports" ]; then + gateway_ports='{"30051/tcp":[{"HostIp":"0.0.0.0","HostPort":"8080"}]}' + fi + local gateway_image="${NEMOCLAW_FAKE_GATEWAY_IMAGE:-ghcr.io/nvidia/openshell/cluster:0.0.37}" + cat >"$bin_dir/docker" <> "\$case_dir/docker-calls.log" +format="" +if [ "\${1:-}" = "inspect" ] || { [ "\${1:-}" = "container" ] && [ "\${2:-}" = "inspect" ]; }; then + while [ "\$#" -gt 0 ]; do + if [ "\${1:-}" = "--format" ]; then + shift + format="\${1:-}" + break + fi + shift + done + case "\$format" in + '{{.State.Running}}'|"'{{.State.Running}}'") + printf '%s\n' '$gateway_running' + exit 0 + ;; + '{{json .NetworkSettings.Ports}}'|"'{{json .NetworkSettings.Ports}}'") + printf '%s\n' '$gateway_ports' + exit 0 + ;; + '{{.Config.Image}}'|"'{{.Config.Image}}'") + printf '%s\n' '$gateway_image' + exit 0 + ;; + esac +fi +printf 'unexpected docker args: %s\n' "\$*" >&2 +exit 9 +SH + chmod +x "$bin_dir/docker" +} + +run_backup_case() { + local name="$1" + shift + CASE_DIR="$WORK_ROOT/$name" + local home="$CASE_DIR/home" + local bin_dir="$CASE_DIR/bin" + mkdir -p "$home" "$bin_dir" + export TMPDIR="$CASE_DIR" + : >"$CASE_DIR/openshell-calls.log" + : >"$CASE_DIR/docker-calls.log" + write_registry "$home" + write_fake_openshell "$bin_dir" + write_fake_docker "$bin_dir" + + local output="$CASE_DIR/command.out" + HOME="$home" \ + PATH="$bin_dir:$PATH" \ + NEMOCLAW_FAKE_CASE_DIR="$CASE_DIR" \ + TMPDIR="$CASE_DIR" \ + NEMOCLAW_FAKE_GATEWAY_RUNNING="${NEMOCLAW_FAKE_GATEWAY_RUNNING:-}" \ + NEMOCLAW_FAKE_GATEWAY_PORTS="${NEMOCLAW_FAKE_GATEWAY_PORTS:-}" \ + NEMOCLAW_FAKE_GATEWAY_IMAGE="${NEMOCLAW_FAKE_GATEWAY_IMAGE:-}" \ + NEMOCLAW_DISABLE_GATEWAY_DRIFT_PREFLIGHT="${NEMOCLAW_DISABLE_GATEWAY_DRIFT_PREFLIGHT:-0}" \ + "$@" >"$output" 2>&1 + return $? +} + +# Host-process / Docker-driver gateway: there is no openshell-cluster-* container, +# so docker inspect always fails. The gateway version comes from probing the +# gateway binary recorded in the runtime marker. +write_fake_docker_no_cluster() { + local bin_dir="$1" + cat >"$bin_dir/docker" <<'SH' +#!/usr/bin/env bash +set -uo pipefail +printf '%s\n' "$*" >> "$NEMOCLAW_FAKE_CASE_DIR/docker-calls.log" +if [ "${1:-}" = "inspect" ] || { [ "${1:-}" = "container" ] && [ "${2:-}" = "inspect" ]; }; then + printf 'Error: No such object\n' >&2 + exit 1 +fi +exit 0 +SH + chmod +x "$bin_dir/docker" +} + +write_fake_gateway_binary() { + local bin_dir="$1" + local version="${2:-0.0.43}" + # --version prints the (drifted) version; any other invocation sleeps so the + # script can run it as a long-lived process whose PID seeds a live marker. + cat >"$bin_dir/openshell-gateway" <"$state_dir/runtime.json" <"$CASE_DIR/openshell-calls.log" + : >"$CASE_DIR/docker-calls.log" + write_registry "$home" + write_fake_openshell "$bin_dir" + write_fake_docker_no_cluster "$bin_dir" + write_fake_gateway_binary "$bin_dir" "${NEMOCLAW_FAKE_GATEWAY_BIN_VERSION:-0.0.43}" + if [ "${NEMOCLAW_E2E_SKIP_MARKER:-0}" = "1" ]; then + # No marker: exercise the marker-less fallback resolver (sibling of the + # resolved openshell binary on PATH). + : + elif [ "${NEMOCLAW_E2E_LIVE_MARKER:-0}" = "1" ]; then + # Live marker: start the gateway as a long-lived process and seed the marker + # with its PID, so the detector trusts marker.gatewayBin via the liveness + # check rather than the fallback resolver. + "$bin_dir/openshell-gateway" serve & + LIVE_GATEWAY_PID=$! + write_host_process_marker "$home" "$bin_dir/openshell-gateway" "$LIVE_GATEWAY_PID" + else + # Stale marker (dead PID): the detector must ignore marker.gatewayBin and + # fall back to live resolution. + write_host_process_marker "$home" "$bin_dir/openshell-gateway" + fi + + local output="$CASE_DIR/command.out" + HOME="$home" \ + PATH="$bin_dir:$PATH" \ + NEMOCLAW_FAKE_CASE_DIR="$CASE_DIR" \ + TMPDIR="$CASE_DIR" \ + NEMOCLAW_DISABLE_GATEWAY_DRIFT_PREFLIGHT="${NEMOCLAW_DISABLE_GATEWAY_DRIFT_PREFLIGHT:-0}" \ + "$@" >"$output" 2>&1 + return $? +} + +assert_contains() { + local file="$1" pattern="$2" description="$3" + if grep -qiE "$pattern" "$file"; then + pass "$description" + else + fail "$description (missing pattern: $pattern)" + fi +} + +assert_not_contains() { + local file="$1" pattern="$2" description="$3" + if grep -qiE "$pattern" "$file"; then + fail "$description (unexpected pattern: $pattern)" + else + pass "$description" + fi +} + +section "Prepare CLI build" +cd "$REPO_ROOT" +load_shell_path +if [ ! -d node_modules ]; then + npm ci --ignore-scripts || fail "npm ci failed" +fi +npm run build:cli || fail "CLI build failed" + +section "Protobuf mismatch from sandbox list fails closed" +set +e +NEMOCLAW_FAKE_GATEWAY_RUNNING=false \ + NEMOCLAW_FAKE_GATEWAY_IMAGE=ghcr.io/nvidia/openshell/cluster:0.0.37 \ + run_backup_case protobuf-mismatch \ + node "$REPO_ROOT/bin/nemoclaw.js" backup-all +rc=$? +set -e +if [ "$rc" -ne 0 ]; then + pass "backup-all exits non-zero on protobuf mismatch" +else + info "backup-all exited 0; checking that it did not silently treat the RPC failure as stopped" +fi +assert_contains "$CASE_DIR/command.out" 'protobuf|schema mismatch|invalid wire type|Skipping '\''?alpha'\''? \(not running\)' "protobuf failure is not silently swallowed" +assert_contains "$CASE_DIR/command.out" 'No sandbox data was changed|Refusing to trust OpenShell sandbox state' "fail-closed no-mutation guidance is printed" +assert_not_contains "$CASE_DIR/command.out" "Skipping '?alpha'? \\(not running\\)" "running sandbox is not misclassified as stopped" +assert_not_contains "$CASE_DIR/command.out" 'Backup complete' "backup does not proceed after unsafe state RPC" + +section "Patched stale gateway image fails before sandbox list" +set +e +NEMOCLAW_FAKE_GATEWAY_IMAGE=nemoclaw-cluster:0.0.36-fuse-overlayfs-aa8b8487 \ + run_backup_case patched-image-drift \ + node "$REPO_ROOT/bin/nemoclaw.js" backup-all +rc=$? +set -e +[ "$rc" -ne 0 ] || fail "backup-all unexpectedly succeeded with stale patched gateway image" +pass "backup-all exits non-zero on stale patched gateway image" +assert_contains "$CASE_DIR/command.out" 'schema preflight failed|gateway schema preflight failed|image.*does not match|Running gateway image' "gateway image drift preflight is surfaced" +assert_contains "$CASE_DIR/command.out" '0\.0\.37' "installed OpenShell version is reported" +assert_contains "$CASE_DIR/command.out" 'nemoclaw-cluster:0\.0\.36-fuse-overlayfs-aa8b8487|0\.0\.36' "patched stale gateway image/version is reported" +if grep -qx 'sandbox list' "$CASE_DIR/openshell-calls.log"; then + fail "sandbox list was called despite preflight image drift" +fi +pass "preflight image drift blocks sandbox list" + +section "Host-process gateway binary drift fails before sandbox list (backup-all, live marker)" +set +e +NEMOCLAW_E2E_LIVE_MARKER=1 \ + run_host_process_case host-process-backup \ + node "$REPO_ROOT/bin/nemoclaw.js" backup-all +rc=$? +set -e +[ "$rc" -ne 0 ] || fail "backup-all unexpectedly succeeded with host-process gateway binary drift" +pass "backup-all exits non-zero on host-process gateway binary drift" +assert_contains "$CASE_DIR/command.out" 'schema preflight failed|gateway schema preflight failed|Running gateway binary' "host-process gateway drift preflight is surfaced" +assert_contains "$CASE_DIR/command.out" '0\.0\.37' "installed OpenShell version is reported" +assert_contains "$CASE_DIR/command.out" 'Running gateway binary.*0\.0\.43' "running host-process gateway binary/version is reported" +assert_contains "$CASE_DIR/command.out" 'No sandbox data was changed|Refusing to trust OpenShell sandbox state' "fail-closed no-mutation guidance is printed" +assert_not_contains "$CASE_DIR/command.out" 'Running gateway image' "host-process drift does not claim a cluster image" +if grep -qx 'sandbox list' "$CASE_DIR/openshell-calls.log"; then + fail "sandbox list was called despite host-process preflight drift" +fi +pass "preflight host-process drift blocks sandbox list" + +section "Host-process gateway binary drift fails before sandbox list (upgrade-sandboxes)" +set +e +run_host_process_case host-process-upgrade \ + node "$REPO_ROOT/bin/nemoclaw.js" upgrade-sandboxes --check +rc=$? +set -e +[ "$rc" -ne 0 ] || fail "upgrade-sandboxes unexpectedly succeeded with host-process gateway binary drift" +pass "upgrade-sandboxes exits non-zero on host-process gateway binary drift" +assert_contains "$CASE_DIR/command.out" 'schema preflight failed|gateway schema preflight failed|Running gateway binary' "host-process gateway drift preflight is surfaced for upgrade-sandboxes" +assert_contains "$CASE_DIR/command.out" 'Running gateway binary.*0\.0\.43' "running host-process gateway binary/version is reported for upgrade-sandboxes" +if grep -qx 'sandbox list' "$CASE_DIR/openshell-calls.log"; then + fail "sandbox list was called despite host-process preflight drift (upgrade-sandboxes)" +fi +pass "preflight host-process drift blocks sandbox list for upgrade-sandboxes" + +section "Host-process gateway binary drift detected via fallback resolver (no runtime marker)" +set +e +NEMOCLAW_E2E_SKIP_MARKER=1 \ + run_host_process_case host-process-no-marker \ + node "$REPO_ROOT/bin/nemoclaw.js" backup-all +rc=$? +set -e +[ "$rc" -ne 0 ] || fail "backup-all unexpectedly succeeded with host-process drift and no runtime marker" +pass "backup-all exits non-zero on host-process gateway binary drift without a runtime marker" +assert_contains "$CASE_DIR/command.out" 'schema preflight failed|gateway schema preflight failed|Running gateway binary' "host-process gateway drift preflight is surfaced without a marker" +assert_contains "$CASE_DIR/command.out" 'Running gateway binary.*0\.0\.43' "fallback-resolved gateway binary/version is reported" +if grep -qx 'sandbox list' "$CASE_DIR/openshell-calls.log"; then + fail "sandbox list was called despite host-process preflight drift (no marker)" +fi +pass "preflight host-process drift (fallback resolver) blocks sandbox list" + +section "Stale runtime marker does not false-positive when the live gateway matches the CLI" +# A dead-PID marker points at a separate old binary, but the gateway that +# recovery would actually launch (sibling of openshell on PATH) matches the +# installed CLI. The preflight must NOT flag host-process drift; it should pass +# and let the (reactive) sandbox-list path run. +CASE_DIR="$WORK_ROOT/host-process-stale-marker" +stale_home="$CASE_DIR/home" +stale_bin="$CASE_DIR/bin" +stale_old="$CASE_DIR/old-install" +mkdir -p "$stale_home" "$stale_bin" "$stale_old" +: >"$CASE_DIR/openshell-calls.log" +: >"$CASE_DIR/docker-calls.log" +write_registry "$stale_home" +write_fake_openshell "$stale_bin" +write_fake_docker_no_cluster "$stale_bin" +# Sibling gateway on PATH matches the fake CLI version (0.0.37): no real drift. +write_fake_gateway_binary "$stale_bin" "0.0.37" +# Separate stale binary (0.0.43) referenced by a dead-PID marker. +write_fake_gateway_binary "$stale_old" "0.0.43" +write_host_process_marker "$stale_home" "$stale_old/openshell-gateway" 999999 +set +e +HOME="$stale_home" \ + PATH="$stale_bin:$PATH" \ + NEMOCLAW_FAKE_CASE_DIR="$CASE_DIR" \ + TMPDIR="$CASE_DIR" \ + NEMOCLAW_DISABLE_GATEWAY_DRIFT_PREFLIGHT=0 \ + node "$REPO_ROOT/bin/nemoclaw.js" backup-all >"$CASE_DIR/command.out" 2>&1 +set -e +assert_not_contains "$CASE_DIR/command.out" 'Running gateway binary.*0\.0\.43' "stale marker binary is not used to fabricate drift" +if grep -qx 'sandbox list' "$CASE_DIR/openshell-calls.log"; then + pass "preflight passes (no false positive) and proceeds to sandbox list" +else + fail "preflight blocked sandbox list on a stale marker even though the live gateway matches the CLI" +fi + +section "Summary" +pass "Gateway drift preflight regression guard completed" diff --git a/test/e2e-vpn/test-gateway-health-honest.sh b/test/e2e-vpn/test-gateway-health-honest.sh new file mode 100755 index 00000000000..e884cad838d --- /dev/null +++ b/test/e2e-vpn/test-gateway-health-honest.sh @@ -0,0 +1,234 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Coverage guard for issue #3111 — "Docker-driver gateway is healthy" +# must not be logged when the gateway binary failed to start. +# +# Background: PR #3001 introduced a Linux Docker-driver gateway managed by +# onboard.ts:startGateway(). On Ubuntu 22.04, the shipped openshell-gateway +# binary is linked against GLIBC 2.38/2.39 and crashes immediately on a +# 22.04 host (GLIBC 2.35). NemoClaw still reports "✓ Docker-driver gateway +# is healthy" because: +# - the detached child becomes a zombie, so isPidAlive(childPid) returns +# true (the pid remains in the process table until the parent reaps it); +# - registerDockerDriverGatewayEndpoint() is metadata-only (openshell +# gateway add --local) and succeeds without any TCP probe; +# - isGatewayHealthy() reads openshell status / gateway info strings, +# not a live health probe — so cached / metadata-only output satisfies +# the check. +# +# This test is platform-independent: instead of exercising the GLIBC path +# (which requires a 22.04 runner we don't have in CI) it substitutes the +# gateway binary with a shim that crashes immediately with the same +# GLIBC-style error on stderr. Any onboard that treats a crashed child as +# healthy fails this test. The fix for #3111 must make startGateway verify +# the child is actually alive (not a zombie) and that the endpoint serves +# a real TCP probe before declaring "healthy". +# +# Expected result on main (bug present): FAIL — the test asserts onboard +# must NOT print "Docker-driver gateway is healthy" when the binary +# crashed; current code does print it, so the assertion fails. +# Expected result after fix: PASS — onboard surfaces the crash and exits +# non-zero. +# +# Related: #3111, PR #3001 + +set -euo pipefail + +LOG_FILE="/tmp/nemoclaw-e2e-gateway-health-honest.log" +START_LOG="/tmp/nemoclaw-e2e-gateway-health-honest-start.log" +GATEWAY_LOG="/tmp/nemoclaw-e2e-gateway-health-honest-process.log" +exec > >(tee "$LOG_FILE") 2>&1 + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +pass() { echo -e "${GREEN}[PASS]${NC} $1"; } +info() { echo -e "${YELLOW}[INFO]${NC} $1"; } +diag() { echo -e "${YELLOW}[DIAG]${NC} $1"; } +fail() { + echo -e "${RED}[FAIL]${NC} $1" >&2 + diag "start log tail:" + tail -80 "$START_LOG" 2>/dev/null || true + diag "gateway process log tail:" + tail -80 "$GATEWAY_LOG" 2>/dev/null || true + diag "onboard gateway log tail (where sabotage stderr lands):" + tail -80 "${STATE_DIR}/openshell-gateway.log" 2>/dev/null || true + diag "openshell status: $(openshell status 2>&1 || true)" + diag "gateway info: $(openshell gateway info -g nemoclaw 2>&1 || true)" + diag "pid file: $(cat "${PID_FILE:-/dev/null}" 2>/dev/null || echo missing)" + exit 1 +} + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +STATE_DIR="${NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR:-$HOME/.local/state/nemoclaw/openshell-docker-gateway}" +PID_FILE="${STATE_DIR}/openshell-gateway.pid" +SABOTAGE_BIN="${STATE_DIR}/openshell-gateway-sabotage" +CHILD_PID="" + +load_shell_path() { + if [ -f "$HOME/.bashrc" ]; then + # shellcheck source=/dev/null + source "$HOME/.bashrc" 2>/dev/null || true + fi + export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" + if [ -s "$NVM_DIR/nvm.sh" ]; then + # shellcheck source=/dev/null + . "$NVM_DIR/nvm.sh" + fi + if [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]]; then + export PATH="$HOME/.local/bin:$PATH" + fi +} + +cleanup_pid() { + local pid="$1" + [ -n "$pid" ] || return 0 + if kill -0 "$pid" 2>/dev/null; then + kill "$pid" 2>/dev/null || true + sleep 1 + kill -9 "$pid" 2>/dev/null || true + fi + # Reap any zombies left over by the test + wait "$pid" 2>/dev/null || true +} + +cleanup() { + set +e + if [ -f "$PID_FILE" ]; then + CHILD_PID="$(tr -d '[:space:]' <"$PID_FILE")" + fi + cleanup_pid "$CHILD_PID" + openshell gateway remove nemoclaw >/dev/null 2>&1 || true + rm -f "$PID_FILE" "$SABOTAGE_BIN" +} +trap cleanup EXIT + +cd "$REPO_ROOT" +load_shell_path + +info "Preparing CLI build and OpenShell binaries" +if [ ! -d node_modules ]; then + npm ci --ignore-scripts +fi +npm run build:cli +bash scripts/install-openshell.sh +load_shell_path + +command -v openshell >/dev/null 2>&1 || fail "openshell not found after install" +command -v openshell-gateway >/dev/null 2>&1 || fail "openshell-gateway not found after install" + +# Start from a clean slate: no prior gateway metadata, no pid file. +mkdir -p "$STATE_DIR" +chmod 700 "$STATE_DIR" +rm -f "$PID_FILE" "$START_LOG" "$GATEWAY_LOG" +openshell gateway remove nemoclaw >/dev/null 2>&1 || true + +info "Installing sabotage gateway binary that simulates the #3111 GLIBC crash" +cat >"$SABOTAGE_BIN" <<'SHIM' +#!/usr/bin/env bash +# Simulates the Ubuntu 22.04 GLIBC-2.38/2.39 failure mode reported in #3111. +# The real binary dies at the dynamic-linker stage before main() runs; we +# mirror that by emitting the same stderr fragment and exiting non-zero +# before opening any TCP port. +printf '%s\n' "$(basename "$0"): /lib/x86_64-linux-gnu/libc.so.6: version \`GLIBC_2.38' not found (required by $(basename "$0"))" >&2 +printf '%s\n' "$(basename "$0"): /lib/x86_64-linux-gnu/libc.so.6: version \`GLIBC_2.39' not found (required by $(basename "$0"))" >&2 +exit 127 +SHIM +chmod 755 "$SABOTAGE_BIN" + +info "Invoking startGateway() with the sabotaged binary" +# startGateway() with exitOnFailure:true calls process.exit(1) when it +# concludes the gateway failed. A correctly-behaved onboard MUST either: +# (a) exit non-zero, OR +# (b) print "failed to start" / a surface error message, +# and MUST NOT print "Docker-driver gateway is healthy". +set +e +NEMOCLAW_OPENSHELL_GATEWAY_BIN="$SABOTAGE_BIN" \ + NEMOCLAW_HEALTH_POLL_COUNT="${NEMOCLAW_HEALTH_POLL_COUNT:-10}" \ + NEMOCLAW_HEALTH_POLL_INTERVAL="${NEMOCLAW_HEALTH_POLL_INTERVAL:-1}" \ + node <<'NODE' 2>&1 | tee "$START_LOG" +const { startGateway } = require("./dist/lib/onboard"); + +startGateway(null) + .then(() => { + console.log("__onboard_startGateway_returned_successfully__"); + process.exit(0); + }) + .catch((error) => { + console.error("__onboard_startGateway_threw__"); + console.error(error && error.stack ? error.stack : error); + process.exit(3); + }); +NODE +NODE_EXIT=$? +set -e + +info "node exit code: ${NODE_EXIT}" + +# ── Pre-assertion: prove the sabotage path was actually exercised ─── +# Without this guard, an unrelated setup failure (module-not-found, +# missing env, stale dist/, etc.) could produce a $START_LOG that +# happens to lack the 'healthy' string and thereby false-green the +# primary assertion. We require positive evidence that the sabotage +# shim ran. +# +# The sabotage shim writes its GLIBC-style stderr to the gateway log +# file opened by onboard.ts:startGatewayWithOptions at +# $STATE_DIR/openshell-gateway.log (NOT to the start log, which only +# captures node's stdout/stderr). That gateway log is the authoritative +# source of truth for "did our binary get exec'd". +GATEWAY_ONBOARD_LOG="${STATE_DIR}/openshell-gateway.log" +if ! grep -qE 'GLIBC_2\.3(8|9)|openshell-gateway-sabotage' "$GATEWAY_ONBOARD_LOG" 2>/dev/null; then + fail "Sabotage markers (GLIBC_2.38/2.39 or 'openshell-gateway-sabotage') not observed in gateway log ${GATEWAY_ONBOARD_LOG} — the test may have failed before the sabotaged gateway was invoked, so the assertions below cannot be trusted. Inspect $START_LOG and $GATEWAY_ONBOARD_LOG above for the real cause." +fi +pass "Sabotage shim was invoked as expected (GLIBC/sabotage markers present in gateway log)" + +# ── Primary assertion ──────────────────────────────────────────────── +# This is the bug from #3111. Onboard printed "healthy" while the child +# process was a crashed zombie and had never served a real connection. +if grep -q "✓ Docker-driver gateway is healthy" "$START_LOG" \ + || grep -q "Docker-driver gateway is healthy" "$START_LOG"; then + fail "Onboard reported '✓ Docker-driver gateway is healthy' although the gateway binary crashed on startup (#3111 false-positive health check)" +fi +pass "Onboard did not falsely log 'Docker-driver gateway is healthy' when the binary crashed" + +# ── Corroborating assertion 1: non-zero exit ───────────────────────── +# startGateway(null) uses exitOnFailure:true → the node process MUST exit +# non-zero when the gateway truly failed to start. Exit 0 means onboard +# silently accepted the crashed gateway as success. +if [ "$NODE_EXIT" -eq 0 ] || grep -q "__onboard_startGateway_returned_successfully__" "$START_LOG"; then + fail "startGateway() resolved successfully despite a crashed binary — onboard would have proceeded to inference setup against a dead gateway" +fi +pass "startGateway() did not resolve successfully with a crashed binary (node exit=${NODE_EXIT})" + +# ── Corroborating assertion 2: user-visible failure surfaced ───────── +# Deliberately narrow: excludes generic 'not found' because an unrelated +# module-not-found (e.g. stale dist/) would satisfy the match without +# proving the gateway-failure code path was exercised. The Pre-assertion +# above already proves the sabotage ran, but this stays narrow anyway. +if ! grep -qiE "failed to start|gateway.*(crash|exit|error)|__onboard_startGateway_threw__" "$START_LOG"; then + fail "Onboard did not surface any gateway failure indicator to the user" +fi +pass "Onboard surfaced a user-visible gateway failure message" + +# ── Corroborating assertion 3: no live gateway process ─────────────── +if [ -f "$PID_FILE" ]; then + LINGERING_PID="$(tr -d '[:space:]' <"$PID_FILE")" + if [ -n "$LINGERING_PID" ] && kill -0 "$LINGERING_PID" 2>/dev/null; then + # A live pid that is *not* a zombie would mean onboard somehow kept + # something alive. Zombies are acceptable as a transient artifact. + STATE="$(ps -p "$LINGERING_PID" -o state= 2>/dev/null | tr -d ' ')" + if [ "$STATE" != "Z" ] && [ -n "$STATE" ]; then + fail "A non-zombie gateway pid (${LINGERING_PID}, state=${STATE}) is still alive after a simulated crash" + fi + fi +fi +pass "No live (non-zombie) gateway process is running after the simulated crash" + +echo "" +pass "#3111 coverage guard green: onboard correctly surfaces a crashed gateway" diff --git a/test/e2e-vpn/test-gpu-double-onboard.sh b/test/e2e-vpn/test-gpu-double-onboard.sh new file mode 100755 index 00000000000..d7abae87c64 --- /dev/null +++ b/test/e2e-vpn/test-gpu-double-onboard.sh @@ -0,0 +1,579 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# GPU Double-Onboard E2E: Ollama proxy token consistency after re-onboard. +# +# Reproduces the exact scenario from issue #2553 — the Ollama proxy token +# divergence bug where re-running onboard left the proxy running with a +# different token than what was persisted to disk, causing silent HTTP 401 +# on all inference. +# +# Flow: +# 1. Prerequisites — Docker, nvidia-smi, env vars +# 2. Install Ollama binary (do NOT start it — onboard handles that) +# 3. First onboard — install.sh --non-interactive with NEMOCLAW_PROVIDER=ollama +# 4. Verify sandbox, proxy, token file, inference through sandbox +# 5. Second onboard (re-onboard) — nemoclaw onboard --non-interactive --yes +# 6. Token consistency verification (the core of this test): +# - Read ~/.nemoclaw/ollama-proxy-token +# - Verify proxy accepts that token (not 401) +# - Verify inference through sandbox succeeds (not 401) +# 7. Destroy and cleanup +# +# Key differences from test-gpu-e2e.sh: +# - Adds a second onboard + token consistency check +# - Uses nemoclaw onboard CLI directly for re-onboard (not install.sh) +# - Distinct sandbox name e2e-gpu-double-onboard +# +# Key differences from test-double-onboard.sh: +# - Uses NEMOCLAW_PROVIDER=ollama (real GPU inference) +# - Tests token consistency explicitly +# - Runs on NVKS ephemeral GPU runner (L40G) +# +# Prerequisites: +# - NVIDIA GPU with drivers (nvidia-smi works) +# - Docker +# - NEMOCLAW_NON_INTERACTIVE=1 +# - NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 +# - Internet access (ollama.com for install, registry.ollama.ai for model pull) +# +# Usage: +# NEMOCLAW_NON_INTERACTIVE=1 NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ +# bash test/e2e-vpn/test-gpu-double-onboard.sh + +# ShellCheck cannot see EXIT trap invocations of cleanup helpers in this E2E script. +# shellcheck disable=SC2317 +set -uo pipefail + +export NEMOCLAW_E2E_DEFAULT_TIMEOUT=1800 +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +# shellcheck source=test/e2e-vpn/e2e-timeout.sh +source "${SCRIPT_DIR}/e2e-timeout.sh" + +PASS=0 +FAIL=0 +SKIP=0 +TOTAL=0 + +pass() { + ((PASS++)) + ((TOTAL++)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} +fail() { + ((FAIL++)) + ((TOTAL++)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} +# shellcheck disable=SC2329 +skip() { + ((SKIP++)) + ((TOTAL++)) + printf '\033[33m SKIP: %s\033[0m\n' "$1" +} +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } + +# Parse chat completion response — handles both content and reasoning_content +parse_chat_content() { + python3 -c " +import json, sys +try: + r = json.load(sys.stdin) + c = r['choices'][0]['message'] + content = c.get('content') or c.get('reasoning_content') or c.get('reasoning') or '' + print(content.strip()) +except Exception as e: + print(f'PARSE_ERROR: {e}', file=sys.stderr) + sys.exit(1) +" +} + +# Determine repo root +if [ -d /workspace ] && [ -f /workspace/install.sh ]; then + REPO="/workspace" +elif [ -f "$(cd "$(dirname "$0")/../.." && pwd)/install.sh" ]; then + REPO="$(cd "$(dirname "$0")/../.." && pwd)" +else + echo "ERROR: Cannot find repo root." + exit 1 +fi + +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-gpu-double-onboard}" +TEST_LOG="/tmp/nemoclaw-gpu-double-onboard-test.log" +INSTALL_LOG="/tmp/nemoclaw-gpu-double-onboard-install.log" +REONBOARD_LOG="/tmp/nemoclaw-gpu-double-onboard-reonboard.log" +PROXY_PORT="${NEMOCLAW_OLLAMA_PROXY_PORT:-11435}" +TOKEN_FILE="$HOME/.nemoclaw/ollama-proxy-token" + +# Enforce Ollama provider — this script only tests local GPU inference. +export NEMOCLAW_PROVIDER="${NEMOCLAW_PROVIDER:-ollama}" +if [ "$NEMOCLAW_PROVIDER" != "ollama" ]; then + echo "ERROR: NEMOCLAW_PROVIDER must be 'ollama' for GPU double-onboard E2E (got: $NEMOCLAW_PROVIDER)" + exit 1 +fi + +exec > >(tee -a "$TEST_LOG") 2>&1 + +# Best-effort cleanup on any exit (prevents dirty state on reused runners) +# shellcheck disable=SC2329 # invoked via trap +cleanup() { + info "Running exit cleanup..." + if command -v nemoclaw >/dev/null 2>&1; then + nemoclaw "$SANDBOX_NAME" destroy --yes 2>/dev/null || true + fi + if command -v openshell >/dev/null 2>&1; then + openshell sandbox delete "$SANDBOX_NAME" 2>/dev/null || true + openshell gateway destroy -g nemoclaw 2>/dev/null || true + fi + pkill -f "ollama serve" 2>/dev/null || true + pkill -f "ollama-auth-proxy" 2>/dev/null || true +} +trap cleanup EXIT + +# ══════════════════════════════════════════════════════════════════ +# Phase 0: Pre-cleanup +# ══════════════════════════════════════════════════════════════════ +section "Phase 0: Pre-cleanup" +info "Destroying any leftover sandbox/gateway from previous runs..." +if command -v nemoclaw >/dev/null 2>&1; then + nemoclaw "$SANDBOX_NAME" destroy --yes 2>/dev/null || true +fi +if command -v openshell >/dev/null 2>&1; then + openshell sandbox delete "$SANDBOX_NAME" 2>/dev/null || true + openshell gateway destroy -g nemoclaw 2>/dev/null || true +fi +pkill -f "ollama serve" 2>/dev/null || true +pkill -f "ollama-auth-proxy" 2>/dev/null || true +sleep 2 +pass "Pre-cleanup complete" + +# ══════════════════════════════════════════════════════════════════ +# Phase 1: Prerequisites +# ══════════════════════════════════════════════════════════════════ +section "Phase 1: Prerequisites" + +if docker info >/dev/null 2>&1; then + pass "Docker is running" +else + fail "Docker is not running — cannot continue" + exit 1 +fi + +if nvidia-smi >/dev/null 2>&1; then + VRAM_MB=$(nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits 2>/dev/null | head -1) + pass "nvidia-smi works (GPU VRAM: ${VRAM_MB:-unknown} MB)" +else + fail "nvidia-smi failed — no NVIDIA GPU available" + exit 1 +fi + +if [ "${NEMOCLAW_NON_INTERACTIVE:-}" != "1" ]; then + fail "NEMOCLAW_NON_INTERACTIVE=1 is required" + exit 1 +fi + +if [ "${NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE:-}" != "1" ]; then + fail "NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 is required for non-interactive install" + exit 1 +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 2: Install Ollama binary +# ══════════════════════════════════════════════════════════════════ +section "Phase 2: Install Ollama binary" + +# Only install the binary — do NOT start Ollama or pull models. +# The nemoclaw onboard flow handles startup and model pull itself. +if command -v ollama >/dev/null 2>&1; then + pass "Ollama already installed: $(ollama --version 2>/dev/null || echo unknown)" +else + info "Installing Ollama..." + if curl -fsSL https://ollama.com/install.sh | sh 2>&1; then + pass "Ollama installed: $(ollama --version 2>/dev/null || echo unknown)" + else + fail "Ollama installation failed" + exit 1 + fi +fi + +# If the Ollama installer started a system service, stop it so onboard +# can restart Ollama on loopback and expose only the authenticated proxy to containers. +if curl -sf http://127.0.0.1:11434/api/tags >/dev/null 2>&1; then + info "Ollama service is running — attempting to stop for clean onboard..." + systemctl --user stop ollama 2>/dev/null || true + systemctl stop ollama 2>/dev/null || true + pkill -f "ollama serve" 2>/dev/null || true + sleep 2 + + if curl -sf http://127.0.0.1:11434/api/tags >/dev/null 2>&1; then + info "Could not stop existing Ollama — onboard will use it as-is" + else + pass "Existing Ollama stopped — port 11434 is free for onboard" + fi +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 3: First onboard — install.sh --non-interactive +# ══════════════════════════════════════════════════════════════════ +section "Phase 3: First onboard (install.sh --non-interactive)" + +cd "$REPO" || { + fail "Could not cd to repo root: $REPO" + exit 1 +} + +info "Running install.sh --non-interactive with NEMOCLAW_PROVIDER=ollama..." +info "Onboard will start Ollama, pull the model, and create the sandbox." + +bash install.sh --non-interactive >"$INSTALL_LOG" 2>&1 & +install_pid=$! +tail -f "$INSTALL_LOG" --pid=$install_pid 2>/dev/null & +tail_pid=$! +wait $install_pid +install_exit=$? +kill $tail_pid 2>/dev/null || true +wait $tail_pid 2>/dev/null || true + +# Source shell profile to pick up nvm/PATH changes +if [ -f "$HOME/.bashrc" ]; then + source "$HOME/.bashrc" 2>/dev/null || true +fi +export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" +[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" +if [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]]; then + export PATH="$HOME/.local/bin:$PATH" +fi + +if [ $install_exit -eq 0 ]; then + pass "install.sh completed (exit 0)" +else + fail "install.sh failed (exit $install_exit)" + info "Last 30 lines of install log:" + tail -30 "$INSTALL_LOG" + exit 1 +fi + +if command -v nemoclaw >/dev/null 2>&1; then + pass "nemoclaw on PATH: $(command -v nemoclaw)" +else + fail "nemoclaw not found on PATH after install" + exit 1 +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 4: Verify first onboard +# ══════════════════════════════════════════════════════════════════ +section "Phase 4: Verify first onboard" + +# 4a: Sandbox exists +if list_output=$(nemoclaw list 2>&1); then + if echo "$list_output" | grep -Fq -- "$SANDBOX_NAME"; then + pass "nemoclaw list contains '${SANDBOX_NAME}'" + else + fail "nemoclaw list does not contain '${SANDBOX_NAME}'" + fi +else + fail "nemoclaw list failed: ${list_output:0:200}" +fi + +# 4b: Status ok +if nemoclaw "$SANDBOX_NAME" status >/dev/null 2>&1; then + pass "nemoclaw ${SANDBOX_NAME} status exits 0" +else + fail "nemoclaw ${SANDBOX_NAME} status failed" +fi + +# 4c: Ollama is running and reachable +if curl -sf http://127.0.0.1:11434/api/tags >/dev/null 2>&1; then + pass "Ollama running on 127.0.0.1:11434" +else + fail "Ollama not running — onboard should have started it" +fi + +# 4d: Auth proxy is running. After #3338 an alive proxy answers 401 on /api/tags +# without a Bearer token, so we accept any HTTP response as proof of life. +PROXY_LIVE_STATUS=$(curl -s -o /dev/null -w "%{http_code}" --connect-timeout 3 \ + "http://127.0.0.1:${PROXY_PORT}/api/tags" 2>/dev/null) || PROXY_LIVE_STATUS="000" +if [[ "$PROXY_LIVE_STATUS" =~ ^[1-9][0-9]{2}$ ]]; then + pass "Auth proxy running on :${PROXY_PORT} (HTTP $PROXY_LIVE_STATUS)" +else + fail "Auth proxy not running on :${PROXY_PORT}" +fi + +# 4e: Token file exists with correct permissions +if [ -f "$TOKEN_FILE" ]; then + pass "Proxy token persisted at $TOKEN_FILE" + PERMS=$(stat -c "%a" "$TOKEN_FILE" 2>/dev/null || stat -f "%Lp" "$TOKEN_FILE" 2>/dev/null) + if [ "$PERMS" = "600" ]; then + pass "Token file permissions: 600" + else + fail "Token file permissions: expected 600, got $PERMS" + fi +else + fail "Proxy token file missing after first onboard" +fi + +# 4f: Record the first-onboard token for later comparison +TOKEN_AFTER_FIRST="" +if [ -f "$TOKEN_FILE" ]; then + TOKEN_AFTER_FIRST=$(tr -d '[:space:]' <"$TOKEN_FILE") + info "Token after first onboard: ${TOKEN_AFTER_FIRST:0:8}..." +fi + +# 4g: Verify proxy accepts first-onboard token +if [ -n "$TOKEN_AFTER_FIRST" ]; then + FIRST_AUTH_STATUS=$(curl -s -o /dev/null -w "%{http_code}" \ + -H "Authorization: Bearer $TOKEN_AFTER_FIRST" \ + "http://127.0.0.1:${PROXY_PORT}/v1/models" 2>/dev/null) || FIRST_AUTH_STATUS="000" + if [ "$FIRST_AUTH_STATUS" = "200" ]; then + pass "Proxy accepts first-onboard token (200)" + else + fail "Proxy rejects first-onboard token (status: $FIRST_AUTH_STATUS)" + fi +fi + +# 4h: Determine model for inference tests +CONFIGURED_MODEL="${NEMOCLAW_MODEL:-}" +if [ -z "$CONFIGURED_MODEL" ]; then + CONFIGURED_MODEL=$(curl -sf http://127.0.0.1:11434/api/tags 2>/dev/null \ + | python3 -c "import json,sys; m=json.load(sys.stdin).get('models',[]); print(m[0]['name'] if m else '')" 2>/dev/null || echo "") +fi +if [ -n "$CONFIGURED_MODEL" ]; then + info "Model for inference tests: $CONFIGURED_MODEL" +else + fail "No models found in Ollama" +fi + +# 4i: First-onboard inference through sandbox +info "Testing inference through sandbox after first onboard..." +ssh_config="$(mktemp)" +sandbox_response="" + +if openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config" 2>/dev/null; then + sandbox_response=$(run_with_timeout 120 ssh -F "$ssh_config" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" \ + "curl -s --max-time 90 https://inference.local/v1/chat/completions \ + -H 'Content-Type: application/json' \ + -d '{\"model\":\"$CONFIGURED_MODEL\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with exactly one word: PONG\"}],\"max_tokens\":200}'" \ + 2>&1) || true +else + fail "openshell sandbox ssh-config failed" +fi +rm -f "$ssh_config" + +if [ -n "$sandbox_response" ]; then + sandbox_content=$(echo "$sandbox_response" | parse_chat_content 2>/dev/null) || true + if echo "$sandbox_content" | grep -qi "PONG"; then + pass "First-onboard sandbox inference succeeded" + else + fail "First-onboard sandbox inference: expected PONG, got: ${sandbox_content:0:200}" + fi +else + fail "First-onboard sandbox inference: no response" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 5: Second onboard (re-onboard) +# ══════════════════════════════════════════════════════════════════ +section "Phase 5: Second onboard (re-onboard via nemoclaw onboard)" + +info "Running nemoclaw onboard --non-interactive --yes with NEMOCLAW_RECREATE_SANDBOX=1..." +info "This exercises the exact code path from issue #2553:" +info " startOllamaAuthProxy() → killStaleProxy() → token generation → persistProxyToken()" + +export NEMOCLAW_RECREATE_SANDBOX=1 +nemoclaw onboard --non-interactive --yes >"$REONBOARD_LOG" 2>&1 & +reonboard_pid=$! +tail -f "$REONBOARD_LOG" --pid=$reonboard_pid 2>/dev/null & +tail_pid=$! +wait $reonboard_pid +reonboard_exit=$? +kill $tail_pid 2>/dev/null || true +wait $tail_pid 2>/dev/null || true + +if [ $reonboard_exit -eq 0 ]; then + pass "Re-onboard completed (exit 0)" +else + fail "Re-onboard failed (exit $reonboard_exit)" + info "Last 30 lines of re-onboard log:" + tail -30 "$REONBOARD_LOG" + exit 1 +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 6: Token consistency verification (core of this test) +# ══════════════════════════════════════════════════════════════════ +section "Phase 6: Token consistency verification (#2553 regression check)" + +info "This is the exact check that would have caught the token divergence bug." +info "After re-onboard, the token on disk MUST match what the running proxy accepts." + +# 6a: Token file still exists +if [ -f "$TOKEN_FILE" ]; then + pass "Proxy token file exists after re-onboard" +else + fail "Proxy token file missing after re-onboard" + exit 1 +fi + +# 6b: Read the post-re-onboard token +TOKEN_AFTER_SECOND=$(tr -d '[:space:]' <"$TOKEN_FILE") +info "Token after re-onboard: ${TOKEN_AFTER_SECOND:0:8}..." + +# 6c: Token file permissions preserved +PERMS=$(stat -c "%a" "$TOKEN_FILE" 2>/dev/null || stat -f "%Lp" "$TOKEN_FILE" 2>/dev/null) +if [ "$PERMS" = "600" ]; then + pass "Token file permissions preserved: 600" +else + fail "Token file permissions: expected 600, got $PERMS" +fi + +# 6d: Auth proxy is running after re-onboard. Same "any HTTP response = alive" +# pattern as 4d — /api/tags now requires auth per #3338. +PROXY_LIVE_STATUS=$(curl -s -o /dev/null -w "%{http_code}" --connect-timeout 3 \ + "http://127.0.0.1:${PROXY_PORT}/api/tags" 2>/dev/null) || PROXY_LIVE_STATUS="000" +if [[ "$PROXY_LIVE_STATUS" =~ ^[1-9][0-9]{2}$ ]]; then + pass "Auth proxy running on :${PROXY_PORT} after re-onboard (HTTP $PROXY_LIVE_STATUS)" +else + fail "Auth proxy not running after re-onboard" +fi + +# 6e: THE CRITICAL CHECK — proxy accepts the persisted token (not 401) +# This is the exact failure mode from #2553: the proxy was running with +# a NEW token in memory, but the OLD token was persisted to disk. +TOKEN_AUTH_STATUS=$(curl -s -o /dev/null -w "%{http_code}" \ + -H "Authorization: Bearer $TOKEN_AFTER_SECOND" \ + "http://127.0.0.1:${PROXY_PORT}/v1/models" 2>/dev/null) || TOKEN_AUTH_STATUS="000" +if [ "$TOKEN_AUTH_STATUS" = "200" ]; then + pass "Proxy accepts persisted token after re-onboard (200 — not 401)" +else + fail "PROXY TOKEN DIVERGENCE DETECTED (#2553 regression)" + fail "Token on disk does not match running proxy (status: $TOKEN_AUTH_STATUS)" + info "This is the exact bug from #2553 — the proxy has a different token than what's on disk." +fi + +# 6f: Proxy rejects unauthenticated requests (sanity check) +UNAUTH_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST \ + "http://127.0.0.1:${PROXY_PORT}/api/generate" -d '{}' 2>/dev/null) || UNAUTH_STATUS="000" +if [ "$UNAUTH_STATUS" = "401" ]; then + pass "Proxy rejects unauthenticated POST after re-onboard (401)" +else + fail "Proxy should reject unauthenticated POST, got $UNAUTH_STATUS" +fi + +# 6g: Proxy rejects a wrong token (sanity check) +WRONG_STATUS=$(curl -s -o /dev/null -w "%{http_code}" \ + -H "Authorization: Bearer wrong-token-$(date +%s)" \ + -X POST "http://127.0.0.1:${PROXY_PORT}/api/generate" -d '{}' 2>/dev/null) || WRONG_STATUS="000" +if [ "$WRONG_STATUS" = "401" ]; then + pass "Proxy rejects wrong token after re-onboard (401)" +else + fail "Proxy should reject wrong token, got $WRONG_STATUS" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 7: Inference through sandbox after re-onboard +# ══════════════════════════════════════════════════════════════════ +section "Phase 7: Inference through sandbox after re-onboard" + +info "Verifying end-to-end inference still works after re-onboard..." +info "Path: sandbox → openshell gateway → auth proxy (:${PROXY_PORT}) → Ollama GPU (:11434)" + +ssh_config="$(mktemp)" +sandbox_response="" + +if openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config" 2>/dev/null; then + sandbox_response=$(run_with_timeout 120 ssh -F "$ssh_config" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" \ + "curl -s --max-time 90 https://inference.local/v1/chat/completions \ + -H 'Content-Type: application/json' \ + -d '{\"model\":\"$CONFIGURED_MODEL\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with exactly one word: PONG\"}],\"max_tokens\":200}'" \ + 2>&1) || true +else + fail "openshell sandbox ssh-config failed after re-onboard" +fi +rm -f "$ssh_config" + +if [ -n "$sandbox_response" ]; then + sandbox_content=$(echo "$sandbox_response" | parse_chat_content 2>/dev/null) || true + if echo "$sandbox_content" | grep -qi "PONG"; then + pass "Sandbox inference after re-onboard succeeded" + info "Full path proven: sandbox → gateway → auth proxy (:${PROXY_PORT}) → Ollama GPU (:11434)" + else + # Check if the failure is specifically a 401 (token divergence) + if echo "$sandbox_response" | grep -q "401"; then + fail "SANDBOX INFERENCE RETURNED 401 — token divergence (#2553 regression)" + else + fail "Sandbox inference after re-onboard: expected PONG, got: ${sandbox_content:0:200}" + fi + fi +else + fail "Sandbox inference after re-onboard: no response" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 8: Destroy and cleanup +# ══════════════════════════════════════════════════════════════════ +section "Phase 8: Destroy and cleanup" + +info "Destroying sandbox ${SANDBOX_NAME}..." +nemoclaw "$SANDBOX_NAME" destroy --yes 2>&1 | tail -5 || true + +# Verify against the registry file directly (see test-gpu-e2e.sh comment). +registry_file="${HOME}/.nemoclaw/sandboxes.json" +if [ -f "$registry_file" ] && grep -Fq "\"${SANDBOX_NAME}\"" "$registry_file"; then + fail "Sandbox ${SANDBOX_NAME} still in registry after destroy" +else + pass "Sandbox ${SANDBOX_NAME} removed from registry" +fi + +openshell gateway destroy -g nemoclaw 2>/dev/null || true + +info "Stopping Ollama..." +pkill -f "ollama serve" 2>/dev/null || true +pkill -f "ollama-auth-proxy" 2>/dev/null || true +pass "Cleanup complete" + +# ══════════════════════════════════════════════════════════════════ +# Summary +# ══════════════════════════════════════════════════════════════════ +echo "" +echo "========================================" +echo " GPU Double-Onboard E2E Results (Ollama Token Consistency):" +echo " Passed: $PASS" +echo " Failed: $FAIL" +echo " Skipped: $SKIP" +echo " Total: $TOTAL" +echo "========================================" +echo "" +echo " What this tested (issue #2553 regression):" +echo " - GPU detection (nvidia-smi)" +echo " - Ollama binary install" +echo " - First onboard: install.sh → Ollama + auth proxy + sandbox + inference" +echo " - Second onboard (re-onboard): nemoclaw onboard --non-interactive --yes" +echo " - TOKEN CONSISTENCY: persisted token matches running proxy after re-onboard" +echo " - Proxy auth enforcement: accept correct token, reject unauth + wrong token" +echo " - End-to-end inference through sandbox after re-onboard" +echo " - Destroy + cleanup" +echo "" + +if [ "$FAIL" -eq 0 ]; then + printf '\n\033[1;32m GPU DOUBLE-ONBOARD E2E PASSED — Ollama proxy token consistency verified.\033[0m\n' + exit 0 +else + printf '\n\033[1;31m %d test(s) failed.\033[0m\n' "$FAIL" + exit 1 +fi diff --git a/test/e2e-vpn/test-gpu-e2e.sh b/test/e2e-vpn/test-gpu-e2e.sh new file mode 100755 index 00000000000..1f4789d4f50 --- /dev/null +++ b/test/e2e-vpn/test-gpu-e2e.sh @@ -0,0 +1,780 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# GPU E2E: Ollama local inference — follows the real user flow. +# +# Mirrors what a user with a GPU would actually do: +# 1. Install Ollama binary +# 2. Run the NemoClaw installer with NEMOCLAW_PROVIDER=ollama +# 3. Onboard starts Ollama (127.0.0.1:11434) + auth proxy (:11435), pulls model, creates sandbox +# 4. Verify inference works through the sandbox +# 5. Destroy + uninstall +# +# The test does NOT pre-start Ollama or pre-pull models — onboard handles that. +# +# Prerequisites: +# - NVIDIA GPU with drivers (nvidia-smi works) +# - Docker +# - NEMOCLAW_NON_INTERACTIVE=1 +# - NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 +# - Internet access (ollama.com for install, registry.ollama.ai for model pull) +# - No existing Ollama service on port 11434 (ephemeral runners are ideal) +# +# Environment variables: +# NEMOCLAW_NON_INTERACTIVE=1 — required +# NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 — required for non-interactive install/onboard +# NEMOCLAW_SANDBOX_NAME — sandbox name (default: e2e-gpu-ollama) +# NEMOCLAW_RECREATE_SANDBOX=1 — recreate sandbox if it exists +# NEMOCLAW_MODEL — model for onboard (default: auto-selected by onboard) +# SKIP_UNINSTALL — set to 1 to skip uninstall (debugging) +# +# Usage: +# NEMOCLAW_NON_INTERACTIVE=1 NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 bash test/e2e-vpn/test-gpu-e2e.sh + +# ShellCheck cannot see EXIT trap invocations of cleanup helpers in this E2E script. +# shellcheck disable=SC2317 +set -uo pipefail + +PASS=0 +FAIL=0 +SKIP=0 +TOTAL=0 + +pass() { + ((PASS++)) + ((TOTAL++)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} +fail() { + ((FAIL++)) + ((TOTAL++)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} +skip() { + ((SKIP++)) + ((TOTAL++)) + printf '\033[33m SKIP: %s\033[0m\n' "$1" +} +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } + +# Parse chat completion response — handles both content and reasoning_content +parse_chat_content() { + python3 -c " +import json, sys +try: + r = json.load(sys.stdin) + c = r['choices'][0]['message'] + # Reasoning models (nemotron-3-nano) may put output in 'reasoning' or + # 'reasoning_content' instead of 'content'. Check all fields. + content = c.get('content') or c.get('reasoning_content') or c.get('reasoning') or '' + print(content.strip()) +except Exception as e: + print(f'PARSE_ERROR: {e}', file=sys.stderr) + sys.exit(1) +" +} + +# Determine repo root +if [ -d /workspace ] && [ -f /workspace/install.sh ]; then + REPO="/workspace" +elif [ -f "$(cd "$(dirname "$0")/../.." && pwd)/install.sh" ]; then + REPO="$(cd "$(dirname "$0")/../.." && pwd)" +else + echo "ERROR: Cannot find repo root." + exit 1 +fi + +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-gpu-ollama}" +TEST_LOG="/tmp/nemoclaw-gpu-e2e-test.log" +INSTALL_LOG="/tmp/nemoclaw-gpu-e2e-install.log" + +# Enforce Ollama provider — this script only tests local GPU inference. +export NEMOCLAW_PROVIDER="${NEMOCLAW_PROVIDER:-ollama}" +if [ "$NEMOCLAW_PROVIDER" != "ollama" ]; then + echo "ERROR: NEMOCLAW_PROVIDER must be 'ollama' for GPU E2E (got: $NEMOCLAW_PROVIDER)" + exit 1 +fi + +exec > >(tee -a "$TEST_LOG") 2>&1 + +# Best-effort cleanup on any exit (prevents dirty state on reused runners) +# shellcheck disable=SC2329 # invoked via trap +cleanup() { + info "Running exit cleanup..." + if command -v nemoclaw >/dev/null 2>&1; then + nemoclaw "$SANDBOX_NAME" destroy --yes 2>/dev/null || true + fi + if command -v openshell >/dev/null 2>&1; then + openshell sandbox delete "$SANDBOX_NAME" 2>/dev/null || true + openshell gateway destroy -g nemoclaw 2>/dev/null || true + fi + pkill -f "ollama serve" 2>/dev/null || true + pkill -f "ollama-auth-proxy" 2>/dev/null || true +} +trap cleanup EXIT + +# ══════════════════════════════════════════════════════════════════ +# Phase 0: Pre-cleanup +# ══════════════════════════════════════════════════════════════════ +section "Phase 0: Pre-cleanup" +info "Destroying any leftover sandbox/gateway from previous runs..." +if command -v nemoclaw >/dev/null 2>&1; then + nemoclaw "$SANDBOX_NAME" destroy --yes 2>/dev/null || true +fi +if command -v openshell >/dev/null 2>&1; then + openshell sandbox delete "$SANDBOX_NAME" 2>/dev/null || true + openshell gateway destroy -g nemoclaw 2>/dev/null || true +fi +pass "Pre-cleanup complete" + +# ══════════════════════════════════════════════════════════════════ +# Phase 1: Prerequisites +# ══════════════════════════════════════════════════════════════════ +section "Phase 1: Prerequisites" + +if docker info >/dev/null 2>&1; then + pass "Docker is running" +else + fail "Docker is not running — cannot continue" + exit 1 +fi + +if nvidia-smi >/dev/null 2>&1; then + VRAM_MB=$(nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits 2>/dev/null | head -1) + pass "nvidia-smi works (GPU VRAM: ${VRAM_MB:-unknown} MB)" +else + fail "nvidia-smi failed — no NVIDIA GPU available" + exit 1 +fi + +if [ "${NEMOCLAW_NON_INTERACTIVE:-}" != "1" ]; then + fail "NEMOCLAW_NON_INTERACTIVE=1 is required" + exit 1 +fi + +if [ "${NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE:-}" != "1" ]; then + fail "NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 is required for non-interactive install" + exit 1 +fi + +# Verify port 11434 is free (onboard needs to start Ollama on 127.0.0.1:11434) +if curl -sf http://127.0.0.1:11434/api/tags >/dev/null 2>&1; then + info "WARNING: Something is already listening on port 11434." + info "Onboard may not be able to start Ollama." + info "On ephemeral runners this should not happen." +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 2: Install Ollama binary +# ══════════════════════════════════════════════════════════════════ +section "Phase 2: Install Ollama binary" + +# Only install the binary — do NOT start Ollama or pull models. +# The nemoclaw onboard flow handles startup and model pull itself. +if command -v ollama >/dev/null 2>&1; then + pass "Ollama already installed: $(ollama --version 2>/dev/null || echo unknown)" +else + info "Installing Ollama..." + if curl -fsSL https://ollama.com/install.sh | sh 2>&1; then + pass "Ollama installed: $(ollama --version 2>/dev/null || echo unknown)" + else + fail "Ollama installation failed" + exit 1 + fi +fi + +# If the Ollama installer started a system service, stop it so onboard +# can restart Ollama on loopback and expose only the authenticated proxy to containers. +# This needs the ollama process to be owned by our user, or systemctl access. +if curl -sf http://127.0.0.1:11434/api/tags >/dev/null 2>&1; then + info "Ollama service is running — attempting to stop for clean onboard..." + # Try systemctl first (works if user has permissions) + systemctl --user stop ollama 2>/dev/null || true + systemctl stop ollama 2>/dev/null || true + # Try direct kill (works if process is owned by our user) + pkill -f "ollama serve" 2>/dev/null || true + sleep 2 + + if curl -sf http://127.0.0.1:11434/api/tags >/dev/null 2>&1; then + info "Could not stop existing Ollama — onboard will use it as-is" + else + pass "Existing Ollama stopped — port 11434 is free for onboard" + fi +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 3: Install NemoClaw and onboard with Ollama +# ══════════════════════════════════════════════════════════════════ +section "Phase 3: Install NemoClaw and onboard with Ollama" + +cd "$REPO" || { + fail "Could not cd to repo root: $REPO" + exit 1 +} + +info "Running install.sh --non-interactive with NEMOCLAW_PROVIDER=ollama..." +info "Onboard will start Ollama, pull the model, and create the sandbox." + +bash install.sh --non-interactive >"$INSTALL_LOG" 2>&1 & +install_pid=$! +tail -f "$INSTALL_LOG" --pid=$install_pid 2>/dev/null & +tail_pid=$! +wait $install_pid +install_exit=$? +kill $tail_pid 2>/dev/null || true +wait $tail_pid 2>/dev/null || true + +# Source shell profile to pick up nvm/PATH changes +if [ -f "$HOME/.bashrc" ]; then + source "$HOME/.bashrc" 2>/dev/null || true +fi +export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" +[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" +if [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]]; then + export PATH="$HOME/.local/bin:$PATH" +fi + +if [ $install_exit -eq 0 ]; then + pass "install.sh completed (exit 0)" +else + fail "install.sh failed (exit $install_exit)" + info "Last 30 lines of install log:" + tail -30 "$INSTALL_LOG" + exit 1 +fi + +if command -v nemoclaw >/dev/null 2>&1; then + pass "nemoclaw on PATH: $(command -v nemoclaw)" +else + fail "nemoclaw not found on PATH after install" + exit 1 +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 4: Verify Ollama-based onboard +# ══════════════════════════════════════════════════════════════════ +section "Phase 4: Verify Ollama-based onboard" + +# 4a: Sandbox exists +if list_output=$(nemoclaw list 2>&1); then + if echo "$list_output" | grep -Fq -- "$SANDBOX_NAME"; then + pass "nemoclaw list contains '${SANDBOX_NAME}'" + else + fail "nemoclaw list does not contain '${SANDBOX_NAME}'" + fi +else + fail "nemoclaw list failed: ${list_output:0:200}" +fi + +# 4b: Status ok +if nemoclaw "$SANDBOX_NAME" status >/dev/null 2>&1; then + pass "nemoclaw ${SANDBOX_NAME} status exits 0" +else + fail "nemoclaw ${SANDBOX_NAME} status failed" +fi + +# 4c: Direct sandbox GPU is enabled by default on NVIDIA hosts +if status_output=$(nemoclaw "$SANDBOX_NAME" status 2>&1); then + if echo "$status_output" | grep -Fq "Sandbox GPU: enabled"; then + pass "Sandbox GPU is enabled by default" + else + fail "Sandbox GPU is not enabled in status output" + fi + # #4231: status must report proven CUDA usability, not a bare "enabled". On a + # working GPU host the onboarding cuInit proof passes, so status should carry + # the "(CUDA verified)" suffix rather than "(CUDA unverified)" or a failure. + if echo "$status_output" | grep -Fq "CUDA verified"; then + pass "Sandbox GPU status reports CUDA verified" + elif echo "$status_output" | grep -Eq "CUDA unverified|last CUDA proof failed"; then + fail "Sandbox GPU status shows CUDA not proven on a working GPU host" + else + skip "Sandbox GPU CUDA proof state not present in status output" + fi +else + fail "Could not read sandbox GPU status" +fi + +# 4d: Direct sandbox GPU proofs. Onboard performs these immediately after the +# Docker GPU patch and before continuing; assert that proof instead of +# re-running OpenShell exec after the full OpenClaw setup. +if grep -Fq "GPU proof passed: nvidia-smi when available" "$INSTALL_LOG"; then + pass "Onboard GPU proof passed: nvidia-smi when available" +else + fail "Onboard GPU proof missing: nvidia-smi when available" +fi + +if grep -Fq "GPU proof passed: /proc//task//comm write" "$INSTALL_LOG"; then + pass "Onboard GPU proof passed: /proc/self/task//comm write" +else + fail "Onboard GPU proof missing: /proc comm write" +fi + +if grep -Fq "GPU proof passed: cuInit(0) via libcuda.so.1" "$INSTALL_LOG"; then + pass "Onboard GPU proof passed: cuInit(0)" +else + fail "Onboard GPU proof missing: cuInit(0)" +fi + +# 4d.1: GPU sandbox local-inference reachability gate (#4509). Onboard must +# prove the OpenClaw agent runtime can reach the local inference backend from +# inside the sandbox's own network namespace — the context the agent's LLM +# client uses — before declaring success. PR #4609 probed this with `docker +# exec` against the recreated `--network host` container, whose main namespace +# is the host's, so a probe there passed while the agent (in OpenShell's +# isolated sandbox netns) still got ECONNREFUSED. The gate now probes via +# `openshell sandbox exec`, so this proof reflects the real runtime path. +if grep -Fq "Docker GPU mode selected" "$INSTALL_LOG"; then + if grep -Fq "GPU sandbox runtime reached local inference" "$INSTALL_LOG"; then + pass "Onboard proved local inference reachable from the sandbox runtime (#4509)" + else + fail "Onboard did not prove sandbox-runtime local inference reachability (#4509 gate missing)" + fi +else + skip "Docker GPU patch recreate not exercised; sandbox-runtime inference gate not asserted" +fi +# If host networking was opted into, onboard must downgrade it to the +# OpenShell-managed bridge path for local inference (host loopback is not +# reachable from the sandbox network namespace). +if [ "${NEMOCLAW_DOCKER_GPU_PATCH_NETWORK:-}" = "host" ]; then + if grep -Fq "keeps OpenShell bridge networking for local inference" "$INSTALL_LOG"; then + pass "Host-network opt-in downgraded to bridge for local inference (#4509)" + else + fail "Host-network opt-in was NOT downgraded for local inference (#4509)" + fi +fi + +# 4e: Inference provider is ollama-local +if inf_check=$(openshell inference get 2>&1); then + if echo "$inf_check" | grep -qi "ollama"; then + pass "Inference provider is Ollama-based" + else + fail "Inference provider is not ollama — got: ${inf_check:0:200}" + fi +else + fail "openshell inference get failed: ${inf_check:0:200}" +fi + +# 4f: Ollama is running and reachable +if curl -sf http://127.0.0.1:11434/api/tags >/dev/null 2>&1; then + pass "Ollama running on 127.0.0.1:11434 (started by onboard)" +else + fail "Ollama not running — onboard should have started it" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 4.5: Auth proxy verification (PR #1922) +# ══════════════════════════════════════════════════════════════════ +section "Phase 4.5: Auth proxy verification" + +PROXY_PORT="${NEMOCLAW_OLLAMA_PROXY_PORT:-11435}" +TOKEN_FILE="$HOME/.nemoclaw/ollama-proxy-token" + +# 4.5a: Token file persisted by onboard +if [ -f "$TOKEN_FILE" ]; then + pass "Proxy token persisted at $TOKEN_FILE" +else + fail "Proxy token file missing — onboard did not persist token" +fi + +# 4.5b: Token file permissions +if [ -f "$TOKEN_FILE" ]; then + PERMS=$(stat -c "%a" "$TOKEN_FILE" 2>/dev/null || stat -f "%Lp" "$TOKEN_FILE" 2>/dev/null) + if [ "$PERMS" = "600" ]; then + pass "Token file permissions: 600" + else + fail "Token file permissions: expected 600, got $PERMS" + fi +fi + +# 4.5c: Auth proxy is running on proxy port. Since #3338 made /api/tags require +# a Bearer token, treat any HTTP response (including 401) as proof of life — +# we only fail when nothing answers at all. +PROXY_LIVE_STATUS=$(curl -s -o /dev/null -w "%{http_code}" --connect-timeout 3 \ + "http://127.0.0.1:${PROXY_PORT}/api/tags" 2>/dev/null) || PROXY_LIVE_STATUS="000" +if [[ "$PROXY_LIVE_STATUS" =~ ^[1-9][0-9]{2}$ ]]; then + pass "Auth proxy running on :${PROXY_PORT} (HTTP $PROXY_LIVE_STATUS)" +else + fail "Auth proxy not running on :${PROXY_PORT} — onboard should have started it" +fi + +# 4.5d: Proxy rejects unauthenticated requests to protected endpoints +PROXY_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST \ + "http://127.0.0.1:${PROXY_PORT}/api/generate" -d '{}' 2>/dev/null) || PROXY_STATUS="000" +if [ "$PROXY_STATUS" = "401" ]; then + pass "Auth proxy rejects unauthenticated POST (401)" +else + fail "Auth proxy should return 401 for unauthenticated POST, got $PROXY_STATUS" +fi + +# 4.5e: Proxy accepts correct token +if [ -f "$TOKEN_FILE" ]; then + PROXY_TOKEN=$(tr -d '[:space:]' <"$TOKEN_FILE") + PROXY_AUTH="Bearer $PROXY_TOKEN" + PROXY_STATUS=$(curl -s -o /dev/null -w "%{http_code}" \ + -H "Authorization: $PROXY_AUTH" \ + -X POST "http://127.0.0.1:${PROXY_PORT}/api/generate" \ + -d '{"model":"test","prompt":"test","stream":false}' 2>/dev/null) || PROXY_STATUS="000" + if [ "$PROXY_STATUS" != "401" ]; then + pass "Auth proxy accepts correct token (status: $PROXY_STATUS)" + else + fail "Auth proxy rejected the persisted token" + fi +fi + +# 4.5f: Container can reach proxy through host.openshell.internal. We only +# care that the network path works — an authenticated-but-401 response is +# still proof of reachability (#3338 requires auth on /api/tags). +if grep -Fq "Docker-driver GPU patch active" "$INSTALL_LOG"; then + skip "Generic Docker bridge proxy reachability skipped; Docker GPU patch uses OpenShell-managed network path" +else + CONTAINER_REACH_STATUS=$(docker run --rm \ + --add-host "host.openshell.internal:host-gateway" \ + curlimages/curl:8.10.1 \ + -s -o /dev/null -w "%{http_code}" \ + --connect-timeout 5 --max-time 10 \ + "http://host.openshell.internal:${PROXY_PORT}/api/tags" 2>/dev/null) || CONTAINER_REACH_STATUS="000" + if [[ "$CONTAINER_REACH_STATUS" =~ ^[1-9][0-9]{2}$ ]]; then + pass "Container reachable: host.openshell.internal:${PROXY_PORT} (HTTP $CONTAINER_REACH_STATUS)" + else + fail "Container cannot reach proxy at host.openshell.internal:${PROXY_PORT}" + fi +fi + +# 4.5g: Proxy recovery — kill and restart from persisted token +info "Testing proxy recovery (kill + restart from persisted token)..." +PROXY_PID_BEFORE=$(lsof -ti ":${PROXY_PORT}" 2>/dev/null | head -1) || true +if [ -n "$PROXY_PID_BEFORE" ] && [ -f "$TOKEN_FILE" ]; then + PROXY_CMD=$(ps -p "$PROXY_PID_BEFORE" -o args= 2>/dev/null) || true + if echo "$PROXY_CMD" | grep -q "ollama-auth-proxy"; then + kill "$PROXY_PID_BEFORE" 2>/dev/null || true + sleep 2 + # Verify proxy is dead. After #3338 an alive proxy returns 401 on + # /api/tags without auth, so curl -sf would fail either way; we need + # the http_code itself: only 000 (no answer at all) means dead. + DEAD_STATUS=$(curl -s -o /dev/null -w "%{http_code}" --connect-timeout 2 \ + "http://127.0.0.1:${PROXY_PORT}/api/tags" 2>/dev/null) || DEAD_STATUS="000" + if [[ "$DEAD_STATUS" =~ ^[1-9][0-9]{2}$ ]]; then + fail "Proxy still alive after kill (HTTP $DEAD_STATUS)" + else + info "Proxy confirmed dead — restarting from persisted token..." + fi + # Restart from persisted token (simulates what ensureOllamaAuthProxy does + # on sandbox connect after a host reboot) + RECOVERED_TOKEN=$(tr -d '[:space:]' <"$TOKEN_FILE") + OLLAMA_PROXY_TOKEN="$RECOVERED_TOKEN" \ + OLLAMA_PROXY_PORT="$PROXY_PORT" \ + OLLAMA_BACKEND_PORT=11434 \ + node "$(dirname "$0")/../../scripts/ollama-auth-proxy.js" >/dev/null 2>&1 & + sleep 2 + RECOVERED_LIVE_STATUS=$(curl -s -o /dev/null -w "%{http_code}" --connect-timeout 3 \ + "http://127.0.0.1:${PROXY_PORT}/api/tags" 2>/dev/null) || RECOVERED_LIVE_STATUS="000" + if [[ "$RECOVERED_LIVE_STATUS" =~ ^[1-9][0-9]{2}$ ]]; then + pass "Proxy recovered from persisted token after kill (HTTP $RECOVERED_LIVE_STATUS)" + else + fail "Proxy did not restart from persisted token" + fi + # Verify the recovered proxy accepts the original token + RECOVER_AUTH="Bearer $RECOVERED_TOKEN" + RECOVER_STATUS=$(curl -s -o /dev/null -w "%{http_code}" \ + -H "Authorization: $RECOVER_AUTH" \ + -X POST "http://127.0.0.1:${PROXY_PORT}/api/generate" \ + -d '{"model":"test","prompt":"test","stream":false}' 2>/dev/null) || RECOVER_STATUS="000" + if [ "$RECOVER_STATUS" != "401" ]; then + pass "Recovered proxy accepts persisted token (status: $RECOVER_STATUS)" + else + fail "Recovered proxy rejected persisted token" + fi + else + skip "Proxy recovery: PID on :${PROXY_PORT} is not ollama-auth-proxy" + fi +else + skip "Proxy recovery: no proxy PID or no token file" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 5: Local inference through sandbox +# ══════════════════════════════════════════════════════════════════ +section "Phase 5: Local inference through sandbox" + +# Determine the model to test. Prefer NEMOCLAW_MODEL (set by workflow), then +# fall back to querying Ollama's /api/tags (handles auto-selection by onboard). +CONFIGURED_MODEL="${NEMOCLAW_MODEL:-}" +if [ -n "$CONFIGURED_MODEL" ]; then + # Verify the expected model is actually available in Ollama + if curl -sf http://127.0.0.1:11434/api/tags 2>/dev/null \ + | python3 -c "import json,sys; m=[x['name'] for x in json.load(sys.stdin).get('models',[])]; sys.exit(0 if '$CONFIGURED_MODEL' in m or any('$CONFIGURED_MODEL' in x for x in m) else 1)" 2>/dev/null; then + info "Using NEMOCLAW_MODEL: $CONFIGURED_MODEL (confirmed in Ollama)" + else + info "NEMOCLAW_MODEL=$CONFIGURED_MODEL not found in Ollama tags — querying available models" + CONFIGURED_MODEL="" + fi +fi +if [ -z "$CONFIGURED_MODEL" ]; then + CONFIGURED_MODEL=$(curl -sf http://127.0.0.1:11434/api/tags 2>/dev/null \ + | python3 -c "import json,sys; m=json.load(sys.stdin).get('models',[]); print(m[0]['name'] if m else '')" 2>/dev/null || echo "") + if [ -n "$CONFIGURED_MODEL" ]; then + info "Auto-detected Ollama model: $CONFIGURED_MODEL" + else + fail "No models found in Ollama" + fi +fi + +# 5a: Direct Ollama inference (host-side, OpenAI-compatible) +info "[LOCAL] Direct Ollama test → 127.0.0.1:11434/v1/chat/completions..." +direct_response=$(curl -s --max-time 120 \ + -X POST http://127.0.0.1:11434/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d "{ + \"model\": \"$CONFIGURED_MODEL\", + \"messages\": [{\"role\": \"user\", \"content\": \"Reply with exactly one word: PONG\"}], + \"max_tokens\": 200 + }" 2>/dev/null) || true + +if [ -n "$direct_response" ]; then + direct_content=$(echo "$direct_response" | parse_chat_content 2>/dev/null) || true + if echo "$direct_content" | grep -qi "PONG"; then + pass "[LOCAL] Direct Ollama: model responded with PONG" + else + fail "[LOCAL] Direct Ollama: expected PONG, got: ${direct_content:0:200}" + fi +else + fail "[LOCAL] Direct Ollama: empty response" +fi + +# 5b: Inference through the sandbox → OpenShell route → Ollama, proven from the +# ACTUAL OpenClaw runtime context (#4509). This MUST go through `openshell +# sandbox exec` — the agent runs in OpenShell's isolated sandbox network +# namespace, so a `docker exec` probe against the recreated container (whose +# main namespace is the host's under `--network host`) does NOT exercise the +# path the agent uses and previously masked the reopened ECONNREFUSED. OpenClaw +# is wired to the OpenShell-managed inference endpoint (inference.local), never +# a direct container loopback URL. +SANDBOX_INFERENCE_URL="https://inference.local/v1/chat/completions" +info "[LOCAL] Sandbox inference test (via openshell sandbox exec) → ${SANDBOX_INFERENCE_URL} → Ollama on GPU..." +sandbox_probe_failure="" +sandbox_response="" +TIMEOUT_CMD="" +command -v timeout >/dev/null 2>&1 && TIMEOUT_CMD="timeout 120" +sandbox_payload=$(python3 -c 'import json, sys; print(json.dumps({"model": sys.argv[1], "messages": [{"role": "user", "content": "Reply with exactly one word: PONG"}], "max_tokens": 200}))' "$CONFIGURED_MODEL") +sandbox_curl_cmd=$(printf "curl -skS --max-time 90 %q -H %q -d %q" \ + "$SANDBOX_INFERENCE_URL" \ + "Content-Type: application/json" \ + "$sandbox_payload") + +run_sandbox_inference_probe() { + sandbox_probe_failure="" + sandbox_response="" + # Always exercise the real OpenClaw runtime context (the sandbox's own + # network namespace) so a host-only reachability path can never mask an + # agent-side ECONNREFUSED (#4509). + local probe_status + sandbox_response=$($TIMEOUT_CMD openshell sandbox exec -n "$SANDBOX_NAME" -- sh -lc "$sandbox_curl_cmd" 2>&1) + probe_status=$? + if [ "$probe_status" -ne 0 ]; then + if [ "$probe_status" -eq 124 ]; then + sandbox_probe_failure="sandbox inference probe timed out (openshell sandbox exec)" + else + sandbox_probe_failure="openshell sandbox exec failed (status ${probe_status}): ${sandbox_response:0:200}" + fi + fi +} + +pong_ok=false +sandbox_content="" +for sandbox_attempt in 1 2 3; do + run_sandbox_inference_probe + if [ -n "$sandbox_probe_failure" ]; then + break + fi + if [ -n "$sandbox_response" ]; then + sandbox_content=$(echo "$sandbox_response" | parse_chat_content 2>/dev/null) || true + if echo "$sandbox_content" | grep -qi "PONG"; then + pong_ok=true + break + fi + info "Sandbox inference attempt ${sandbox_attempt}/3: got '${sandbox_content:0:80}'" + info "Sandbox inference raw response (first 400 chars): ${sandbox_response:0:400}" + else + info "Sandbox inference attempt ${sandbox_attempt}/3: empty response" + fi + [ "$sandbox_attempt" -lt 3 ] || break + sleep 5 +done + +if [ -n "$sandbox_probe_failure" ]; then + fail "[LOCAL] Sandbox inference: ${sandbox_probe_failure}" +elif $pong_ok; then + pass "[LOCAL] Sandbox inference: Ollama responded through sandbox" + info "Full path proven: sandbox → ${SANDBOX_INFERENCE_URL} → Ollama GPU (:11434)" +elif [ -n "$sandbox_response" ]; then + fail "[LOCAL] Sandbox inference: expected PONG after 3 attempts, got: ${sandbox_content:0:200}" + info "Sandbox inference final raw response (first 800 chars): ${sandbox_response:0:800}" +else + fail "[LOCAL] Sandbox inference: no response from ${SANDBOX_INFERENCE_URL} inside sandbox" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 5.5: OpenClaw TUI first-turn compaction guard (#5468) +# ══════════════════════════════════════════════════════════════════ +# Local Ollama small-context models (e.g. qwen2.5:0.5b) are floored to a 16k +# runtime window. With OpenClaw 2026.5.x's default 20k compaction reserve that +# leaves only ~8k of first-turn prompt budget, so the very first user turn +# overflowed and preemptive auto-compaction (no prior history to compact) failed +# with "Auto-compaction could not recover this turn". The fix bakes a +# context-aware agents.defaults.compaction reserve into openclaw.json. This phase +# guards both the baked config and the real first-turn TUI outcome. +section "Phase 5.5: OpenClaw TUI first-turn compaction guard (#5468)" + +# 5.5a: The baked openclaw.json must carry a small-context compaction reserve +# policy whenever the served window is small enough to need it (<= 28k). The +# expected reserve is recomputed from the config's own contextWindow/maxTokens +# so the assertion tracks the policy exactly (reserve = min(maxTokens, +# contextWindow - 8000)) instead of a hard-coded budget. +tui_config=$(openshell sandbox exec -n "$SANDBOX_NAME" -- sh -lc 'cat /sandbox/.openclaw/openclaw.json' 2>/dev/null) +if echo "$tui_config" | python3 -c " +import json, sys +cfg = json.load(sys.stdin) +defaults = cfg.get('agents', {}).get('defaults', {}) +comp = defaults.get('compaction') +window = max_tokens = None +for provider in cfg['models']['providers'].values(): + model = provider['models'][0] + window = model.get('contextWindow') + max_tokens = model.get('maxTokens') +# Only small windows need the policy; larger windows keep OpenClaw's default. +if window is not None and window <= 28000: + assert isinstance(comp, dict), 'missing compaction policy for small window' + expected = min(max_tokens, max(0, window - 8000)) + assert comp.get('reserveTokens') == expected, f'reserveTokens {comp.get(\"reserveTokens\")} != {expected}' + assert comp.get('reserveTokensFloor') == expected, 'reserveTokensFloor mismatch' +sys.exit(0) +" 2>/dev/null; then + pass "[#5468] Baked openclaw.json carries the small-context compaction reserve policy" +else + fail "[#5468] Baked openclaw.json missing/incorrect small-context compaction policy" +fi + +# 5.5b: Drive the real OpenClaw TUI first turn and assert preemptive +# auto-compaction does not block the reply. Requires `expect`; skip cleanly if +# it is unavailable so the rest of the GPU lane still runs. The harness waits +# for the gateway to connect before sending (so a slow host cannot drop the +# keystroke), treats a healthy reply ("streaming") as success, and fails — not +# passes — on a dropped turn, an early EOF/crash, or an inconclusive timeout, so +# a turn that never ran can never be scored as a pass. +if command -v expect >/dev/null 2>&1; then + TUI_CAPTURE="/tmp/nemoclaw-5468-tui-capture.log" + : >"$TUI_CAPTURE" + TUI_TIMEOUT_SEC="${NEMOCLAW_5468_TUI_TIMEOUT_SEC:-240}" + tui_expect_script=$(mktemp "${TMPDIR:-/tmp}/nemoclaw-5468-tui.XXXXXX") + cat >"$tui_expect_script" </dev/null 2>&1 + tui_rc=$? + rm -f "$tui_expect_script" + if grep -qiE "could not recover this turn|context limit exceeded" "$TUI_CAPTURE"; then + fail "[#5468] OpenClaw TUI first turn blocked by preemptive auto-compaction" + info "TUI capture (first 800 chars): $(tr -d '\000' <"$TUI_CAPTURE" | head -c 800)" + elif [ "$tui_rc" -eq 0 ]; then + pass "[#5468] OpenClaw TUI first turn produced a reply without auto-compaction failure" + else + fail "[#5468] OpenClaw TUI first turn did not complete (rc=$tui_rc) — see capture" + info "TUI capture (first 800 chars): $(tr -d '\000' <"$TUI_CAPTURE" | head -c 800)" + fi +else + skip "[#5468] expect not installed — TUI first-turn compaction guard not exercised" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 6: Destroy and uninstall +# ══════════════════════════════════════════════════════════════════ +section "Phase 6: Destroy and uninstall" + +# 6a: Destroy sandbox +info "Destroying sandbox ${SANDBOX_NAME}..." +nemoclaw "$SANDBOX_NAME" destroy --yes 2>&1 | tail -5 || true + +# Verify against the registry file directly. `nemoclaw list` triggers +# gateway recovery which can restart a destroyed gateway and re-import stale +# sandbox entries — that's a separate issue (#TBD), so avoid it here. +registry_file="${HOME}/.nemoclaw/sandboxes.json" +if [ -f "$registry_file" ] && grep -Fq "\"${SANDBOX_NAME}\"" "$registry_file"; then + fail "Sandbox ${SANDBOX_NAME} still in registry after destroy" +else + pass "Sandbox ${SANDBOX_NAME} removed from registry" +fi + +openshell gateway destroy -g nemoclaw 2>/dev/null || true + +# 6b: Uninstall with --delete-models (Ollama-specific flag) +if [ "${SKIP_UNINSTALL:-}" = "1" ]; then + skip "Uninstall skipped (SKIP_UNINSTALL=1)" +else + info "Running uninstall.sh --yes --delete-models..." + if bash "$REPO/uninstall.sh" --yes --delete-models 2>&1 | tail -20; then + pass "uninstall.sh --delete-models completed" + else + fail "uninstall.sh failed" + fi + + if [ -d "$HOME/.nemoclaw" ]; then + fail "$HOME/.nemoclaw directory still exists after uninstall" + else + pass "$HOME/.nemoclaw removed" + fi +fi + +# 6c: Stop Ollama (started by onboard) +info "Stopping Ollama..." +pkill -f "ollama serve" 2>/dev/null || true +pass "Cleanup complete" + +# ══════════════════════════════════════════════════════════════════ +# Summary +# ══════════════════════════════════════════════════════════════════ +echo "" +echo "========================================" +echo " GPU E2E Results (Ollama Local Inference):" +echo " Passed: $PASS" +echo " Failed: $FAIL" +echo " Skipped: $SKIP" +echo " Total: $TOTAL" +echo "========================================" +echo "" +echo " What this tested (real user flow):" +echo " - GPU detection (nvidia-smi)" +echo " - Ollama binary install" +echo " - install.sh --non-interactive with NEMOCLAW_PROVIDER=ollama" +echo " - Onboard: starts Ollama on 127.0.0.1, starts auth proxy, pulls model, creates sandbox" +echo " - Auth proxy: token persistence, auth reject/accept, container reachability, recovery" +echo " - Local inference: direct + sandbox → gateway → auth proxy → Ollama on GPU" +echo " - Destroy + uninstall --delete-models" +echo "" + +if [ "$FAIL" -eq 0 ]; then + printf '\n\033[1;32m GPU E2E PASSED — Ollama local inference verified end-to-end.\033[0m\n' + exit 0 +else + printf '\n\033[1;31m %d test(s) failed.\033[0m\n' "$FAIL" + exit 1 +fi diff --git a/test/e2e-vpn/test-hermes-discord-e2e.sh b/test/e2e-vpn/test-hermes-discord-e2e.sh new file mode 100755 index 00000000000..3163084821b --- /dev/null +++ b/test/e2e-vpn/test-hermes-discord-e2e.sh @@ -0,0 +1,666 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Hermes Discord E2E: onboard --agent hermes with Discord enabled, then verify +# the Hermes sandbox has the schema, placeholder/token isolation, and native +# OpenShell WebSocket Gateway rewrite path required by NVIDIA/NemoClaw#3032. +# +# Uses a fake Discord token by default. The fake token should never appear in +# /sandbox/.hermes/config.yaml, /sandbox/.hermes/.env, sandbox env, sandbox +# process args, or sandbox filesystem. The sandbox should hold only the +# OpenShell resolver placeholder. Gateway proof uses a hermetic fake Discord +# Gateway on the host, not a local in-sandbox facade or live Discord token. +# +# Environment variables: +# NEMOCLAW_NON_INTERACTIVE=1 - required +# NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 - required +# NEMOCLAW_AGENT=hermes - auto-set if not already set +# NEMOCLAW_POLICY_TIER=open - auto-set if not already set +# NEMOCLAW_SANDBOX_NAME - sandbox name (default: e2e-hermes-discord) +# NEMOCLAW_RECREATE_SANDBOX=1 - auto-set +# NEMOCLAW_FRESH=1 - auto-set to discard interrupted onboard sessions +# NEMOCLAW_OPENSHELL_BIN - optional OpenShell binary under test +# NVIDIA_API_KEY - required for Hermes onboarding +# DISCORD_BOT_TOKEN - defaults to a fake token +# DISCORD_SERVER_IDS - defaults to a fake snowflake +# DISCORD_ALLOWED_IDS - defaults to a fake snowflake +# DISCORD_REQUIRE_MENTION - defaults to 0 to verify config propagation +# +# Usage: +# NEMOCLAW_NON_INTERACTIVE=1 NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ +# NVIDIA_API_KEY=... bash test/e2e-vpn/test-hermes-discord-e2e.sh + +set -uo pipefail + +PASS=0 +FAIL=0 +SKIP=0 +TOTAL=0 + +pass() { + ((PASS++)) + ((TOTAL++)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} +fail() { + ((FAIL++)) + ((TOTAL++)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} +skip() { + ((SKIP++)) + ((TOTAL++)) + printf '\033[33m SKIP: %s\033[0m\n' "$1" +} +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } + +run_with_timeout() { + local seconds="$1" + shift + if command -v timeout >/dev/null 2>&1; then + timeout "$seconds" "$@" + elif command -v gtimeout >/dev/null 2>&1; then + gtimeout "$seconds" "$@" + else + "$@" + fi +} + +dump_hermes_discord_diagnostics() { + info "--- Hermes Discord sandbox diagnostics ---" + if ! openshell --version >/dev/null 2>&1; then + info "openshell is not available for sandbox diagnostics" + return + fi + + local sandboxes diag_output diag_script + sandboxes=$(openshell sandbox list 2>&1 || true) + info "openshell sandbox list:" + echo "$sandboxes" | tail -20 | while IFS= read -r line; do + info " $line" + done + + if ! grep -Fq -- "$SANDBOX_NAME" <<<"$sandboxes"; then + info "sandbox '${SANDBOX_NAME}' is not visible to openshell" + return + fi + + diag_script='set +e' + diag_script+='; echo "== hermes config =="; sed -n "1,120p" /sandbox/.hermes/config.yaml 2>&1 || true' + diag_script+='; echo "== hermes env keys =="; cut -d= -f1 /sandbox/.hermes/.env 2>&1 || true' + diag_script+='; echo "== hermes runtime status =="; cat /sandbox/.hermes/gateway_state.json 2>&1 || true' + diag_script+='; echo "== hermes health =="; curl -sf http://localhost:8642/health 2>&1 || true' + diag_script+='; echo "== hermes-related processes =="' + # shellcheck disable=SC2016 # script is intentionally evaluated inside the sandbox + diag_script+='; for p in /proc/[0-9]*; do cmd=$(tr "\000" " " < "$p/cmdline" 2>/dev/null || true); case "$cmd" in *hermes*|*socat*) echo "$(basename "$p") $cmd" ;; esac; done' + diag_script+='; echo "== /tmp/nemoclaw-start.log tail =="; tail -n 80 /tmp/nemoclaw-start.log 2>&1 || true' + diag_script+='; echo "== /tmp/gateway.log tail =="; tail -n 120 /tmp/gateway.log 2>&1 || true' + diag_output=$(openshell sandbox exec -n "$SANDBOX_NAME" -- sh -lc "$diag_script" 2>&1 || true) + + echo "$diag_output" | while IFS= read -r line; do + info " $line" + done + info "--- End Hermes Discord diagnostics ---" +} + +# Run a command inside the sandbox and capture stdout/stderr. +sandbox_exec() { + local cmd="$1" + local ssh_config + ssh_config="$(mktemp)" + openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config" 2>/dev/null + + local result + result=$(run_with_timeout 60 ssh -F "$ssh_config" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" \ + "$cmd" \ + 2>&1) || true + + rm -f "$ssh_config" + echo "$result" +} + +# Run a command inside the sandbox via stdin. This avoids putting sensitive +# values into the remote command line when grepping for leak checks. +sandbox_exec_stdin() { + local cmd="$1" + local ssh_config + ssh_config="$(mktemp)" + openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config" 2>/dev/null + + local result + result=$(run_with_timeout 60 ssh -F "$ssh_config" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" \ + "$cmd" \ + 2>/dev/null) || true + + rm -f "$ssh_config" + echo "$result" +} + +if [ -d /workspace ] && [ -f /workspace/install.sh ]; then + REPO="/workspace" +elif [ -f "$(cd "$(dirname "$0")/../.." && pwd)/install.sh" ]; then + REPO="$(cd "$(dirname "$0")/../.." && pwd)" +else + echo "ERROR: Cannot find repo root." + exit 1 +fi + +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-hermes-discord}" +OPENSHELL_BIN="${NEMOCLAW_OPENSHELL_BIN:-openshell}" +DISCORD_TOKEN="${DISCORD_BOT_TOKEN:-test-fake-discord-token-hermes-e2e}" + +openshell() { + if [ "$OPENSHELL_BIN" = "openshell" ]; then + command openshell "$@" + else + "$OPENSHELL_BIN" "$@" + fi +} +export NEMOCLAW_AGENT="${NEMOCLAW_AGENT:-hermes}" +export NEMOCLAW_POLICY_TIER="${NEMOCLAW_POLICY_TIER:-open}" +export NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" +export NEMOCLAW_RECREATE_SANDBOX=1 +export NEMOCLAW_FRESH=1 +export DISCORD_BOT_TOKEN="$DISCORD_TOKEN" +export DISCORD_SERVER_IDS="${DISCORD_SERVER_IDS:-1491590992753590594}" +export DISCORD_ALLOWED_IDS="${DISCORD_ALLOWED_IDS:-1005536447329222676}" +export DISCORD_REQUIRE_MENTION="${DISCORD_REQUIRE_MENTION:-0}" + +# shellcheck source=test/e2e-vpn/lib/sandbox-teardown.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib/sandbox-teardown.sh" +# shellcheck source=test/e2e-vpn/lib/ci-compatible-inference.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib/ci-compatible-inference.sh" +register_sandbox_for_teardown "$SANDBOX_NAME" +nemoclaw_e2e_configure_compatible_inference || exit 1 + +# shellcheck source=test/e2e-vpn/lib/discord-gateway-proof.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib/discord-gateway-proof.sh" + +section "Phase 0: Prerequisites" + +if docker info >/dev/null 2>&1; then + pass "Docker is running" +else + fail "Docker is not running" + exit 1 +fi + +if ! nemoclaw_e2e_require_hosted_inference_key; then + exit 1 +fi + +if [ "${NEMOCLAW_NON_INTERACTIVE:-}" = "1" ]; then + pass "NEMOCLAW_NON_INTERACTIVE=1" +else + fail "NEMOCLAW_NON_INTERACTIVE=1 is required" + exit 1 +fi + +if [ "${NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE:-}" = "1" ]; then + pass "NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1" +else + fail "NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 is required" + exit 1 +fi + +info "Sandbox name: $SANDBOX_NAME" +info "Agent: $NEMOCLAW_AGENT" +info "Policy tier: $NEMOCLAW_POLICY_TIER" +info "Discord server IDs configured: ${DISCORD_SERVER_IDS}" +info "Discord allowed IDs configured: ${DISCORD_ALLOWED_IDS}" +info "Discord require mention: ${DISCORD_REQUIRE_MENTION}" + +section "Phase 1: Install NemoClaw with Hermes Discord" + +cd "$REPO" || { + fail "Could not cd to repo root: $REPO" + exit 1 +} + +info "Pre-cleanup..." +if command -v nemoclaw >/dev/null 2>&1; then + nemoclaw "$SANDBOX_NAME" destroy --yes 2>/dev/null || true +fi +if openshell --version >/dev/null 2>&1; then + openshell sandbox delete "$SANDBOX_NAME" 2>/dev/null || true + openshell gateway destroy -g nemoclaw 2>/dev/null || true +fi +pass "Pre-cleanup complete" + +INSTALL_LOG="/tmp/nemoclaw-e2e-hermes-discord-install.log" +info "Running install.sh --non-interactive with NEMOCLAW_AGENT=hermes and Discord enabled..." +bash install.sh --non-interactive >"$INSTALL_LOG" 2>&1 & +install_pid=$! +tail -f "$INSTALL_LOG" --pid=$install_pid 2>/dev/null & +tail_pid=$! +wait $install_pid +install_exit=$? +kill $tail_pid 2>/dev/null || true +wait $tail_pid 2>/dev/null || true + +if [ -f "$HOME/.bashrc" ]; then + # shellcheck source=/dev/null + source "$HOME/.bashrc" 2>/dev/null || true +fi +export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" +if [ -s "$NVM_DIR/nvm.sh" ]; then + # shellcheck source=/dev/null + . "$NVM_DIR/nvm.sh" +fi +if [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]]; then + export PATH="$HOME/.local/bin:$PATH" +fi + +if [ $install_exit -eq 0 ]; then + pass "install.sh completed (exit 0)" +else + fail "install.sh failed (exit $install_exit)" + info "Last 40 lines of install log:" + tail -40 "$INSTALL_LOG" 2>/dev/null || true + dump_hermes_discord_diagnostics + exit 1 +fi + +if command -v nemoclaw >/dev/null 2>&1; then + pass "nemoclaw installed at $(command -v nemoclaw)" +else + fail "nemoclaw not found on PATH after install" + exit 1 +fi + +if openshell --version >/dev/null 2>&1; then + pass "openshell installed ($(openshell --version 2>&1 || echo unknown))" +else + fail "openshell not found on PATH after install" + exit 1 +fi + +section "Phase 2: Hermes sandbox and provider" + +if list_output=$(nemoclaw list 2>&1); then + if grep -Fq -- "$SANDBOX_NAME" <<<"$list_output"; then + pass "nemoclaw list contains '${SANDBOX_NAME}'" + else + fail "nemoclaw list does not contain '${SANDBOX_NAME}'" + fi +else + fail "nemoclaw list failed: ${list_output:0:200}" +fi + +if openshell provider get "${SANDBOX_NAME}-discord-bridge" >/dev/null 2>&1; then + pass "Discord provider '${SANDBOX_NAME}-discord-bridge' exists in gateway" +else + fail "Discord provider '${SANDBOX_NAME}-discord-bridge' not found in gateway" +fi + +section "Phase 3: Hermes health" + +hermes_healthy=false +health_response="" +for attempt in $(seq 1 15); do + health_response=$(sandbox_exec "curl -sf http://localhost:8642/health") + if echo "$health_response" | grep -qi '"ok"'; then + hermes_healthy=true + break + fi + info "Health check attempt ${attempt}/15 - waiting 4s..." + sleep 4 +done + +if $hermes_healthy; then + pass "Hermes health probe returned ok with Discord enabled" +else + fail "Hermes health probe did not return ok after 15 attempts" + info "Last response: ${health_response:0:200}" + dump_hermes_discord_diagnostics +fi + +section "Phase 4: Hermes Discord config shape" + +expected_require_mention="true" +if [ "$DISCORD_REQUIRE_MENTION" = "0" ]; then + expected_require_mention="false" +fi +expected_allowed_users="${DISCORD_ALLOWED_IDS// /}" +expected_guild_ids="${DISCORD_SERVER_IDS// /}" + +config_probe=$( + sandbox_exec_stdin "EXPECTED_REQUIRE_MENTION=$expected_require_mention python3 -" <<'PY' +import os +import sys, yaml +with open("/sandbox/.hermes/config.yaml", "r", encoding="utf-8") as f: + text = f.read() +cfg = yaml.safe_load(text) or {} +errors = [] +discord = cfg.get("discord") +if not isinstance(discord, dict): + errors.append("missing top-level discord") +else: + expected = { + "require_mention": os.environ["EXPECTED_REQUIRE_MENTION"] == "true", + "free_response_channels": "", + "allowed_channels": "", + "auto_thread": True, + "reactions": True, + "channel_prompts": {}, + } + for key, value in expected.items(): + if discord.get(key) != value: + errors.append(f"discord.{key}={discord.get(key)!r} expected {value!r}") +platforms = cfg.get("platforms") +if not isinstance(platforms, dict): + errors.append("missing platforms") +else: + discord_platform = platforms.get("discord") + if discord_platform != {"enabled": True}: + errors.append(f"platforms.discord={discord_platform!r} expected enabled true") + if not isinstance(platforms.get("api_server"), dict): + errors.append("platforms.api_server missing") +if "DISCORD_BOT_TOKEN" in text: + errors.append("config.yaml contains DISCORD_BOT_TOKEN") +if errors: + print("FAIL " + "; ".join(errors)) +else: + print("OK") +PY +) + +if [ "$config_probe" = "OK" ]; then + pass "config.yaml uses top-level discord and platforms.discord" +else + fail "config.yaml schema check failed: ${config_probe:0:400}" +fi + +env_probe=$( + sandbox_exec_stdin "EXPECTED_ALLOWED_USERS=$expected_allowed_users EXPECTED_GUILD_IDS=$expected_guild_ids python3 -" <<'PY' +import os +from pathlib import Path +text = Path("/sandbox/.hermes/.env").read_text(encoding="utf-8") +errors = [] +required = [ + "DISCORD_BOT_TOKEN=openshell:resolve:env:DISCORD_BOT_TOKEN", + f"NEMOCLAW_DISCORD_GUILD_IDS={os.environ['EXPECTED_GUILD_IDS']}", + f"DISCORD_ALLOWED_USERS={os.environ['EXPECTED_ALLOWED_USERS']}", +] +for line in required: + if line not in text.splitlines(): + errors.append(f"missing {line}") +if "API_SERVER_PORT=18642" not in text.splitlines(): + errors.append("missing API_SERVER_PORT") +if errors: + print("FAIL " + "; ".join(errors)) +else: + print("OK") +PY +) + +if [ "$env_probe" = "OK" ]; then + pass ".hermes/.env contains Discord placeholder and allowed users" +else + fail ".hermes/.env check failed: ${env_probe:0:400}" +fi + +fake_gateway_ready=0 +if start_fake_discord_gateway "$DISCORD_TOKEN"; then + fake_gateway_ready=1 + pass "Hermetic fake Discord Gateway started on host port ${FAKE_DISCORD_GATEWAY_PORT}" +else + fail "Failed to start hermetic fake Discord Gateway" +fi + +if [ "$fake_gateway_ready" = "1" ] \ + && apply_fake_discord_gateway_policy "$SANDBOX_NAME" "$FAKE_DISCORD_GATEWAY_PORT" >/tmp/nemoclaw-hermes-fake-discord-policy.log 2>&1; then + pass "Applied native WebSocket policy with credential rewrite for Hermes fake Discord Gateway" +else + fail "Failed to apply Hermes fake Discord Gateway policy: $(tail -20 /tmp/nemoclaw-hermes-fake-discord-policy.log 2>/dev/null | tr '\n' ' ' | cut -c1-300)" +fi + +native_gateway_protocol="" +if [ "$fake_gateway_ready" = "1" ]; then + native_gateway_protocol=$(run_fake_discord_gateway_python_client "$FAKE_DISCORD_GATEWAY_PORT" || true) +fi +info "Hermes native Discord Gateway protocol probe: ${native_gateway_protocol:0:400}" +if echo "$native_gateway_protocol" | grep -q "^UPGRADE$" \ + && echo "$native_gateway_protocol" | grep -q "^HELLO$" \ + && echo "$native_gateway_protocol" | grep -q "^IDENTIFY_SENT_PLACEHOLDER$" \ + && echo "$native_gateway_protocol" | grep -q "^READY$" \ + && echo "$native_gateway_protocol" | grep -q "^HEARTBEAT_ACK$"; then + pass "Hermes Python Discord Gateway path reaches READY through native OpenShell WebSocket policy" +elif echo "$native_gateway_protocol" | grep -q "IMPORT_DISCORD_FAILED"; then + fail "Hermes native Gateway probe could not import discord.py: ${native_gateway_protocol:0:300}" +else + fail "Hermes native Gateway protocol probe failed: ${native_gateway_protocol:0:300}" +fi + +fake_gateway_capture_check="" +if [ "$fake_gateway_ready" = "1" ]; then + fake_gateway_capture_check=$(check_fake_discord_gateway_rewrite_capture "$FAKE_DISCORD_GATEWAY_CAPTURE_FILE" "$DISCORD_TOKEN" 2>&1 || true) +fi + +if [ "$fake_gateway_ready" = "1" ] && [ "$fake_gateway_capture_check" = "OK" ]; then + pass "Hermes fake Gateway proved WebSocket placeholder rewrite without logging the raw token" +else + if [ "$fake_gateway_ready" = "1" ]; then + info "Hermes fake Gateway capture check: ${fake_gateway_capture_check:0:300}" + fi + fail "Hermes fake Gateway did not prove WebSocket placeholder rewrite" +fi + +token_file_hits=$(printf '%s' "$DISCORD_TOKEN" | sandbox_exec_stdin 'grep -Fq -f - /sandbox/.hermes/config.yaml /sandbox/.hermes/.env 2>/dev/null && echo LEAK || echo OK') +if [ "$token_file_hits" = "OK" ]; then + pass "Raw Discord token absent from Hermes config.yaml and .env" +else + fail "Raw Discord token found in Hermes config files" +fi + +section "Phase 5: Sandbox token isolation" + +sandbox_env_all=$(sandbox_exec "env 2>/dev/null") +if [ -z "$sandbox_env_all" ]; then + skip "Sandbox environment dump is empty" +elif echo "$sandbox_env_all" | grep -qF "$DISCORD_TOKEN"; then + fail "Raw Discord token found in sandbox environment" +elif echo "$sandbox_env_all" | grep -q "^DISCORD_PROXY="; then + fail "Sandbox environment still contains DISCORD_PROXY bridge setting" +else + pass "Raw Discord token absent from sandbox environment; no DISCORD_PROXY bridge setting" +fi + +sandbox_ps=$(sandbox_exec 'cat /proc/[0-9]*/cmdline 2>/dev/null | tr "\0" "\n"') +if [ -z "$sandbox_ps" ]; then + skip "Sandbox process list is empty" +elif echo "$sandbox_ps" | grep -qF "$DISCORD_TOKEN"; then + fail "Raw Discord token found in sandbox process list" +else + pass "Raw Discord token absent from sandbox process list" +fi + +sandbox_fs_hits=$(printf '%s' "$DISCORD_TOKEN" | sandbox_exec_stdin 'grep -rFlm1 -f - /sandbox /home /etc /tmp /var 2>/dev/null || true') +if [ -n "$sandbox_fs_hits" ]; then + fail "Raw Discord token found on sandbox filesystem: ${sandbox_fs_hits:0:200}" +else + pass "Raw Discord token absent from sandbox filesystem" +fi + +section "Phase 6: Discord REST placeholder egress" + +dc_api=$(sandbox_exec 'NODE_NO_WARNINGS=1 node -e " +const fs = require(\"fs\"); +const https = require(\"https\"); +const env = fs.readFileSync(\"/sandbox/.hermes/.env\", \"utf8\"); +const line = env.split(/\\n/).find((entry) => entry.startsWith(\"DISCORD_BOT_TOKEN=\")); +const token = line ? line.slice(\"DISCORD_BOT_TOKEN=\".length) : \"\"; +if (!token) { + console.log(JSON.stringify({ error: \"missing_token\" })); + process.exit(0); +} +const req = https.request({ + hostname: \"discord.com\", + path: \"/api/v10/users/@me\", + method: \"GET\", + headers: { \"Authorization\": \"Bot \" + token }, +}, (res) => { + let body = \"\"; + res.on(\"data\", (d) => body += d); + res.on(\"end\", () => console.log(JSON.stringify({ + statusCode: res.statusCode, + body: body.slice(0, 200), + }))); +}); +req.on(\"error\", (e) => console.log(JSON.stringify({ error: e.message }))); +req.setTimeout(20000, () => { req.destroy(); console.log(JSON.stringify({ error: \"timeout\" })); }); +req.end(); +"' 2>/dev/null || true) + +info "Discord users/@me response: ${dc_api:0:300}" +dc_status=$(echo "$dc_api" | python3 -c 'import json,sys +lines = [line.strip() for line in sys.stdin if line.strip().startswith("{")] +try: + print(json.loads(lines[-1]).get("statusCode", "") if lines else "") +except Exception: + print("") +' 2>/dev/null || true) +dc_error=$(echo "$dc_api" | python3 -c 'import json,sys +lines = [line.strip() for line in sys.stdin if line.strip().startswith("{")] +try: + print(json.loads(lines[-1]).get("error", "") if lines else "") +except Exception: + print("") +' 2>/dev/null || true) + +if [ "$dc_status" = "200" ]; then + pass "Discord users/@me returned 200 with configured token" +elif [ "$dc_status" = "401" ]; then + pass "Discord users/@me returned 401 - REST path reached Discord; this is not gateway IDENTIFY auth proof" +elif [ "$dc_error" = "timeout" ]; then + skip "Discord API timed out" +elif [ -n "$dc_error" ]; then + fail "Discord API call failed: ${dc_error:0:200}" +else + fail "Unexpected Discord API response: ${dc_api:0:300}" +fi + +section "Phase 7: No local Discord bridge" + +# shellcheck disable=SC2016 # Remote script is intentionally single-quoted for sandbox execution. +facade_residue=$(sandbox_exec 'set +e +env_needle="$(printf "%s%s" "NEMOCLAW_DISCORD_" "FACADE_URL")" +name_needle="$(printf "%s%s" "nemoclaw-discord-" "facade")" +proxy_needle="$(printf "%s" "DISCORD_PROXY")" +decode_needle="$(printf "%s%s%s" "nemoclaw-" "decode" "-proxy")" +if env | grep -q "$env_needle"; then echo ENV_FACADE; fi +if env | grep -q "^${proxy_needle}="; then echo ENV_DISCORD_PROXY; fi +if grep -Fq "$env_needle" /sandbox/.hermes/.env /sandbox/.hermes/config.yaml /tmp/nemoclaw-proxy-env.sh /tmp/gateway.env 2>/dev/null; then echo FILE_FACADE; fi +if grep -Fq "$proxy_needle" /sandbox/.hermes/.env /sandbox/.hermes/config.yaml /tmp/nemoclaw-proxy-env.sh /tmp/gateway.env 2>/dev/null; then echo FILE_DISCORD_PROXY; fi +if find /tmp -maxdepth 1 -type f \( -name "discord-facade.log" -o -name "nemoclaw-discord-facade*" \) 2>/dev/null | grep -q .; then echo FILE_FACADE; fi +if command -v "$decode_needle" >/dev/null 2>&1; then echo BIN_DECODE_PROXY; fi +current_pid="$$" +for p in /proc/[0-9]*; do + pid=$(basename "$p") + [ "$pid" = "$current_pid" ] && continue + cmd=$(tr "\000" " " < "$p/cmdline" 2>/dev/null || true) + case "$cmd" in *"name_needle="*|*"for p in /proc/"*) continue ;; esac + case "$cmd" in *"$name_needle"*) echo PROCESS_FACADE ;; esac + case "$cmd" in *"$decode_needle"*) echo PROCESS_DECODE_PROXY ;; esac +done') +if [ -z "$facade_residue" ]; then + pass "Hermes Discord proof used native WebSocket policy with no local facade, decode proxy, or DISCORD_PROXY residue" +else + fail "Local Discord bridge residue found after native Gateway proof: ${facade_residue:0:300}" + dump_hermes_discord_diagnostics +fi + +section "Phase 8: Gateway-stored credential rebuild" + +# Rebuild with NVIDIA_API_KEY unset so the preflight is forced to reuse the +# gateway-stored inference credential. Catches the Hermes regression that +# motivated the gateway-aware credential check in setupNim + rebuild. + +# Phase 7's fake Discord Gateway leaves a root-owned scratch dir at +# $REPO/.tmp/fake-discord.* via its Docker bind-mount. `nemoclaw rebuild` +# recreates the sandbox by copying $REPO into a build context, which hits +# EACCES on those files because the runner uid cannot read root-owned +# bytes. Tear the container down and `sudo rm` the scratch before the +# rebuild so the build context copy succeeds. +if [ -n "${FAKE_DISCORD_GATEWAY_CONTAINER:-}" ]; then + docker rm -f "$FAKE_DISCORD_GATEWAY_CONTAINER" >/dev/null 2>&1 || true +fi +if [ -d "$REPO/.tmp" ]; then + sudo rm -rf "$REPO/.tmp"/fake-discord.* 2>/dev/null || rm -rf "$REPO/.tmp"/fake-discord.* 2>/dev/null || true +fi + +NVIDIA_API_KEY_BACKUP="${NVIDIA_API_KEY:-}" +NVIDIA_API_KEY_BACKUP="${NVIDIA_API_KEY:-}" +unset NVIDIA_API_KEY NVIDIA_API_KEY +info "NVIDIA_API_KEY and NVIDIA_API_KEY unset; gateway must hold the inference credential" + +HERMES_REBUILD_LOG="/tmp/nc-hermes-rebuild-noenv.log" +if nemoclaw "$SANDBOX_NAME" rebuild --yes >"$HERMES_REBUILD_LOG" 2>&1; then + rebuild_rc=0 +else + rebuild_rc=$? +fi + +if [ "$rebuild_rc" -ne 0 ]; then + fail "Hermes rebuild failed with NVIDIA_API_KEY unset (rc=${rebuild_rc})" + tail -80 "$HERMES_REBUILD_LOG" 2>/dev/null || true +elif grep -q "provider credential not found" "$HERMES_REBUILD_LOG"; then + fail "REGRESSION — rebuild aborted on missing NVIDIA_API_KEY despite gateway-registered credential" +else + pass "Hermes rebuild reused gateway-stored credential without NVIDIA_API_KEY" +fi + +if [ -n "$NVIDIA_API_KEY_BACKUP" ]; then + export NVIDIA_API_KEY="$NVIDIA_API_KEY_BACKUP" +fi +if [ -n "$NVIDIA_API_KEY_BACKUP" ]; then + export NVIDIA_API_KEY="$NVIDIA_API_KEY_BACKUP" +fi +unset NVIDIA_API_KEY_BACKUP +unset NVIDIA_API_KEY_BACKUP + +section "Phase 9: Cleanup" + +if [[ "${NEMOCLAW_E2E_KEEP_SANDBOX:-}" != "1" ]]; then + nemoclaw "$SANDBOX_NAME" destroy --yes 2>&1 | tail -3 || true + openshell gateway destroy -g nemoclaw 2>/dev/null || true +fi + +registry_file="${HOME}/.nemoclaw/sandboxes.json" +if [ -f "$registry_file" ] && grep -Fq "\"${SANDBOX_NAME}\"" "$registry_file"; then + fail "Sandbox ${SANDBOX_NAME} still in registry after destroy" +else + pass "Sandbox ${SANDBOX_NAME} removed" +fi + +echo "" +echo "========================================" +echo " Hermes Discord E2E Results:" +echo " Passed: $PASS" +echo " Failed: $FAIL" +echo " Skipped: $SKIP" +echo " Total: $TOTAL" +echo "========================================" + +if [ "$FAIL" -eq 0 ]; then + printf '\n\033[1;32m Hermes Discord E2E PASSED - schema, placeholder, provider, sandbox boot, and native Gateway rewrite verified.\033[0m\n' + exit 0 +else + printf '\n\033[1;31m %d test(s) failed.\033[0m\n' "$FAIL" + exit 1 +fi diff --git a/test/e2e-vpn/test-hermes-e2e.sh b/test/e2e-vpn/test-hermes-e2e.sh new file mode 100755 index 00000000000..fad0877588c --- /dev/null +++ b/test/e2e-vpn/test-hermes-e2e.sh @@ -0,0 +1,768 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Hermes Agent E2E: install → onboard --agent hermes → verify sandbox → live inference +# +# Proves the COMPLETE Hermes user journey including agent selection, health +# probe verification, and real inference through the sandbox. Uses the same +# install.sh --non-interactive path as the OpenClaw E2E but passes +# NEMOCLAW_AGENT=hermes to select the Hermes agent during onboarding. +# +# Prerequisites: +# - Docker running +# - NVIDIA_API_KEY set for hosted inference +# - Network access to inference.nvidia.com +# +# Environment variables: +# NEMOCLAW_NON_INTERACTIVE=1 — required (enables non-interactive install + onboard) +# NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 — required for non-interactive install/onboard +# NEMOCLAW_AGENT=hermes — auto-set if not already set +# NEMOCLAW_SANDBOX_NAME — sandbox name (default: e2e-hermes) +# NEMOCLAW_RECREATE_SANDBOX=1 — recreate sandbox if it exists from a previous run +# NEMOCLAW_E2E_HERMES_DASHBOARD=1 — validate the built-in Hermes web dashboard end-to-end +# NEMOCLAW_HERMES_DASHBOARD_TUI=1 — enable Hermes' optional in-browser TUI tab during onboard +# NVIDIA_API_KEY — required for hosted inference +# +# Usage: +# NEMOCLAW_NON_INTERACTIVE=1 NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 NVIDIA_API_KEY=... bash test/e2e-vpn/test-hermes-e2e.sh + +set -uo pipefail + +PASS=0 +FAIL=0 +SKIP=0 +TOTAL=0 + +pass() { + ((PASS++)) + ((TOTAL++)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} +fail() { + ((FAIL++)) + ((TOTAL++)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} +skip() { + ((SKIP++)) + ((TOTAL++)) + printf '\033[33m SKIP: %s\033[0m\n' "$1" +} +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } + +dump_hermes_diagnostics() { + info "--- Hermes sandbox diagnostics ---" + if ! command -v openshell >/dev/null 2>&1; then + info "openshell is not available for sandbox diagnostics" + return + fi + + local sandboxes diag_output diag_script + sandboxes=$(openshell sandbox list 2>&1 || true) + info "openshell sandbox list:" + echo "$sandboxes" | tail -20 | while IFS= read -r line; do + info " $line" + done + + if ! grep -Fq -- "$SANDBOX_NAME" <<<"$sandboxes"; then + info "sandbox '${SANDBOX_NAME}' is not visible to openshell" + return + fi + + diag_script='set +e' + diag_script+='; echo "== identity =="; id 2>&1 || true' + diag_script+='; echo "== listening sockets =="; ss -tlnp 2>&1 || ss -tln 2>&1 || true' + diag_script+='; echo "== log and state paths =="; ls -ld /tmp /sandbox/.hermes /sandbox/.hermes/logs 2>&1 || true; ls -l /tmp/nemoclaw-start.log /tmp/gateway.log 2>&1 || true' + diag_script+='; echo "== hermes-related processes =="' + # shellcheck disable=SC2016 # script is intentionally evaluated inside the sandbox + diag_script+='; for p in /proc/[0-9]*; do cmd=$(tr "\000" " " < "$p/cmdline" 2>/dev/null || true); case "$cmd" in *hermes*|*socat*) echo "$(basename "$p") $cmd" ;; esac; done' + diag_script+='; echo "== /tmp/nemoclaw-start.log tail =="; tail -n 80 /tmp/nemoclaw-start.log 2>&1 || true' + diag_script+='; echo "== /tmp/gateway.log tail =="; tail -n 120 /tmp/gateway.log 2>&1 || true' + diag_output=$(openshell sandbox exec -n "$SANDBOX_NAME" -- sh -lc "$diag_script" 2>&1 || true) + + echo "$diag_output" | while IFS= read -r line; do + info " $line" + done + info "--- End Hermes sandbox diagnostics ---" +} + +# Parse chat completion response — handles both content and reasoning_content +# (nemotron-3-super is a reasoning model that may put output in reasoning_content) +parse_chat_content() { + python3 -c " +import json, sys +try: + r = json.load(sys.stdin) + c = r['choices'][0]['message'] + content = c.get('content') or c.get('reasoning_content') or '' + print(content.strip()) +except Exception as e: + print(f'PARSE_ERROR: {e}', file=sys.stderr) + sys.exit(1) +" +} + +is_truthy_env_value() { + case "${1:-}" in + 1 | true | TRUE | yes | YES | on | ON) return 0 ;; + *) return 1 ;; + esac +} + +# shellcheck source=test/e2e-vpn/lib/ci-compatible-inference.sh +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/lib/ci-compatible-inference.sh" + +hermes_dashboard_e2e_enabled() { + is_truthy_env_value "${NEMOCLAW_E2E_HERMES_DASHBOARD:-}" \ + || is_truthy_env_value "${NEMOCLAW_HERMES_DASHBOARD:-}" +} + +http_status_ok() { + case "$1" in + 2?? | 3??) return 0 ;; + *) return 1 ;; + esac +} + +forward_list_has_running_port() { + local sandbox="$1" + local port="$2" + local forward_list="$3" + FORWARD_LIST_TEXT="$forward_list" python3 - "$sandbox" "$port" <<'PY' +import os +import re +import sys + +ANSI_RE = re.compile(r"\x1B(?:\[[0-?]*[ -/]*[@-~]|\][^\x07]*(?:\x07|\x1B\\)|[@-_])") +sandbox = sys.argv[1] +port = sys.argv[2] +for raw_line in os.environ.get("FORWARD_LIST_TEXT", "").splitlines(): + line = ANSI_RE.sub("", raw_line) + parts = line.split() + if len(parts) >= 5 and parts[0] == sandbox and parts[2] == port and parts[-1].lower() in {"running", "active"}: + sys.exit(0) +sys.exit(1) +PY +} + +# Determine repo root +if [ -d /workspace ] && [ -f /workspace/install.sh ]; then + REPO="/workspace" +elif [ -f "$(cd "$(dirname "$0")/../.." && pwd)/install.sh" ]; then + REPO="$(cd "$(dirname "$0")/../.." && pwd)" +else + echo "ERROR: Cannot find repo root." + exit 1 +fi + +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-hermes}" +export NEMOCLAW_AGENT="${NEMOCLAW_AGENT:-hermes}" +nemoclaw_e2e_configure_compatible_inference || exit 1 +HOSTED_INFERENCE_BASE_URL="$(nemoclaw_e2e_hosted_inference_base_url)" +HOSTED_INFERENCE_MODEL="$(nemoclaw_e2e_hosted_inference_model)" +HOSTED_INFERENCE_KEY="$(nemoclaw_e2e_hosted_inference_key)" + +# shellcheck source=test/e2e-vpn/lib/sandbox-teardown.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib/sandbox-teardown.sh" +register_sandbox_for_teardown "$SANDBOX_NAME" + +# Hermes health probe endpoint (from agents/hermes/manifest.yaml) +HERMES_HEALTH_URL="http://localhost:8642/health" +HERMES_DASHBOARD_PORT="${NEMOCLAW_DASHBOARD_PORT:-18789}" +HERMES_DASHBOARD_INTERNAL_PORT="${NEMOCLAW_HERMES_DASHBOARD_INTERNAL_PORT:-19119}" +TIMEOUT_CMD="" + +# ══════════════════════════════════════════════════════════════════ +# Phase 0: Pre-cleanup +# ══════════════════════════════════════════════════════════════════ +section "Phase 0: Pre-cleanup" +info "Destroying any leftover sandbox/gateway from previous runs..." +if command -v nemoclaw >/dev/null 2>&1; then + nemoclaw "$SANDBOX_NAME" destroy --yes 2>/dev/null || true +fi +if command -v openshell >/dev/null 2>&1; then + openshell sandbox delete "$SANDBOX_NAME" 2>/dev/null || true + openshell gateway destroy -g nemoclaw 2>/dev/null || true +fi +pass "Pre-cleanup complete" + +# ══════════════════════════════════════════════════════════════════ +# Phase 1: Prerequisites +# ══════════════════════════════════════════════════════════════════ +section "Phase 1: Prerequisites" + +if docker info >/dev/null 2>&1; then + pass "Docker is running" +else + fail "Docker is not running — cannot continue" + exit 1 +fi + +if ! nemoclaw_e2e_require_hosted_inference_key; then + exit 1 +fi + +if nemoclaw_e2e_probe_hosted_inference; then + pass "Network access to ${HOSTED_INFERENCE_BASE_URL}" +else + fail "Cannot reach ${HOSTED_INFERENCE_BASE_URL}" + exit 1 +fi + +if [ "${NEMOCLAW_NON_INTERACTIVE:-}" != "1" ]; then + fail "NEMOCLAW_NON_INTERACTIVE=1 is required" + exit 1 +fi + +if [ "${NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE:-}" != "1" ]; then + fail "NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 is required for non-interactive install" + exit 1 +fi + +# Verify agents/hermes/ exists in repo +if [ -d "$REPO/agents/hermes" ] && [ -f "$REPO/agents/hermes/manifest.yaml" ]; then + pass "agents/hermes/ directory and manifest.yaml exist" +else + fail "agents/hermes/ not found — is the hermes-agent-support branch checked out?" + exit 1 +fi + +info "NEMOCLAW_AGENT=${NEMOCLAW_AGENT}" + +# ══════════════════════════════════════════════════════════════════ +# Phase 2: Install nemoclaw (non-interactive mode, --agent hermes) +# ══════════════════════════════════════════════════════════════════ +section "Phase 2: Install nemoclaw (non-interactive mode, agent=hermes)" + +cd "$REPO" || { + fail "Could not cd to repo root: $REPO" + exit 1 +} + +info "Running install.sh --non-interactive with NEMOCLAW_AGENT=hermes..." +info "This installs Node.js, openshell, NemoClaw, and runs onboard with Hermes agent." +info "Expected duration: 10-15 minutes on first run (Hermes base image build)." + +INSTALL_LOG="/tmp/nemoclaw-e2e-hermes-install.log" +# Write to a file instead of piping through tee. openshell's background +# port-forward inherits pipe file descriptors, which prevents tee from exiting. +# Use tail -f in the background for real-time output in CI logs. +bash install.sh --non-interactive >"$INSTALL_LOG" 2>&1 & +install_pid=$! +tail -f "$INSTALL_LOG" --pid=$install_pid 2>/dev/null & +tail_pid=$! +wait $install_pid +install_exit=$? +kill $tail_pid 2>/dev/null || true +wait $tail_pid 2>/dev/null || true + +# Source shell profile to pick up nvm/PATH changes from install.sh +if [ -f "$HOME/.bashrc" ]; then + # shellcheck source=/dev/null + source "$HOME/.bashrc" 2>/dev/null || true +fi +# Ensure nvm is loaded in current shell +export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" +if [ -s "$NVM_DIR/nvm.sh" ]; then + # shellcheck source=/dev/null + . "$NVM_DIR/nvm.sh" +fi +# Ensure ~/.local/bin is on PATH (openshell may be installed there in non-interactive mode) +if [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]]; then + export PATH="$HOME/.local/bin:$PATH" +fi + +if [ $install_exit -eq 0 ]; then + pass "install.sh completed (exit 0)" +else + fail "install.sh failed (exit $install_exit)" + dump_hermes_diagnostics + exit 1 +fi + +# Verify nemoclaw is on PATH +if command -v nemoclaw >/dev/null 2>&1; then + pass "nemoclaw installed at $(command -v nemoclaw)" +else + fail "nemoclaw not found on PATH after install" + exit 1 +fi + +# Verify openshell was installed +if command -v openshell >/dev/null 2>&1; then + pass "openshell installed ($(openshell --version 2>&1 || echo unknown))" +else + fail "openshell not found on PATH after install" + exit 1 +fi + +if nemoclaw --help >/dev/null 2>&1; then + pass "nemoclaw --help exits 0" +else + fail "nemoclaw --help failed" +fi + +if hermes_dashboard_e2e_enabled; then + if grep -Fq "Deployment verified — gateway and dashboard are healthy." "$INSTALL_LOG"; then + pass "Install output confirms Hermes gateway and dashboard health" + else + fail "Install output did not confirm Hermes gateway and dashboard health" + fi + + if grep -Fq "Hermes Agent Dashboard" "$INSTALL_LOG" \ + && grep -Fq "http://127.0.0.1:${HERMES_DASHBOARD_PORT}/" "$INSTALL_LOG"; then + pass "Install output advertises Hermes web dashboard on ${HERMES_DASHBOARD_PORT}" + else + fail "Install output did not advertise Hermes web dashboard on ${HERMES_DASHBOARD_PORT}" + fi +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 3: Sandbox verification (Hermes-specific) +# ══════════════════════════════════════════════════════════════════ +section "Phase 3: Sandbox verification (Hermes)" + +# 3a: nemoclaw list +if list_output=$(nemoclaw list 2>&1); then + if grep -Fq -- "$SANDBOX_NAME" <<<"$list_output"; then + pass "nemoclaw list contains '${SANDBOX_NAME}'" + else + fail "nemoclaw list does not contain '${SANDBOX_NAME}'" + fi +else + fail "nemoclaw list failed: ${list_output:0:200}" +fi + +# 3b: nemoclaw status +if status_output=$(nemoclaw "$SANDBOX_NAME" status 2>&1); then + pass "nemoclaw ${SANDBOX_NAME} status exits 0" +else + fail "nemoclaw ${SANDBOX_NAME} status failed: ${status_output:0:200}" +fi + +# 3c: Session records agent=hermes +session_file="$HOME/.nemoclaw/onboard-session.json" +if [ -f "$session_file" ]; then + if grep -qE '"agent"\s*:\s*"hermes"' "$session_file"; then + pass "Onboard session records agent=hermes" + else + fail "Onboard session does not contain agent=hermes" + info "Session contents: $(head -20 "$session_file" 2>/dev/null)" + fi +else + fail "Session file not found: $session_file" +fi + +# 3d: Inference must be configured by onboard +if inf_check=$(openshell inference get 2>&1); then + expected_provider="$(nemoclaw_e2e_expected_route_provider)" + expected_model="" + if nemoclaw_e2e_using_compatible_inference; then + expected_model="$HOSTED_INFERENCE_MODEL" + fi + if nemoclaw_e2e_inference_output_matches "$inf_check" "$expected_provider" "$expected_model"; then + pass "Inference configured via onboard (${expected_provider})" + else + inf_check_plain="$(printf '%s' "$inf_check" | nemoclaw_e2e_strip_ansi)" + fail "Inference not configured - onboard did not set up ${expected_provider}: ${inf_check_plain:0:200}" + fi +else + fail "openshell inference get failed: ${inf_check:0:200}" +fi + +# 3e: Policy presets applied +if policy_output=$(openshell policy get --full "$SANDBOX_NAME" 2>&1); then + if grep -qi "network_policies" <<<"$policy_output"; then + pass "Policy applied to sandbox" + else + fail "No network policy found on sandbox" + fi +else + fail "openshell policy get failed: ${policy_output:0:200}" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 4: Hermes agent health verification +# ══════════════════════════════════════════════════════════════════ +section "Phase 4: Hermes agent health" + +# 4a: Health probe via SSH into sandbox +info "Checking Hermes health probe at ${HERMES_HEALTH_URL} inside sandbox..." +ssh_config="$(mktemp)" +hermes_healthy=false + +if openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config" 2>/dev/null; then + TIMEOUT_CMD="" + command -v timeout >/dev/null 2>&1 && TIMEOUT_CMD="timeout 60" + command -v gtimeout >/dev/null 2>&1 && TIMEOUT_CMD="gtimeout 60" + + # Retry health check — Hermes may still be starting + for attempt in $(seq 1 15); do + health_response=$($TIMEOUT_CMD ssh -F "$ssh_config" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" \ + "curl -sf ${HERMES_HEALTH_URL}" \ + 2>&1) || true + + if echo "$health_response" | grep -qi '"ok"'; then + hermes_healthy=true + break + fi + info "Health check attempt ${attempt}/15 — waiting 4s..." + sleep 4 + done + + if $hermes_healthy; then + pass "Hermes health probe returned ok" + info "Response: ${health_response:0:200}" + else + fail "Hermes health probe did not return ok after 15 attempts" + info "Last response: ${health_response:0:200}" + fi +else + fail "Could not get SSH config for sandbox ${SANDBOX_NAME}" +fi + +# 4b: Verify Hermes binary exists in sandbox +if openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config" 2>/dev/null; then + hermes_version=$($TIMEOUT_CMD ssh -F "$ssh_config" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" \ + "hermes --version 2>&1 || echo MISSING" \ + 2>&1) || true + + if echo "$hermes_version" | grep -qi "MISSING\|not found\|No such file"; then + fail "Hermes binary not found in sandbox" + else + pass "Hermes binary found in sandbox: ${hermes_version:0:100}" + fi +fi + +# 4c: Verify Hermes config integrity (config hash check) +config_hash_check=$($TIMEOUT_CMD ssh -F "$ssh_config" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" \ + "test -f /sandbox/.hermes/config.yaml && echo EXISTS || echo MISSING" \ + 2>&1) || true + +if echo "$config_hash_check" | grep -q "EXISTS"; then + pass "Hermes config.yaml exists at /sandbox/.hermes/config.yaml" +else + fail "Hermes config.yaml not found at /sandbox/.hermes/config.yaml" +fi + +# 4d: Verify config directory is writable (mutable default) +writable_check=$($TIMEOUT_CMD ssh -F "$ssh_config" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" \ + "touch /sandbox/.hermes/test-write 2>&1 && echo WRITABLE && rm -f /sandbox/.hermes/test-write || echo READ_ONLY" \ + 2>&1) || true + +if echo "$writable_check" | grep -q "WRITABLE"; then + pass "Hermes config directory is writable (mutable default)" +elif echo "$writable_check" | grep -q "READ_ONLY"; then + fail "Hermes config directory is read-only — should be writable by default" +else + skip "Could not determine config directory mutability: ${writable_check:0:100}" +fi + +# 4e: Verify writable data directory exists +data_dir_check=$($TIMEOUT_CMD ssh -F "$ssh_config" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" \ + "test -d /sandbox/.hermes && echo EXISTS || echo MISSING" \ + 2>&1) || true + +if echo "$data_dir_check" | grep -q "EXISTS"; then + pass "Hermes config/state directory exists at /sandbox/.hermes" +else + fail "Hermes config/state directory not found at /sandbox/.hermes" +fi + +if hermes_dashboard_e2e_enabled; then + section "Phase 4f: Hermes web dashboard" + + registry_check=$( + python3 - "$SANDBOX_NAME" "$HERMES_DASHBOARD_PORT" "$HERMES_DASHBOARD_INTERNAL_PORT" <<'PY' 2>&1 +import json +import os +import sys + +sandbox_name = sys.argv[1] +public_port = int(sys.argv[2]) +internal_port = int(sys.argv[3]) +registry_path = os.path.join(os.path.expanduser("~"), ".nemoclaw", "sandboxes.json") +with open(registry_path, encoding="utf-8") as fh: + registry = json.load(fh) +sandbox = (registry.get("sandboxes") or {}).get(sandbox_name) +errors = [] +if sandbox is None: + errors.append(f"{sandbox_name} missing from registry") +elif sandbox.get("agent") != "hermes": + errors.append(f"agent={sandbox.get('agent')!r}") +else: + if sandbox.get("dashboardPort") != public_port: + errors.append(f"dashboardPort={sandbox.get('dashboardPort')!r} expected {public_port!r}") + # The trusted main workflow may still set the legacy optional-dashboard flag + # while testing this PR head. That can add hermesDashboard* metadata for the + # compatibility forward, but the built-in dashboard contract is still proved + # by dashboardPort plus the host/internal probes below. +if errors: + print("; ".join(errors)) + sys.exit(1) +print("ok") +PY + ) + if [ "$registry_check" = "ok" ]; then + pass "Registry records Hermes API and optional dashboard ports separately" + else + fail "Registry did not record Hermes dashboard metadata: ${registry_check:0:240}" + fi + + forward_list=$(openshell forward list 2>&1 || true) + if forward_list_has_running_port "$SANDBOX_NAME" "8642" "$forward_list"; then + pass "OpenShell forward list shows Hermes API port 8642 running" + else + fail "OpenShell forward list does not show Hermes API port 8642 running" + info "forward list: ${forward_list:0:300}" + fi + if forward_list_has_running_port "$SANDBOX_NAME" "$HERMES_DASHBOARD_PORT" "$forward_list"; then + pass "OpenShell forward list shows Hermes dashboard port ${HERMES_DASHBOARD_PORT} running" + else + fail "OpenShell forward list does not show Hermes dashboard port ${HERMES_DASHBOARD_PORT} running" + info "forward list: ${forward_list:0:300}" + fi + + dashboard_body="$(mktemp)" + dashboard_url="http://127.0.0.1:${HERMES_DASHBOARD_PORT}/" + dashboard_code="000" + for attempt in $(seq 1 20); do + dashboard_code=$(curl -sS -L --max-time 10 -o "$dashboard_body" -w "%{http_code}" "$dashboard_url" 2>/dev/null || echo "000") + if http_status_ok "$dashboard_code" && [ -s "$dashboard_body" ]; then + break + fi + info "Dashboard host probe attempt ${attempt}/20 returned HTTP ${dashboard_code:-000}; waiting 4s..." + sleep 4 + done + if http_status_ok "$dashboard_code" && [ -s "$dashboard_body" ]; then + pass "Hermes web dashboard responds from host on ${dashboard_url} (HTTP ${dashboard_code})" + else + fail "Hermes web dashboard did not respond from host on ${dashboard_url} (HTTP ${dashboard_code:-000})" + fi + + api_health=$(curl -sf --max-time 10 "http://127.0.0.1:8642/health" 2>&1 || true) + if echo "$api_health" | grep -qi '"ok"'; then + pass "Hermes API health remains on port 8642" + else + fail "Hermes API health did not respond on port 8642: ${api_health:0:160}" + fi + + if [ ! -s "$ssh_config" ]; then + openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config" 2>/dev/null || true + fi + if [ -s "$ssh_config" ]; then + dashboard_internal_code=$($TIMEOUT_CMD ssh -F "$ssh_config" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" \ + "code=\$(curl -sS -L --max-time 10 -o /tmp/hermes-dashboard-e2e-body -w '%{http_code}' http://127.0.0.1:${HERMES_DASHBOARD_INTERNAL_PORT}/ 2>/dev/null || echo 000); if test -s /tmp/hermes-dashboard-e2e-body; then echo \"\$code\"; else echo \"EMPTY:\$code\"; fi" \ + 2>&1) || true + + if http_status_ok "$dashboard_internal_code"; then + pass "Hermes dashboard process responds inside sandbox on ${HERMES_DASHBOARD_INTERNAL_PORT}" + else + fail "Hermes dashboard process did not respond inside sandbox on ${HERMES_DASHBOARD_INTERNAL_PORT}: ${dashboard_internal_code:0:160}" + fi + else + fail "Could not get SSH config for in-sandbox Hermes dashboard probe" + fi + rm -f "$dashboard_body" +fi + +rm -f "$ssh_config" + +# ══════════════════════════════════════════════════════════════════ +# Phase 5: Live inference — the real proof +# ══════════════════════════════════════════════════════════════════ +section "Phase 5: Live inference" + +# ── Test 5a: Direct hosted inference endpoint ── +info "[LIVE] Direct API test → ${HOSTED_INFERENCE_BASE_URL}..." +api_response=$(curl -s --max-time 30 \ + -X POST "${HOSTED_INFERENCE_BASE_URL}/chat/completions" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $HOSTED_INFERENCE_KEY" \ + -d "$(printf '{"model":"%s","messages":[{"role":"user","content":"Reply with exactly one word: PONG"}],"max_tokens":100}' "$HOSTED_INFERENCE_MODEL")" 2>/dev/null) || true + +if [ -n "$api_response" ]; then + api_content=$(echo "$api_response" | parse_chat_content 2>/dev/null) || true + if grep -qi "PONG" <<<"$api_content"; then + pass "[LIVE] Direct API: model responded with PONG" + else + fail "[LIVE] Direct API: expected PONG, got: ${api_content:0:200}" + fi +else + fail "[LIVE] Direct API: empty response from curl" +fi + +# ── Test 5b: Inference through the sandbox (THE definitive test) ── +# Routing-layer check, not a Hermes/openclaw check. The HTTP request is made +# by curl from inside the sandbox; nothing in this path exercises the Hermes +# agent runtime or openclaw's HTTP client. See NemoClaw #2490 for the +# openclaw 4.9 SSRF regression that was invisible to assertions of this shape. +info "[ROUTING] inference.local DNS + OpenShell proxy reachable from Hermes sandbox..." +ssh_config="$(mktemp)" +sandbox_response="" + +if openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config" 2>/dev/null; then + # Use timeout if available (Linux, Homebrew), fall back to plain ssh + TIMEOUT_CMD="" + command -v timeout >/dev/null 2>&1 && TIMEOUT_CMD="timeout 90" + command -v gtimeout >/dev/null 2>&1 && TIMEOUT_CMD="gtimeout 90" + sandbox_response=$($TIMEOUT_CMD ssh -F "$ssh_config" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" \ + "curl -s --max-time 60 https://inference.local/v1/chat/completions \ + -H 'Content-Type: application/json' \ + -d '{\"model\":\"$HOSTED_INFERENCE_MODEL\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with exactly one word: PONG\"}],\"max_tokens\":100}'" \ + 2>&1) || true +fi +rm -f "$ssh_config" + +if [ -n "$sandbox_response" ]; then + sandbox_content=$(echo "$sandbox_response" | parse_chat_content 2>/dev/null) || true + if grep -qi "PONG" <<<"$sandbox_content"; then + pass "[ROUTING] inference.local: OpenShell routed curl to the hosted inference endpoint and returned PONG" + info "Routing path proven: sandbox curl → DNS forwarder → gateway proxy → hosted inference endpoint (does not exercise the Hermes agent runtime or openclaw HTTP client)" + else + fail "[ROUTING] inference.local: expected PONG, got: ${sandbox_content:0:200}" + fi +else + fail "[ROUTING] inference.local: no response from inference.local inside Hermes sandbox" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 6: NemoClaw CLI operations (Hermes-specific) +# ══════════════════════════════════════════════════════════════════ +section "Phase 6: NemoClaw CLI operations (Hermes)" + +# ── Test 6a: nemoclaw logs ── +info "Testing sandbox log retrieval..." +logs_output=$(nemoclaw "$SANDBOX_NAME" logs 2>&1) || true +if [ -n "$logs_output" ]; then + pass "nemoclaw logs: produced output ($(echo "$logs_output" | wc -l | tr -d ' ') lines)" +else + fail "nemoclaw logs: no output" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 7: OpenClaw regression (ensure default agent path still works) +# ══════════════════════════════════════════════════════════════════ +section "Phase 7: OpenClaw regression check" + +# Verify that the agent-defs module can still load the openclaw manifest +info "Verifying OpenClaw agent manifest is still loadable..." +openclaw_check=$(node -e " + const { loadAgent, listAgents } = require('$REPO/bin/lib/agent-defs'); + const agents = listAgents(); + console.log('agents:', agents.join(', ')); + const oc = loadAgent('openclaw'); + console.log('openclaw_display:', oc.displayName); + console.log('openclaw_port:', oc.forwardPort); + const h = loadAgent('hermes'); + console.log('hermes_display:', h.displayName); + console.log('hermes_port:', h.forwardPort); +" 2>&1) || true + +if echo "$openclaw_check" | grep -q "openclaw_display:.*OpenClaw"; then + pass "OpenClaw agent manifest loads correctly" +else + fail "OpenClaw agent manifest failed to load" + info "Output: ${openclaw_check:0:300}" +fi + +if echo "$openclaw_check" | grep -q "hermes_display:.*Hermes"; then + pass "Hermes agent manifest loads correctly" +else + fail "Hermes agent manifest failed to load" + info "Output: ${openclaw_check:0:300}" +fi + +if echo "$openclaw_check" | grep -q "agents:.*openclaw.*hermes\|agents:.*hermes.*openclaw"; then + pass "Both agents listed by listAgents()" +else + fail "listAgents() did not return both openclaw and hermes" + info "Output: ${openclaw_check:0:300}" +fi + +# ══════════════════════════════════════════════════════════════════ +# Optional Phase 7b: Security posture regression checks +# ══════════════════════════════════════════════════════════════════ +if [ "${NEMOCLAW_E2E_SECURITY_POSTURE:-}" = "1" ]; then + # shellcheck source=test/e2e-vpn/lib/security-posture-assertions.sh + . "$(dirname "${BASH_SOURCE[0]}")/lib/security-posture-assertions.sh" + security_posture_assertions_run "$SANDBOX_NAME" "hermes" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 8: Cleanup +# ══════════════════════════════════════════════════════════════════ +section "Phase 8: Cleanup" + +[[ "${NEMOCLAW_E2E_KEEP_SANDBOX:-}" = "1" ]] || nemoclaw "$SANDBOX_NAME" destroy --yes 2>&1 | tail -3 || true +openshell gateway destroy -g nemoclaw 2>/dev/null || true + +# Verify against the registry file directly. `nemoclaw list` triggers +# gateway recovery which can restart a destroyed gateway and re-import stale +# sandbox entries — that's a separate issue, so avoid it here. +registry_file="${HOME}/.nemoclaw/sandboxes.json" +if [ -f "$registry_file" ] && grep -Fq "\"${SANDBOX_NAME}\"" "$registry_file"; then + fail "Sandbox ${SANDBOX_NAME} still in registry after destroy" +else + pass "Sandbox ${SANDBOX_NAME} removed" +fi + +# ══════════════════════════════════════════════════════════════════ +# Summary +# ══════════════════════════════════════════════════════════════════ +echo "" +echo "========================================" +echo " Hermes Agent E2E Results:" +echo " Passed: $PASS" +echo " Failed: $FAIL" +echo " Skipped: $SKIP" +echo " Total: $TOTAL" +echo "========================================" + +if [ "$FAIL" -eq 0 ]; then + printf '\n\033[1;32m Hermes E2E PASSED — agent selection + inference verified end-to-end.\033[0m\n' + exit 0 +else + printf '\n\033[1;31m %d test(s) failed.\033[0m\n' "$FAIL" + exit 1 +fi diff --git a/test/e2e-vpn/test-hermes-inference-switch.sh b/test/e2e-vpn/test-hermes-inference-switch.sh new file mode 100755 index 00000000000..464237882c9 --- /dev/null +++ b/test/e2e-vpn/test-hermes-inference-switch.sh @@ -0,0 +1,623 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Hermes inference switch E2E. +# +# Installs NemoClaw with Hermes, switches the running sandbox with +# `nemohermes inference set`, verifies OpenShell and Hermes config state, and +# sends live requests after the switch without restarting Hermes. +# +# Prerequisites: +# - Docker running +# - NVIDIA_API_KEY set for hosted inference +# - NEMOCLAW_NON_INTERACTIVE=1 +# - NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 + +# Do not use errexit because this test records pass/fail counts and exits +# explicitly after critical failures or at the final summary. +set -uo pipefail + +PASS=0 +FAIL=0 +SKIP=0 +TOTAL=0 + +pass() { + ((PASS++)) + ((TOTAL++)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} +fail() { + ((FAIL++)) + ((TOTAL++)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} +skip() { + ((SKIP++)) + ((TOTAL++)) + printf '\033[33m SKIP: %s\033[0m\n' "$1" +} +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } + +is_transient_live_http_code() { + case "${1:-}" in + 502 | 503 | 504) return 0 ;; + *) return 1 ;; + esac +} + +http_status_from_response() { + sed -n 's/^__NEMOCLAW_HTTP_STATUS__=//p' <<<"$1" | tail -1 +} + +http_body_from_response() { + sed '/^__NEMOCLAW_HTTP_STATUS__=/d' <<<"$1" +} + +parse_chat_content() { + python3 -c " +import json, sys +try: + r = json.load(sys.stdin) + c = r['choices'][0]['message'] + content = c.get('content') or c.get('reasoning_content') or c.get('reasoning') or '' + print(content.strip()) +except Exception as e: + print(f'PARSE_ERROR: {e}', file=sys.stderr) + sys.exit(1) +" +} + +hermes_gateway_pid() { + # shellcheck disable=SC2016 # awk runs inside the sandbox. + openshell sandbox exec --name "$SANDBOX_NAME" -- sh -lc \ + 'ps -eo pid=,comm=,args= 2>/dev/null | awk '"'"'$2 != "sh" && $2 != "bash" && $2 != "awk" && $0 ~ /hermes/ && $0 ~ /gateway run/ { print $1; exit }'"'"'' \ + 2>/dev/null || true +} + +get_route_output() { + local output + if output=$(openshell inference get -g nemoclaw 2>&1); then + printf '%s\n' "$output" + return 0 + fi + openshell inference get 2>&1 +} + +strip_ansi() { + python3 -c 'import re, sys; sys.stdout.write(re.sub(r"\x1b\[[0-9;]*m", "", sys.stdin.read()))' +} + +assert_route() { + local output plain_output + if ! output=$(get_route_output); then + fail "OpenShell inference get failed: ${output:0:240}" + return + fi + plain_output=$(printf '%s' "$output" | strip_ansi) + + if grep -Fq "Provider: ${SWITCH_PROVIDER}" <<<"$plain_output" \ + && grep -Fq "Model: ${SWITCH_MODEL}" <<<"$plain_output"; then + pass "OpenShell route points at ${SWITCH_PROVIDER} / ${SWITCH_MODEL}" + else + fail "OpenShell route did not switch to ${SWITCH_PROVIDER} / ${SWITCH_MODEL}: ${plain_output:0:400}" + fi +} + +assert_registry_session() { + local probe + probe=$( + SANDBOX_NAME="$SANDBOX_NAME" EXPECTED_PROVIDER="$SWITCH_PROVIDER" EXPECTED_MODEL="$SWITCH_MODEL" python3 - <<'PY' +import json +import os +from pathlib import Path + +home = Path.home() +name = os.environ["SANDBOX_NAME"] +provider = os.environ["EXPECTED_PROVIDER"] +model = os.environ["EXPECTED_MODEL"] +errors = [] + +registry_path = home / ".nemoclaw" / "sandboxes.json" +try: + registry = json.loads(registry_path.read_text(encoding="utf-8")) + sandbox = (registry.get("sandboxes") or {}).get(name) +except Exception as exc: + sandbox = None + errors.append(f"could not read registry: {exc}") + +if not sandbox: + errors.append(f"sandbox {name} missing from registry") +else: + if sandbox.get("agent") != "hermes": + errors.append(f"registry agent={sandbox.get('agent')!r}") + if sandbox.get("provider") != provider: + errors.append(f"registry provider={sandbox.get('provider')!r}") + if sandbox.get("model") != model: + errors.append(f"registry model={sandbox.get('model')!r}") + +session_path = home / ".nemoclaw" / "onboard-session.json" +try: + session = json.loads(session_path.read_text(encoding="utf-8")) +except Exception as exc: + session = None + errors.append(f"could not read onboard session: {exc}") + +if session is not None: + if not isinstance(session, dict) or not session: + errors.append("onboard session is empty or invalid") + else: + if session.get("sandboxName") != name: + errors.append(f"session sandboxName={session.get('sandboxName')!r}") + if session.get("agent") != "hermes": + errors.append(f"session agent={session.get('agent')!r}") + if session.get("provider") != provider: + errors.append(f"session provider={session.get('provider')!r}") + if session.get("model") != model: + errors.append(f"session model={session.get('model')!r}") + +if errors: + print("; ".join(errors)) + raise SystemExit(1) +print("OK") +PY + ) || { + fail "Registry/session were not updated for switch: ${probe:0:400}" + return + } + pass "Registry and onboard session record the switched Hermes provider/model" +} + +assert_hermes_health() { + local health_response attempt + for attempt in 1 2 3 4 5; do + health_response=$(openshell sandbox exec --name "$SANDBOX_NAME" -- \ + curl -sf --max-time 10 http://localhost:8642/health 2>&1) || true + if grep -qi '"ok"' <<<"$health_response"; then + pass "Hermes health endpoint returns ok" + return + fi + [ "$attempt" -ge 5 ] || sleep 4 + done + fail "Hermes health endpoint did not return ok: ${health_response:0:240}" +} + +assert_hermes_config() { + local config probe + config=$(openshell sandbox exec --name "$SANDBOX_NAME" -- cat /sandbox/.hermes/config.yaml 2>&1) || { + fail "Could not read /sandbox/.hermes/config.yaml: ${config:0:240}" + return + } + + # Keep this parser dependency-free for the E2E runner: it only reads the + # simple model block and should move to PyYAML if nested or multiline values + # become relevant. + probe=$( + CONFIG_TEXT="$config" EXPECTED_MODEL="$SWITCH_MODEL" EXPECTED_INFERENCE_API="$SWITCH_INFERENCE_API" python3 - <<'PY' +import os +import re + +text = os.environ["CONFIG_TEXT"] +expected = os.environ["EXPECTED_MODEL"] +expected_api = os.environ["EXPECTED_INFERENCE_API"] +errors = [] +expected_base = "https://inference.local" if expected_api == "anthropic-messages" else "https://inference.local/v1" +expected_api_mode = { + "anthropic-messages": "anthropic_messages", + "openai-responses": "codex_responses", +}.get(expected_api) + +model = {} +in_model = False +for line in text.splitlines(): + if re.match(r"^model:\s*$", line): + in_model = True + continue + if in_model and re.match(r"^[A-Za-z0-9_-]+:", line): + break + if in_model: + match = re.match(r"^\s+([A-Za-z0-9_-]+):\s*(.*?)\s*$", line) + if match: + value = match.group(2).strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'": + value = value[1:-1] + model[match.group(1)] = value + +if model.get("default") != expected: + errors.append(f"model.default={model.get('default')!r}") +if model.get("base_url") != expected_base: + errors.append(f"model.base_url={model.get('base_url')!r}") +if model.get("provider") != "custom": + errors.append(f"model.provider={model.get('provider')!r}") +if expected_api_mode: + if model.get("api_mode") != expected_api_mode: + errors.append(f"model.api_mode={model.get('api_mode')!r}") +elif "api_mode" in model: + errors.append(f"stale model.api_mode={model.get('api_mode')!r}") +api_key = model.get("api_key") +if not isinstance(api_key, str) or not api_key.startswith("sk-"): + errors.append(f"model.api_key={api_key!r}") + +if re.search(r"(?ms)^models:\s*\n(?:[ \t].*\n)*?[ \t]+providers:", text): + errors.append("OpenClaw-style models.providers block present") + +if errors: + print("; ".join(errors)) + raise SystemExit(1) +print("OK") +PY + ) || { + fail "Hermes config.yaml was not patched correctly: ${probe:0:400}" + return + } + pass "Hermes config.yaml model block uses ${SWITCH_MODEL} via inference.local" +} + +assert_hermes_hashes() { + local strict_check compat_check perms_probe + strict_check=$(openshell sandbox exec --name "$SANDBOX_NAME" -- sh -lc \ + 'sha256sum -c /etc/nemoclaw/hermes.config-hash --status && echo OK' 2>&1 || true) + if grep -qx "OK" <<<"$strict_check"; then + pass "Hermes strict config hash matches config.yaml and .env" + else + fail "Hermes strict config hash check failed: ${strict_check:0:240}" + fi + + compat_check=$(openshell sandbox exec --name "$SANDBOX_NAME" -- sh -lc \ + 'sha256sum -c /sandbox/.hermes/.config-hash --status && echo OK' 2>&1 || true) + if grep -qx "OK" <<<"$compat_check"; then + pass "Hermes compatibility config hash matches config.yaml and .env" + else + fail "Hermes compatibility config hash check failed: ${compat_check:0:240}" + fi + + perms_probe=$(openshell sandbox exec --name "$SANDBOX_NAME" -- sh -lc \ + "stat -c '%u %a' /etc/nemoclaw/hermes.config-hash" 2>&1 || true) + if PERMS_PROBE="$perms_probe" python3 - <<'PY'; then +import os +import sys + +parts = os.environ.get("PERMS_PROBE", "").split() +if len(parts) != 2: + raise SystemExit(1) +uid = int(parts[0]) +mode = int(parts[1], 8) +if uid != 0 or mode & 0o222: + raise SystemExit(1) +PY + pass "Hermes strict hash is root-owned and not writable" + else + fail "Hermes strict hash permissions are wrong: ${perms_probe:0:120}" + fi +} + +assert_env_hash_unchanged() { + local after + after=$(openshell sandbox exec --name "$SANDBOX_NAME" -- sha256sum /sandbox/.hermes/.env 2>/dev/null | awk '{print $1}') || true + if [ -n "$ENV_HASH_BEFORE" ] && [ "$after" = "$ENV_HASH_BEFORE" ]; then + pass "Hermes .env was not rewritten by inference set" + else + fail "Hermes .env hash changed during inference set (${ENV_HASH_BEFORE:-missing} -> ${after:-missing})" + fi +} + +check_inference_local() { + local payload payload_arg response rc content attempt last_fail http_code body remote transient=0 + payload=$(SWITCH_MODEL="$SWITCH_MODEL" SWITCH_INFERENCE_API="$SWITCH_INFERENCE_API" python3 -c ' +import json +import os +if os.environ["SWITCH_INFERENCE_API"] == "anthropic-messages": + print(json.dumps({ + "model": os.environ["SWITCH_MODEL"], + "messages": [{"role": "user", "content": "Reply with exactly one word: PONG"}], + "max_tokens": 32, + })) +else: + print(json.dumps({ + "model": os.environ["SWITCH_MODEL"], + "messages": [{"role": "user", "content": "Reply with exactly one word: PONG"}], + "max_tokens": 100, + })) +') + payload_arg="$(printf '%q' "$payload")" + if [ "$SWITCH_INFERENCE_API" = "anthropic-messages" ]; then + remote="tmp=\$(mktemp); code=\$(curl -sS -o \"\$tmp\" -w '%{http_code}' --max-time 90 https://inference.local/v1/messages -H 'Content-Type: application/json' -H 'anthropic-version: 2023-06-01' -d $payload_arg); rc=\$?; cat \"\$tmp\"; rm -f \"\$tmp\"; printf '\n__NEMOCLAW_HTTP_STATUS__=%s\n' \"\${code:-000}\"; exit \"\$rc\"" + else + remote="tmp=\$(mktemp); code=\$(curl -sS -o \"\$tmp\" -w '%{http_code}' --max-time 90 https://inference.local/v1/chat/completions -H 'Content-Type: application/json' -d $payload_arg); rc=\$?; cat \"\$tmp\"; rm -f \"\$tmp\"; printf '\n__NEMOCLAW_HTTP_STATUS__=%s\n' \"\${code:-000}\"; exit \"\$rc\"" + fi + last_fail="" + + for attempt in 1 2 3; do + rc=0 + transient=0 + response=$(openshell sandbox exec --name "$SANDBOX_NAME" -- sh -lc "$remote" 2>&1) || rc=$? + http_code=$(http_status_from_response "$response") + [ -n "$http_code" ] || http_code="000" + body=$(http_body_from_response "$response") + + if [ "$rc" -ne 0 ]; then + [ "$rc" -eq 28 ] && transient=1 + last_fail="curl failed with exit ${rc}; HTTP ${http_code}: ${body:0:300}" + elif is_transient_live_http_code "$http_code"; then + transient=1 + last_fail="transient HTTP ${http_code}: ${body:0:300}" + elif [ "$http_code" != "200" ]; then + last_fail="HTTP ${http_code}: ${body:0:300}" + else + if [ "$SWITCH_INFERENCE_API" = "anthropic-messages" ]; then + content=$(printf '%s' "$body" | parse_anthropic_content 2>/dev/null) || content="" + else + content=$(printf '%s' "$body" | parse_chat_content 2>/dev/null) || content="" + fi + if grep -qi "PONG" <<<"$content"; then + pass "Hermes sandbox inference.local returned PONG with ${SWITCH_MODEL}" + return + fi + last_fail="expected PONG, got ${content:0:300}" + fi + + [ "$attempt" -ge 3 ] || { + info "Hermes inference.local attempt ${attempt}/3 failed: ${last_fail}" + sleep 5 + } + done + + if [ "$transient" -eq 1 ]; then + skip "Hermes sandbox inference.local transient failure after switch; route/config checks already passed" + else + fail "Hermes sandbox inference.local did not work after switch: ${last_fail}" + fi +} + +check_hermes_api_chat() { + local payload payload_arg response rc content remote attempt last_fail http_code body transient=0 + payload=$(SWITCH_MODEL="$SWITCH_MODEL" python3 -c ' +import json +import os +print(json.dumps({ + "model": os.environ["SWITCH_MODEL"], + "messages": [{"role": "user", "content": "Reply with exactly one word: PONG"}], + "max_tokens": 100, +})) +') + payload_arg="$(printf '%q' "$payload")" + remote="set -a; [ ! -f /sandbox/.hermes/.env ] || . /sandbox/.hermes/.env; set +a; tmp=\$(mktemp); if [ -n \"\${API_SERVER_KEY:-}\" ]; then code=\$(curl -sS -o \"\$tmp\" -w '%{http_code}' --max-time 120 http://localhost:8642/v1/chat/completions -H 'Content-Type: application/json' -H \"Authorization: Bearer \${API_SERVER_KEY}\" -d $payload_arg); else code=\$(curl -sS -o \"\$tmp\" -w '%{http_code}' --max-time 120 http://localhost:8642/v1/chat/completions -H 'Content-Type: application/json' -d $payload_arg); fi; rc=\$?; cat \"\$tmp\"; rm -f \"\$tmp\"; printf '\n__NEMOCLAW_HTTP_STATUS__=%s\n' \"\${code:-000}\"; exit \"\$rc\"" + last_fail="" + + for attempt in 1 2 3; do + rc=0 + transient=0 + response=$(openshell sandbox exec --name "$SANDBOX_NAME" -- sh -lc "$remote" 2>&1) || rc=$? + http_code=$(http_status_from_response "$response") + [ -n "$http_code" ] || http_code="000" + body=$(http_body_from_response "$response") + + if [ "$rc" -ne 0 ]; then + [ "$rc" -eq 28 ] && transient=1 + last_fail="Hermes API curl failed with exit ${rc}; HTTP ${http_code}: ${body:0:300}" + elif is_transient_live_http_code "$http_code"; then + transient=1 + last_fail="transient HTTP ${http_code}: ${body:0:300}" + elif [ "$http_code" != "200" ]; then + last_fail="HTTP ${http_code}: ${body:0:300}" + else + content=$(printf '%s' "$body" | parse_chat_content 2>/dev/null) || content="" + if grep -qi "PONG" <<<"$content"; then + pass "Hermes API chat works after inference switch" + return + fi + last_fail="expected PONG from Hermes API, got ${content:0:300}; response=${body:0:300}" + fi + + [ "$attempt" -ge 3 ] || { + info "Hermes API chat attempt ${attempt}/3 failed: ${last_fail}" + sleep 5 + } + done + + if [ "$transient" -eq 1 ]; then + skip "Hermes API chat transient failure after switch; route/config checks already passed" + else + fail "Hermes API chat did not work after switch: ${last_fail}" + fi +} + +if [ -d /workspace ] && [ -f /workspace/install.sh ]; then + REPO="/workspace" +elif [ -f "$(cd "$(dirname "$0")/../.." && pwd)/install.sh" ]; then + REPO="$(cd "$(dirname "$0")/../.." && pwd)" +else + echo "ERROR: Cannot find repo root." + exit 1 +fi + +E2E_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=test/e2e-vpn/lib/inference-switch-retry.sh +. "${E2E_DIR}/lib/inference-switch-retry.sh" +# shellcheck source=test/e2e-vpn/lib/anthropic-switch-provider.sh +. "${E2E_DIR}/lib/anthropic-switch-provider.sh" +# shellcheck source=test/e2e-vpn/lib/ci-compatible-inference.sh +. "${E2E_DIR}/lib/ci-compatible-inference.sh" +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-hermes-inference-switch}" +if nemoclaw_e2e_using_compatible_inference; then + SWITCH_PROVIDER="${NEMOCLAW_SWITCH_PROVIDER:-$(nemoclaw_e2e_expected_route_provider)}" + SWITCH_MODEL="${NEMOCLAW_SWITCH_MODEL:-$(nemoclaw_e2e_hosted_inference_model)}" +else + SWITCH_PROVIDER="${NEMOCLAW_SWITCH_PROVIDER:-compatible-endpoint}" + SWITCH_MODEL="${NEMOCLAW_SWITCH_MODEL:-z-ai/glm-5.1}" +fi +SWITCH_INFERENCE_API="${NEMOCLAW_SWITCH_INFERENCE_API:-openai-completions}" +# shellcheck disable=SC2034 # consumed by sourced anthropic-switch-provider.sh +SWITCH_ENDPOINT_URL="${NEMOCLAW_SWITCH_ENDPOINT_URL:-}" +# shellcheck disable=SC2034 # consumed by sourced anthropic-switch-provider.sh +SWITCH_MOCK_ANTHROPIC="${NEMOCLAW_SWITCH_MOCK_ANTHROPIC:-0}" +# shellcheck disable=SC2034 # consumed by sourced anthropic-switch-provider.sh +SWITCH_MOCK_PORT="${NEMOCLAW_SWITCH_MOCK_PORT:-18766}" +INSTALL_LOG="/tmp/nemoclaw-e2e-hermes-inference-switch-install.log" +ENV_HASH_BEFORE="" + +export NEMOCLAW_AGENT="${NEMOCLAW_AGENT:-hermes}" + +# shellcheck source=test/e2e-vpn/lib/sandbox-teardown.sh +. "${E2E_DIR}/lib/sandbox-teardown.sh" +trap 'stop_mock_anthropic_switch_provider; _nemoclaw_sandbox_teardown' EXIT +# shellcheck source=test/e2e-vpn/lib/install-path-refresh.sh +. "${E2E_DIR}/lib/install-path-refresh.sh" +register_sandbox_for_teardown "$SANDBOX_NAME" +nemoclaw_e2e_configure_compatible_inference || exit 1 + +section "Phase 0: Pre-cleanup" +if command -v nemohermes >/dev/null 2>&1; then + nemohermes "$SANDBOX_NAME" destroy --yes 2>/dev/null || true +elif command -v nemoclaw >/dev/null 2>&1; then + NEMOCLAW_AGENT=hermes nemoclaw "$SANDBOX_NAME" destroy --yes 2>/dev/null || true +fi +if command -v openshell >/dev/null 2>&1; then + openshell sandbox delete "$SANDBOX_NAME" 2>/dev/null || true + openshell gateway destroy -g nemoclaw 2>/dev/null || true +fi +pass "Pre-cleanup complete" + +section "Phase 1: Prerequisites" +if docker info >/dev/null 2>&1; then + pass "Docker is running" +else + fail "Docker is not running" + exit 1 +fi + +if ! nemoclaw_e2e_require_hosted_inference_key; then + exit 1 +fi + +if [ "${NEMOCLAW_NON_INTERACTIVE:-}" = "1" ]; then + pass "NEMOCLAW_NON_INTERACTIVE=1" +else + fail "NEMOCLAW_NON_INTERACTIVE=1 is required" + exit 1 +fi + +if [ "${NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE:-}" = "1" ]; then + pass "Third-party software acceptance is set" +else + fail "NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 is required" + exit 1 +fi + +section "Phase 2: Install and onboard Hermes" +cd "$REPO" || { + fail "Could not cd to repo root: $REPO" + exit 1 +} + +export NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" +export NEMOCLAW_RECREATE_SANDBOX="${NEMOCLAW_RECREATE_SANDBOX:-1}" + +info "Running install.sh --non-interactive for Hermes sandbox ${SANDBOX_NAME}..." +bash install.sh --non-interactive --yes-i-accept-third-party-software >"$INSTALL_LOG" 2>&1 & +install_pid=$! +tail -f "$INSTALL_LOG" --pid=$install_pid 2>/dev/null & +tail_pid=$! +wait "$install_pid" +install_exit=$? +kill "$tail_pid" 2>/dev/null || true +wait "$tail_pid" 2>/dev/null || true + +nemoclaw_refresh_install_env +export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" +# shellcheck source=/dev/null +[ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" +nemoclaw_ensure_local_bin_on_path + +if [ "$install_exit" -eq 0 ]; then + pass "install.sh completed" +else + fail "install.sh failed (exit ${install_exit})" + tail -80 "$INSTALL_LOG" || true + exit 1 +fi + +command -v nemohermes >/dev/null 2>&1 || { + fail "nemohermes not found on PATH" + exit 1 +} +command -v openshell >/dev/null 2>&1 || { + fail "openshell not found on PATH" + exit 1 +} +pass "nemohermes and openshell are on PATH" +assert_hermes_health +ensure_compatible_anthropic_switch_provider || exit 1 + +section "Phase 3: Switch inference" +pid_before="$(hermes_gateway_pid)" +ENV_HASH_BEFORE=$(openshell sandbox exec --name "$SANDBOX_NAME" -- sha256sum /sandbox/.hermes/.env 2>/dev/null | awk '{print $1}') || true + +info "Switching Hermes to ${SWITCH_PROVIDER} / ${SWITCH_MODEL} with nemohermes inference set..." +switch_output=$(run_inference_set_with_retry nemohermes inference set --provider "$SWITCH_PROVIDER" --model "$SWITCH_MODEL") +switch_rc=$? +if [ "$switch_rc" -eq 0 ]; then + pass "nemohermes inference set completed without --sandbox" +else + fail "nemohermes inference set failed (exit ${switch_rc}): ${switch_output:0:500}" + exit 1 +fi + +pid_after="$(hermes_gateway_pid)" +if [ -n "$pid_before" ] && [ -n "$pid_after" ]; then + if [ "$pid_before" = "$pid_after" ]; then + pass "Hermes gateway process stayed running during switch" + else + fail "Hermes gateway process changed during switch (${pid_before} -> ${pid_after})" + fi +else + skip "Could not capture Hermes gateway PID before and after switch" +fi + +assert_hermes_health +assert_route +assert_hermes_config +assert_env_hash_unchanged +assert_hermes_hashes +assert_registry_session + +section "Phase 4: Live requests after switch" +check_inference_local +check_hermes_api_chat + +section "Phase 5: Cleanup" +if [ "${NEMOCLAW_E2E_KEEP_SANDBOX:-}" != "1" ]; then + nemohermes "$SANDBOX_NAME" destroy --yes 2>&1 | tail -3 || true + openshell gateway destroy -g nemoclaw 2>/dev/null || true + + registry_file="${HOME}/.nemoclaw/sandboxes.json" + if [ -f "$registry_file" ] && grep -Fq "\"${SANDBOX_NAME}\"" "$registry_file"; then + fail "Sandbox ${SANDBOX_NAME} still in registry after destroy" + else + pass "Sandbox ${SANDBOX_NAME} removed" + fi +else + skip "Sandbox ${SANDBOX_NAME} kept; removal check skipped" +fi + +echo "" +echo "========================================" +echo " Hermes inference switch E2E Results:" +echo " Passed: $PASS" +echo " Failed: $FAIL" +echo " Skipped: $SKIP" +echo " Total: $TOTAL" +echo "========================================" + +if [ "$FAIL" -eq 0 ]; then + printf '\n\033[1;32m Hermes inference switch E2E PASSED.\033[0m\n' + exit 0 +fi + +printf '\n\033[1;31m %d test(s) failed.\033[0m\n' "$FAIL" +exit 1 diff --git a/test/e2e-vpn/test-hermes-root-entrypoint-smoke.sh b/test/e2e-vpn/test-hermes-root-entrypoint-smoke.sh new file mode 100755 index 00000000000..81e9269ecd1 --- /dev/null +++ b/test/e2e-vpn/test-hermes-root-entrypoint-smoke.sh @@ -0,0 +1,202 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Hermes root-entrypoint smoke test: +# - builds the real Hermes sandbox image unless NEMOCLAW_HERMES_TEST_IMAGE is set +# - starts the container as root via /usr/local/bin/nemoclaw-start +# - verifies Hermes health, privilege separation, PID-file layout, and sticky +# config protection +# - repeats startup from a legacy image/state shape with gateway.pid as a +# runtime symlink + +set -euo pipefail + +LOG_PATH="${NEMOCLAW_HERMES_ROOT_ENTRYPOINT_LOG:-/tmp/nemoclaw-hermes-root-entrypoint-smoke.log}" +: >"$LOG_PATH" +exec > >(tee -a "$LOG_PATH") 2>&1 + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +pass() { echo -e "${GREEN}[PASS]${NC} $1"; } +info() { echo -e "${YELLOW}[INFO]${NC} $1"; } + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +RUN_ID="${GITHUB_RUN_ID:-local}-$$" +IMAGE="${NEMOCLAW_HERMES_TEST_IMAGE:-nemoclaw-hermes-root-entrypoint-smoke:${RUN_ID}}" +BASE_IMAGE="nemoclaw-hermes-root-entrypoint-base:${RUN_ID}" +containers=() + +dump_container() { + local container="$1" + docker inspect "$container" >/dev/null 2>&1 || return 0 + echo -e "${YELLOW}[DIAG]${NC} --- ${container} diagnostics ---" >&2 + docker ps -a --filter "name=^/${container}$" --format 'table {{.Names}}\t{{.Status}}\t{{.Image}}' >&2 || true + docker logs "$container" >&2 || true + docker exec "$container" sh -lc \ + 'set +e; echo "== identity =="; id; echo "== hermes tree =="; ls -ld /sandbox/.hermes /sandbox/.hermes/runtime /sandbox/.hermes/logs /sandbox/.hermes/logs/curator /sandbox/.hermes/hooks /sandbox/.hermes/image_cache /sandbox/.hermes/audio_cache 2>&1; ls -l /sandbox/.hermes/gateway.pid /sandbox/.hermes/runtime/gateway.pid /sandbox/.hermes/config.yaml 2>&1; echo "== processes =="; ps -eo user=,pid=,args= | grep -E "hermes|socat" | grep -v grep; echo "== start log =="; tail -n 120 /tmp/nemoclaw-start.log 2>&1; echo "== gateway log =="; tail -n 160 /tmp/gateway.log 2>&1' \ + >&2 || true + echo -e "${YELLOW}[DIAG]${NC} --- end ${container} diagnostics ---" >&2 +} + +fail() { + echo -e "${RED}[FAIL]${NC} $1" >&2 + for container in "${containers[@]}"; do + dump_container "$container" + done + exit 1 +} + +cleanup() { + for container in "${containers[@]}"; do + docker rm -f "$container" >/dev/null 2>&1 || true + done +} +trap cleanup EXIT + +require_docker() { + command -v docker >/dev/null 2>&1 || fail "docker is required" + docker info >/dev/null 2>&1 || fail "docker daemon is not available" +} + +build_image_if_needed() { + if [ -n "${NEMOCLAW_HERMES_TEST_IMAGE:-}" ]; then + info "Using prebuilt Hermes image ${IMAGE}" + docker image inspect "$IMAGE" >/dev/null 2>&1 || fail "prebuilt image not found: ${IMAGE}" + return 0 + fi + + info "Building Hermes base image ${BASE_IMAGE}" + docker build -f "${REPO_ROOT}/agents/hermes/Dockerfile.base" -t "$BASE_IMAGE" "$REPO_ROOT" \ + || fail "failed to build Hermes base image" + + info "Building Hermes production image ${IMAGE}" + docker build -f "${REPO_ROOT}/agents/hermes/Dockerfile" \ + --build-arg "BASE_IMAGE=${BASE_IMAGE}" \ + -t "$IMAGE" \ + "$REPO_ROOT" \ + || fail "failed to build Hermes production image" +} + +wait_for_health() { + local container="$1" + local body="" + local running="" + + for _attempt in $(seq 1 90); do + if body="$(docker exec "$container" sh -lc 'curl -sf --max-time 2 http://127.0.0.1:8642/health' 2>/dev/null)"; then + echo "$body" + printf '%s\n' "$body" | grep -Eq '"status"[[:space:]]*:[[:space:]]*"ok"' \ + || fail "${container}: health response did not report status ok: ${body}" + printf '%s\n' "$body" | grep -Eq '"platform"[[:space:]]*:[[:space:]]*"hermes-agent"' \ + || fail "${container}: health response did not report Hermes platform: ${body}" + return 0 + fi + + running="$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || true)" + [ "$running" = "true" ] || fail "${container}: container exited before health became ready" + sleep 2 + done + + fail "${container}: Hermes health did not become ready" +} + +assert_container_sh() { + local container="$1" + local message="$2" + local command="$3" + docker exec "$container" sh -lc "$command" >/dev/null || fail "${container}: ${message}" +} + +assert_container_sh_fails() { + local container="$1" + local message="$2" + local command="$3" + if docker exec "$container" sh -lc "$command" >/dev/null 2>&1; then + fail "${container}: ${message}" + fi +} + +assert_gateway_log_clean() { + local container="$1" + assert_container_sh "$container" "gateway log contains PID race failure" \ + "! grep -F 'PID file race lost' /tmp/gateway.log" + assert_container_sh "$container" "gateway log contains config load failure" \ + "! grep -F 'Could not load config.yaml' /tmp/gateway.log" +} + +assert_runtime_layout() { + local container="$1" + + assert_container_sh "$container" "Hermes config root mode is not 3770" \ + "[ \"\$(stat -c '%a' /sandbox/.hermes)\" = '3770' ]" + assert_container_sh "$container" "required Hermes v0.14 directories are missing" \ + "for dir in hooks image_cache audio_cache logs/curator; do test -d \"/sandbox/.hermes/\$dir\"; done" + assert_container_sh "$container" "gateway user cannot write required Hermes v0.14 directories" \ + "gosu gateway sh -lc 'for dir in hooks image_cache audio_cache logs/curator; do p=\"/sandbox/.hermes/\$dir/.nemoclaw-write-test\"; : >\"\$p\" && rm -f \"\$p\"; done'" + assert_container_sh "$container" "gateway.pid is not a regular top-level file" \ + "test -f /sandbox/.hermes/gateway.pid && test ! -L /sandbox/.hermes/gateway.pid" + assert_container_sh_fails "$container" "gateway user was able to remove config.yaml" \ + "gosu gateway rm /sandbox/.hermes/config.yaml" + assert_container_sh "$container" "config.yaml disappeared after gateway remove attempt" \ + "test -f /sandbox/.hermes/config.yaml" +} + +assert_gateway_process() { + local container="$1" + assert_container_sh "$container" "Hermes gateway process is not running as gateway user" \ + "ps -eo user=,args= | awk '\$1 == \"gateway\" && (index(\$0, \"hermes gateway run\") || index(\$0, \"hermes.real gateway run\")) { found = 1 } END { exit found ? 0 : 1 }'" + assert_container_sh "$container" "start log does not show gateway privilege separation" \ + "grep -F \"hermes gateway launched as 'gateway' user\" /tmp/nemoclaw-start.log" +} + +run_clean_variant() { + local container="nemoclaw-hermes-root-clean-${RUN_ID}" + info "Starting clean root-entrypoint container ${container}" + docker run -d --name "$container" "$IMAGE" /usr/local/bin/nemoclaw-start >/dev/null \ + || fail "failed to start clean root-entrypoint container" + containers+=("$container") + + wait_for_health "$container" >/dev/null + assert_gateway_process "$container" + assert_gateway_log_clean "$container" + assert_runtime_layout "$container" + pass "Clean root-entrypoint startup reached Hermes health" +} + +run_legacy_variant() { + local container="nemoclaw-hermes-root-legacy-${RUN_ID}" + local legacy_bootstrap + legacy_bootstrap='set -euo pipefail +rm -f /sandbox/.hermes/gateway.pid +printf "stale pid\n" >/sandbox/.hermes/runtime/gateway.pid +printf "stale lock\n" >/sandbox/.hermes/runtime/gateway.lock +ln -s runtime/gateway.pid /sandbox/.hermes/gateway.pid +chmod 750 /sandbox/.hermes +rm -rf /sandbox/.hermes/hooks /sandbox/.hermes/image_cache /sandbox/.hermes/audio_cache /sandbox/.hermes/logs/curator +exec /usr/local/bin/nemoclaw-start /usr/local/bin/nemoclaw-start' + + info "Starting legacy-layout root-entrypoint container ${container}" + docker run -d --name "$container" --entrypoint /bin/bash "$IMAGE" -lc "$legacy_bootstrap" >/dev/null \ + || fail "failed to start legacy-layout root-entrypoint container" + containers+=("$container") + + wait_for_health "$container" >/dev/null + assert_gateway_process "$container" + assert_gateway_log_clean "$container" + assert_runtime_layout "$container" + assert_container_sh "$container" "legacy gateway.pid symlink migration was not logged" \ + "grep -F 'Removing unsafe stale Hermes legacy PID file symlink' /tmp/nemoclaw-start.log" + pass "Legacy gateway.pid symlink/state migrated and booted" +} + +require_docker +build_image_if_needed +run_clean_variant +run_legacy_variant + +pass "Hermes root-entrypoint smoke passed" diff --git a/test/e2e-vpn/test-hermes-sandbox-secret-boundary.sh b/test/e2e-vpn/test-hermes-sandbox-secret-boundary.sh new file mode 100755 index 00000000000..6cc2947e278 --- /dev/null +++ b/test/e2e-vpn/test-hermes-sandbox-secret-boundary.sh @@ -0,0 +1,416 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Hermes sandbox secret-boundary smoke: +# - builds the real Hermes sandbox image unless NEMOCLAW_HERMES_TEST_IMAGE is set +# - inspects the built Hermes image for raw secret-shaped .env values +# - verifies remote platform toolsets preserve Hermes capabilities +# - verifies managed tool gateway images keep auth out of sandbox env/config +# - proves startup rejects newly introduced raw secret-shaped .env values + +set -euo pipefail + +LOG_PATH="${NEMOCLAW_HERMES_SECRET_BOUNDARY_LOG:-/tmp/nemoclaw-hermes-sandbox-secret-boundary.log}" +: >"$LOG_PATH" +exec > >(tee -a "$LOG_PATH") 2>&1 + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +pass() { echo -e "${GREEN}[PASS]${NC} $1"; } +info() { echo -e "${YELLOW}[INFO]${NC} $1"; } +fail() { + echo -e "${RED}[FAIL]${NC} $1" >&2 + exit 1 +} + +shell_quote() { + local value="$1" + printf "'%s'" "${value//\'/\'\\\'\'}" +} + +require_docker() { + command -v docker >/dev/null 2>&1 || fail "docker is required" + docker info >/dev/null 2>&1 || fail "docker daemon is not available" +} + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +RUN_ID="${GITHUB_RUN_ID:-local}-$$" +IMAGE="${NEMOCLAW_HERMES_TEST_IMAGE:-nemoclaw-hermes-secret-boundary:${RUN_ID}}" +BASE_IMAGE_FROM_ENV="${NEMOCLAW_HERMES_BASE_IMAGE:-${HERMES_BASE_IMAGE:-}}" +BASE_IMAGE="${BASE_IMAGE_FROM_ENV:-nemoclaw-hermes-secret-boundary-base:${RUN_ID}}" +MANAGED_IMAGE="${NEMOCLAW_HERMES_MANAGED_TEST_IMAGE:-nemoclaw-hermes-secret-boundary-managed:${RUN_ID}}" +MANAGED_PRESETS_B64="$( + python3 - <<'PY' +import base64 +import json + +print( + base64.b64encode( + json.dumps( + ["nous-web", "nous-audio", "nous-browser", "nous-image", "nous-code"], + separators=(",", ":"), + ).encode("utf-8") + ).decode("ascii") +) +PY +)" + +build_image_if_needed() { + if [ -n "${NEMOCLAW_HERMES_TEST_IMAGE:-}" ]; then + info "Using prebuilt Hermes image ${IMAGE}" + docker image inspect "$IMAGE" >/dev/null 2>&1 || fail "prebuilt image not found: ${IMAGE}" + return 0 + fi + + if [ -z "$BASE_IMAGE_FROM_ENV" ]; then + info "Building Hermes base image ${BASE_IMAGE}" + docker build -f "${REPO_ROOT}/agents/hermes/Dockerfile.base" -t "$BASE_IMAGE" "$REPO_ROOT" \ + || fail "failed to build Hermes base image" + else + info "Using configured Hermes base image ${BASE_IMAGE}" + fi + + info "Building Hermes production image ${IMAGE}" + docker build -f "${REPO_ROOT}/agents/hermes/Dockerfile" \ + --build-arg "BASE_IMAGE=${BASE_IMAGE}" \ + -t "$IMAGE" \ + "$REPO_ROOT" \ + || fail "failed to build Hermes production image" +} + +build_managed_image_if_needed() { + if [ -n "${NEMOCLAW_HERMES_MANAGED_TEST_IMAGE:-}" ]; then + info "Using prebuilt managed-tool Hermes image ${MANAGED_IMAGE}" + docker image inspect "$MANAGED_IMAGE" >/dev/null 2>&1 \ + || fail "prebuilt managed-tool image not found: ${MANAGED_IMAGE}" + return 0 + fi + + if [ -z "$BASE_IMAGE_FROM_ENV" ] && ! docker image inspect "$BASE_IMAGE" >/dev/null 2>&1; then + info "Building Hermes base image ${BASE_IMAGE} for managed-tool variant" + docker build -f "${REPO_ROOT}/agents/hermes/Dockerfile.base" -t "$BASE_IMAGE" "$REPO_ROOT" \ + || fail "failed to build Hermes base image for managed-tool variant" + else + info "Using Hermes base image ${BASE_IMAGE} for managed-tool variant" + fi + + info "Building Hermes managed-tool production image ${MANAGED_IMAGE}" + docker build -f "${REPO_ROOT}/agents/hermes/Dockerfile" \ + --build-arg "BASE_IMAGE=${BASE_IMAGE}" \ + --build-arg "NEMOCLAW_HERMES_TOOL_GATEWAY_BROKER=1" \ + --build-arg "NEMOCLAW_HERMES_TOOL_GATEWAY_PRESETS_B64=${MANAGED_PRESETS_B64}" \ + -t "$MANAGED_IMAGE" \ + "$REPO_ROOT" \ + || fail "failed to build Hermes managed-tool production image" +} + +inspect_image_boundary() { + local image="$1" + info "Inspecting Hermes sandbox boundary in ${image}" + docker run --rm --entrypoint python3 "$image" - <<'PY' +import re +import sys +from pathlib import Path + +secret_key_re = re.compile(r"(^|_)(TOKEN|KEY|SECRET|PASSWORD|CREDENTIAL|API)(_|$)") +slack_alias_re = re.compile(r"^(xoxb|xapp)-OPENSHELL-RESOLVE-ENV-[A-Z0-9_]+$") +allowed_nonsecret_keys = {"API_SERVER_HOST", "API_SERVER_PORT"} +allowed_literals = {"", "[STRIPPED_BY_MIGRATION]"} +required_remote_toolsets = { + "web", + "browser", + "terminal", + "file", + "code_execution", + "vision", + "image_gen", + "skills", + "todo", + "memory", + "session_search", + "delegation", + "cronjob", + "nemoclaw", + "audio", +} + + +def unquote(value: str) -> str: + value = value.strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in ("'", '"'): + return value[1:-1] + return value + + +def env_violations(path: Path) -> list[str]: + violations: list[str] = [] + for lineno, raw_line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): + stripped = raw_line.strip() + if not stripped or stripped.startswith("#") or "=" not in stripped: + continue + if stripped.startswith("export "): + stripped = stripped[len("export ") :].lstrip() + key, value = stripped.split("=", 1) + key = key.strip() + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key): + continue + if key in allowed_nonsecret_keys: + continue + if not secret_key_re.search(key): + continue + value = unquote(value) + if ( + value in allowed_literals + or value.startswith("openshell:resolve:env:") + or slack_alias_re.fullmatch(value) + ): + continue + violations.append(f"{key} line {lineno}") + return violations + + +def parse_platform_toolsets(text: str) -> dict[str, list[str]]: + toolsets: dict[str, list[str]] = {} + in_block = False + block_indent = 0 + current: str | None = None + for raw_line in text.splitlines(): + stripped = raw_line.strip() + if not stripped or stripped.startswith("#"): + continue + indent = len(raw_line) - len(raw_line.lstrip(" ")) + if stripped == "platform_toolsets:": + in_block = True + block_indent = indent + continue + if not in_block: + continue + if indent <= block_indent and not stripped.startswith("- "): + break + key_match = re.fullmatch(r"([A-Za-z0-9_-]+):(?:\s*\[\])?", stripped) + if key_match: + current = key_match.group(1) + toolsets[current] = [] + continue + if stripped.startswith("- ") and current: + toolsets[current].append(unquote(stripped[2:])) + return toolsets + + +env_path = Path("/sandbox/.hermes/.env") +config_path = Path("/sandbox/.hermes/config.yaml") +if env_path.is_symlink(): + print(f"{env_path} is a symlink", file=sys.stderr) + sys.exit(1) +if not env_path.is_file(): + print(f"{env_path} missing", file=sys.stderr) + sys.exit(1) +if not config_path.is_file(): + print(f"{config_path} missing", file=sys.stderr) + sys.exit(1) + +violations = env_violations(env_path) +if violations: + print("raw secret-shaped Hermes .env values:", ", ".join(violations), file=sys.stderr) + sys.exit(1) + +toolsets = parse_platform_toolsets(config_path.read_text(encoding="utf-8")) +api_server_toolsets = set(toolsets.get("api_server", [])) +if not api_server_toolsets: + print("platform_toolsets.api_server missing", file=sys.stderr) + sys.exit(1) +missing = sorted(required_remote_toolsets - api_server_toolsets) +if missing: + print(f"platform_toolsets.api_server missing expected Hermes toolsets: {missing}", file=sys.stderr) + sys.exit(1) +if "no_mcp" in api_server_toolsets: + print("platform_toolsets.api_server unexpectedly disables default MCP servers with no_mcp", file=sys.stderr) + sys.exit(1) +PY + pass "Built Hermes image has no raw secret-shaped .env values and preserves remote toolsets" +} + +inspect_managed_tool_boundary() { + local image="$1" + info "Inspecting Hermes managed-tool gateway boundary in ${image}" + docker run --rm --entrypoint python3 "$image" - <<'PY' +import re +import sys +from pathlib import Path + +secret_key_re = re.compile(r"(^|_)(TOKEN|KEY|SECRET|PASSWORD|CREDENTIAL|API)(_|$)") +slack_alias_re = re.compile(r"^(xoxb|xapp)-OPENSHELL-RESOLVE-ENV-[A-Z0-9_]+$") +allowed_nonsecret_keys = {"API_SERVER_HOST", "API_SERVER_PORT"} +allowed_literals = {"", "[STRIPPED_BY_MIGRATION]"} +required_env_lines = { + "NEMOCLAW_HERMES_TOOL_GATEWAY_BROKER=1", + "FIRECRAWL_GATEWAY_URL=http://host.openshell.internal:11436/firecrawl", + "OPENAI_AUDIO_GATEWAY_URL=http://host.openshell.internal:11436/openai-audio", + "BROWSER_USE_GATEWAY_URL=http://host.openshell.internal:11436/browser-use", + "FAL_QUEUE_GATEWAY_URL=http://host.openshell.internal:11436/fal-queue", + "MODAL_GATEWAY_URL=http://host.openshell.internal:11436/modal", +} +required_config_fragments = [ + "backend: firecrawl", + "provider: openai", + "cloud_provider: browser-use", + "image_gen:", + "backend: modal", + "modal_mode: managed", + "tts:", +] + + +def unquote(value: str) -> str: + value = value.strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in ("'", '"'): + return value[1:-1] + return value + + +def env_violations(path: Path) -> list[str]: + violations: list[str] = [] + for lineno, raw_line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): + stripped = raw_line.strip() + if not stripped or stripped.startswith("#") or "=" not in stripped: + continue + if stripped.startswith("export "): + stripped = stripped[len("export ") :].lstrip() + key, value = stripped.split("=", 1) + key = key.strip() + if key in allowed_nonsecret_keys: + continue + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key): + continue + if not secret_key_re.search(key): + continue + value = unquote(value) + if ( + value in allowed_literals + or value.startswith("openshell:resolve:env:") + or slack_alias_re.fullmatch(value) + ): + continue + violations.append(f"{key} line {lineno}") + return violations + + +env_path = Path("/sandbox/.hermes/.env") +config_path = Path("/sandbox/.hermes/config.yaml") +if not env_path.is_file() or env_path.is_symlink(): + print(f"{env_path} missing, not a file, or unsafe symlink", file=sys.stderr) + sys.exit(1) +if not config_path.is_file(): + print(f"{config_path} missing", file=sys.stderr) + sys.exit(1) + +env_text = env_path.read_text(encoding="utf-8") +config_text = config_path.read_text(encoding="utf-8") +env_lines = set(env_text.splitlines()) +violations = env_violations(env_path) +if violations: + print("raw secret-shaped managed-tool .env values:", ", ".join(violations), file=sys.stderr) + sys.exit(1) + +missing_env = sorted(required_env_lines - env_lines) +missing_config = [fragment for fragment in required_config_fragments if fragment not in config_text] +for forbidden in ( + "TOOL_GATEWAY_USER_TOKEN", + "NEMOCLAW_HERMES_TOOL_GATEWAY_REFRESH_TOKEN=", + "raw-refresh-token", +): + if forbidden in env_text or forbidden in config_text: + print(f"managed-tool sandbox config contains forbidden token surface: {forbidden}", file=sys.stderr) + sys.exit(1) + +if missing_env: + print("managed-tool .env missing expected gateway lines: " + ", ".join(missing_env), file=sys.stderr) + sys.exit(1) +if missing_config: + print("managed-tool config.yaml missing expected fragments: " + ", ".join(missing_config), file=sys.stderr) + sys.exit(1) +PY + pass "Managed-tool Hermes image keeps gateway auth out of sandbox while preserving tool config" +} + +assert_startup_rejects_env_entry() { + local assignment="$1" + local key="$2" + local value="$3" + local quoted_assignment output script + + quoted_assignment="$(shell_quote "$assignment")" + script="set -euo pipefail; printf '%s\n' ${quoted_assignment} >> /sandbox/.hermes/.env; exec /usr/local/bin/nemoclaw-start true" + + info "Verifying Hermes startup rejects ${key}" + if output="$(docker run --rm --user sandbox --entrypoint /bin/bash "$IMAGE" -lc "$script" 2>&1)"; then + printf '%s\n' "$output" + fail "Hermes startup accepted ${key}" + fi + printf '%s\n' "$output" | grep -F "raw secret-shaped values" >/dev/null \ + || fail "Hermes startup rejection did not mention raw secret-shaped values" + printf '%s\n' "$output" | grep -F "$key" >/dev/null \ + || fail "Hermes startup rejection did not name ${key}" + if printf '%s\n' "$output" | grep -F "$value" >/dev/null; then + fail "Hermes startup rejection printed the raw value for ${key}" + fi + pass "Hermes startup rejects ${key} without echoing its value" +} + +assert_startup_rejects_runtime_env_entry() { + local assignment="$1" + local key="$2" + local value="$3" + local output + + info "Verifying Hermes startup rejects runtime env ${key}" + if output="$(docker run --rm --user sandbox --env "$assignment" --entrypoint /usr/local/bin/nemoclaw-start "$IMAGE" true 2>&1)"; then + printf '%s\n' "$output" + fail "Hermes startup accepted runtime env ${key}" + fi + printf '%s\n' "$output" | grep -F "process environment" >/dev/null \ + || fail "Hermes startup rejection did not mention process environment" + printf '%s\n' "$output" | grep -F "$key" >/dev/null \ + || fail "Hermes startup rejection did not name ${key}" + if printf '%s\n' "$output" | grep -F "$value" >/dev/null; then + fail "Hermes startup rejection printed the raw value for runtime env ${key}" + fi + pass "Hermes startup rejects runtime env ${key} without echoing its value" +} + +require_docker +build_image_if_needed +docker image inspect "$IMAGE" >/dev/null 2>&1 || fail "image not found: ${IMAGE}" +build_managed_image_if_needed +docker image inspect "$MANAGED_IMAGE" >/dev/null 2>&1 || fail "image not found: ${MANAGED_IMAGE}" + +inspect_image_boundary "$IMAGE" +inspect_managed_tool_boundary "$MANAGED_IMAGE" +RAW_SECRET_SENTINEL="SENTINEL_RAW_SECRET_VALUE" +assert_startup_rejects_env_entry \ + "DEVTEST_API_TOKEN=${RAW_SECRET_SENTINEL}" \ + "DEVTEST_API_TOKEN" \ + "$RAW_SECRET_SENTINEL" +assert_startup_rejects_env_entry \ + "INTERNAL_API=${RAW_SECRET_SENTINEL}" \ + "INTERNAL_API" \ + "$RAW_SECRET_SENTINEL" +assert_startup_rejects_env_entry \ + "OPENAI_API_KEY=sk-OPENSHELL-PROXY-REWRITE" \ + "OPENAI_API_KEY" \ + "sk-OPENSHELL-PROXY-REWRITE" +assert_startup_rejects_runtime_env_entry \ + "DEVTEST_API_TOKEN=${RAW_SECRET_SENTINEL}" \ + "DEVTEST_API_TOKEN" \ + "$RAW_SECRET_SENTINEL" +assert_startup_rejects_runtime_env_entry \ + "NEMOCLAW_HERMES_TOOL_GATEWAY_REFRESH_TOKEN=raw-refresh-token" \ + "NEMOCLAW_HERMES_TOOL_GATEWAY_REFRESH_TOKEN" \ + "raw-refresh-token" + +pass "Hermes sandbox secret-boundary smoke passed" diff --git a/test/e2e-vpn/test-hermes-slack-e2e.sh b/test/e2e-vpn/test-hermes-slack-e2e.sh new file mode 100755 index 00000000000..7f8e96359dc --- /dev/null +++ b/test/e2e-vpn/test-hermes-slack-e2e.sh @@ -0,0 +1,663 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Hermes Slack E2E: onboard --agent hermes with Slack enabled, then verify +# the Hermes sandbox keeps the Hermes-specific Slack policy and can reach the +# Slack API through the Python/OpenShell placeholder path. +# +# Uses fake Slack tokens by default. Fake tokens should appear only where the +# sandbox runtime needs them for OpenShell env resolution, not in Hermes config +# files, logs, or process arguments. +# +# Environment variables: +# NEMOCLAW_NON_INTERACTIVE=1 - required +# NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 - required +# NEMOCLAW_AGENT=hermes - auto-set if not already set +# NEMOCLAW_POLICY_TIER=open - auto-set if not already set +# NEMOCLAW_SANDBOX_NAME - sandbox name (default: e2e-hermes-slack) +# NEMOCLAW_RECREATE_SANDBOX=1 - auto-set +# NVIDIA_API_KEY - required for Hermes onboarding +# SLACK_BOT_TOKEN - defaults to a fake xoxb- token +# SLACK_APP_TOKEN - defaults to a fake xapp- token +# +# Usage: +# NEMOCLAW_NON_INTERACTIVE=1 NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ +# NVIDIA_API_KEY=... bash test/e2e-vpn/test-hermes-slack-e2e.sh + +set -uo pipefail + +PASS=0 +FAIL=0 +SKIP=0 +TOTAL=0 + +pass() { + ((PASS++)) + ((TOTAL++)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} +fail() { + ((FAIL++)) + ((TOTAL++)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} +skip() { + ((SKIP++)) + ((TOTAL++)) + printf '\033[33m SKIP: %s\033[0m\n' "$1" +} +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } +is_fake_slack_token() { + case "${1:-}" in + xoxb-fake-* | xoxb-test-* | xapp-fake-* | xapp-test-*) return 0 ;; + *) return 1 ;; + esac +} + +run_with_timeout() { + local seconds="$1" + shift + if command -v timeout >/dev/null 2>&1; then + timeout "$seconds" "$@" + elif command -v gtimeout >/dev/null 2>&1; then + gtimeout "$seconds" "$@" + else + "$@" + fi +} + +dump_hermes_slack_diagnostics() { + info "--- Hermes Slack sandbox diagnostics ---" + if ! command -v openshell >/dev/null 2>&1; then + info "openshell is not available for sandbox diagnostics" + return + fi + + local sandboxes diag_output diag_script + sandboxes=$(openshell sandbox list 2>&1 || true) + info "openshell sandbox list:" + echo "$sandboxes" | tail -20 | while IFS= read -r line; do + info " $line" + done + + if ! grep -Fq -- "$SANDBOX_NAME" <<<"$sandboxes"; then + info "sandbox '${SANDBOX_NAME}' is not visible to openshell" + return + fi + + diag_script='set +e' + diag_script+='; echo "== hermes config =="; sed -n "1,120p" /sandbox/.hermes/config.yaml 2>&1 || true' + diag_script+='; echo "== hermes env keys =="; cut -d= -f1 /sandbox/.hermes/.env 2>&1 || true' + diag_script+='; echo "== hermes health =="; curl -sf http://localhost:8642/health 2>&1 || true' + diag_script+='; echo "== hermes-related processes =="' + # shellcheck disable=SC2016 + diag_script+='; for p in /proc/[0-9]*; do cmd=$(tr "\000" " " < "$p/cmdline" 2>/dev/null || true); case "$cmd" in *hermes*|*socat*) echo "$(basename "$p") $cmd" ;; esac; done' + diag_script+='; echo "== /tmp/nemoclaw-start.log tail =="; tail -n 80 /tmp/nemoclaw-start.log 2>&1 || true' + diag_script+='; echo "== /tmp/gateway.log tail =="; tail -n 120 /tmp/gateway.log 2>&1 || true' + diag_output=$(openshell sandbox exec -n "$SANDBOX_NAME" -- sh -lc "$diag_script" 2>&1 || true) + + echo "$diag_output" | while IFS= read -r line; do + info " $line" + done + info "--- End Hermes Slack diagnostics ---" +} + +sandbox_exec() { + local cmd="$1" + local ssh_config + ssh_config="$(mktemp)" + openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config" 2>/dev/null + + local result + result=$(run_with_timeout 60 ssh -F "$ssh_config" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" \ + "$cmd" \ + 2>&1) || true + + rm -f "$ssh_config" + echo "$result" +} + +sandbox_exec_stdin() { + local cmd="$1" + local ssh_config + ssh_config="$(mktemp)" + openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config" 2>/dev/null + + local result + result=$(run_with_timeout 60 ssh -F "$ssh_config" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" \ + "$cmd" \ + 2>/dev/null) || true + + rm -f "$ssh_config" + echo "$result" +} + +if [ -d /workspace ] && [ -f /workspace/install.sh ]; then + REPO="/workspace" +elif [ -f "$(cd "$(dirname "$0")/../.." && pwd)/install.sh" ]; then + REPO="$(cd "$(dirname "$0")/../.." && pwd)" +else + echo "ERROR: Cannot find repo root." + exit 1 +fi + +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-hermes-slack}" +SLACK_BOT="${SLACK_BOT_TOKEN:-xoxb-test-hermes-slack-token}" +SLACK_APP="${SLACK_APP_TOKEN:-xapp-test-hermes-slack-app-token}" +export NEMOCLAW_AGENT="${NEMOCLAW_AGENT:-hermes}" +export NEMOCLAW_POLICY_TIER="${NEMOCLAW_POLICY_TIER:-open}" +export NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" +export NEMOCLAW_RECREATE_SANDBOX=1 +export SLACK_BOT_TOKEN="$SLACK_BOT" +export SLACK_APP_TOKEN="$SLACK_APP" +if [ -z "${NEMOCLAW_SKIP_SLACK_AUTH_VALIDATION:-}" ] \ + && { is_fake_slack_token "$SLACK_BOT" || is_fake_slack_token "$SLACK_APP"; }; then + export NEMOCLAW_SKIP_SLACK_AUTH_VALIDATION=1 + info "Skipping onboarding Slack auth validation for fake-token E2E" +fi + +# shellcheck source=test/e2e-vpn/lib/sandbox-teardown.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib/sandbox-teardown.sh" +# shellcheck source=test/e2e-vpn/lib/ci-compatible-inference.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib/ci-compatible-inference.sh" +register_sandbox_for_teardown "$SANDBOX_NAME" +nemoclaw_e2e_configure_compatible_inference || exit 1 + +section "Phase 0: Prerequisites" + +if docker info >/dev/null 2>&1; then + pass "Docker is running" +else + fail "Docker is not running" + exit 1 +fi + +if ! nemoclaw_e2e_require_hosted_inference_key; then + exit 1 +fi + +if [ "${NEMOCLAW_NON_INTERACTIVE:-}" = "1" ]; then + pass "NEMOCLAW_NON_INTERACTIVE=1" +else + fail "NEMOCLAW_NON_INTERACTIVE=1 is required" + exit 1 +fi + +if [ "${NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE:-}" = "1" ]; then + pass "NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1" +else + fail "NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 is required" + exit 1 +fi + +info "Sandbox name: $SANDBOX_NAME" +info "Agent: $NEMOCLAW_AGENT" +info "Policy tier: $NEMOCLAW_POLICY_TIER" + +section "Phase 1: Install NemoClaw with Hermes Slack" + +cd "$REPO" || { + fail "Could not cd to repo root: $REPO" + exit 1 +} + +info "Pre-cleanup..." +if command -v nemoclaw >/dev/null 2>&1; then + nemoclaw "$SANDBOX_NAME" destroy --yes 2>/dev/null || true +fi +if command -v openshell >/dev/null 2>&1; then + openshell sandbox delete "$SANDBOX_NAME" 2>/dev/null || true + openshell provider delete "${SANDBOX_NAME}-slack-bridge" 2>/dev/null || true + openshell provider delete "${SANDBOX_NAME}-slack-app" 2>/dev/null || true + openshell gateway destroy -g nemoclaw 2>/dev/null || true +fi +pass "Pre-cleanup complete" + +INSTALL_LOG="/tmp/nemoclaw-e2e-hermes-slack-install.log" +info "Running install.sh --non-interactive with NEMOCLAW_AGENT=hermes and Slack enabled..." +bash install.sh --non-interactive >"$INSTALL_LOG" 2>&1 & +install_pid=$! +tail -f "$INSTALL_LOG" --pid=$install_pid 2>/dev/null & +tail_pid=$! +wait $install_pid +install_exit=$? +kill $tail_pid 2>/dev/null || true +wait $tail_pid 2>/dev/null || true + +if [ -f "$HOME/.bashrc" ]; then + # shellcheck source=/dev/null + source "$HOME/.bashrc" 2>/dev/null || true +fi +export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" +if [ -s "$NVM_DIR/nvm.sh" ]; then + # shellcheck source=/dev/null + . "$NVM_DIR/nvm.sh" +fi +if [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]]; then + export PATH="$HOME/.local/bin:$PATH" +fi + +if [ $install_exit -eq 0 ]; then + pass "install.sh completed (exit 0)" +else + fail "install.sh failed (exit $install_exit)" + info "Last 40 lines of install log:" + tail -40 "$INSTALL_LOG" 2>/dev/null || true + dump_hermes_slack_diagnostics + exit 1 +fi + +if command -v nemoclaw >/dev/null 2>&1; then + pass "nemoclaw installed at $(command -v nemoclaw)" +else + fail "nemoclaw not found on PATH after install" + exit 1 +fi + +if command -v openshell >/dev/null 2>&1; then + pass "openshell installed ($(openshell --version 2>&1 || echo unknown))" +else + fail "openshell not found on PATH after install" + exit 1 +fi + +section "Phase 2: Hermes sandbox and Slack providers" + +if list_output=$(nemoclaw list 2>&1); then + if grep -Fq -- "$SANDBOX_NAME" <<<"$list_output"; then + pass "nemoclaw list contains '${SANDBOX_NAME}'" + else + fail "nemoclaw list does not contain '${SANDBOX_NAME}'" + fi +else + fail "nemoclaw list failed: ${list_output:0:200}" +fi + +if openshell provider get "${SANDBOX_NAME}-slack-bridge" >/dev/null 2>&1; then + pass "Slack bot provider '${SANDBOX_NAME}-slack-bridge' exists in gateway" +else + fail "Slack bot provider '${SANDBOX_NAME}-slack-bridge' not found in gateway" +fi + +if openshell provider get "${SANDBOX_NAME}-slack-app" >/dev/null 2>&1; then + pass "Slack app provider '${SANDBOX_NAME}-slack-app' exists in gateway" +else + fail "Slack app provider '${SANDBOX_NAME}-slack-app' not found in gateway" +fi + +section "Phase 3: Hermes health" + +hermes_healthy=false +health_response="" +for attempt in $(seq 1 15); do + health_response=$(sandbox_exec "curl -sf http://localhost:8642/health") + if echo "$health_response" | grep -qi '"ok"'; then + hermes_healthy=true + break + fi + info "Health check attempt ${attempt}/15 - waiting 4s..." + sleep 4 +done + +if $hermes_healthy; then + pass "Hermes health probe returned ok with Slack enabled" +else + fail "Hermes health probe did not return ok after 15 attempts" + info "Last response: ${health_response:0:200}" + dump_hermes_slack_diagnostics +fi + +section "Phase 4: Hermes Slack config shape" + +config_probe=$( + sandbox_exec_stdin "python3 -" <<'PY' +import sys +from pathlib import Path +try: + import yaml +except Exception as exc: + print(f"FAIL cannot import yaml: {exc}") + sys.exit(0) + +config_text = Path("/sandbox/.hermes/config.yaml").read_text(encoding="utf-8") +cfg = yaml.safe_load(config_text) or {} +errors = [] +platforms = cfg.get("platforms") +if not isinstance(platforms, dict): + errors.append("platforms map missing or not a mapping") +else: + slack = platforms.get("slack") + if not isinstance(slack, dict): + errors.append("platforms.slack missing or not a mapping") + elif slack.get("enabled") is not True: + errors.append(f"platforms.slack.enabled is not true ({slack!r})") +if "SLACK_BOT_TOKEN" in config_text or "SLACK_APP_TOKEN" in config_text: + errors.append("config.yaml contains Slack token env keys") +if errors: + print("FAIL " + "; ".join(errors)) +else: + print("OK") +PY +) + +if [ "$config_probe" = "OK" ]; then + pass "config.yaml enables platforms.slack and contains no Slack token keys" +else + fail "config.yaml check failed: ${config_probe:0:400}" +fi + +env_probe=$( + sandbox_exec_stdin "python3 -" <<'PY' +from pathlib import Path +text = Path("/sandbox/.hermes/.env").read_text(encoding="utf-8") +lines = set(text.splitlines()) +required = { + "SLACK_BOT_TOKEN=xoxb-OPENSHELL-RESOLVE-ENV-SLACK_BOT_TOKEN", + "SLACK_APP_TOKEN=xapp-OPENSHELL-RESOLVE-ENV-SLACK_APP_TOKEN", + "API_SERVER_PORT=18642", +} +missing = sorted(required - lines) +if missing: + print("FAIL missing " + ", ".join(missing)) +else: + print("OK") +PY +) + +if [ "$env_probe" = "OK" ]; then + pass ".hermes/.env contains Slack SDK-shaped resolver placeholders" +else + fail ".hermes/.env check failed: ${env_probe:0:400}" +fi + +secret_boundary_probe=$( + sandbox_exec_stdin "python3 -" <<'PY' +import re +from pathlib import Path + +secret_key_re = re.compile(r"(^|_)(TOKEN|KEY|SECRET|PASSWORD|CREDENTIAL|API)(_|$)") +slack_alias_re = re.compile(r"^(xoxb|xapp)-OPENSHELL-RESOLVE-ENV-[A-Z0-9_]+$") +allowed_nonsecret_keys = {"API_SERVER_HOST", "API_SERVER_PORT"} +allowed_literals = {"", "[STRIPPED_BY_MIGRATION]"} +env_path = Path("/sandbox/.hermes/.env") + + +def unquote(value: str) -> str: + value = value.strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in ("'", '"'): + return value[1:-1] + return value + + +if env_path.is_symlink(): + print("FAIL .hermes/.env is a symlink") + raise SystemExit +if not env_path.is_file(): + print("FAIL .hermes/.env missing") + raise SystemExit + +violations = [] +for lineno, raw_line in enumerate(env_path.read_text(encoding="utf-8").splitlines(), 1): + stripped = raw_line.strip() + if not stripped or stripped.startswith("#") or "=" not in stripped: + continue + if stripped.startswith("export "): + stripped = stripped[len("export ") :].lstrip() + key, value = stripped.split("=", 1) + key = key.strip() + if key in allowed_nonsecret_keys: + continue + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", key): + continue + if not secret_key_re.search(key): + continue + value = unquote(value) + if ( + value in allowed_literals + or value.startswith("openshell:resolve:env:") + or slack_alias_re.fullmatch(value) + ): + continue + violations.append(f"{key} line {lineno}") + +if violations: + print("FAIL raw secret-shaped Hermes .env values: " + ", ".join(violations)) +else: + print("OK") +PY +) + +if [ "$secret_boundary_probe" = "OK" ]; then + pass "Hermes Slack .env contains only resolver placeholders for secret-shaped keys" +else + fail "Hermes Slack secret-boundary scan failed: ${secret_boundary_probe:0:400}" +fi + +token_file_hits=$(printf '%s\n%s\n' "$SLACK_BOT" "$SLACK_APP" | sandbox_exec_stdin 'grep -Fq -f - /sandbox/.hermes/config.yaml /sandbox/.hermes/.env /tmp/nemoclaw-start.log /tmp/gateway.log 2>/dev/null && echo LEAK || echo OK') +if [ "$token_file_hits" = "OK" ]; then + pass "Raw Slack tokens absent from Hermes config files and logs" +else + fail "Raw Slack token found in Hermes config files or logs" +fi + +sandbox_ps=$(sandbox_exec 'cat /proc/[0-9]*/cmdline 2>/dev/null | tr "\0" "\n"') +if [ -z "$sandbox_ps" ]; then + skip "Sandbox process list is empty" +elif echo "$sandbox_ps" | grep -qF "$SLACK_BOT" || echo "$sandbox_ps" | grep -qF "$SLACK_APP"; then + fail "Raw Slack token found in sandbox process list" +else + pass "Raw Slack tokens absent from sandbox process list" +fi + +section "Phase 5: Hermes Slack policy" + +if policy_output=$(openshell policy get --full "$SANDBOX_NAME" 2>&1); then + slack_block=$(awk ' + /^ slack:/ { in_slack = 1; print; next } + in_slack && /^ [A-Za-z0-9_-]+:/ { exit } + in_slack { print } + ' <<<"$policy_output") + + if [ -n "$slack_block" ]; then + pass "Sandbox policy contains Slack network policy" + else + fail "Sandbox policy missing Slack network policy" + fi + + if echo "$slack_block" | grep -Fq "/usr/local/bin/hermes" \ + && echo "$slack_block" | grep -Fq "/usr/bin/python3*" \ + && echo "$slack_block" | grep -Fq "/opt/hermes/.venv/bin/python"; then + pass "Slack policy is scoped to Hermes and Python binaries" + else + fail "Slack policy missing Hermes/Python binary allowlist" + fi + + if echo "$slack_block" | grep -Fq "/usr/local/bin/node" \ + || echo "$slack_block" | grep -Fq "/usr/bin/node"; then + fail "Slack policy was replaced by or widened to Node" + else + pass "Slack policy does not allow Node" + fi + + if echo "$slack_block" | grep -Fq "wss-primary.slack.com" \ + && echo "$slack_block" | grep -Fq "wss-backup.slack.com"; then + pass "Slack policy includes Socket Mode websocket hosts" + else + fail "Slack policy missing Socket Mode websocket hosts" + fi + + if echo "$slack_block" | grep -Fq "request_body_credential_rewrite: true"; then + pass "Slack REST policy enables OpenShell request-body credential rewrite" + else + fail "Slack policy missing request_body_credential_rewrite for REST alias rewrite" + fi +else + fail "openshell policy get failed: ${policy_output:0:200}" +fi + +# shellcheck disable=SC2016 +bridge_residue=$(sandbox_exec 'set +e +decode_needle="$(printf "%s%s%s" "nemoclaw-" "decode" "-proxy")" +preload_needle="$(printf "%s" "/opt/nemoclaw-hermes-discord-preload")" +if env | grep -Fq "$preload_needle"; then echo ENV_PYTHON_PRELOAD; fi +if grep -Fq "$preload_needle" /tmp/nemoclaw-proxy-env.sh /sandbox/.hermes/.env /sandbox/.hermes/config.yaml 2>/dev/null; then echo FILE_PYTHON_PRELOAD; fi +if command -v "$decode_needle" >/dev/null 2>&1; then echo BIN_DECODE_PROXY; fi +current_pid="$$" +for p in /proc/[0-9]*; do + pid=$(basename "$p") + [ "$pid" = "$current_pid" ] && continue + cmd=$(tr "\000" " " < "$p/cmdline" 2>/dev/null || true) + case "$cmd" in *"$decode_needle"*) echo PROCESS_DECODE_PROXY ;; esac +done') +if [ -z "$bridge_residue" ]; then + pass "Hermes Slack sandbox has no decode proxy or Python placeholder-normalization preload" +else + fail "Hermes Slack bridge residue found: ${bridge_residue:0:300}" + dump_hermes_slack_diagnostics +fi + +section "Phase 6: Slack alias egress from Python" + +slack_probe=$( + sandbox_exec_stdin 'sh -lc ". /tmp/nemoclaw-proxy-env.sh 2>/dev/null || true; if [ -x /opt/hermes/.venv/bin/python ]; then exec /opt/hermes/.venv/bin/python -; fi; exec python3 -" 2>&1' <<'PY' +import json +import http.client +import socket +import ssl +import sys +import urllib.error +import urllib.request + +TLS_CONTEXT = ssl._create_unverified_context() + +def call(label, path, env_key, allowed_errors): + prefix = { + "SLACK_BOT_TOKEN": "xoxb", + "SLACK_APP_TOKEN": "xapp", + }[env_key] + token = f"{prefix}-OPENSHELL-RESOLVE-ENV-{env_key}" + req = urllib.request.Request( + f"https://slack.com/api/{path}", + data=b"", + method="POST", + headers={ + "Authorization": f"Bearer {token}", + "Content-Type": "application/x-www-form-urlencoded", + }, + ) + try: + # The assertion here is placeholder substitution + Slack egress. CA + # wiring is covered separately by proxy-env tests and can vary by + # OpenShell proxy runner, so this probe does not make TLS trust the + # signal. + with urllib.request.urlopen(req, timeout=30, context=TLS_CONTEXT) as resp: + status = resp.status + body = resp.read().decode("utf-8", errors="replace") + except socket.timeout: + print(f"TIMEOUT {label}") + return False + except urllib.error.URLError as exc: + reason = str(getattr(exc, "reason", exc)) + if "timed out" in reason.lower(): + print(f"TIMEOUT {label}: {reason}") + return False + print(f"ERROR {label}: {reason}") + return False + except Exception as exc: + reason = f"{type(exc).__name__}: {exc}" + if isinstance(exc, http.client.RemoteDisconnected) or "timed out" in reason.lower(): + print(f"TIMEOUT {label}: {reason}") + return False + print(f"ERROR {label}: {reason}") + return False + + print(json.dumps({"label": label, "status": status, "body": body[:300]})) + try: + parsed = json.loads(body) + except Exception as exc: + print(f"FAIL {label}: non-json body {exc}") + return False + error = parsed.get("error") + if status == 200 and (parsed.get("ok") is True or error in allowed_errors): + print(f"OK {label}: {error or 'ok'}") + return True + print(f"FAIL {label}: status={status} error={error!r}") + return False + +ok = True +ok = call("auth.test", "auth.test", "SLACK_BOT_TOKEN", {"invalid_auth", "not_authed"}) and ok +ok = call( + "apps.connections.open", + "apps.connections.open", + "SLACK_APP_TOKEN", + {"invalid_auth", "not_authed", "not_allowed_token_type"}, +) and ok +sys.exit(0 if ok else 2) +PY +) + +info "Slack Python probe response: ${slack_probe:0:500}" +if echo "$slack_probe" | grep -q "^OK auth.test:" \ + && echo "$slack_probe" | grep -q "^OK apps.connections.open:"; then + pass "Slack API reached from Python through OpenShell alias substitution" +elif echo "$slack_probe" | grep -q "^TIMEOUT"; then + skip "Slack API timed out" +elif echo "$slack_probe" | grep -qE "^(FAIL|ERROR)"; then + fail "Slack Python API probe failed: ${slack_probe:0:400}" + dump_hermes_slack_diagnostics +else + fail "Unexpected Slack Python API response: ${slack_probe:0:400}" +fi + +section "Phase 7: Cleanup" + +if [[ "${NEMOCLAW_E2E_KEEP_SANDBOX:-}" != "1" ]]; then + nemoclaw "$SANDBOX_NAME" destroy --yes 2>&1 | tail -3 || true + openshell gateway destroy -g nemoclaw 2>/dev/null || true +fi + +registry_file="${HOME}/.nemoclaw/sandboxes.json" +if [ -f "$registry_file" ] && grep -Fq "\"${SANDBOX_NAME}\"" "$registry_file"; then + fail "Sandbox ${SANDBOX_NAME} still in registry after destroy" +else + pass "Sandbox ${SANDBOX_NAME} removed" +fi + +if openshell provider get "${SANDBOX_NAME}-slack-app" >/dev/null 2>&1; then + fail "Slack app provider still exists after destroy" + openshell provider delete "${SANDBOX_NAME}-slack-app" 2>/dev/null || true +else + pass "Slack app provider removed" +fi + +echo "" +echo "========================================" +echo " Hermes Slack E2E Results:" +echo " Passed: $PASS" +echo " Failed: $FAIL" +echo " Skipped: $SKIP" +echo " Total: $TOTAL" +echo "========================================" + +if [ "$FAIL" -eq 0 ]; then + printf '\n\033[1;32m Hermes Slack E2E PASSED - policy, placeholder, provider, and sandbox boot verified.\033[0m\n' + exit 0 +else + printf '\n\033[1;31m %d test(s) failed.\033[0m\n' "$FAIL" + exit 1 +fi diff --git a/test/e2e-vpn/test-inference-routing.sh b/test/e2e-vpn/test-inference-routing.sh new file mode 100755 index 00000000000..a34013c63e0 --- /dev/null +++ b/test/e2e-vpn/test-inference-routing.sh @@ -0,0 +1,716 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# ============================================================================= +# test-inference-routing.sh +# NemoClaw Inference Routing E2E Tests +# +# Validates inference routing through the OpenShell gateway proxy for +# multiple providers, credential isolation, and error classification. +# +# Covers: +# TC-INF-02: OpenAI provider end-to-end inference (requires OPENAI_API_KEY) +# TC-INF-03: Anthropic provider end-to-end inference (requires ANTHROPIC_API_KEY) +# TC-INF-05: Credential isolation inside sandbox (requires NVIDIA_API_KEY) +# TC-INF-06: Invalid API key → classified "credential" error (PR-safe) +# TC-INF-07: Unreachable endpoint → classified "transport" error (PR-safe) +# TC-INF-09: Custom OpenAI-compatible endpoint (requires NEMOCLAW_ENDPOINT_URL + COMPATIBLE_API_KEY) +# +# TC-INF-06 and TC-INF-07 are PR-safe (no real API keys needed). +# TC-INF-02, TC-INF-03, TC-INF-05, TC-INF-09 skip gracefully when +# their required API keys are not set. +# +# Prerequisites: +# - NemoClaw installed (nemoclaw on PATH) +# - Docker running +# - openshell on PATH +# ============================================================================= + +set -euo pipefail + +# ── Overall timeout ────────────────────────────────────────────────────────── +export NEMOCLAW_E2E_DEFAULT_TIMEOUT=1200 +SCRIPT_DIR_TIMEOUT="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +# shellcheck source=test/e2e-vpn/e2e-timeout.sh +source "${SCRIPT_DIR_TIMEOUT}/e2e-timeout.sh" + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +CYAN='\033[0;36m' +NC='\033[0m' + +PASS=0 +FAIL=0 +SKIP=0 +TOTAL=0 + +LOG_FILE="test-inference-routing-$(date +%Y%m%d-%H%M%S).log" + +# Safe literal string replacement for redacting secrets in log output. +redact_stream() { + local secret="${1:-}" + SECRET_TO_REDACT="$secret" python3 -c ' +import os, sys +secret = os.environ.get("SECRET_TO_REDACT", "") +data = sys.stdin.read() +sys.stdout.write(data.replace(secret, "REDACTED") if secret else data) +' +} + +# Log a timestamped message to stdout and the log file. +log() { echo -e "${CYAN}[$(date +%H:%M:%S)]${NC} $*" | tee -a "$LOG_FILE"; } +# Record a passing test assertion. +pass() { + ((PASS += 1)) + ((TOTAL += 1)) + echo -e "${GREEN} PASS${NC} $1" | tee -a "$LOG_FILE" +} +# Record a failing test assertion with a reason. +fail() { + ((FAIL += 1)) + ((TOTAL += 1)) + echo -e "${RED} FAIL${NC} $1 — $2" | tee -a "$LOG_FILE" +} +# Record a skipped test with a reason. +skip() { + ((SKIP += 1)) + ((TOTAL += 1)) + echo -e "${YELLOW} SKIP${NC} $1 — $2" | tee -a "$LOG_FILE" +} + +# ── Resolve repo root ──────────────────────────────────────────────────────── +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +if [ -f "$SCRIPT_DIR/../../install.sh" ]; then + REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +elif [ -f "./install.sh" ]; then + REPO_ROOT="$(pwd)" +else + echo "ERROR: Cannot find install.sh — run from the repo root or test/e2e-vpn/" + exit 1 +fi + +# ── Install NemoClaw if not present ────────────────────────────────────────── +install_nemoclaw() { + if command -v nemoclaw &>/dev/null; then + log "nemoclaw already installed: $(nemoclaw --version 2>/dev/null || echo 'unknown')" + return 0 + fi + + log "=== Installing NemoClaw via install.sh ===" + + # Use a dummy key so install.sh doesn't prompt — the key will fail + # validation, but install.sh only needs it for the onboard step which + # we control separately in each test case. + NVIDIA_API_KEY="nvapi-DUMMY-FOR-INSTALL" \ + NEMOCLAW_NON_INTERACTIVE=1 \ + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ + bash "$REPO_ROOT/install.sh" --non-interactive --yes-i-accept-third-party-software \ + 2>&1 | tee -a "$LOG_FILE" || true + + # Source shell profile to pick up PATH changes + if [ -f "$HOME/.bashrc" ]; then + # shellcheck source=/dev/null + source "$HOME/.bashrc" 2>/dev/null || true + fi + export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" + if [ -s "$NVM_DIR/nvm.sh" ]; then + # shellcheck source=/dev/null + . "$NVM_DIR/nvm.sh" + fi + if [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]]; then + export PATH="$HOME/.local/bin:$PATH" + fi + + # Install may fail at onboard (bad key) but CLI should still be available + if ! command -v nemoclaw &>/dev/null; then + echo -e "${RED}FATAL: nemoclaw not found on PATH after install${NC}" + exit 1 + fi + + log "nemoclaw installed: $(nemoclaw --version 2>/dev/null || echo 'unknown')" + + # Clean up any sandbox the installer might have partially created + rm -f "$HOME/.nemoclaw/onboard.lock" 2>/dev/null || true +} + +# ── Pre-flight ─────────────────────────────────────────────────────────────── +preflight() { + log "=== Pre-flight checks ===" + + if ! docker info &>/dev/null; then + echo -e "${RED}ERROR: Docker is not running.${NC}" + exit 1 + fi + log "Docker is running" + + install_nemoclaw + + log "nemoclaw: $(nemoclaw --version 2>/dev/null || echo 'unknown')" + log "timeout: $TIMEOUT_CMD" + log "Pre-flight complete" + echo "" +} + +# ── Sandbox helpers ─────────────────────────────────────────────────────────── +SANDBOX_NAME="e2e-inf-cred" + +# Execute a command inside the sandbox via nemoclaw connect. +sandbox_exec() { + local cmd="$1" + local ssh_cfg + ssh_cfg="$(mktemp)" + if ! openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_cfg" 2>/dev/null; then + log " [sandbox_exec] Failed to get SSH config" + rm -f "$ssh_cfg" + echo "" + return 1 + fi + local result ssh_exit=0 + result=$(run_with_timeout 60 ssh -F "$ssh_cfg" \ + -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" "$cmd" 2>&1) || ssh_exit=$? + rm -f "$ssh_cfg" + if [[ $ssh_exit -ne 0 ]]; then + log " [sandbox_exec] SSH command failed (exit $ssh_exit)" + fi + echo "$result" + return $ssh_exit +} + +# ============================================================================= +# TC-INF-05: Credential not visible inside sandbox +# ============================================================================= +test_inf_05_credential_isolation() { + log "=== TC-INF-05: Credential Isolation ===" + + # Determine the real API key to search for + local real_key="${NVIDIA_API_KEY:-}" + if [[ -z "$real_key" ]]; then + skip "TC-INF-05" "NVIDIA_API_KEY not set — cannot test credential isolation" + return + fi + + # Always recreate to avoid stale state hiding credential plumbing regressions. + # Unconditional destroy catches not-ready sandboxes that `nemoclaw list` misses. + log " Preflight: destroying any existing '$SANDBOX_NAME' sandbox..." + nemoclaw "$SANDBOX_NAME" destroy --yes 2>/dev/null || true + + log " Onboarding sandbox '$SANDBOX_NAME' for credential test..." + rm -f "$HOME/.nemoclaw/onboard.lock" 2>/dev/null || true + local onboard_exit=0 + NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" \ + NEMOCLAW_NON_INTERACTIVE=1 \ + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ + NEMOCLAW_POLICY_TIER="open" \ + nemoclaw onboard --non-interactive --yes-i-accept-third-party-software \ + 2>&1 | redact_stream "$real_key" | tee -a "$LOG_FILE" || onboard_exit=$? + if [[ $onboard_exit -ne 0 ]]; then + fail "TC-INF-05: Setup" "Onboard failed (exit $onboard_exit)" + return + fi + + # Capture sandbox environment and process list once + log " Capturing sandbox environment..." + local sandbox_env + sandbox_env=$(sandbox_exec "env 2>/dev/null") || true + if [[ -z "$sandbox_env" ]]; then + fail "TC-INF-05: Setup" "Could not capture sandbox environment (SSH failure)" + return + fi + + log " Capturing sandbox process list..." + local sandbox_ps ps_exit=0 + sandbox_ps=$(sandbox_exec "ps aux 2>/dev/null || ps -ef 2>/dev/null") || ps_exit=$? + + # TC-INF-05a: Real API key not in environment variables + if echo "$sandbox_env" | grep -qF "$real_key"; then + fail "TC-INF-05a: Env vars" "Real API key found in sandbox environment" + else + pass "TC-INF-05a: Real API key absent from sandbox environment" + fi + + # TC-INF-05b: Real API key not in process list + if [[ $ps_exit -ne 0 || -z "$sandbox_ps" ]]; then + skip "TC-INF-05b: Process list" "ps not available in hardened sandbox" + elif echo "$sandbox_ps" | grep -qF "$real_key"; then + fail "TC-INF-05b: Process list" "Real API key found in sandbox process list" + else + pass "TC-INF-05b: Real API key absent from sandbox process list" + fi + + # TC-INF-05c: Real API key not on filesystem + # Pass key via base64 to avoid shell escaping issues and command-line exposure + log " Scanning sandbox filesystem..." + local key_b64 + key_b64=$(printf '%s' "$real_key" | base64 | tr -d '\n') + local fs_scan + fs_scan=$(sandbox_exec "node -e \" +const fs = require('fs'); +const { execSync } = require('child_process'); +const key = Buffer.from('$key_b64', 'base64').toString('utf8'); +if (!key) { console.log('NO_KEY_PROVIDED'); process.exit(0); } +try { + const out = execSync('find /sandbox /home /tmp -type f -size -1M 2>/dev/null | head -200', { encoding: 'utf8' }); + const files = out.trim().split('\\n').filter(Boolean); + for (const f of files) { + try { + const content = fs.readFileSync(f, 'utf8'); + if (content.includes(key)) { console.log('FOUND:' + f); } + } catch {} + } + console.log('SCAN_DONE'); +} catch { console.log('SCAN_ERROR'); } +\"") || true + + if echo "$fs_scan" | grep -q "FOUND:"; then + local found_files + found_files=$(echo "$fs_scan" | grep "FOUND:" | sed 's/FOUND://') + fail "TC-INF-05c: Filesystem" "Real API key found in: $found_files" + elif echo "$fs_scan" | grep -q "NO_KEY_PROVIDED"; then + fail "TC-INF-05c: Filesystem" "Key was not passed to the scanner" + elif echo "$fs_scan" | grep -q "SCAN_DONE"; then + pass "TC-INF-05c: Real API key absent from sandbox filesystem" + else + fail "TC-INF-05c: Filesystem" "Scan failed: ${fs_scan:0:200}" + fi + + # TC-INF-05d: Placeholder token IS present in environment + local placeholder + placeholder=$(sandbox_exec "printenv NVIDIA_API_KEY 2>/dev/null || true") || true + if [[ -n "$placeholder" && "$placeholder" != "$real_key" ]]; then + pass "TC-INF-05d: Placeholder token present in sandbox (not the real key)" + elif [[ "$placeholder" == "$real_key" ]]; then + fail "TC-INF-05d: Placeholder" "Sandbox has the REAL key, not a placeholder" + else + skip "TC-INF-05d: Placeholder" "NVIDIA_API_KEY not set in sandbox (placeholder injection may not be active)" + fi +} + +# ============================================================================= +# TC-INF-06: Invalid API key → classified error message +# ============================================================================= +test_inf_06_invalid_api_key() { + log "=== TC-INF-06: Invalid API Key → Classified Error ===" + + rm -f "$HOME/.nemoclaw/onboard.lock" 2>/dev/null || true + + local invalid_api_key="nvapi-INTENTIONALLY-INVALID-KEY-FOR-E2E-TEST" # gitleaks:allow + local output exit_code=0 + output=$(NVIDIA_API_KEY="$invalid_api_key" \ + NEMOCLAW_NON_INTERACTIVE=1 \ + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ + NEMOCLAW_SANDBOX_NAME="e2e-invalid-key" \ + run_with_timeout 120 nemoclaw onboard --non-interactive --yes-i-accept-third-party-software \ + 2>&1) || exit_code=$? + + # 1. Exit code should be non-zero (onboard should fail) + if [[ $exit_code -eq 0 ]]; then + fail "TC-INF-06: Exit code" "Onboard succeeded with invalid key (expected failure)" + return + fi + pass "TC-INF-06: Onboard failed as expected (exit $exit_code)" + + # 2. Output should contain a classified error keyword + if echo "$output" | grep -qiE "authorization|credential|invalid|401|Unauthorized|api[._-]key"; then + pass "TC-INF-06: Output contains classified error message" + else + fail "TC-INF-06: Error classification" "No classified error keyword found in output" + log " First 10 lines of output:" + echo "$output" | head -10 | while IFS= read -r line; do log " $line"; done + fi + + # 3. Output should NOT contain a raw Node.js stack trace + local stack_count + stack_count=$(echo "$output" | grep -cE "at Object\.|at Module\.|at node:internal|at process\." || true) + if [[ $stack_count -gt 0 ]]; then + fail "TC-INF-06: Stack trace" "Raw Node.js stack trace found ($stack_count lines)" + else + pass "TC-INF-06: No raw stack trace in output" + fi + + # 4. The invalid API key should not appear in plain text in output + if echo "$output" | grep -qF "INTENTIONALLY-INVALID-KEY-FOR-E2E-TEST"; then + fail "TC-INF-06: Key exposure" "Invalid API key visible in plain text in output" + else + pass "TC-INF-06: API key not exposed in output" + fi + + # 5. Sandbox should not be left running after a failed onboard. + # The product may transiently create then roll back the sandbox during + # onboard; the important invariant is that no active sandbox remains. + if nemoclaw "e2e-invalid-key" status 2>/dev/null | grep -qiE "running|ready"; then + fail "TC-INF-06: Sandbox cleanup" "Sandbox 'e2e-invalid-key' is still running after failed onboard" + nemoclaw "e2e-invalid-key" destroy --yes 2>/dev/null || true + else + pass "TC-INF-06: No active sandbox left behind (correct)" + # Clean up any stale registry entry + nemoclaw "e2e-invalid-key" destroy --yes 2>/dev/null || true + fi + + rm -f "$HOME/.nemoclaw/onboard.lock" 2>/dev/null || true +} + +# ============================================================================= +# TC-INF-07: Unreachable endpoint → classified error message +# ============================================================================= +test_inf_07_unreachable_endpoint() { + log "=== TC-INF-07: Unreachable Endpoint → Classified Error ===" + + rm -f "$HOME/.nemoclaw/onboard.lock" 2>/dev/null || true + + # Use an RFC 2606 invalid domain — deterministic DNS failure across runners + local output exit_code=0 + output=$(NVIDIA_API_KEY="nvapi-valid-format-but-fake-key-1234567890" \ + NEMOCLAW_NON_INTERACTIVE=1 \ + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ + NEMOCLAW_SANDBOX_NAME="e2e-unreachable" \ + NEMOCLAW_PROVIDER="custom" \ + NEMOCLAW_ENDPOINT_URL="https://nemoclaw-e2e.invalid/v1" \ + NEMOCLAW_MODEL="test-model" \ + COMPATIBLE_API_KEY="fake-key-for-unreachable-test" \ + run_with_timeout 120 nemoclaw onboard --non-interactive --yes-i-accept-third-party-software \ + 2>&1) || exit_code=$? + + # 1. Exit code should be non-zero + if [[ $exit_code -eq 0 ]]; then + fail "TC-INF-07: Exit code" "Onboard succeeded with unreachable endpoint (expected failure)" + return + fi + pass "TC-INF-07: Onboard failed as expected (exit $exit_code)" + + # 2. Output should contain transport/connection error keywords + if echo "$output" | grep -qiE "unreachable|timeout|connect|ECONNREFUSED|ETIMEDOUT|ENETUNREACH|EHOSTUNREACH|ENOTFOUND|EAI_AGAIN|No route to host|transport|network|endpoint|dns"; then + pass "TC-INF-07: Output contains transport error classification" + else + fail "TC-INF-07: Error classification" "No transport error keyword found" + log " First 10 lines of output:" + echo "$output" | head -10 | while IFS= read -r line; do log " $line"; done + fi + + # 3. No raw stack trace + local stack_count + stack_count=$(echo "$output" | grep -cE "at Object\.|at Module\.|at node:internal|at process\." || true) + if [[ $stack_count -gt 0 ]]; then + fail "TC-INF-07: Stack trace" "Raw Node.js stack trace found ($stack_count lines)" + else + pass "TC-INF-07: No raw stack trace in output" + fi + + # 4. Sandbox should not be left running after a failed onboard. + # The product may transiently create then roll back the sandbox during + # onboard; the important invariant is that no active sandbox remains. + if nemoclaw "e2e-unreachable" status 2>/dev/null | grep -qiE "running|ready"; then + fail "TC-INF-07: Sandbox cleanup" "Sandbox 'e2e-unreachable' is still running after failed onboard" + nemoclaw "e2e-unreachable" destroy --yes 2>/dev/null || true + else + pass "TC-INF-07: No active sandbox left behind (correct)" + # Clean up any stale registry entry + nemoclaw "e2e-unreachable" destroy --yes 2>/dev/null || true + fi + + rm -f "$HOME/.nemoclaw/onboard.lock" 2>/dev/null || true +} + +# ============================================================================= +# TC-INF-02: OpenAI provider end-to-end inference +# ============================================================================= +test_inf_02_openai() { + log "=== TC-INF-02: OpenAI Provider Inference ===" + + local api_key="${OPENAI_API_KEY:-}" + if [[ -z "$api_key" ]]; then + skip "TC-INF-02" "OPENAI_API_KEY not set" + return + fi + + local sbx_name="e2e-openai" + local model="${NEMOCLAW_OPENAI_MODEL:-gpt-4o-mini}" + rm -f "$HOME/.nemoclaw/onboard.lock" 2>/dev/null || true + + log " Preflight: destroying any existing '$sbx_name' sandbox..." + nemoclaw "$sbx_name" destroy --yes 2>/dev/null || true + + log " Onboarding with OpenAI provider, model: $model" + local onboard_exit=0 + NEMOCLAW_SANDBOX_NAME="$sbx_name" \ + NEMOCLAW_NON_INTERACTIVE=1 \ + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ + NEMOCLAW_POLICY_TIER="open" \ + NEMOCLAW_PROVIDER="openai" \ + NEMOCLAW_MODEL="$model" \ + OPENAI_API_KEY="$api_key" \ + run_with_timeout 300 nemoclaw onboard --non-interactive --yes-i-accept-third-party-software \ + 2>&1 | redact_stream "$api_key" | tee -a "$LOG_FILE" || onboard_exit=$? + + if [[ $onboard_exit -ne 0 ]]; then + fail "TC-INF-02: Onboard" "Onboard with OpenAI failed (exit $onboard_exit)" + return + fi + pass "TC-INF-02: Onboard with OpenAI succeeded" + + local ssh_cfg + ssh_cfg="$(mktemp)" + if ! openshell sandbox ssh-config "$sbx_name" >"$ssh_cfg" 2>/dev/null; then + fail "TC-INF-02: SSH" "Could not get SSH config for sandbox" + rm -f "$ssh_cfg" + return + fi + + log " Sending test prompt through sandbox inference proxy..." + local response + response=$(run_with_timeout 90 ssh -F "$ssh_cfg" \ + -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 -o LogLevel=ERROR \ + "openshell-${sbx_name}" \ + "curl -s --max-time 60 https://inference.local/v1/chat/completions \ + -H 'Content-Type: application/json' \ + -d '{\"model\":\"$model\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with exactly one word: PONG\"}],\"max_tokens\":50}'" \ + 2>&1) || true + rm -f "$ssh_cfg" + + log " Response: ${response:0:300}" + + local content + content=$(echo "$response" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d['choices'][0]['message']['content'])" 2>/dev/null) || true + + if [[ -n "$content" ]] && echo "$content" | grep -qi "PONG"; then + pass "TC-INF-02: OpenAI inference response received through sandbox proxy" + elif [[ -n "$content" ]]; then + pass "TC-INF-02: OpenAI response received (content: ${content:0:100})" + else + fail "TC-INF-02: Inference" "No valid response from OpenAI through sandbox: ${response:0:200}" + fi + + nemoclaw "$sbx_name" destroy --yes 2>/dev/null || true + rm -f "$HOME/.nemoclaw/onboard.lock" 2>/dev/null || true +} + +# ============================================================================= +# TC-INF-03: Anthropic provider end-to-end inference +# ============================================================================= +test_inf_03_anthropic() { + log "=== TC-INF-03: Anthropic Provider Inference ===" + + local api_key="${ANTHROPIC_API_KEY:-}" + if [[ -z "$api_key" ]]; then + skip "TC-INF-03" "ANTHROPIC_API_KEY not set" + return + fi + + local sbx_name="e2e-anthropic" + local model="${NEMOCLAW_ANTHROPIC_MODEL:-claude-sonnet-4-6}" + rm -f "$HOME/.nemoclaw/onboard.lock" 2>/dev/null || true + + log " Preflight: destroying any existing '$sbx_name' sandbox..." + nemoclaw "$sbx_name" destroy --yes 2>/dev/null || true + + log " Onboarding with Anthropic provider, model: $model" + local onboard_exit=0 + NEMOCLAW_SANDBOX_NAME="$sbx_name" \ + NEMOCLAW_NON_INTERACTIVE=1 \ + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ + NEMOCLAW_POLICY_TIER="open" \ + NEMOCLAW_PROVIDER="anthropic" \ + NEMOCLAW_MODEL="$model" \ + ANTHROPIC_API_KEY="$api_key" \ + run_with_timeout 300 nemoclaw onboard --non-interactive --yes-i-accept-third-party-software \ + 2>&1 | redact_stream "$api_key" | tee -a "$LOG_FILE" || onboard_exit=$? + + if [[ $onboard_exit -ne 0 ]]; then + fail "TC-INF-03: Onboard" "Onboard with Anthropic failed (exit $onboard_exit)" + return + fi + pass "TC-INF-03: Onboard with Anthropic succeeded" + + local ssh_cfg + ssh_cfg="$(mktemp)" + if ! openshell sandbox ssh-config "$sbx_name" >"$ssh_cfg" 2>/dev/null; then + fail "TC-INF-03: SSH" "Could not get SSH config for sandbox" + rm -f "$ssh_cfg" + return + fi + + log " Sending test prompt through sandbox inference proxy (Anthropic Messages API)..." + local response + response=$(run_with_timeout 90 ssh -F "$ssh_cfg" \ + -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 -o LogLevel=ERROR \ + "openshell-${sbx_name}" \ + "curl -s --max-time 60 https://inference.local/v1/messages \ + -H 'Content-Type: application/json' \ + -d '{\"model\":\"$model\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with exactly one word: PONG\"}],\"max_tokens\":50}'" \ + 2>&1) || true + rm -f "$ssh_cfg" + + log " Response: ${response:0:300}" + + local content + content=$(printf '%s' "$response" | python3 -c " +import sys, json +d = json.load(sys.stdin) +# Anthropic Messages API returns content as array of blocks +if 'content' in d and isinstance(d['content'], list): + print(''.join(part.get('text', '') for part in d['content'] if isinstance(part, dict))) +# Fallback: OpenAI-compatible format (gateway may translate) +elif 'choices' in d: + print(d['choices'][0]['message']['content']) +" 2>/dev/null) || true + + if [[ -n "$content" ]] && echo "$content" | grep -qi "PONG"; then + pass "TC-INF-03: Anthropic inference response received through sandbox proxy" + elif [[ -n "$content" ]]; then + pass "TC-INF-03: Anthropic response received (content: ${content:0:100})" + else + fail "TC-INF-03: Inference" "No valid response from Anthropic through sandbox: ${response:0:200}" + fi + + nemoclaw "$sbx_name" destroy --yes 2>/dev/null || true + rm -f "$HOME/.nemoclaw/onboard.lock" 2>/dev/null || true +} + +# ============================================================================= +# TC-INF-09: Custom OpenAI-compatible endpoint inference +# ============================================================================= +test_inf_09_compatible_endpoint() { + log "=== TC-INF-09: Custom OpenAI-Compatible Endpoint ===" + + local endpoint_url="${NEMOCLAW_ENDPOINT_URL:-}" + local endpoint_model="${NEMOCLAW_COMPAT_MODEL:-}" + local endpoint_key="${COMPATIBLE_API_KEY:-}" + + if [[ -z "$endpoint_url" || -z "$endpoint_model" || -z "$endpoint_key" ]]; then + skip "TC-INF-09" "Missing NEMOCLAW_ENDPOINT_URL, NEMOCLAW_COMPAT_MODEL, or COMPATIBLE_API_KEY" + return + fi + + local sbx_name="e2e-compat-ep" + rm -f "$HOME/.nemoclaw/onboard.lock" 2>/dev/null || true + + log " Preflight: destroying any existing '$sbx_name' sandbox..." + nemoclaw "$sbx_name" destroy --yes 2>/dev/null || true + + log " Onboarding with compatible endpoint: $endpoint_url" + log " Model: $endpoint_model" + local onboard_exit=0 + NEMOCLAW_SANDBOX_NAME="$sbx_name" \ + NEMOCLAW_NON_INTERACTIVE=1 \ + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ + NEMOCLAW_POLICY_TIER="open" \ + NEMOCLAW_PROVIDER="custom" \ + NEMOCLAW_ENDPOINT_URL="$endpoint_url" \ + NEMOCLAW_MODEL="$endpoint_model" \ + COMPATIBLE_API_KEY="$endpoint_key" \ + run_with_timeout 300 nemoclaw onboard --non-interactive --yes-i-accept-third-party-software \ + 2>&1 | redact_stream "$endpoint_key" | tee -a "$LOG_FILE" || onboard_exit=$? + + if [[ $onboard_exit -ne 0 ]]; then + fail "TC-INF-09: Onboard" "Onboard with compatible endpoint failed (exit $onboard_exit)" + return + fi + pass "TC-INF-09: Onboard with compatible endpoint succeeded" + + # Get SSH config for the sandbox + local ssh_cfg + ssh_cfg="$(mktemp)" + if ! openshell sandbox ssh-config "$sbx_name" >"$ssh_cfg" 2>/dev/null; then + fail "TC-INF-09: SSH" "Could not get SSH config for sandbox" + rm -f "$ssh_cfg" + return + fi + + # Send a prompt through the inference proxy inside the sandbox + log " Sending test prompt through sandbox inference proxy..." + local response + response=$(run_with_timeout 90 ssh -F "$ssh_cfg" \ + -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 -o LogLevel=ERROR \ + "openshell-${sbx_name}" \ + "curl -s --max-time 60 https://inference.local/v1/chat/completions \ + -H 'Content-Type: application/json' \ + -d '{\"model\":\"$endpoint_model\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with exactly one word: PONG\"}],\"max_tokens\":50}'" \ + 2>&1) || true + rm -f "$ssh_cfg" + + log " Response: ${response:0:300}" + + local content + content=$(echo "$response" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d['choices'][0]['message']['content'])" 2>/dev/null) || true + + if [[ -n "$content" ]] && echo "$content" | grep -qi "PONG"; then + pass "TC-INF-09: Inference response received through sandbox proxy" + elif [[ -n "$content" ]]; then + pass "TC-INF-09: Inference response received (content: ${content:0:100})" + elif [[ -n "$response" ]]; then + fail "TC-INF-09: Inference" "Got response but could not extract content: ${response:0:200}" + else + fail "TC-INF-09: Inference" "No response from inference.local" + fi + + nemoclaw "$sbx_name" destroy --yes 2>/dev/null || true + rm -f "$HOME/.nemoclaw/onboard.lock" 2>/dev/null || true +} + +# ── Teardown ───────────────────────────────────────────────────────────────── +teardown() { + # Do not unlink ~/.nemoclaw/onboard.lock: see rationale in + # test/e2e-vpn/lib/sandbox-teardown.sh — the lock is PID-ownership-aware + # and onboard cleans up stale locks itself. + set +e + nemoclaw "$SANDBOX_NAME" destroy --yes 2>/dev/null || true + nemoclaw "e2e-openai" destroy --yes 2>/dev/null || true + nemoclaw "e2e-anthropic" destroy --yes 2>/dev/null || true + nemoclaw "e2e-invalid-key" destroy --yes 2>/dev/null || true + nemoclaw "e2e-unreachable" destroy --yes 2>/dev/null || true + nemoclaw "e2e-compat-ep" destroy --yes 2>/dev/null || true + set -e +} + +# ── Summary ────────────────────────────────────────────────────────────────── +summary() { + echo "" + echo "============================================================" + echo " NemoClaw Inference Routing E2E Results" + echo "============================================================" + echo -e " ${GREEN}PASS: $PASS${NC}" + echo -e " ${RED}FAIL: $FAIL${NC}" + echo -e " ${YELLOW}SKIP: $SKIP${NC}" + echo " TOTAL: $TOTAL" + echo "============================================================" + echo " Log: $LOG_FILE" + echo "============================================================" + echo "" + + if [[ $FAIL -gt 0 ]]; then + exit 1 + fi + exit 0 +} + +# ── Main ───────────────────────────────────────────────────────────────────── +main() { + echo "" + echo "============================================================" + echo " NemoClaw Inference Routing E2E Tests" + echo " $(date)" + echo "============================================================" + echo "" + + preflight + + test_inf_02_openai + test_inf_03_anthropic + test_inf_05_credential_isolation + test_inf_06_invalid_api_key + test_inf_07_unreachable_endpoint + test_inf_09_compatible_endpoint + + trap - EXIT + teardown + summary +} + +trap teardown EXIT +main "$@" diff --git a/test/e2e-vpn/test-issue-2478-crash-loop-recovery.sh b/test/e2e-vpn/test-issue-2478-crash-loop-recovery.sh new file mode 100755 index 00000000000..ebe95622eba --- /dev/null +++ b/test/e2e-vpn/test-issue-2478-crash-loop-recovery.sh @@ -0,0 +1,782 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Long-running e2e regression for NVIDIA/NemoClaw#2478 — gateway crash-loop +# recovery when a sandboxed library throws on init. +# +# STAYS_IN_PR_UNTIL_SHIP — delete this file before merging the fix once +# the soak has produced a clean run on a real DGX Spark / Brev instance. +# Tracking removal in the PR description, not here, so the file does not +# silently outlive the issue it was written for. +# +# What this test exercises (the fix from #2478): +# +# The sandbox ships a chain of NODE_OPTIONS=--require preloads (sandbox +# safety-net, ciao networkInterfaces guard, slack guard, http-proxy fix, +# ws-proxy fix, nemotron fix). They are emitted into +# /tmp/nemoclaw-proxy-env.sh at sandbox-start and reach the gateway via +# ~/.bashrc on the FIRST start. Before #2478 the gateway recovery path +# (laptop sleep, health-monitor restart, manual `nemoclaw connect`) +# silently swallowed sourcing errors with `2>/dev/null` and never asserted +# that NODE_OPTIONS actually contained the guards. A stale or missing +# proxy-env.sh therefore left the respawned gateway naked, and any library +# that threw during init (ciao mDNS being the trigger documented in the +# issue) crashed the gateway in a loop forever. +# +# This test: +# +# 1. Onboards a sandbox normally. +# 2. Verifies the *initial* gateway has the safety-net + ciao guard active +# (via /proc//environ on the gateway PID). +# 3. Crash-recovery loop (NORMAL): kill the gateway 5x, each time triggers +# `nemoclaw connect --probe-only` (which calls +# recoverSandboxProcesses), and checks the respawned gateway still has +# guards in NODE_OPTIONS. +# 4. Negative case: removes /tmp/nemoclaw-proxy-env.sh, kills the gateway, +# triggers recovery — expects the new "[gateway-recovery] WARNING" +# line in gateway.log instead of silent guard loss. +# 5. Soak: leaves the sandbox idle for $NEMOCLAW_E2E_SOAK_SECONDS +# (default 300) so the health-monitor restart cadence (~4 min in prod) +# gets at least one chance to fire, then asserts the gateway has not +# crash-looped in the meantime (PID stable OR exactly one clean +# respawn, no churn). +# +# Prerequisites: +# - Docker running +# - node + curl for the local OpenAI-compatible mock endpoint +# +# Environment variables: +# NEMOCLAW_NON_INTERACTIVE=1 — required +# NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 — required +# NEMOCLAW_E2E_USE_COMPAT_MOCK — use local mock provider (default: 1) +# NEMOCLAW_COMPAT_MOCK_PORT — local mock endpoint port (default: 18191) +# NEMOCLAW_COMPAT_MODEL — local mock model id (default: test-model) +# NEMOCLAW_COMPAT_MOCK_LOG — local mock server log path +# NEMOCLAW_SANDBOX_NAME — sandbox name (default: e2e-2478) +# NEMOCLAW_E2E_INSTALL_LOG — install log path for CI artifacts +# NEMOCLAW_E2E_TIMEOUT_SECONDS — overall timeout (default: 1500) +# NEMOCLAW_E2E_CRASH_CYCLES — crash-recover cycles (default: 5) +# NEMOCLAW_E2E_SOAK_SECONDS — idle soak window (default: 300) +# NVIDIA_API_KEY — required only with NEMOCLAW_E2E_USE_COMPAT_MOCK=0 +# NVIDIA_API_KEY — legacy fallback for NVIDIA_API_KEY +# +# Usage: +# NEMOCLAW_NON_INTERACTIVE=1 \ +# NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ +# bash test/e2e-vpn/test-issue-2478-crash-loop-recovery.sh + +# ShellCheck cannot see EXIT trap invocations of cleanup helpers in this E2E script. +# shellcheck disable=SC2317,SC2329 +set -uo pipefail + +export NEMOCLAW_E2E_DEFAULT_TIMEOUT=1500 +SCRIPT_DIR_TIMEOUT="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +# shellcheck source=test/e2e-vpn/e2e-timeout.sh +source "${SCRIPT_DIR_TIMEOUT}/e2e-timeout.sh" + +PASS=0 +FAIL=0 +TOTAL=0 + +pass() { + ((PASS++)) + ((TOTAL++)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} +fail() { + ((FAIL++)) + ((TOTAL++)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } + +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-2478}" +CRASH_CYCLES="${NEMOCLAW_E2E_CRASH_CYCLES:-5}" +SOAK_SECONDS="${NEMOCLAW_E2E_SOAK_SECONDS:-300}" +DASHBOARD_PORT="${NEMOCLAW_DASHBOARD_PORT:-18789}" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +# shellcheck source=test/e2e-vpn/lib/openai-compatible-api-proof.sh +source "${SCRIPT_DIR}/lib/openai-compatible-api-proof.sh" +USE_COMPAT_MOCK="${NEMOCLAW_E2E_USE_COMPAT_MOCK:-1}" +COMPAT_MOCK_PORT="${NEMOCLAW_COMPAT_MOCK_PORT:-18191}" +COMPAT_MODEL="${NEMOCLAW_COMPAT_MODEL:-test-model}" +COMPATIBLE_KEY="${COMPATIBLE_API_KEY:-nemoclaw-e2e-compatible-key}" +COMPAT_MOCK_LOG="${NEMOCLAW_COMPAT_MOCK_LOG:-/tmp/nemoclaw-2478-compatible-endpoint.log}" + +# ── Helpers ────────────────────────────────────────────────────── + +host_ip_for_sandbox() { + local ip_addr + if command -v ip >/dev/null 2>&1; then + ip_addr="$(ip route get 1 2>/dev/null | awk '{for (i=1;i<=NF;i++) if ($i=="src") {print $(i+1); exit}}')" + if [ -n "$ip_addr" ]; then + echo "$ip_addr" + return + fi + fi + if command -v hostname >/dev/null 2>&1; then + for ip_addr in $(hostname -I 2>/dev/null); do + case "$ip_addr" in + 127.* | ::1) ;; + *) + echo "$ip_addr" + return + ;; + esac + done + fi + echo "127.0.0.1" +} + +stop_compat_mock() { + stop_fake_openai_compatible_api +} +trap stop_compat_mock EXIT + +start_compat_mock() { + export FAKE_OPENAI_HOST="0.0.0.0" + export FAKE_OPENAI_PORT="$COMPAT_MOCK_PORT" + export FAKE_OPENAI_MODEL="$COMPAT_MODEL" + export FAKE_OPENAI_API_KEY="$COMPATIBLE_KEY" + export FAKE_OPENAI_REQUIRE_AUTH="1" + export FAKE_OPENAI_CHAT_CONTENT="PONG from issue-2478 mock" + export FAKE_OPENAI_RESPONSE_TEXT="PONG from issue-2478 mock" + export FAKE_OPENAI_LOG="$COMPAT_MOCK_LOG" + start_fake_openai_compatible_api || return 1 +} + +# Run a command inside the sandbox via openshell sandbox exec. Returns +# stdout; non-zero exit prints stderr but does not abort the test. +sandbox_exec() { + openshell sandbox exec --name "$SANDBOX_NAME" -- "$@" 2>&1 +} + +# Get the current OpenClaw gateway PID inside the sandbox, or empty string. +# OpenClaw v0.0.44/2026.5.18 can show the long-running process as plain +# `openclaw` rather than the older `openclaw-gateway` argv. Match the process +# table directly so readiness does not depend on the legacy rename. +gateway_pid() { + local out + # shellcheck disable=SC2016 # Single-quoted body runs inside the sandbox shell. + out="$(sandbox_exec sh -c 'pid="$(ps -eo pid=,comm=,args= 2>/dev/null | awk '\''($2 == "openclaw" && $0 ~ /gateway/) || $0 ~ /openclaw[ -]gateway/ { print $1 }'\'' | sort -n | head -n 1)"; if [ -z "$pid" ]; then pid="$(ps -eo pid=,comm=,args= 2>/dev/null | awk '\''$2 == "openclaw" { print $1 }'\'' | sort -n | head -n 1)"; fi; printf "%s\n" "$pid"')" + printf '%s\n' "$out" | awk '/^[0-9]+$/ { print; exit }' +} + +# Read /tmp/nemoclaw-proxy-env.sh — the single source of truth for the +# NODE_OPTIONS guard chain that the recovery script sources before +# launching the gateway. Owned root:root 444, readable by sandbox user. +proxy_env_contents() { + sandbox_exec sh -c "cat /tmp/nemoclaw-proxy-env.sh 2>/dev/null" +} + +gateway_log_line_count() { + sandbox_exec sh -c "wc -l < /tmp/gateway.log 2>/dev/null || printf '0\n'" \ + | awk '/^[0-9]+$/ { print; found=1; exit } END { if (!found) print 0 }' +} + +gateway_log_marker() { + local label="$1" + local marker + marker="NEMOCLAW_E2E_LOG_MARKER_${label}_$(date +%s)_$$" + if sandbox_exec sh -c "printf '%s\n' '$marker' >> /tmp/gateway.log 2>/dev/null && grep -Fq '$marker' /tmp/gateway.log 2>/dev/null" >/dev/null; then + printf '%s\n' "$marker" + return + fi + gateway_log_line_count +} + +gateway_log_after_boundary() { + local boundary="${1:-0}" + if [[ "$boundary" =~ ^[0-9]+$ ]]; then + local current_lines + current_lines="$(gateway_log_line_count)" + if [ "$current_lines" -lt "$boundary" ]; then + boundary=0 + fi + sandbox_exec sh -c "cat /tmp/gateway.log 2>/dev/null" | tail -n +"$((boundary + 1))" + return + fi + + local out status + out="$(sandbox_exec sh -c "awk -v marker='$boundary' 'found { print } index(\$0, marker) { found=1; next } END { if (!found) exit 42 }' /tmp/gateway.log 2>/dev/null")" + status=$? + if [ "$status" -eq 0 ]; then + printf '%s\n' "$out" + return + fi + + # If the gateway rewrote or rotated /tmp/gateway.log, the marker is gone and + # the whole current file is fresh relative to the boundary. + sandbox_exec sh -c "cat /tmp/gateway.log 2>/dev/null" +} + +# Returns 0 if the gateway has the library guard chain active, 1 otherwise. +# /proc//environ is unreadable across non-ancestor process trees due +# to kernel.yama.ptrace_scope=1, so the maintained E2E fixture treats +# /tmp/nemoclaw-proxy-env.sh as the source of truth: recovery sources that file +# before launching the gateway, and the rest of this test separately proves the +# gateway PID is alive plus inference.local keeps serving. Do not require fresh +# gateway.log preload lines here; a recovered gateway can already have a valid +# guard chain without producing new activation lines after our marker. +gateway_guards_active() { + local pid="$1" + local timeout="${2:-30}" + local elapsed=0 + + if [ -z "$pid" ]; then + return 1 + fi + + while [ "$elapsed" -lt "$timeout" ]; do + local env_contents + env_contents="$(proxy_env_contents)" + if echo "$env_contents" | grep -q 'nemoclaw-sandbox-safety-net' \ + && echo "$env_contents" | grep -q 'nemoclaw-ciao-network-guard'; then + if [ -n "$(gateway_pid)" ]; then + return 0 + fi + echo " [guards] proxy-env.sh has guard exports but gateway no longer running" + return 1 + fi + sleep 3 + elapsed=$((elapsed + 3)) + done + + echo " [guards] proxy-env.sh missing safety-net or ciao guard exports within ${timeout}s" + return 1 +} + +# Tail gateway.log from inside the sandbox (last N lines). +gateway_log_tail() { + sandbox_exec sh -c "tail -n ${1:-50} /tmp/gateway.log 2>/dev/null" +} + +# Verify the gateway is actually serving its inference API, not just alive +# as a process. A NemoClaw user reported on #2478 that pre-fix the ciao +# crash left `https://inference.local/v1/models` returning empty — i.e. +# their deployed model "disappeared" from the user's perspective. This +# helper closes that loop so we prove the recovery preserves the +# user-visible service surface, not just the OS process. Polls up to $1 +# seconds (default 30) since the new gateway needs ~1-3s to bind after +# launch. +gateway_serves_inference() { + local timeout="${1:-30}" + local elapsed=0 + local out="" + while [ "$elapsed" -lt "$timeout" ]; do + out="$(sandbox_exec sh -c 'curl -sf --max-time 5 https://inference.local/v1/models 2>/dev/null')" + # OpenAI-compatible /v1/models response — top-level "data" array, plus + # entries with "object" or "id". Match any of the three to be tolerant + # of provider-specific shapes (VPN NVIDIA inference vs. local Ollama). + case "$out" in + *'"data"'* | *'"object"'* | *'"id"'*) return 0 ;; + esac + sleep 3 + elapsed=$((elapsed + 3)) + done + echo " [inference] /v1/models did not return a usable response within ${timeout}s" + echo " [inference] last response: ${out:0:200}" + return 1 +} + +# Dump diagnostic snapshot for triage when an environ read or guard +# assertion fails. Helps distinguish wrong-PID matching, gateway-not-running, +# and cross-namespace /proc visibility issues. +gateway_diagnostics() { + local pid="${1:-}" + echo " --- gateway diagnostics ---" + echo " [exec context: whoami / hostname / pwd / pid namespace]" + # shellcheck disable=SC2016 # intentional: expand inside sandbox, not host + sandbox_exec sh -c 'echo "user=$(whoami) host=$(hostname) pwd=$(pwd) pid_ns=$(readlink /proc/self/ns/pid 2>/dev/null)"' | sed 's/^/ /' + echo " [pgrep -af '[o]penclaw' (any openclaw process)]" + sandbox_exec sh -c "pgrep -af '[o]penclaw' || echo '(no matches)'" | sed 's/^/ /' + echo " [ps auxf (full tree, top 40 lines)]" + sandbox_exec sh -c "ps auxf 2>/dev/null | head -40 || ps -ef 2>/dev/null | head -40" | sed 's/^/ /' + echo " [ls /tmp (gateway.log presence + size)]" + sandbox_exec sh -c "ls -la /tmp/gateway.log /tmp/auto-pair.log /tmp/openclaw-* 2>&1 | head -20" | sed 's/^/ /' + echo " [tail /tmp/gateway.log -n 60]" + sandbox_exec sh -c "tail -n 60 /tmp/gateway.log 2>&1 || echo '(no gateway.log)'" | sed 's/^/ /' + echo " [nemoclaw status]" + nemoclaw "$SANDBOX_NAME" status 2>&1 | head -30 | sed 's/^/ /' + echo " [openshell sandbox list]" + openshell sandbox list 2>&1 | head -20 | sed 's/^/ /' || true + if [ -n "$pid" ]; then + echo " [reported pid: $pid]" + echo " [/proc/${pid} listing]" + sandbox_exec sh -c "ls -la /proc/${pid}/ 2>&1 | head -8 || echo '(cannot list)'" | sed 's/^/ /' + echo " [/proc/${pid}/cmdline]" + sandbox_exec sh -c "cat /proc/${pid}/cmdline 2>&1 | tr '\\0' ' '; echo" | sed 's/^/ /' + echo " [/proc/${pid}/status (uid/state)]" + sandbox_exec sh -c "grep -E '^(Name|State|Uid|Pid|PPid):' /proc/${pid}/status 2>&1" | sed 's/^/ /' + fi + echo " ---------------------------" +} + +run_probe_only_or_fail() { + local context="$1" + local probe_out + probe_out="$(mktemp)" + if ! timeout 60 nemoclaw "$SANDBOX_NAME" connect --probe-only >"$probe_out" 2>&1; then + fail "${context}: connect --probe-only exited nonzero" + sed 's/^/ /' "$probe_out" + rm -f "$probe_out" + gateway_diagnostics "" + exit 1 + fi + rm -f "$probe_out" +} + +# Returns 0 when the current OpenClaw runtime has crossed the same readiness +# surface used by newer gateway E2Es: ready log, local /health, or healthy host +# status. This avoids failing on the old PID-name-only probe when OpenClaw is +# already serving. +gateway_runtime_ready() { + if sandbox_exec sh -c "grep -Fq '[gateway] ready' /tmp/gateway.log 2>/dev/null"; then + return 0 + fi + local health_code + health_code="$(sandbox_exec sh -c "curl -so /dev/null -w '%{http_code}' --max-time 3 http://localhost:${DASHBOARD_PORT}/health 2>/dev/null" | tr -d '[:space:]')" || true + if [ "$health_code" = "200" ]; then + return 0 + fi + local status_output + status_output="$(timeout 20 nemoclaw "$SANDBOX_NAME" status 2>&1)" || true + if echo "$status_output" | grep -Eiq '\b(healthy|ready)\b'; then + return 0 + fi + if echo "$status_output" | grep -Eiq '\brunning\b' \ + && ! echo "$status_output" | grep -Eiq '\bnot[[:space:]]+running\b'; then + return 0 + fi + return 1 +} + +# Wait until gateway PID is non-empty and runtime-ready (or timeout). Echoes +# pid, returns 0/1. +wait_for_gateway_up() { + local timeout="${1:-30}" + local elapsed=0 pid="" + while [ "$elapsed" -lt "$timeout" ]; do + pid="$(gateway_pid)" + if [ -n "$pid" ] && gateway_runtime_ready; then + echo "$pid" + return 0 + fi + sleep 2 + elapsed=$((elapsed + 2)) + done + echo "" + return 1 +} + +# ══════════════════════════════════════════════════════════════════ +# Phase 0: Preflight +# ══════════════════════════════════════════════════════════════════ +section "Phase 0: Preflight" + +if ! docker info >/dev/null 2>&1; then + fail "Docker is not running" + exit 1 +fi +pass "Docker running" + +if [ "$USE_COMPAT_MOCK" = "1" ]; then + if ! command -v node >/dev/null 2>&1; then + fail "node is required for the compatible endpoint mock" + exit 1 + fi + if ! node --experimental-strip-types -e "" >/dev/null 2>&1; then + fail "node with --experimental-strip-types support is required for the compatible endpoint mock" + exit 1 + fi + if ! command -v curl >/dev/null 2>&1; then + fail "curl is required for the compatible endpoint mock readiness probe" + exit 1 + fi + pass "Compatible endpoint mock prerequisites available" +else + if [ -z "${NVIDIA_API_KEY:-}" ] && [ -n "${NVIDIA_API_KEY:-}" ]; then + export NVIDIA_API_KEY="$NVIDIA_API_KEY" + fi + if [ -z "${NVIDIA_API_KEY:-}" ] || [[ "${NVIDIA_API_KEY}" != nvapi-* ]]; then + fail "NVIDIA_API_KEY not set or invalid" + exit 1 + fi + pass "NVIDIA_API_KEY set" +fi + +if [ "${NEMOCLAW_NON_INTERACTIVE:-}" != "1" ] || [ "${NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE:-}" != "1" ]; then + fail "NEMOCLAW_NON_INTERACTIVE=1 and NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 are required" + exit 1 +fi +pass "Required env vars set" + +# ══════════════════════════════════════════════════════════════════ +# Phase 1: Pre-cleanup + onboard +# ══════════════════════════════════════════════════════════════════ +section "Phase 1: Pre-cleanup + onboard" + +if command -v nemoclaw >/dev/null 2>&1; then + nemoclaw "$SANDBOX_NAME" destroy --yes 2>/dev/null || true +fi +if command -v openshell >/dev/null 2>&1; then + openshell sandbox delete "$SANDBOX_NAME" 2>/dev/null || true +fi + +cd "$REPO_ROOT" || { + fail "cd $REPO_ROOT" + exit 1 +} + +install_env=( + env + NEMOCLAW_NON_INTERACTIVE=1 + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 + NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" + NEMOCLAW_RECREATE_SANDBOX=1 +) + +if [ "$USE_COMPAT_MOCK" = "1" ]; then + COMPAT_HOST="$(host_ip_for_sandbox)" + COMPAT_ENDPOINT_URL="http://${COMPAT_HOST}:${COMPAT_MOCK_PORT}/v1" + if start_compat_mock; then + pass "Compatible endpoint mock listening at ${COMPAT_ENDPOINT_URL}" + else + fail "Compatible endpoint mock did not become ready; see ${COMPAT_MOCK_LOG}" + cat "$COMPAT_MOCK_LOG" + exit 1 + fi + install_env+=( + COMPATIBLE_API_KEY="$COMPATIBLE_KEY" + NEMOCLAW_PROVIDER=custom + NEMOCLAW_ENDPOINT_URL="$COMPAT_ENDPOINT_URL" + NEMOCLAW_MODEL="$COMPAT_MODEL" + ) +fi + +INSTALL_LOG="${NEMOCLAW_E2E_INSTALL_LOG:-/tmp/nemoclaw-e2e-install.log}" +: >"$INSTALL_LOG" || { + fail "Cannot write install log at $INSTALL_LOG" + exit 1 +} +"${install_env[@]}" bash install.sh --non-interactive >"$INSTALL_LOG" 2>&1 + +install_exit=$? +if [ $install_exit -ne 0 ]; then + fail "install.sh failed (exit $install_exit). Last 30 lines:" + tail -30 "$INSTALL_LOG" + info "Full install log retained at $INSTALL_LOG" + exit 1 +fi +pass "install.sh + onboard completed" + +# Pick up PATH changes +[ -f "$HOME/.bashrc" ] && { source "$HOME/.bashrc" 2>/dev/null || true; } +export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" +[ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" +[ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]] && export PATH="$HOME/.local/bin:$PATH" + +if ! command -v nemoclaw >/dev/null 2>&1; then + fail "nemoclaw not on PATH after install" + exit 1 +fi +pass "nemoclaw on PATH" + +# ══════════════════════════════════════════════════════════════════ +# Phase 2: Verify initial gateway has the guard chain +# ══════════════════════════════════════════════════════════════════ +section "Phase 2: Initial gateway has guard chain" + +INIT_PID="$(wait_for_gateway_up 60)" +if [ -z "$INIT_PID" ]; then + fail "Gateway never came up after onboard" + gateway_diagnostics "" + exit 1 +fi +pass "Gateway up (pid=$INIT_PID)" + +if gateway_guards_active "$INIT_PID" 30; then + pass "Initial gateway has guard chain configured (proxy-env exports present)" +else + fail "Initial gateway missing library guard chain — fix is not deployed?" + gateway_diagnostics "$INIT_PID" + exit 1 +fi + +if gateway_serves_inference 30; then + pass "Initial gateway serves inference API (https://inference.local/v1/models responds)" +else + fail "Initial gateway alive but not serving inference — recovery is incomplete from user POV" + gateway_diagnostics "$INIT_PID" + exit 1 +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 3: Crash-recovery loop ($CRASH_CYCLES cycles) +# ══════════════════════════════════════════════════════════════════ +section "Phase 3: Crash-recovery loop ($CRASH_CYCLES cycles)" + +prev_pid="$INIT_PID" +for cycle in $(seq 1 "$CRASH_CYCLES"); do + info "Cycle $cycle/$CRASH_CYCLES — killing gateway pid=$prev_pid" + guard_log_start="$(gateway_log_marker "cycle_${cycle}")" + sandbox_exec sh -c "kill -9 $prev_pid 2>/dev/null; sleep 1" >/dev/null + + # Trigger recovery via the actual operator probe path: + # `nemoclaw connect --probe-only` calls + # checkAndRecoverSandboxProcesses() -> recoverSandboxProcesses() without + # opening an interactive SSH session. Bound it with `timeout` so a hang in + # CLI internals cannot eat the whole 30-min job budget. + run_probe_only_or_fail "Cycle $cycle after gateway kill" + + if ! sandbox_exec sh -c 'test -s /tmp/gateway.log'; then + fail "Cycle $cycle: connect --probe-only did not leave /tmp/gateway.log evidence" + gateway_diagnostics "" + exit 1 + fi + + new_pid="$(wait_for_gateway_up 45)" + if [ -z "$new_pid" ]; then + fail "Cycle $cycle: gateway did not respawn within 45s" + gateway_log_tail 60 + exit 1 + fi + if [ "$new_pid" = "$prev_pid" ]; then + fail "Cycle $cycle: PID unchanged ($new_pid) — kill did not land" + exit 1 + fi + pass "Cycle $cycle: gateway respawned (pid $prev_pid → $new_pid)" + + if gateway_guards_active "$new_pid" 30 "$guard_log_start"; then + pass "Cycle $cycle: respawned gateway retains guard chain configuration (proxy-env exports present)" + else + fail "Cycle $cycle: respawned gateway LOST guard chain — recovery hardening regressed" + gateway_diagnostics "$new_pid" + gateway_log_tail 80 + exit 1 + fi + + if gateway_serves_inference 30; then + pass "Cycle $cycle: respawned gateway serves inference API" + else + fail "Cycle $cycle: gateway up + guards active but inference API not serving" + gateway_diagnostics "$new_pid" + gateway_log_tail 80 + exit 1 + fi + + prev_pid="$new_pid" +done + +# ══════════════════════════════════════════════════════════════════ +# Phase 4: Negative case — env file missing → warning logged +# ══════════════════════════════════════════════════════════════════ +section "Phase 4: Negative case — proxy-env.sh missing surfaces a warning" + +# Snapshot proxy-env.sh contents so we can restore after the test. +# Capture as base64 from inside the sandbox so the round-trip is byte- +# faithful — `$(cat ...)` would strip trailing newlines and break the +# eventual size verification by ~2 bytes. We also pull the original size +# separately so the post-restore wc -c can be compared exactly. +SNAPSHOT_B64="$(sandbox_exec sh -c 'base64 < /tmp/nemoclaw-proxy-env.sh' | tr -d '[:space:]')" +SNAPSHOT_SIZE="$(sandbox_exec sh -c 'wc -c < /tmp/nemoclaw-proxy-env.sh' | tr -d '[:space:]')" +if [ -z "$SNAPSHOT_B64" ] || [ -z "$SNAPSHOT_SIZE" ] || [ "$SNAPSHOT_SIZE" -eq 0 ]; then + fail "proxy-env.sh is empty/missing already — cannot run negative case" + exit 1 +fi +info "Snapshotted proxy-env.sh ($SNAPSHOT_SIZE bytes, ${#SNAPSHOT_B64}-char base64)" + +# Remove proxy-env.sh, kill the entire openclaw process tree, trigger +# recovery, expect WARNING. We must kill the launcher AND the gateway — +# pkill -9 -f '[o]penclaw' takes them all out so the launcher's watchdog +# can't silently respawn the gateway before nemoclaw status runs the +# recovery script (which is the only path that emits the warning). +negative_guard_log_start="$(gateway_log_marker "negative")" +sandbox_exec sh -c 'rm -f /tmp/nemoclaw-proxy-env.sh' >/dev/null +sandbox_exec sh -c "pkill -9 -f '[o]penclaw' 2>/dev/null; sleep 2; pgrep -af '[o]penclaw' || echo ALL_DEAD" >/dev/null +run_probe_only_or_fail "Negative case after proxy-env removal" + +# The new gateway.log should contain the [gateway-recovery] WARNING line and +# recovery should have attempted a real gateway respawn. +warn_seen=false +for _ in 1 2 3 4 5; do + if gateway_log_tail 100 | grep -q '\[gateway-recovery\] WARNING'; then + warn_seen=true + break + fi + sleep 3 +done +if $warn_seen; then + pass "Recovery emitted [gateway-recovery] WARNING when proxy-env.sh missing" +else + fail "Recovery silently launched without warning (regression of #2478 fix)" + gateway_log_tail 100 +fi +NEGATIVE_PID="$(wait_for_gateway_up 45)" +if [ -z "$NEGATIVE_PID" ]; then + fail "Recovery warning was logged, but gateway did not respawn within 45s" + gateway_diagnostics "" + exit 1 +fi +info "Negative-case recovery respawned gateway pid=$NEGATIVE_PID" + +# ── #2701 contract assertion ───────────────────────────────────────── +# After recovery, the guard chain MUST be restored. Today this fails on +# `main`: recovery emits the WARNING above and then launches the gateway +# naked, leaving /tmp/nemoclaw-proxy-env.sh absent. On aarch64 / DGX Spark +# this triggers the @homebridge/ciao crash loop documented in #2701; on +# x86 the gateway boots fine but the guard chain is still missing, which +# is the failure shape this assertion catches. +# +# Once the #2701 fix lands, recovery re-emits the chain before launching +# and this assertion flips green. Will fail on origin/main as of 2026-06-09. +if gateway_guards_active "$NEGATIVE_PID" 30 "$negative_guard_log_start"; then + pass "#2701: recovery restored guard chain (proxy-env.sh + safety-net + ciao)" +else + fail "#2701: recovery did NOT restore guard chain — gateway respawned naked (DGX Spark crash-loop scenario)" + gateway_diagnostics "$NEGATIVE_PID" + # Do not exit 1 yet — we still want Phase 4's restore + Phase 5 soak to + # run so the artifact bundle is comparable to historical runs. Defer the + # failure decision to the test-level fail counter at the end of the file. +fi + +# Restore proxy-env.sh by base64-injecting the snapshot via argv. `openshell +# sandbox exec` does not pipe stdin from the caller through to the subshell, +# so a `printf | sandbox_exec sh -c 'cat > file'` would leave an empty file. +# Encoding into the command argv sidesteps the stdin gap entirely. +# +# After the #2701 fix, recovery may already have restored a minimal hardened +# proxy-env from trusted preload sources. In that case the sandbox user cannot +# necessarily overwrite the root-owned 0444 file with the original snapshot; +# accept the recovered file if it still proves the guard chain is active. This +# keeps the test focused on the user-visible contract instead of byte-identical +# cleanup of an implementation detail. +sandbox_exec sh -c "echo '$SNAPSHOT_B64' | base64 -d > /tmp/nemoclaw-proxy-env.sh && chmod 444 /tmp/nemoclaw-proxy-env.sh" >/dev/null + +restored_size="$(sandbox_exec sh -c 'wc -c < /tmp/nemoclaw-proxy-env.sh' | tr -d '[:space:]')" +if [ "$restored_size" = "$SNAPSHOT_SIZE" ]; then + info "proxy-env.sh restored from snapshot (${restored_size} bytes verified)" +elif gateway_guards_active "$NEGATIVE_PID" 5 "$negative_guard_log_start"; then + info "proxy-env.sh retained recovered guard env (${restored_size} bytes; original snapshot was ${SNAPSHOT_SIZE} bytes)" +else + fail "proxy-env.sh restore failed and recovered guard env is not active: expected $SNAPSHOT_SIZE bytes, got '${restored_size}'" + exit 1 +fi + +# Kill the current negative-case gateway, then trigger recovery to bring the +# gateway back with guards intact from either the snapshot or recovered env file. +restore_guard_log_start="$(gateway_log_marker "restore")" +sandbox_exec sh -c "pkill -9 -f '[o]penclaw' 2>/dev/null; sleep 2; pgrep -af '[o]penclaw' || echo ALL_DEAD" >/dev/null +run_probe_only_or_fail "Guard restore recovery" +SOAK_START_PID="$(wait_for_gateway_up 30)" +if [ -z "$SOAK_START_PID" ]; then + fail "Gateway not up entering soak phase" + gateway_diagnostics "" + exit 1 +fi +# Confirm the restored gateway has guards back in place — otherwise the +# soak measures a crash-looping gateway, not steady-state recovery. +if ! gateway_guards_active "$SOAK_START_PID" 30 "$restore_guard_log_start"; then + fail "Gateway up but guards not active entering soak — restore did not take" + gateway_diagnostics "$SOAK_START_PID" + exit 1 +fi +if ! gateway_serves_inference 30; then + fail "Gateway alive + guards active but inference API not serving entering soak" + gateway_diagnostics "$SOAK_START_PID" + exit 1 +fi +pass "Gateway healthy with guards active and inference API serving (pid=$SOAK_START_PID)" + +# ══════════════════════════════════════════════════════════════════ +# Phase 5: Soak — verify no crash-loop over $SOAK_SECONDS +# ══════════════════════════════════════════════════════════════════ +section "Phase 5: Soak ($SOAK_SECONDS s) — detect crash-loop regression" + +info "Sleeping ${SOAK_SECONDS}s while observing gateway. Health-monitor restart" +info "cadence is ~240s in prod, so a $SOAK_SECONDS s window catches at least one cycle." + +# Sample PID every 15s + probe the inference endpoint every 60s. Count +# distinct PIDs, empty PID samples (gateway down), and inference-endpoint +# failures. The endpoint probe is the user-facing signal — pre-fix the +# ciao crash made `inference.local/v1/models` go silent for the user +# even though the underlying OS process state was variously alive/dead. +declare -a SAMPLES=() +empty_samples=0 +inference_probes=0 +inference_failures=0 +elapsed=0 +INTERVAL=15 +while [ "$elapsed" -lt "$SOAK_SECONDS" ]; do + cur="$(gateway_pid)" + SAMPLES+=("$cur") + [ -z "$cur" ] && empty_samples=$((empty_samples + 1)) + if [ $((elapsed % 60)) -eq 0 ]; then + inference_probes=$((inference_probes + 1)) + if ! gateway_serves_inference 5; then + inference_failures=$((inference_failures + 1)) + fi + fi + sleep "$INTERVAL" + elapsed=$((elapsed + INTERVAL)) +done + +# Distinct non-empty PIDs. +distinct=$(printf '%s\n' "${SAMPLES[@]}" | grep -v '^$' | sort -u | wc -l | tr -d ' ') +total_samples=${#SAMPLES[@]} + +info "Soak summary: ${total_samples} samples, ${distinct} distinct PID(s), ${empty_samples} empty observations, ${inference_failures}/${inference_probes} inference probes failed" + +# Crash-loop signature: many distinct PIDs (>2 over 5min = bad). One respawn +# (distinct=2) is acceptable if health-monitor fires once. Empty samples >1 +# indicate the gateway was actually down for >15s, which is also bad. +if [ "$distinct" -le 2 ] && [ "$empty_samples" -le 1 ]; then + pass "No crash-loop detected during soak ($distinct distinct PIDs, $empty_samples empty samples)" +else + fail "Crash-loop signature: $distinct distinct PIDs and $empty_samples empty samples in ${SOAK_SECONDS}s" + printf ' PID samples: %s\n' "${SAMPLES[*]}" + gateway_log_tail 120 +fi + +# Inference-API availability: this is the user-facing failure surface from +# the #2478 comment ("deployed model not available because curl returns +# nothing"). Zero failures across the soak proves recovery preserves the +# user-visible service, not just the OS process. +if [ "$inference_failures" -eq 0 ]; then + pass "Inference API available throughout soak ($inference_probes/$inference_probes probes succeeded)" +else + fail "Inference API unavailable during soak ($inference_failures/$inference_probes probes failed)" + gateway_log_tail 120 +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 6: Cleanup +# ══════════════════════════════════════════════════════════════════ +section "Phase 6: Cleanup" + +[[ "${NEMOCLAW_E2E_KEEP_SANDBOX:-}" = "1" ]] || nemoclaw "$SANDBOX_NAME" destroy --yes >/dev/null 2>&1 || true + +# ══════════════════════════════════════════════════════════════════ +# Summary +# ══════════════════════════════════════════════════════════════════ +echo "" +echo "========================================" +echo " Issue #2478 crash-loop recovery e2e:" +echo " Passed: $PASS" +echo " Failed: $FAIL" +echo " Total: $TOTAL" +echo "========================================" + +if [ "$FAIL" -eq 0 ]; then + printf '\n\033[1;32m PASS — gateway recovery preserves library guards under repeated kill-respawn and idle soak.\033[0m\n' + exit 0 +else + printf '\n\033[1;31m %d test(s) failed.\033[0m\n' "$FAIL" + exit 1 +fi diff --git a/test/e2e-vpn/test-issue-4434-tui-unreachable-inference.sh b/test/e2e-vpn/test-issue-4434-tui-unreachable-inference.sh new file mode 100755 index 00000000000..ab559055949 --- /dev/null +++ b/test/e2e-vpn/test-issue-4434-tui-unreachable-inference.sh @@ -0,0 +1,229 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Opt-in live repro for #4434: +# openclaw tui must show a visible error, and stop the active spinner, when +# the NVIDIA endpoint is unreachable from the sandbox. +# +# This mutates host firewall state. Run only on a Linux Docker host you control: +# +# NEMOCLAW_ISSUE_4434_LIVE=1 NVIDIA_API_KEY=... \ +# bash test/e2e-vpn/test-issue-4434-tui-unreachable-inference.sh + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=test/e2e-vpn/lib/ci-compatible-inference.sh +. "${SCRIPT_DIR}/lib/ci-compatible-inference.sh" + +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-issue-4434-tui-unreachable}" +INSTALL_LOG="${E2E_ISSUE_4434_INSTALL_LOG:-/tmp/nemoclaw-e2e-issue-4434-install.log}" +CAPTURE_DIR="${NEMOCLAW_ISSUE_4434_CAPTURE_DIR:-$(mktemp -d "${TMPDIR:-/tmp}/nemoclaw-issue-4434.XXXXXX")}" +CAPTURE_FILE="${CAPTURE_DIR}/openclaw-tui-capture.log" +PLAIN_CAPTURE_FILE="${CAPTURE_DIR}/openclaw-tui-capture.plain.log" +TUI_TIMEOUT_SEC="${NEMOCLAW_ISSUE_4434_TUI_TIMEOUT_SEC:-180}" +VISIBLE_ERROR_RE="error|failed|timeout|timed out|unavailable|fetch failed|ETIMEDOUT|ECONN|upstream" +SPINNER_CONNECTED_RE="flibbertigibbeting|[0-9]+m[[:space:]][0-9]+s[[:space:]]*\\|[[:space:]]*connected" +STATUS_LINE_RE="(connecting|gateway connected|connected|sending|running|flibbertigibbeting).*\\|[[:space:]]*(connected|error)" +BLOCKED_IPS=("75.2.113.119" "99.83.136.103") +INSERTED_IPS=() +CLEANUP_SANDBOX=0 + +info() { printf '[issue-4434] %s\n' "$*"; } +fail() { + printf '[issue-4434] FAIL: %s\n' "$*" >&2 + printf '[issue-4434] capture: %s\n' "$CAPTURE_FILE" >&2 + exit 1 +} + +cleanup_firewall() { + local ip + for ip in "${INSERTED_IPS[@]}"; do + sudo iptables -D DOCKER-USER -d "$ip" -j DROP >/dev/null 2>&1 || true + done +} + +cleanup_sandbox() { + if [ "$CLEANUP_SANDBOX" != "1" ]; then + return + fi + if [ "${NEMOCLAW_E2E_SKIP_CLEANUP:-0}" = "1" ]; then + return + fi + SANDBOX_NAME="$SANDBOX_NAME" bash "${SCRIPT_DIR}/e2e-cloud-experimental/cleanup.sh" --verify >/dev/null 2>&1 || true +} + +cleanup() { + cleanup_firewall + cleanup_sandbox +} +trap cleanup EXIT + +if [ "${NEMOCLAW_ISSUE_4434_LIVE:-0}" != "1" ]; then + info "skipping: set NEMOCLAW_ISSUE_4434_LIVE=1 to run the privileged live repro" + exit 0 +fi + +if nemoclaw_e2e_using_compatible_inference; then + info "skipping: hosted compatible inference is gateway-managed; this repro only blocks sandbox egress" + exit 0 +fi + +if [ "$(uname -s)" != "Linux" ]; then + fail "Linux host required for DOCKER-USER iptables repro" +fi +for command in docker sudo expect curl timeout perl; do + command -v "$command" >/dev/null 2>&1 || fail "missing required command: $command" +done +docker info >/dev/null 2>&1 || fail "Docker is not running" +sudo -n true >/dev/null 2>&1 || fail "passwordless sudo is required for non-interactive iptables cleanup" +nemoclaw_e2e_configure_compatible_inference || fail "hosted CI inference could not be configured" +nemoclaw_e2e_require_hosted_inference_key || exit 1 + +mkdir -p "$CAPTURE_DIR" +CLEANUP_SANDBOX=1 + +export NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" +export E2E_CLOUD_ONBOARD_INSTALL_LOG="$INSTALL_LOG" +export NEMOCLAW_E2E_KEEP_SANDBOX=1 +export NEMOCLAW_NON_INTERACTIVE="${NEMOCLAW_NON_INTERACTIVE:-1}" +export NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE="${NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE:-1}" +export NEMOCLAW_RECREATE_SANDBOX="${NEMOCLAW_RECREATE_SANDBOX:-1}" +export NEMOCLAW_CLOUD_EXPERIMENTAL_MODEL="${NEMOCLAW_CLOUD_EXPERIMENTAL_MODEL:-nvidia/nemotron-3-super-120b-a12b}" + +info "onboarding sandbox ${SANDBOX_NAME} with ${NEMOCLAW_CLOUD_EXPERIMENTAL_MODEL}" +bash "${SCRIPT_DIR}/test-cloud-onboard-e2e.sh" + +# Pick up PATH changes from the installer in this shell. +# shellcheck source=test/e2e-vpn/lib/install-path-refresh.sh +. "${SCRIPT_DIR}/lib/install-path-refresh.sh" +nemoclaw_refresh_install_env +nemoclaw_ensure_local_bin_on_path +export PATH="/usr/local/bin:${HOME}/.local/bin:${PATH}" +for command in nemoclaw openshell; do + command -v "$command" >/dev/null 2>&1 || fail "missing installed command after onboard: $command" +done + +openclaw_version="$(openshell sandbox exec --name "$SANDBOX_NAME" -- openclaw --version 2>&1 || true)" +info "sandbox OpenClaw version: ${openclaw_version}" +if ! grep -q "2026.5.27" <<<"$openclaw_version"; then + fail "expected sandbox OpenClaw 2026.5.27" +fi + +status_log="${CAPTURE_DIR}/nemoclaw-status-before-block.log" +if ! nemoclaw "$SANDBOX_NAME" status >"$status_log" 2>&1; then + fail "nemoclaw ${SANDBOX_NAME} status failed before firewall block" +fi +if ! grep -Eiq "inference.*healthy|healthy.*inference" "$status_log"; then + if grep -Eiq "Inference:[[:space:]]*not probed" "$status_log"; then + info "status skipped inference reachability; probing inference.local directly" + else + fail "pre-block status did not report healthy or not-probed inference" + fi +fi + +route_log="${CAPTURE_DIR}/openshell-inference-before-block.log" +if ! route_output=$(openshell inference get 2>&1); then + printf '%s\n' "$route_output" >"$route_log" + fail "openshell inference get failed before firewall block" +fi +printf '%s\n' "$route_output" >"$route_log" +expected_provider="$(nemoclaw_e2e_expected_route_provider)" +expected_model="$(nemoclaw_e2e_hosted_inference_model)" +if ! nemoclaw_e2e_inference_output_matches "$route_output" "$expected_provider" "$expected_model"; then + route_plain="$(printf '%s' "$route_output" | nemoclaw_e2e_strip_ansi)" + fail "pre-block OpenShell route was not ${expected_provider} / ${expected_model}: ${route_plain:0:240}" +fi + +preblock_probe_log="${CAPTURE_DIR}/inference-local-before-block.log" +preblock_payload="$(printf '{"model":"%s","messages":[{"role":"user","content":"Reply with OK."}],"max_tokens":8}' "$expected_model")" +preblock_payload_arg="$(printf '%q' "$preblock_payload")" +if ! timeout 90 openshell sandbox exec --name "$SANDBOX_NAME" -- sh -lc \ + "curl -sf --max-time 60 https://inference.local/v1/chat/completions -H 'Content-Type: application/json' -d $preblock_payload_arg >/dev/null" \ + >"$preblock_probe_log" 2>&1; then + fail "inference.local was not reachable from inside the sandbox before firewall block" +fi + +connect_probe_log="${CAPTURE_DIR}/nemoclaw-connect-probe-before-block.log" +if ! nemoclaw "$SANDBOX_NAME" connect --probe-only >"$connect_probe_log" 2>&1; then + fail "nemoclaw ${SANDBOX_NAME} connect --probe-only failed before firewall block" +fi + +info "installing DOCKER-USER DROP rules for NVIDIA endpoint IPs" +for ip in "${BLOCKED_IPS[@]}"; do + sudo iptables -I DOCKER-USER -d "$ip" -j DROP + INSERTED_IPS+=("$ip") +done + +block_probe_log="${CAPTURE_DIR}/blocked-endpoint-probe.log" +set +e +timeout 25 openshell sandbox exec --name "$SANDBOX_NAME" -- sh -lc \ + 'curl -sk --connect-timeout 5 --max-time 12 https://inference.nvidia.com/v1/models >/tmp/issue4434-models.out 2>&1' \ + >"$block_probe_log" 2>&1 +block_probe_rc=$? +set -e +if [ "$block_probe_rc" -eq 0 ]; then + fail "inference.nvidia.com was still reachable from inside the sandbox after firewall block" +fi +info "sandbox endpoint block verified (probe exit ${block_probe_rc})" + +info "launching openclaw tui through OpenShell sandbox exec --tty" +set +e +env \ + NEMOCLAW_ISSUE_4434_SANDBOX="$SANDBOX_NAME" \ + NEMOCLAW_ISSUE_4434_CAPTURE="$CAPTURE_FILE" \ + NEMOCLAW_ISSUE_4434_TUI_TIMEOUT="$TUI_TIMEOUT_SEC" \ + expect >"${CAPTURE_DIR}/expect.log" 2>&1 <<'EXPECT' +set timeout $env(NEMOCLAW_ISSUE_4434_TUI_TIMEOUT) +set sandbox $env(NEMOCLAW_ISSUE_4434_SANDBOX) +set capture $env(NEMOCLAW_ISSUE_4434_CAPTURE) +log_file -a $capture +spawn openshell sandbox exec --name $sandbox --tty -- sh -lc {export TERM=xterm-256color; cd /sandbox; openclaw tui} +sleep 10 +send -- "hello\r" +expect { + -nocase -re {(error|failed|timeout|timed out|unavailable|fetch failed|ETIMEDOUT|ECONN|upstream)} { + sleep 5 + send "\003" + sleep 1 + send "\003" + exit 0 + } + timeout { + send "\003" + sleep 1 + send "\003" + exit 20 + } + eof { exit 21 } +} +EXPECT +expect_rc=$? +set -e + +perl -pe 's/\x1b\][^\a]*(?:\a|\x1b\\)//g; s/\x1b\[[0-9;?]*[ -\/]*[@-~]//g; s/\r/\n/g' \ + "$CAPTURE_FILE" >"$PLAIN_CAPTURE_FILE" + +if ! grep -Eiq "$VISIBLE_ERROR_RE" "$PLAIN_CAPTURE_FILE"; then + if grep -Eiq "$SPINNER_CONNECTED_RE" "$PLAIN_CAPTURE_FILE"; then + fail "matched #4434 signature: spinner plus connected status with no visible error" + fi + fail "TUI did not surface a visible inference error before the timeout window" +fi +if [ "$expect_rc" -ne 0 ]; then + fail "expect harness exited ${expect_rc} even though an error-looking capture was found" +fi +last_status_line="$(grep -E "$STATUS_LINE_RE" "$PLAIN_CAPTURE_FILE" | tail -1 || true)" +if [ -z "$last_status_line" ]; then + fail "TUI capture did not include a recognizable final status line" +fi +if ! grep -Eiq "\\|[[:space:]]*error\\b" <<<"$last_status_line"; then + if grep -Eiq "$SPINNER_CONNECTED_RE" <<<"$last_status_line"; then + fail "TUI capture still ends with active connected spinner after the visible error" + fi + fail "TUI capture did not end with a visible error status after the failed run" +fi + +info "PASS: openclaw tui surfaced a visible unreachable-inference error and stopped the spinner" +info "capture: ${PLAIN_CAPTURE_FILE}" diff --git a/test/e2e-vpn/test-issue-4462-scope-upgrade-approval.sh b/test/e2e-vpn/test-issue-4462-scope-upgrade-approval.sh new file mode 100755 index 00000000000..97e746c47fb --- /dev/null +++ b/test/e2e-vpn/test-issue-4462-scope-upgrade-approval.sh @@ -0,0 +1,1111 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Issue #4462 E2E: +# +# Build a real NemoClaw/OpenClaw sandbox, create a low-scope CLI device +# approval, trigger the later `openclaw agent` operator.write scope upgrade, and +# then run in one of two modes: +# +# approval Approve the pending request through the fixed proxy-env guard, +# verify the request is no longer pending, and verify the next +# `openclaw agent` turn stays on the gateway path. +# legacy-repro Characterize the old gateway-pinned approve path. Current +# OpenClaw builds may return a #4462 failure, return a replacement +# request id, time out, succeed cleanly, or apply approval before +# reporting failure. If the request remains pending, recover +# through the fixed proxy-env guard so the sandbox is not left +# dirty. This mode is diagnostic, not the fix gate. +# +# Prerequisites: +# - Docker running +# - NVIDIA_API_KEY set +# - NEMOCLAW_NON_INTERACTIVE=1 +# - NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 + +# shellcheck disable=SC2016 +# SC2016: remote sandbox scripts intentionally expand inside the sandbox. + +set -uo pipefail + +export NEMOCLAW_E2E_DEFAULT_TIMEOUT="${NEMOCLAW_E2E_DEFAULT_TIMEOUT:-2700}" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +# shellcheck source=test/e2e-vpn/e2e-timeout.sh +. "${SCRIPT_DIR}/e2e-timeout.sh" + +PASS=0 +FAIL=0 +TOTAL=0 + +pass() { + ((PASS++)) + ((TOTAL++)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} + +fail() { + ((FAIL++)) + ((TOTAL++)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} + +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} + +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } + +finish_success() { + section "Summary" + echo "" + printf ' Total: %d | \033[32mPass: %d\033[0m | \033[31mFail: %d\033[0m\n' \ + "$TOTAL" "$PASS" "$FAIL" + echo "" + echo "$1" + exit 0 +} + +if [ -d /workspace ] && [ -f /workspace/install.sh ]; then + REPO="/workspace" +elif [ -f "$(cd "${SCRIPT_DIR}/../.." && pwd)/install.sh" ]; then + REPO="$(cd "${SCRIPT_DIR}/../.." && pwd)" +else + echo "ERROR: Cannot find repo root." >&2 + exit 1 +fi + +E2E_DIR="${SCRIPT_DIR}" +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-issue-4462-scope-upgrade}" +OPENSHELL_BIN="${NEMOCLAW_OPENSHELL_BIN:-openshell}" +TEST_MODE="${NEMOCLAW_4462_MODE:-approval}" +case "$TEST_MODE" in + approval | legacy-repro) ;; + *) + fail "Unknown NEMOCLAW_4462_MODE=${TEST_MODE}; expected approval or legacy-repro" + exit 1 + ;; +esac +INSTALL_LOG="${NEMOCLAW_4462_INSTALL_LOG:-/tmp/nemoclaw-e2e-issue-4462-scope-upgrade-install.log}" +APPROVAL_LOG="${NEMOCLAW_4462_APPROVAL_LOG:-/tmp/nemoclaw-issue-4462-scope-upgrade-approval.log}" +AGENT_LOG="${NEMOCLAW_4462_AGENT_LOG:-/tmp/nemoclaw-issue-4462-scope-upgrade-agent.log}" +STATE_LOG="${NEMOCLAW_4462_STATE_LOG:-/tmp/nemoclaw-issue-4462-scope-upgrade-state.log}" +INSTALL_TIMEOUT_SECONDS="${NEMOCLAW_E2E_INSTALL_TIMEOUT_SECONDS:-1800}" + +AUTO_PAIR_FAST_DEADLINE_DEFAULT="3" +AUTO_PAIR_DEADLINE_DEFAULT="30" +AUTO_PAIR_SLOW_INTERVAL_DEFAULT="600" +AUTO_PAIR_RUN_TIMEOUT_DEFAULT="10" +if [ "$TEST_MODE" = "legacy-repro" ]; then + AUTO_PAIR_FAST_DEADLINE_DEFAULT="1" + AUTO_PAIR_DEADLINE_DEFAULT="12" + AUTO_PAIR_SLOW_INTERVAL_DEFAULT="1" + AUTO_PAIR_RUN_TIMEOUT_DEFAULT="2" +fi +AUTO_PAIR_FAST_DEADLINE_SECS="${NEMOCLAW_4462_AUTO_PAIR_FAST_DEADLINE_SECS:-${NEMOCLAW_AUTO_PAIR_FAST_DEADLINE_SECS:-$AUTO_PAIR_FAST_DEADLINE_DEFAULT}}" +AUTO_PAIR_DEADLINE_SECS="${NEMOCLAW_4462_AUTO_PAIR_DEADLINE_SECS:-${NEMOCLAW_AUTO_PAIR_DEADLINE_SECS:-$AUTO_PAIR_DEADLINE_DEFAULT}}" +AUTO_PAIR_SLOW_INTERVAL_SECS="${NEMOCLAW_4462_AUTO_PAIR_SLOW_INTERVAL_SECS:-${NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS:-$AUTO_PAIR_SLOW_INTERVAL_DEFAULT}}" +AUTO_PAIR_RUN_TIMEOUT_SECS="${NEMOCLAW_4462_AUTO_PAIR_RUN_TIMEOUT_SECS:-${NEMOCLAW_AUTO_PAIR_RUN_TIMEOUT_SECS:-$AUTO_PAIR_RUN_TIMEOUT_DEFAULT}}" +# Current onboard finalization may warm up and approve the CLI operator.read/write +# scope-upgrade before this E2E inspects the intermediate low-scope state. That +# is an acceptable final authorization state only if operator.admin is absent; +# the script must still prove the final agent turn stays on the gateway path. +# In legacy-repro mode, a preapproved state or an agent trigger that succeeds +# without producing a pending upgrade leaves no gateway-pinned approve request +# to characterize, so the final result calls that out explicitly instead of +# claiming the legacy approval behavior was exercised. +SCOPE_UPGRADE_ALREADY_SATISFIED=0 +LEGACY_SCOPE_UPGRADE_NOT_REPRODUCED=0 + +# shellcheck source=test/e2e-vpn/lib/sandbox-teardown.sh +. "${E2E_DIR}/lib/sandbox-teardown.sh" +# shellcheck source=test/e2e-vpn/lib/install-path-refresh.sh +. "${E2E_DIR}/lib/install-path-refresh.sh" +# shellcheck source=test/e2e-vpn/lib/openclaw-json.sh +. "${E2E_DIR}/lib/openclaw-json.sh" +register_sandbox_for_teardown "$SANDBOX_NAME" + +quote_for_remote_sh() { + local value="${1:-}" + printf "'%s'" "$(printf '%s' "$value" | sed "s/'/'\\\\''/g")" +} + +sandbox_exec_sh_script() { + local seconds="$1" + local script="$2" + shift 2 + local encoded remote_cmd arg + encoded="$(printf '%s' "$script" | base64 | tr -d '\n')" + remote_cmd="tmp=\$(mktemp); trap 'rm -f \"\$tmp\"' EXIT; printf %s $(quote_for_remote_sh "$encoded") | base64 -d > \"\$tmp\"; bash \"\$tmp\"" + for arg in "$@"; do + remote_cmd+=" $(quote_for_remote_sh "$arg")" + done + run_with_timeout "$seconds" "$OPENSHELL_BIN" sandbox exec --name "$SANDBOX_NAME" -- sh -lc "$remote_cmd" +} + +extract_json_doc() { + python3 -c ' +import json +import sys + +raw = sys.stdin.read() +decoder = json.JSONDecoder() +for idx, char in enumerate(raw): + if char != "{": + continue + try: + doc, _end = decoder.raw_decode(raw[idx:]) + except Exception: + continue + print(json.dumps(doc, sort_keys=True)) + raise SystemExit(0) +raise SystemExit(1) +' +} + +json_field() { + local field="$1" + python3 -c ' +import json +import sys + +field = sys.argv[1] +doc = json.load(sys.stdin) +value = doc +for part in field.split("."): + if not isinstance(value, dict): + value = None + break + value = value.get(part) +if isinstance(value, (dict, list)): + print(json.dumps(value, sort_keys=True)) +elif value is not None: + print(value) +' "$field" +} + +extract_scope_request_id_from_output() { + sed -nE 's/.*requestId: ([[:alnum:]_-]+).*/\1/p' | head -1 +} + +device_state_json() { + local output rc + output=$(sandbox_exec_sh_script 60 ' +set -u +if [ -r /tmp/nemoclaw-proxy-env.sh ]; then + # shellcheck source=/dev/null + . /tmp/nemoclaw-proxy-env.sh +fi +python3 - <<'"'"'PY'"'"' +import json +import os +from pathlib import Path + +root = Path(os.environ.get("OPENCLAW_STATE_DIR") or "/sandbox/.openclaw") / "devices" + +def load(name): + path = root / name + try: + value = json.loads(path.read_text(encoding="utf-8")) + except FileNotFoundError: + return {} + if not isinstance(value, dict): + return {} + return value + +pending = load("pending.json") +paired = load("paired.json") +print(json.dumps({ + "pending": list(pending.values()), + "paired": list(paired.values()), + "paths": { + "pending": str(root / "pending.json"), + "paired": str(root / "paired.json"), + }, +}, sort_keys=True)) +PY +' 2>&1) + rc=$? + if [ "$rc" -ne 0 ]; then + printf '%s\n' "$output" + return "$rc" + fi + printf '%s\n' "$output" | extract_json_doc +} + +summarize_device_state() { + local state_doc + state_doc="$(cat)" + OPENCLAW_4462_DEVICE_STATE="$state_doc" python3 - <<'PY' +import json +import os +import sys + +raw = os.environ.get("OPENCLAW_4462_DEVICE_STATE") or "{}" +doc = json.loads(raw) +pending = doc.get("pending") or [] +paired = doc.get("paired") or [] + +def norm(value): + return str(value or "").strip() + +def is_cli(entry): + mode = norm(entry.get("clientMode")).lower() + client = norm(entry.get("clientId")).lower() + return mode == "cli" or "cli" in client + +def scope_list(entry, *keys): + out = [] + seen = set() + for key in keys: + for scope in entry.get(key) or []: + scope = norm(scope) + if scope and scope not in seen: + out.append(scope) + seen.add(scope) + return out + +def fmt(values): + return ",".join(values) if values else "-" + +print(f"pending={len(pending)} paired={len(paired)}") +for label, rows in (("pending", pending), ("paired", paired)): + for row in rows: + if not isinstance(row, dict) or not is_cli(row): + continue + request_id = row.get("requestId") or "-" + device_id = row.get("deviceId") or "-" + approved = scope_list(row, "approvedScopes") + if label == "paired": + approved = approved or scope_list(row, "scopes") + requested = scope_list(row, "scopes", "requestedScopes") + print( + f"{label}: pendingCount={len(pending)} requestId={request_id} " + f"deviceId={device_id} approvedScopes={fmt(approved)} " + f"requestedScopes={fmt(requested)}" + ) +PY +} + +select_cli_request() { + local kind="$1" + python3 -c ' +import json +import sys + +kind = sys.argv[1] +doc = json.load(sys.stdin) +pending = [p for p in doc.get("pending") or [] if isinstance(p, dict)] +paired = [p for p in doc.get("paired") or [] if isinstance(p, dict)] + +def norm(value): + return str(value or "").strip() + +def is_cli(entry): + return norm(entry.get("clientMode")).lower() == "cli" or "cli" in norm(entry.get("clientId")).lower() + +def roles(entry): + out = set() + role = norm(entry.get("role")) + if role: + out.add(role) + for role in entry.get("roles") or []: + role = norm(role) + if role: + out.add(role) + return out + +def scopes(entry): + return {norm(scope) for scope in (entry.get("scopes") or []) if norm(scope)} + +def approved_scopes(entry): + return {norm(scope) for scope in (entry.get("approvedScopes") or entry.get("scopes") or []) if norm(scope)} + +paired_by_device = {norm(item.get("deviceId")): item for item in paired if norm(item.get("deviceId"))} + +for req in sorted(pending, key=lambda item: item.get("ts") or 0, reverse=True): + if not is_cli(req) or not norm(req.get("requestId")): + continue + paired_entry = paired_by_device.get(norm(req.get("deviceId"))) + requested = scopes(req) + approved = approved_scopes(paired_entry or {}) + if kind == "new" and not paired_entry: + print(req["requestId"]) + raise SystemExit(0) + if kind == "scope-upgrade" and paired_entry and roles(req).issubset(roles(paired_entry) or roles(req)): + # OpenClaw 2026.5.27 may create a follow-on operator.admin request after + # the write/read request has already been applied. Keep this selector + # focused on the NemoClaw gateway-mode upgrade contract. + if {"operator.write", "operator.read"}.intersection(requested) and not requested.issubset(approved): + print(req["requestId"]) + raise SystemExit(0) +raise SystemExit(1) +' "$kind" +} + +select_cli_paired_without_write() { + python3 -c ' +import json +import sys + +doc = json.load(sys.stdin) +paired = [p for p in doc.get("paired") or [] if isinstance(p, dict)] + +def norm(value): + return str(value or "").strip() + +def is_cli(entry): + return norm(entry.get("clientMode")).lower() == "cli" or "cli" in norm(entry.get("clientId")).lower() + +def scopes(entry): + return {norm(scope) for scope in (entry.get("approvedScopes") or entry.get("scopes") or []) if norm(scope)} + +for device in sorted(paired, key=lambda item: item.get("approvedAtMs") or 0, reverse=True): + if not is_cli(device): + continue + approved = scopes(device) + if "operator.pairing" in approved and "operator.write" not in approved and "operator.admin" not in approved: + print(norm(device.get("deviceId")) or "cli-device") + raise SystemExit(0) +raise SystemExit(1) +' +} + +select_cli_paired_with_agent_scopes() { + python3 -c ' +import json +import sys + +doc = json.load(sys.stdin) +paired = [p for p in doc.get("paired") or [] if isinstance(p, dict)] + +def norm(value): + return str(value or "").strip() + +def is_cli(entry): + return norm(entry.get("clientMode")).lower() == "cli" or "cli" in norm(entry.get("clientId")).lower() + +def scopes(entry): + return {norm(scope) for scope in (entry.get("approvedScopes") or entry.get("scopes") or []) if norm(scope)} + +for device in sorted(paired, key=lambda item: item.get("approvedAtMs") or 0, reverse=True): + if not is_cli(device): + continue + approved = scopes(device) + if {"operator.write", "operator.read"}.issubset(approved): + print(norm(device.get("deviceId")) or "cli-device") + raise SystemExit(0) +raise SystemExit(1) +' +} + +select_cli_paired_with_admin() { + python3 -c ' +import json +import sys + +doc = json.load(sys.stdin) +paired = [p for p in doc.get("paired") or [] if isinstance(p, dict)] + +def norm(value): + return str(value or "").strip() + +def is_cli(entry): + return norm(entry.get("clientMode")).lower() == "cli" or "cli" in norm(entry.get("clientId")).lower() + +def scopes(entry): + return {norm(scope) for scope in (entry.get("approvedScopes") or entry.get("scopes") or []) if norm(scope)} + +for device in sorted(paired, key=lambda item: item.get("approvedAtMs") or 0, reverse=True): + if is_cli(device) and "operator.admin" in scopes(device): + print(norm(device.get("deviceId")) or "cli-device") + raise SystemExit(0) +raise SystemExit(1) +' +} + +approve_request() { + local request_id="$1" + local label="$2" + local allow_already_approved="${3:-0}" + local output rc approve_json approved_id before_url before_port before_token after_url after_port after_token approve_env state_after_approve approved_after_approve pending_after_approve + output=$(sandbox_exec_sh_script 90 ' + set -u + request_id="$1" + real_openclaw="$(command -v openclaw || true)" + if [ -z "$real_openclaw" ]; then + echo "missing real openclaw binary" >&2 + exit 2 + fi + if [ ! -r /tmp/nemoclaw-proxy-env.sh ]; then + echo "missing /tmp/nemoclaw-proxy-env.sh" >&2 + exit 2 + fi + # shellcheck source=/dev/null + . /tmp/nemoclaw-proxy-env.sh + probe_dir="$(mktemp -d /tmp/nemoclaw-approve-env.XXXXXX)" + probe_log="$probe_dir/env.log" + cat >"$probe_dir/openclaw" <<'"'"'PROBESH'"'"' +#!/bin/sh +token_state="$([ "${OPENCLAW_GATEWAY_TOKEN+x}" = x ] && printf set || printf unset)" +printf "__APPROVE_SUBPROCESS_ENV__=%s:%s:%s\n" "${OPENCLAW_GATEWAY_URL-unset}" "${OPENCLAW_GATEWAY_PORT-unset}" "$token_state" >>"$NEMOCLAW_4462_APPROVE_ENV_LOG" +exec "$NEMOCLAW_4462_REAL_OPENCLAW" "$@" +PROBESH + chmod +x "$probe_dir/openclaw" + export NEMOCLAW_4462_REAL_OPENCLAW="$real_openclaw" + export NEMOCLAW_4462_APPROVE_ENV_LOG="$probe_log" + PATH="$probe_dir:$PATH" + printf "__URL_BEFORE__=%s\n" "${OPENCLAW_GATEWAY_URL-unset}" + printf "__PORT_BEFORE__=%s\n" "${OPENCLAW_GATEWAY_PORT-unset}" + printf "__TOKEN_BEFORE__=%s\n" "$([ "${OPENCLAW_GATEWAY_TOKEN+x}" = x ] && printf set || printf unset)" + set +e + approve_output="$(openclaw devices approve "$request_id" --json 2>&1)" + approve_rc=$? + set -e + printf "__APPROVE_RC__=%s\n" "$approve_rc" + printf "__APPROVE_OUTPUT_BEGIN__\n%s\n__APPROVE_OUTPUT_END__\n" "$approve_output" + if [ -r "$probe_log" ]; then + cat "$probe_log" + else + printf "__APPROVE_SUBPROCESS_ENV__=missing:missing:missing\n" + fi + printf "__URL_AFTER__=%s\n" "${OPENCLAW_GATEWAY_URL-unset}" + printf "__PORT_AFTER__=%s\n" "${OPENCLAW_GATEWAY_PORT-unset}" + printf "__TOKEN_AFTER__=%s\n" "$([ "${OPENCLAW_GATEWAY_TOKEN+x}" = x ] && printf set || printf unset)" + rm -rf "$probe_dir" + exit "$approve_rc" + ' "$request_id" 2>&1) + rc=$? + { + printf '=== approve %s request=%s rc=%s ===\n' "$label" "$request_id" "$rc" + printf '%s\n' "$output" + } >>"$APPROVAL_LOG" + if [ "$rc" -ne 0 ]; then + if [ "$allow_already_approved" = "1" ]; then + state_after_approve="$(device_state_json 2>&1)" || state_after_approve="" + if [ -n "$state_after_approve" ]; then + printf '=== state after failed approve %s request=%s ===\n%s\n' "$label" "$request_id" "$state_after_approve" >>"$STATE_LOG" + approved_after_approve=$(printf '%s' "$state_after_approve" | select_cli_paired_with_agent_scopes 2>/dev/null) || approved_after_approve="" + pending_after_approve=$(printf '%s' "$state_after_approve" | select_cli_request scope-upgrade 2>/dev/null) || pending_after_approve="" + if [ -n "$approved_after_approve" ] && [ -z "$pending_after_approve" ]; then + pass "${label}: request was already approved when fixed approve retried (${approved_after_approve})" + return 0 + fi + fi + fi + fail "${label}: openclaw devices approve failed for ${request_id}: ${output:0:500}" + return 1 + fi + before_url=$(sed -n 's/^__URL_BEFORE__=//p' <<<"$output" | tail -1) + before_port=$(sed -n 's/^__PORT_BEFORE__=//p' <<<"$output" | tail -1) + before_token=$(sed -n 's/^__TOKEN_BEFORE__=//p' <<<"$output" | tail -1) + after_url=$(sed -n 's/^__URL_AFTER__=//p' <<<"$output" | tail -1) + after_port=$(sed -n 's/^__PORT_AFTER__=//p' <<<"$output" | tail -1) + after_token=$(sed -n 's/^__TOKEN_AFTER__=//p' <<<"$output" | tail -1) + approve_env=$(sed -n 's/^__APPROVE_SUBPROCESS_ENV__=//p' <<<"$output" | tail -1) + if [[ "$before_url" != ws://127.0.0.1:* ]] && [[ "$before_url" != ws://localhost:* ]]; then + fail "${label}: proxy env did not expose a loopback OPENCLAW_GATEWAY_URL before approve (${before_url:-empty})" + return 1 + fi + if [ -z "$before_port" ] || [ "$before_port" = "unset" ] || [ "$before_token" != "set" ]; then + fail "${label}: proxy env did not expose OPENCLAW_GATEWAY_PORT/TOKEN before approve (port=${before_port:-empty} token_state=${before_token:-empty})" + return 1 + fi + if [ "$after_url" != "$before_url" ]; then + fail "${label}: devices approve leaked OPENCLAW_GATEWAY_URL mutation into caller shell (${before_url} -> ${after_url})" + return 1 + fi + if [ "$after_port" != "$before_port" ] || [ "$after_token" != "$before_token" ]; then + fail "${label}: devices approve leaked gateway port/token mutation into caller shell (port ${before_port} -> ${after_port}; token changed=$([ "$after_token" != "$before_token" ] && printf yes || printf no))" + return 1 + fi + if [ "$approve_env" != "unset:unset:unset" ]; then + fail "${label}: devices approve subprocess retained gateway env (${approve_env:-empty})" + return 1 + fi + approve_json=$(sed -n '/^__APPROVE_OUTPUT_BEGIN__$/,/^__APPROVE_OUTPUT_END__$/p' <<<"$output" | sed '1d;$d' | extract_json_doc 2>/dev/null) || approve_json="" + if [ -z "$approve_json" ]; then + fail "${label}: approve output did not contain JSON: ${output:0:500}" + return 1 + fi + approved_id=$(printf '%s' "$approve_json" | json_field requestId) + if [ "$approved_id" != "$request_id" ]; then + fail "${label}: approve returned requestId=${approved_id:-empty}, expected ${request_id}" + return 1 + fi + pass "${label}: openclaw devices approve ${request_id} --json succeeded with caller gateway URL preserved" +} + +legacy_gateway_pinned_approval_characterization() { + local request_id="$1" + local output legacy_rc before_url legacy_approve_output legacy_failure_request_id state pending_after approved_after recovery_request_id + output=$(sandbox_exec_sh_script 90 ' +set -u +request_id="$1" +if [ ! -r /tmp/nemoclaw-proxy-env.sh ]; then + echo "missing /tmp/nemoclaw-proxy-env.sh" >&2 + exit 2 +fi +# shellcheck source=/dev/null +. /tmp/nemoclaw-proxy-env.sh +printf "__URL_FOR_LEGACY_APPROVE__=%s\n" "${OPENCLAW_GATEWAY_URL-unset}" +OPENCLAW_4462_REQUEST_ID="$request_id" python3 - <<'"'"'PY'"'"' +import os +import subprocess + +request_id = os.environ["OPENCLAW_4462_REQUEST_ID"] +env = os.environ.copy() +try: + proc = subprocess.run( + ["openclaw", "devices", "approve", request_id, "--json"], + capture_output=True, + text=True, + timeout=20, + env=env, + ) + print(f"__LEGACY_APPROVE_RC__={proc.returncode}") + print("__LEGACY_APPROVE_OUTPUT_BEGIN__") + if proc.stdout: + print(proc.stdout, end="") + if proc.stderr: + print(proc.stderr, end="") + print("\n__LEGACY_APPROVE_OUTPUT_END__") +except subprocess.TimeoutExpired as exc: + print("__LEGACY_APPROVE_RC__=124") + print("__LEGACY_APPROVE_OUTPUT_BEGIN__") + if exc.stdout: + print(exc.stdout if isinstance(exc.stdout, str) else exc.stdout.decode(), end="") + if exc.stderr: + print(exc.stderr if isinstance(exc.stderr, str) else exc.stderr.decode(), end="") + print("\nTIMEOUT waiting for gateway-pinned devices approve") + print("__LEGACY_APPROVE_OUTPUT_END__") +PY +printf "__URL_AFTER_LEGACY_APPROVE__=%s\n" "${OPENCLAW_GATEWAY_URL-unset}" +exit 0 +' "$request_id" 2>&1) + { + printf '=== legacy gateway-pinned approve request=%s ===\n' "$request_id" + printf '%s\n' "$output" + } >>"$APPROVAL_LOG" + before_url=$(sed -n 's/^__URL_FOR_LEGACY_APPROVE__=//p' <<<"$output" | tail -1) + if [[ "$before_url" != ws://127.0.0.1:* ]] && [[ "$before_url" != ws://localhost:* ]]; then + fail "legacy characterization did not run with gateway URL pinned (${before_url:-empty})" + return 1 + fi + legacy_rc=$(sed -n 's/^__LEGACY_APPROVE_RC__=//p' <<<"$output" | tail -1) + if [ -z "$legacy_rc" ]; then + fail "legacy characterization did not report approve rc: ${output:0:500}" + return 1 + fi + legacy_approve_output=$(sed -n '/^__LEGACY_APPROVE_OUTPUT_BEGIN__$/,/^__LEGACY_APPROVE_OUTPUT_END__$/p' <<<"$output" | sed '1d;$d') + if [ "$legacy_rc" = "0" ]; then + pass "legacy gateway-pinned devices approve now exits successfully" + elif [ "$legacy_rc" = "124" ]; then + pass "legacy gateway-pinned devices approve timed out before approval could complete" + elif grep -Fq "GatewayClientRequestError" <<<"$legacy_approve_output" \ + && grep -Fq "scope upgrade pending approval" <<<"$legacy_approve_output"; then + legacy_failure_request_id=$(printf '%s' "$legacy_approve_output" | extract_scope_request_id_from_output) || legacy_failure_request_id="" + if [ -z "$legacy_failure_request_id" ]; then + fail "legacy gateway-pinned devices approve did not report a requestId: ${legacy_approve_output:0:500}" + return 1 + fi + if [ "$legacy_failure_request_id" = "$request_id" ]; then + pass "legacy gateway-pinned devices approve returns the #4462 pending-scope failure for the requested id" + else + pass "legacy gateway-pinned devices approve returns the #4462 pending-scope failure for replacement id ${legacy_failure_request_id}" + fi + else + pass "legacy gateway-pinned devices approve returned nonzero without the known #4462 signature" + fi + + state="$(device_state_json 2>&1)" || { + fail "Could not read OpenClaw device state after legacy approve failure: ${state:0:500}" + return 1 + } + printf '=== state after legacy gateway-pinned approve failure ===\n%s\n' "$state" >>"$STATE_LOG" + pending_after=$(printf '%s' "$state" | select_cli_request scope-upgrade 2>/dev/null) || pending_after="" + approved_after=$(printf '%s' "$state" | select_cli_paired_with_agent_scopes 2>/dev/null) || approved_after="" + if [ -n "$pending_after" ]; then + pass "legacy gateway-pinned approve leaves the CLI scope-upgrade request pending" + recovery_request_id="$pending_after" + approve_request "$recovery_request_id" "recovery after legacy characterization" 1 || return 1 + pass "fixed devices approve path recovers the pending legacy request" + return 0 + fi + if [ -n "$approved_after" ]; then + pass "legacy gateway-pinned approve returned failure after applying the scope upgrade (${approved_after})" + return 0 + fi + fail "legacy gateway-pinned characterization left neither pending nor approved CLI scope-upgrade state: $(printf '%s' "$state" | summarize_device_state)" + return 1 +} + +wait_for_auto_pair_watcher_inactive() { + local output rc + for _attempt in 1 2 3 4 5 6 7 8 9 10 11 12; do + output=$(sandbox_exec_sh_script 20 ' +set -u +find_auto_pair_pids() { + for proc in /proc/[0-9]*; do + pid="${proc##*/}" + [ "$pid" = "$$" ] && continue + [ -r "$proc/cmdline" ] || continue + cmd="$(tr "\000" " " <"$proc/cmdline" 2>/dev/null || true)" + case "$cmd" in + *"python3 -"*) + fd1="$(readlink "$proc/fd/1" 2>/dev/null || true)" + fd2="$(readlink "$proc/fd/2" 2>/dev/null || true)" + case "${fd1} ${fd2}" in + *"/tmp/auto-pair.log"*) printf "%s\n" "$pid" ;; + esac + ;; + esac + done | sort -u +} +if [ -r /tmp/auto-pair.log ]; then + if grep -F "[auto-pair] watcher deadline reached" /tmp/auto-pair.log >/dev/null; then + echo "__AUTO_PAIR_WATCHER__=deadline-reached" + tail -20 /tmp/auto-pair.log + exit 0 + fi + pids="$(find_auto_pair_pids)" + if [ -z "$pids" ]; then + echo "__AUTO_PAIR_WATCHER__=inactive" + tail -20 /tmp/auto-pair.log || true + exit 0 + fi + echo "__AUTO_PAIR_WATCHER__=still-waiting" + printf "__AUTO_PAIR_PIDS__=%s\n" "$(printf "%s" "$pids" | tr "\n" " ")" + tail -20 /tmp/auto-pair.log || true +else + echo "__AUTO_PAIR_WATCHER__=missing-log" +fi +exit 1 +' 2>&1) + rc=$? + printf '=== auto-pair watcher inactivity probe rc=%s ===\n%s\n' "$rc" "$output" >>"$STATE_LOG" + if [ "$rc" -eq 0 ]; then + pass "auto-pair watcher reached its deadline before legacy scope-upgrade trigger" + return 0 + fi + sleep 2 + done + output=$(sandbox_exec_sh_script 30 ' +set -u +find_auto_pair_pids() { + for proc in /proc/[0-9]*; do + pid="${proc##*/}" + [ "$pid" = "$$" ] && continue + [ -r "$proc/cmdline" ] || continue + cmd="$(tr "\000" " " <"$proc/cmdline" 2>/dev/null || true)" + case "$cmd" in + *"python3 -"*) + fd1="$(readlink "$proc/fd/1" 2>/dev/null || true)" + fd2="$(readlink "$proc/fd/2" 2>/dev/null || true)" + case "${fd1} ${fd2}" in + *"/tmp/auto-pair.log"*) printf "%s\n" "$pid" ;; + esac + ;; + esac + done | sort -u +} +pids="$(find_auto_pair_pids)" +if [ -z "$pids" ]; then + echo "__AUTO_PAIR_WATCHER__=inactive-before-stop" + exit 0 +fi +printf "__AUTO_PAIR_STOPPING_PIDS__=%s\n" "$(printf "%s" "$pids" | tr "\n" " ")" +kill $pids 2>/dev/null || true +sleep 2 +remaining="$(find_auto_pair_pids)" +if [ -n "$remaining" ]; then + printf "__AUTO_PAIR_KILLING_PIDS__=%s\n" "$(printf "%s" "$remaining" | tr "\n" " ")" + kill -KILL $remaining 2>/dev/null || true + sleep 1 +fi +remaining="$(find_auto_pair_pids)" +if [ -n "$remaining" ]; then + printf "__AUTO_PAIR_WATCHER__=still-active pids=%s\n" "$(printf "%s" "$remaining" | tr "\n" " ")" + exit 1 +fi +echo "__AUTO_PAIR_WATCHER__=stopped" +tail -20 /tmp/auto-pair.log 2>/dev/null || true +' 2>&1) + rc=$? + printf '=== auto-pair watcher forced stop rc=%s ===\n%s\n' "$rc" "$output" >>"$STATE_LOG" + if [ "$rc" -eq 0 ]; then + pass "auto-pair watcher is inactive before legacy scope-upgrade trigger" + return 0 + fi + fail "auto-pair watcher was still active before legacy scope-upgrade trigger: ${output:0:500}" + return 1 +} + +section "Phase 0: Preflight" + +if [ -z "${NVIDIA_API_KEY:-}" ]; then + fail "NVIDIA_API_KEY not set" + exit 1 +fi +pass "NVIDIA_API_KEY is set" + +if ! docker info >/dev/null 2>&1; then + fail "Docker is not running" + exit 1 +fi +pass "Docker is running" + +command -v python3 >/dev/null 2>&1 || { + fail "python3 is required" + exit 1 +} +pass "python3 is available" + +info "Repo: ${REPO}" +info "Sandbox name: ${SANDBOX_NAME}" +info "Mode: ${TEST_MODE}" +info "Logs: ${INSTALL_LOG}, ${APPROVAL_LOG}, ${AGENT_LOG}, ${STATE_LOG}" +info "Auto-pair timing: fast=${AUTO_PAIR_FAST_DEADLINE_SECS}s deadline=${AUTO_PAIR_DEADLINE_SECS}s slow=${AUTO_PAIR_SLOW_INTERVAL_SECS}s run-timeout=${AUTO_PAIR_RUN_TIMEOUT_SECS}s" +: >"$APPROVAL_LOG" +: >"$AGENT_LOG" +: >"$STATE_LOG" + +section "Phase 1: Install real NemoClaw/OpenClaw sandbox" + +cd "$REPO" || { + fail "Could not cd to repo root" + exit 1 +} + +info "Pre-cleanup" +if command -v nemoclaw >/dev/null 2>&1; then + run_with_timeout 120 nemoclaw "$SANDBOX_NAME" destroy --yes >/dev/null 2>&1 || true +fi +if command -v "$OPENSHELL_BIN" >/dev/null 2>&1 || [ "$OPENSHELL_BIN" != "openshell" ]; then + run_with_timeout 60 "$OPENSHELL_BIN" sandbox delete "$SANDBOX_NAME" >/dev/null 2>&1 || true + if [[ "${CI:-}" = "true" || "${NEMOCLAW_E2E_DESTROY_GATEWAY:-}" = "1" ]]; then + run_with_timeout 60 "$OPENSHELL_BIN" gateway destroy -g nemoclaw >/dev/null 2>&1 || true + fi +fi +pass "Pre-cleanup complete" + +info "Running install.sh --non-interactive" +( + export NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" + export NEMOCLAW_RECREATE_SANDBOX=1 + export NEMOCLAW_FRESH=1 + export NEMOCLAW_NON_INTERACTIVE=1 + export NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 + export NEMOCLAW_AUTO_PAIR_FAST_DEADLINE_SECS="$AUTO_PAIR_FAST_DEADLINE_SECS" + export NEMOCLAW_AUTO_PAIR_DEADLINE_SECS="$AUTO_PAIR_DEADLINE_SECS" + export NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS="$AUTO_PAIR_SLOW_INTERVAL_SECS" + export NEMOCLAW_AUTO_PAIR_RUN_TIMEOUT_SECS="$AUTO_PAIR_RUN_TIMEOUT_SECS" + run_with_timeout "$INSTALL_TIMEOUT_SECONDS" bash install.sh --non-interactive --yes-i-accept-third-party-software +) >"$INSTALL_LOG" 2>&1 +install_rc=$? + +nemoclaw_refresh_install_env +export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" +# shellcheck source=/dev/null +[ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" +nemoclaw_ensure_local_bin_on_path +hash -r + +if [ "$install_rc" -ne 0 ]; then + fail "install.sh failed with exit ${install_rc}; see ${INSTALL_LOG}" + tail -40 "$INSTALL_LOG" || true + exit 1 +fi +pass "NemoClaw installed and onboarded" + +command -v nemoclaw >/dev/null 2>&1 || { + fail "nemoclaw not found on PATH after install" + exit 1 +} +command -v "$OPENSHELL_BIN" >/dev/null 2>&1 || { + fail "${OPENSHELL_BIN} not found on PATH after install" + exit 1 +} +pass "nemoclaw and openshell are available" + +section "Phase 2: Verify in-sandbox proxy env guard" + +guard_probe=$(sandbox_exec_sh_script 60 ' +set -u +if [ ! -r /tmp/nemoclaw-proxy-env.sh ]; then + echo "MISSING_PROXY_ENV" + exit 2 +fi +# shellcheck source=/dev/null +. /tmp/nemoclaw-proxy-env.sh +printf "OPENCLAW_GATEWAY_URL=%s\n" "${OPENCLAW_GATEWAY_URL-unset}" +type openclaw 2>/dev/null | sed -n "1,12p" +grep -F "unset OPENCLAW_GATEWAY_URL OPENCLAW_GATEWAY_PORT OPENCLAW_GATEWAY_TOKEN; command openclaw" /tmp/nemoclaw-proxy-env.sh >/dev/null \ + && echo "APPROVE_GUARD_PRESENT" +' 2>&1) +guard_rc=$? +printf '%s\n' "$guard_probe" >>"$STATE_LOG" +if [ "$guard_rc" -ne 0 ]; then + fail "Could not source /tmp/nemoclaw-proxy-env.sh: ${guard_probe:0:400}" + exit 1 +fi +if grep -q '^OPENCLAW_GATEWAY_URL=ws://127\.0\.0\.1:' <<<"$guard_probe" \ + && grep -q '^APPROVE_GUARD_PRESENT$' <<<"$guard_probe"; then + pass "proxy env preserves gateway URL and contains devices approve guard" +else + fail "proxy env missing gateway URL or approve guard: ${guard_probe:0:600}" + exit 1 +fi + +section "Phase 3: Establish low-scope CLI device approval" + +info "Creating initial CLI pairing request with openclaw devices list" +initial_list=$(sandbox_exec_sh_script 60 ' +set -u +# shellcheck source=/dev/null +. /tmp/nemoclaw-proxy-env.sh +set +e +openclaw devices list --json +rc=$? +set -e +printf "__LIST_RC__=%s\n" "$rc" >&2 +exit 0 +' 2>&1) +printf '=== initial devices list ===\n%s\n' "$initial_list" >>"$STATE_LOG" + +state="$(device_state_json 2>&1)" || { + fail "Could not read OpenClaw device state after initial list: ${state:0:500}" + exit 1 +} +printf '=== state after initial list ===\n%s\n' "$state" >>"$STATE_LOG" +summary=$(printf '%s' "$state" | summarize_device_state) +info "$summary" + +initial_request_id=$(printf '%s' "$state" | select_cli_request new 2>/dev/null) || initial_request_id="" +if [ -n "$initial_request_id" ]; then + pass "pending low-scope CLI pairing request exists (${initial_request_id})" + approve_request "$initial_request_id" "initial CLI pairing" || exit 1 +else + paired_without_write=$(printf '%s' "$state" | select_cli_paired_without_write 2>/dev/null) || paired_without_write="" + if [ -n "$paired_without_write" ]; then + pass "CLI device is already paired with low scope (${paired_without_write})" + else + paired_with_agent_scopes=$(printf '%s' "$state" | select_cli_paired_with_agent_scopes 2>/dev/null) || paired_with_agent_scopes="" + paired_with_admin=$(printf '%s' "$state" | select_cli_paired_with_admin 2>/dev/null) || paired_with_admin="" + if [ -n "$paired_with_agent_scopes" ] && [ -z "$paired_with_admin" ]; then + pass "CLI device already has operator.read/operator.write without operator.admin (${paired_with_agent_scopes})" + SCOPE_UPGRADE_ALREADY_SATISFIED=1 + else + fail "No pending or paired low-scope CLI device found after devices list: ${summary}" + exit 1 + fi + fi +fi + +state="$(device_state_json 2>&1)" || { + fail "Could not read OpenClaw device state after initial approval: ${state:0:500}" + exit 1 +} +printf '=== state after initial approval ===\n%s\n' "$state" >>"$STATE_LOG" +if [ "$SCOPE_UPGRADE_ALREADY_SATISFIED" = "1" ]; then + pass "initial approval check skipped because CLI scope-upgrade is already satisfied" +else + paired_without_write=$(printf '%s' "$state" | select_cli_paired_without_write 2>/dev/null) || paired_without_write="" + if [ -n "$paired_without_write" ]; then + pass "CLI device is paired with operator.pairing but not operator.write" + else + fail "Initial approval did not leave a low-scope CLI device: $(printf '%s' "$state" | summarize_device_state)" + exit 1 + fi +fi + +gateway_list=$(sandbox_exec_sh_script 60 ' +set -u +# shellcheck source=/dev/null +. /tmp/nemoclaw-proxy-env.sh +printf "__URL_FOR_LIST__=%s\n" "${OPENCLAW_GATEWAY_URL-unset}" >&2 +openclaw devices list --json +' 2>&1) +gateway_list_rc=$? +printf '=== gateway devices list after initial approval rc=%s ===\n%s\n' "$gateway_list_rc" "$gateway_list" >>"$STATE_LOG" +if [ "$gateway_list_rc" -eq 0 ] && grep -q '^__URL_FOR_LIST__=ws://' <<<"$gateway_list"; then + pass "openclaw devices list observes device state while OPENCLAW_GATEWAY_URL is set" +else + fail "devices list did not work with gateway URL after initial approval: ${gateway_list:0:500}" + exit 1 +fi + +if [ "$TEST_MODE" = "legacy-repro" ] && [ "$SCOPE_UPGRADE_ALREADY_SATISFIED" != "1" ]; then + wait_for_auto_pair_watcher_inactive || exit 1 +fi + +section "Phase 4: Trigger and approve CLI scope upgrade" + +if [ "$SCOPE_UPGRADE_ALREADY_SATISFIED" = "1" ]; then + info "Skipping trigger/approval because CLI operator.read/operator.write scopes were already approved" +else + info "Triggering agent operator.write scope upgrade" + trigger_output=$(sandbox_exec_sh_script 120 ' +set -u +# shellcheck source=/dev/null +. /tmp/nemoclaw-proxy-env.sh +session_id="issue-4462-trigger-$(date +%s)-$$" +rm -f "/sandbox/.openclaw/agents/main/sessions/${session_id}.jsonl.lock" \ + "/sandbox/.openclaw/agents/main/sessions/${session_id}.trajectory.jsonl" 2>/dev/null || true +printf "__URL_FOR_TRIGGER_AGENT__=%s\n" "${OPENCLAW_GATEWAY_URL-unset}" +set +e +openclaw agent --agent main --json --session-id "$session_id" \ + -m "What is 6 multiplied by 7? Reply with only the integer, no extra words." +agent_rc=$? +set -e +printf "__TRIGGER_AGENT_RC__=%s\n" "$agent_rc" +exit 0 +' 2>&1) + printf '=== trigger agent output ===\n%s\n' "$trigger_output" >>"$AGENT_LOG" + + scope_request_id="" + auto_approved_device="" + for _attempt in 1 2 3 4 5; do + state="$(device_state_json 2>&1)" || state="" + if [ -n "$state" ]; then + printf '=== state while waiting for scope upgrade ===\n%s\n' "$state" >>"$STATE_LOG" + scope_request_id=$(printf '%s' "$state" | select_cli_request scope-upgrade 2>/dev/null) || scope_request_id="" + auto_approved_device=$(printf '%s' "$state" | select_cli_paired_with_agent_scopes 2>/dev/null) || auto_approved_device="" + fi + [ -n "$scope_request_id" ] && break + if [ "$TEST_MODE" = "approval" ] && [ -n "$auto_approved_device" ]; then + break + fi + sleep 2 + done + + if [ -z "$scope_request_id" ] && [ "$TEST_MODE" = "legacy-repro" ]; then + scope_request_id=$(printf '%s' "$trigger_output" | extract_scope_request_id_from_output) || scope_request_id="" + fi + + if [ -n "$scope_request_id" ]; then + pass "pending CLI scope-upgrade request exists (${scope_request_id})" + elif [ "$TEST_MODE" = "approval" ] && [ -n "$auto_approved_device" ]; then + pass "auto-pair watcher approved the CLI scope upgrade before pending inspection (${auto_approved_device})" + elif [ "$TEST_MODE" = "legacy-repro" ] \ + && grep -q '^__URL_FOR_TRIGGER_AGENT__=ws://' <<<"$trigger_output" \ + && grep -q '^__TRIGGER_AGENT_RC__=0$' <<<"$trigger_output" \ + && ! grep -Eiq 'EMBEDDED FALLBACK|scope upgrade pending approval|pairing required|fallbackFrom[": ]+gateway|transport[": ]+embedded' <<<"$trigger_output"; then + pass "legacy gateway-pinned scope-upgrade was not reproducible because trigger agent completed through gateway mode" + LEGACY_SCOPE_UPGRADE_NOT_REPRODUCED=1 + else + fail "No pending CLI scope-upgrade request appeared after agent trigger. State: $(printf '%s' "${state:-{}}" | summarize_device_state 2>/dev/null || true). Trigger: ${trigger_output:0:500}" + exit 1 + fi + + if [ "$TEST_MODE" = "legacy-repro" ] && [ "$LEGACY_SCOPE_UPGRADE_NOT_REPRODUCED" != "1" ]; then + legacy_gateway_pinned_approval_characterization "$scope_request_id" || exit 1 + if [ "$FAIL" -gt 0 ]; then + section "Summary" + echo "" + printf ' Total: %d | \033[32mPass: %d\033[0m | \033[31mFail: %d\033[0m\n' \ + "$TOTAL" "$PASS" "$FAIL" + echo "" + echo "RESULT: FAILED - ${FAIL} test(s) failed" + exit 1 + fi + finish_success "RESULT: PASSED - #4462 legacy gateway-pinned approval behavior characterized and final state handled" + fi + + if [ -n "$scope_request_id" ]; then + approve_request "$scope_request_id" "CLI scope upgrade" 1 || exit 1 + else + info "Skipping manual scope-upgrade approval because the auto-pair watcher already granted it" + fi +fi + +state="$(device_state_json 2>&1)" || { + fail "Could not read OpenClaw device state after scope-upgrade approval: ${state:0:500}" + exit 1 +} +printf '=== state after scope-upgrade approval ===\n%s\n' "$state" >>"$STATE_LOG" +pending_after_approval=$(printf '%s' "$state" | select_cli_request scope-upgrade 2>/dev/null) || pending_after_approval="" +paired_with_agent_scopes=$(printf '%s' "$state" | select_cli_paired_with_agent_scopes 2>/dev/null) || paired_with_agent_scopes="" +paired_with_admin=$(printf '%s' "$state" | select_cli_paired_with_admin 2>/dev/null) || paired_with_admin="" +if [ -n "$pending_after_approval" ]; then + fail "Scope-upgrade request is still pending after approval (${pending_after_approval})" + exit 1 +fi +if [ -z "$paired_with_agent_scopes" ] && [ "$LEGACY_SCOPE_UPGRADE_NOT_REPRODUCED" != "1" ]; then + fail "No CLI paired device has operator.write and operator.read after approval: $(printf '%s' "$state" | summarize_device_state)" + exit 1 +fi +if [ -n "$paired_with_admin" ]; then + fail "Unexpected operator.admin approval for CLI device (${paired_with_admin})" + exit 1 +fi +if [ "$SCOPE_UPGRADE_ALREADY_SATISFIED" = "1" ]; then + pass "preapproved CLI scope-upgrade state has operator.write/operator.read without operator.admin" +elif [ "$LEGACY_SCOPE_UPGRADE_NOT_REPRODUCED" = "1" ]; then + pass "legacy repro trigger left no pending scope-upgrade and no operator.admin grant" +else + pass "scope-upgrade approval grants the CLI device operator.write and operator.read without approving operator.admin" +fi + +section "Phase 5: Verify agent stays on gateway path" + +agent_ok=0 +last_agent_detail="" +for attempt in 1 2; do + info "Running approved openclaw agent turn (attempt ${attempt}/2)" + final_output=$(sandbox_exec_sh_script 180 ' +set -u +# shellcheck source=/dev/null +. /tmp/nemoclaw-proxy-env.sh +session_id="issue-4462-fixed-$(date +%s)-$$" +rm -f "/sandbox/.openclaw/agents/main/sessions/${session_id}.jsonl.lock" \ + "/sandbox/.openclaw/agents/main/sessions/${session_id}.trajectory.jsonl" 2>/dev/null || true +printf "__URL_FOR_FINAL_AGENT__=%s\n" "${OPENCLAW_GATEWAY_URL-unset}" +openclaw agent --agent main --json --session-id "$session_id" \ + -m "What is 6 multiplied by 7? Reply with only the integer, no extra words." +' 2>&1) + final_rc=$? + printf '=== final agent attempt %s rc=%s ===\n%s\n' "$attempt" "$final_rc" "$final_output" >>"$AGENT_LOG" + reply=$(printf '%s' "$final_output" | parse_openclaw_agent_text 2>/dev/null) || reply="" + if grep -Eiq 'EMBEDDED FALLBACK|scope upgrade pending approval|pairing required|fallbackFrom[": ]+gateway|transport[": ]+embedded' <<<"$final_output"; then + last_agent_detail="agent output contained fallback or pairing marker: ${final_output:0:500}" + elif [ "$final_rc" -ne 0 ]; then + last_agent_detail="agent exited ${final_rc}: ${final_output:0:500}" + elif ! grep -q '^__URL_FOR_FINAL_AGENT__=ws://' <<<"$final_output"; then + last_agent_detail="agent command did not preserve OPENCLAW_GATEWAY_URL: ${final_output:0:500}" + elif grep -qE '(^|[^0-9])42([^0-9]|$)' <<<"$reply"; then + agent_ok=1 + pass "approved openclaw agent turn answered through gateway mode" + break + else + last_agent_detail="expected reply 42, got reply='${reply:0:200}', raw='${final_output:0:400}'" + fi + sleep 5 +done + +if [ "$agent_ok" -ne 1 ]; then + fail "Final approved agent turn did not prove gateway-mode success: ${last_agent_detail}" + exit 1 +fi + +pass "approved agent output contains no fallback or pairing markers" + +if [ "$FAIL" -gt 0 ]; then + section "Summary" + echo "" + printf ' Total: %d | \033[32mPass: %d\033[0m | \033[31mFail: %d\033[0m\n' \ + "$TOTAL" "$PASS" "$FAIL" + echo "" + echo "RESULT: FAILED - ${FAIL} test(s) failed" + exit 1 +fi + +if [ "$TEST_MODE" = "legacy-repro" ] && [ "$SCOPE_UPGRADE_ALREADY_SATISFIED" = "1" ]; then + finish_success "RESULT: PASSED - #4462 legacy gateway-pinned approval characterization skipped because scope-upgrade was already satisfied; final gateway path verified" +fi +if [ "$TEST_MODE" = "legacy-repro" ] && [ "$LEGACY_SCOPE_UPGRADE_NOT_REPRODUCED" = "1" ]; then + finish_success "RESULT: PASSED - #4462 legacy gateway-pinned approval characterization skipped because trigger agent completed without a pending scope-upgrade; final gateway path verified" +fi + +finish_success "RESULT: PASSED - #4462 CLI scope-upgrade approval stays on the gateway path" diff --git a/test/e2e-vpn/test-jetson-nvmap-gpu.sh b/test/e2e-vpn/test-jetson-nvmap-gpu.sh new file mode 100755 index 00000000000..fc07ebc5fa3 --- /dev/null +++ b/test/e2e-vpn/test-jetson-nvmap-gpu.sh @@ -0,0 +1,323 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Jetson nvmap GPU status E2E: reproduces the EXACT reporter workflow from +# issue #4231 on a Jetson Orin host and proves the fix. +# +# Reporter symptom (NemoClaw v0.0.58, Jetson Orin, reopened after PR #4599): +# - sandbox user groups: uid=998(sandbox) gid=998(sandbox) groups=998(sandbox) +# - /dev/nvmap is `crw-rw---- root video` +# - CUDA fails inside the sandbox with `NvRmMemInitNvmap failed with +# Permission denied`, cuInit(0)=999 +# - `nemoclaw status` still reports "Sandbox GPU: enabled" (misleading) +# +# Root cause: the Jetson Docker GPU recreate did not grant the sandbox user +# membership in the host group (`video`) that owns the Tegra device nodes, so +# CUDA could not open /dev/nvmap even though the devices were mounted. +# +# This test runs the reporter's exact workflow end-to-end: +# 1. Onboard with GPU passthrough (Jetson auto-enables sandbox GPU) +# 2. Inspect the sandbox user's supplementary groups (`id`) +# 3. Inspect /dev/nvmap inside the sandbox (`ls -l`) +# 4. Run the authoritative CUDA usability proof (cuInit(0)) inside the sandbox +# 5. Assert `nemoclaw status` reports "(CUDA verified)" — not a bare/misleading +# "enabled", "(CUDA unverified)", or "(last CUDA proof failed)" +# +# Acceptance gate (#4231): the test passes only when CUDA actually initializes +# in the sandbox (cuInit(0)=0) AND status reflects proven CUDA usability. A +# bare "enabled" is treated as a failure. +# +# Prerequisites: +# - NVIDIA Jetson Orin (or other L4T/Tegra) host with /dev/nvmap present +# - NVIDIA Container Runtime configured for Docker (nvidia-ctk runtime configure) +# - Docker +# - NEMOCLAW_NON_INTERACTIVE=1 +# - NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 +# - A working inference provider (default: ollama; onboard handles startup) +# +# On a non-Jetson host this test SKIPS cleanly (exit 0) so it is safe to wire +# into pipelines that may schedule it on mixed hardware. +# +# Usage: +# NEMOCLAW_NON_INTERACTIVE=1 NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ +# bash test/e2e-vpn/test-jetson-nvmap-gpu.sh + +# ShellCheck cannot see EXIT trap invocations of cleanup helpers in this E2E script. +# shellcheck disable=SC2317 +set -uo pipefail + +PASS=0 +FAIL=0 +SKIP=0 +TOTAL=0 + +pass() { + ((PASS++)) + ((TOTAL++)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} +fail() { + ((FAIL++)) + ((TOTAL++)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} +skip() { + ((SKIP++)) + ((TOTAL++)) + printf '\033[33m SKIP: %s\033[0m\n' "$1" +} +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } + +# Determine repo root +if [ -d /workspace ] && [ -f /workspace/install.sh ]; then + REPO="/workspace" +elif [ -f "$(cd "$(dirname "$0")/../.." && pwd)/install.sh" ]; then + REPO="$(cd "$(dirname "$0")/../.." && pwd)" +else + echo "ERROR: Cannot find repo root." + exit 1 +fi + +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-jetson-nvmap}" +TEST_LOG="/tmp/nemoclaw-jetson-nvmap-e2e-test.log" +INSTALL_LOG="/tmp/nemoclaw-jetson-nvmap-e2e-install.log" +export NEMOCLAW_PROVIDER="${NEMOCLAW_PROVIDER:-ollama}" + +exec > >(tee -a "$TEST_LOG") 2>&1 + +# ══════════════════════════════════════════════════════════════════ +# Phase 0: Jetson hardware gate +# ══════════════════════════════════════════════════════════════════ +section "Phase 0: Jetson hardware gate" + +is_jetson() { + [ -e /dev/nvmap ] && return 0 + [ -f /etc/nv_tegra_release ] && return 0 + if [ -r /proc/device-tree/model ] && grep -qi "jetson\|orin\|tegra" /proc/device-tree/model 2>/dev/null; then + return 0 + fi + return 1 +} + +if ! is_jetson; then + skip "Not a Jetson/Tegra host (/dev/nvmap absent) — reporter workflow requires Jetson hardware" + echo "" + echo " This test reproduces issue #4231 on Jetson Orin. It cannot run on" + echo " non-Jetson hardware. Hermetic regression coverage of the same fix" + echo " (sandbox user → /dev/nvmap group propagation) lives in" + echo " src/lib/onboard/docker-gpu-patch.test.ts." + echo "" + echo " Skipped (exit 0): no Jetson hardware available." + exit 0 +fi +pass "Jetson/Tegra host detected (/dev/nvmap present)" + +HOST_NVMAP_PERMS="$(ls -l /dev/nvmap 2>/dev/null || true)" +HOST_NVMAP_GID="$(stat -c '%g' /dev/nvmap 2>/dev/null || true)" +HOST_NVMAP_GROUP="$(stat -c '%G' /dev/nvmap 2>/dev/null || true)" +info "Host /dev/nvmap: ${HOST_NVMAP_PERMS}" +info "Host /dev/nvmap owning group: ${HOST_NVMAP_GROUP} (gid ${HOST_NVMAP_GID})" + +if [ "${NEMOCLAW_NON_INTERACTIVE:-}" != "1" ]; then + fail "NEMOCLAW_NON_INTERACTIVE=1 is required" + exit 1 +fi +if [ "${NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE:-}" != "1" ]; then + fail "NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 is required for non-interactive onboard" + exit 1 +fi + +# Best-effort cleanup on any exit (prevents dirty state on reused runners) +# shellcheck disable=SC2329 # invoked via trap +cleanup() { + info "Running exit cleanup..." + if command -v nemoclaw >/dev/null 2>&1; then + nemoclaw "$SANDBOX_NAME" destroy --yes 2>/dev/null || true + fi + if command -v openshell >/dev/null 2>&1; then + openshell sandbox delete "$SANDBOX_NAME" 2>/dev/null || true + openshell gateway destroy -g nemoclaw 2>/dev/null || true + fi + pkill -f "ollama serve" 2>/dev/null || true + pkill -f "ollama-auth-proxy" 2>/dev/null || true +} +trap cleanup EXIT + +# ══════════════════════════════════════════════════════════════════ +# Phase 1: Prerequisites +# ══════════════════════════════════════════════════════════════════ +section "Phase 1: Prerequisites" + +if docker info >/dev/null 2>&1; then + pass "Docker is running" +else + fail "Docker is not running — cannot continue" + exit 1 +fi + +# Jetson sandbox GPU uses the NVIDIA Container Runtime (not CDI). +if docker info --format '{{json .Runtimes}}' 2>/dev/null | grep -q '"nvidia"\|nvidia:'; then + pass "Docker NVIDIA runtime detected" +else + fail "Docker NVIDIA runtime not detected — run: sudo nvidia-ctk runtime configure --runtime=docker" + exit 1 +fi + +# Pre-cleanup +if command -v nemoclaw >/dev/null 2>&1; then + nemoclaw "$SANDBOX_NAME" destroy --yes 2>/dev/null || true +fi +if command -v openshell >/dev/null 2>&1; then + openshell sandbox delete "$SANDBOX_NAME" 2>/dev/null || true + openshell gateway destroy -g nemoclaw 2>/dev/null || true +fi + +# Install Ollama binary if the provider needs it (onboard starts it). +if [ "$NEMOCLAW_PROVIDER" = "ollama" ] && ! command -v ollama >/dev/null 2>&1; then + info "Installing Ollama binary..." + curl -fsSL https://ollama.com/install.sh | sh 2>&1 || true + systemctl stop ollama 2>/dev/null || true + pkill -f "ollama serve" 2>/dev/null || true +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 2: Onboard with GPU (reporter workflow) +# ══════════════════════════════════════════════════════════════════ +section "Phase 2: Onboard with GPU passthrough" + +cd "$REPO" || { + fail "Could not cd to repo root: $REPO" + exit 1 +} + +info "Running install.sh --non-interactive (Jetson auto-enables sandbox GPU)..." +bash install.sh --non-interactive >"$INSTALL_LOG" 2>&1 & +install_pid=$! +tail -f "$INSTALL_LOG" --pid=$install_pid 2>/dev/null & +tail_pid=$! +wait $install_pid +install_exit=$? +kill $tail_pid 2>/dev/null || true +wait $tail_pid 2>/dev/null || true + +# Pick up PATH changes from the installer. +if [ -f "$HOME/.bashrc" ]; then + source "$HOME/.bashrc" 2>/dev/null || true +fi +export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" +[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" +if [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]]; then + export PATH="$HOME/.local/bin:$PATH" +fi + +if [ $install_exit -eq 0 ]; then + pass "install.sh completed (exit 0)" +else + fail "install.sh failed (exit $install_exit)" + tail -40 "$INSTALL_LOG" + exit 1 +fi + +if ! command -v nemoclaw >/dev/null 2>&1; then + fail "nemoclaw not found on PATH after install" + exit 1 +fi + +# 2a: The Jetson recreate must announce that it grants the Tegra device-node +# group(s) to the sandbox user (the fix for #4231). +if grep -Fq "Granting sandbox user access to Jetson Tegra GPU device nodes via --group-add" "$INSTALL_LOG"; then + GROUP_ADD_LINE="$(grep -F "Granting sandbox user access to Jetson Tegra GPU device nodes" "$INSTALL_LOG" | head -1)" + pass "Onboard granted Tegra device-node group via --group-add" + info "${GROUP_ADD_LINE}" +else + fail "Onboard did not grant the Tegra device-node group (--group-add) — #4231 fix missing" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 3: In-sandbox device + group + CUDA inspection (reporter workflow) +# ══════════════════════════════════════════════════════════════════ +section "Phase 3: In-sandbox /dev/nvmap, groups, and CUDA proof" + +# 3a: sandbox user supplementary groups — must now include the /dev/nvmap GID. +SANDBOX_ID="$(openshell sandbox exec -n "$SANDBOX_NAME" -- sh -lc 'id' 2>&1)" || true +info "sandbox 'id': ${SANDBOX_ID}" +if [ -n "$HOST_NVMAP_GID" ] && echo "$SANDBOX_ID" | grep -Eq "(^|[(,=])${HOST_NVMAP_GID}([(,) ]|$)"; then + pass "Sandbox user is a member of the /dev/nvmap owning group (gid ${HOST_NVMAP_GID})" +else + fail "Sandbox user is NOT in the /dev/nvmap owning group (gid ${HOST_NVMAP_GID}) — CUDA will be denied" +fi + +# 3b: /dev/nvmap present inside the sandbox. +SANDBOX_NVMAP="$(openshell sandbox exec -n "$SANDBOX_NAME" -- sh -lc 'ls -l /dev/nvmap' 2>&1)" || true +info "sandbox /dev/nvmap: ${SANDBOX_NVMAP}" +if echo "$SANDBOX_NVMAP" | grep -q "/dev/nvmap"; then + pass "/dev/nvmap is present inside the sandbox" +else + fail "/dev/nvmap is not present inside the sandbox" +fi + +# 3c: Authoritative CUDA usability proof — cuInit(0) must return 0. This is the +# exact signal the reporter saw fail (cuInit=999, NvRmMemInitNvmap denied). +CUDA_PROBE='python3 -c '\''import ctypes; lib = ctypes.CDLL("libcuda.so.1"); rc = lib.cuInit(0); print(f"cuInit(0)={rc}"); raise SystemExit(0 if rc == 0 else 1)'\''' +CUDA_OUT="$(openshell sandbox exec -n "$SANDBOX_NAME" -- sh -lc "$CUDA_PROBE" 2>&1)" || true +info "sandbox cuInit probe: ${CUDA_OUT}" +if echo "$CUDA_OUT" | grep -q "cuInit(0)=0"; then + pass "CUDA initialized inside the sandbox (cuInit(0)=0)" +elif echo "$CUDA_OUT" | grep -qi "NvRmMemInitNvmap\|Permission denied"; then + fail "CUDA failed with the reporter's nvmap permission error: ${CUDA_OUT}" +else + fail "CUDA did not initialize inside the sandbox: ${CUDA_OUT}" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 4: nemoclaw status must report proven CUDA usability +# ══════════════════════════════════════════════════════════════════ +section "Phase 4: nemoclaw status CUDA proof state" + +STATUS_OUT="$(nemoclaw "$SANDBOX_NAME" status 2>&1)" || true +echo "$STATUS_OUT" | grep -F "Sandbox GPU:" || true + +if echo "$STATUS_OUT" | grep -Fq "Sandbox GPU: enabled"; then + pass "Status reports Sandbox GPU: enabled (Jetson auto-enable)" +else + fail "Status does not report Sandbox GPU enabled" +fi + +# The core #4231 assertion: status must carry "(CUDA verified)" — a bare +# "enabled", "(CUDA unverified)", or "(last CUDA proof failed)" is the +# misleading state the reporter hit and must NOT pass. +if echo "$STATUS_OUT" | grep -Fq "CUDA verified"; then + pass "Status reports (CUDA verified) — GPU usability is proven, not misleading" +elif echo "$STATUS_OUT" | grep -Eq "last CUDA proof failed"; then + fail "Status shows CUDA proof FAILED — Jetson nvmap access not granted (#4231 unfixed)" +elif echo "$STATUS_OUT" | grep -Fq "CUDA unverified"; then + fail "Status shows CUDA UNVERIFIED — proof did not confirm usability (#4231 misleading state)" +else + fail "Status reports a bare 'enabled' with no CUDA proof state (#4231 misleading status)" +fi + +# ══════════════════════════════════════════════════════════════════ +# Summary +# ══════════════════════════════════════════════════════════════════ +echo "" +echo "========================================" +echo " Jetson nvmap GPU E2E Results (#4231):" +echo " Passed: $PASS" +echo " Failed: $FAIL" +echo " Skipped: $SKIP" +echo " Total: $TOTAL" +echo "========================================" +echo "" + +if [ "$FAIL" -eq 0 ]; then + printf '\n\033[1;32m Jetson nvmap GPU E2E PASSED — CUDA usable + status proven (#4231).\033[0m\n' + exit 0 +else + printf '\n\033[1;31m %d test(s) failed.\033[0m\n' "$FAIL" + exit 1 +fi diff --git a/test/e2e-vpn/test-kimi-inference-compat.sh b/test/e2e-vpn/test-kimi-inference-compat.sh new file mode 100755 index 00000000000..053b36c82ee --- /dev/null +++ b/test/e2e-vpn/test-kimi-inference-compat.sh @@ -0,0 +1,871 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Kimi inference compatibility E2E (#2620 / #3046) +# +# Live path: +# - uses the public VPN NVIDIA inference provider with moonshotai/kimi-k2.6 +# - onboards a fresh sandbox through the managed inference.local route +# - asks Kimi to exercise exec tool calls +# - verifies the NemoClaw Kimi plugin splits it into three exec tool calls +# - verifies the trajectory records exactly those three tool executions +# +# Hermetic fallback: +# - set NEMOCLAW_KIMI_USE_MOCK=1 to use the local OpenAI-compatible mock +# - the mock emits one combined Kimi exec tool call: hostname; date; uptime +# +# Environment: +# NEMOCLAW_SANDBOX_NAME - sandbox name (default: e2e-kimi-compat) +# NVIDIA_API_KEY - public VPN NVIDIA inference key (nvapi-*) +# NEMOCLAW_KIMI_USE_MOCK=1 - use the hermetic mock fallback +# NEMOCLAW_KIMI_MOCK_PORT - mock endpoint port (default: 18146) +# NEMOCLAW_KIMI_MOCK_ENDPOINT_URL - optional endpoint URL for gateway provider +# NEMOCLAW_E2E_KEEP_SANDBOX=1 - keep sandbox for debugging +# +# Usage: +# NEMOCLAW_NON_INTERACTIVE=1 NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ +# bash test/e2e-vpn/test-kimi-inference-compat.sh + +set -uo pipefail + +export NEMOCLAW_E2E_DEFAULT_TIMEOUT=2400 +SCRIPT_DIR_TIMEOUT="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +# shellcheck source=test/e2e-vpn/e2e-timeout.sh +. "${SCRIPT_DIR_TIMEOUT}/e2e-timeout.sh" + +PASS=0 +FAIL=0 +SKIP=0 +TOTAL=0 + +pass() { + ((PASS++)) + ((TOTAL++)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} + +fail() { + ((FAIL++)) + ((TOTAL++)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} + +skip() { + ((SKIP++)) + ((TOTAL++)) + printf '\033[33m SKIP: %s\033[0m\n' "$1" +} + +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} + +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } + +summary() { + echo "" + echo "============================================================" + echo " Kimi Inference Compatibility E2E Results" + echo "============================================================" + echo " PASS: $PASS" + echo " FAIL: $FAIL" + echo " SKIP: $SKIP" + echo " TOTAL: $TOTAL" + echo "============================================================" + if [ "$FAIL" -gt 0 ]; then + exit 1 + fi +} + +quote_for_remote_sh() { + local value="${1:-}" + printf "'%s'" "$(printf '%s' "$value" | sed "s/'/'\\\\''/g")" +} + +sandbox_exec_sh_script() { + local script="$1" + shift + local encoded remote_cmd arg + encoded="$(printf '%s' "$script" | base64 | tr -d '\n')" + remote_cmd="tmp=\$(mktemp); trap 'rm -f \"\$tmp\"' EXIT; printf %s $(quote_for_remote_sh "$encoded") | base64 -d > \"\$tmp\"; sh \"\$tmp\"" + for arg in "$@"; do + remote_cmd+=" $(quote_for_remote_sh "$arg")" + done + openshell sandbox exec --name "$SANDBOX_NAME" -- sh -lc "$remote_cmd" +} + +stop_kimi_mock() { + if [ -n "${KIMI_MOCK_PID:-}" ] && kill -0 "$KIMI_MOCK_PID" 2>/dev/null; then + kill "$KIMI_MOCK_PID" 2>/dev/null || true + wait "$KIMI_MOCK_PID" 2>/dev/null || true + fi + KIMI_MOCK_PID="" +} + +use_kimi_mock() { + [ "${KIMI_USE_MOCK:-0}" = "1" ] +} + +ensure_public_nvidia_api_key() { + if [ -n "${NVIDIA_API_KEY:-}" ] && [[ "${NVIDIA_API_KEY}" == nvapi-* ]]; then + # NemoClaw's VPN NVIDIA inference provider still reads NVIDIA_API_KEY. + # Source the public Kimi credential from NVIDIA_API_KEY, then mirror it only + # for the shared onboarding/provider-registration path. + export NVIDIA_API_KEY="$NVIDIA_API_KEY" + return 0 + fi + return 1 +} + +start_kimi_mock() { + : >"$KIMI_MOCK_LOG" + python3 - "$KIMI_MOCK_PORT" "$KIMI_MODEL" "$KIMI_MOCK_API_KEY" >"$KIMI_MOCK_LOG" 2>&1 <<'PY' & +import json +import sys +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +port = int(sys.argv[1]) +model = sys.argv[2] +api_key = sys.argv[3] + + +def chunk(chunk_id, delta, finish_reason=None): + return { + "id": chunk_id, + "object": "chat.completion.chunk", + "created": int(time.time()), + "model": model, + "choices": [{"index": 0, "delta": delta, "finish_reason": finish_reason}], + } + + +class Handler(BaseHTTPRequestHandler): + def log_message(self, fmt, *args): + return + + def _send_json(self, status, payload): + body = json.dumps(payload).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def _send_sse(self, chunks): + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Cache-Control", "no-cache") + self.end_headers() + for item in chunks: + self.wfile.write(("data: " + json.dumps(item) + "\n\n").encode("utf-8")) + self.wfile.write(b"data: [DONE]\n\n") + + def _auth_ok(self): + return self.headers.get("Authorization", "") == "Bearer " + api_key + + def do_GET(self): + if self.path == "/v1/models": + print("GET /v1/models", flush=True) + self._send_json(200, {"object": "list", "data": [{"id": model, "object": "model"}]}) + return + self._send_json(404, {"error": {"message": "not found"}}) + + def do_POST(self): + length = int(self.headers.get("Content-Length", "0") or "0") + raw = self.rfile.read(length) if length else b"" + try: + payload = json.loads(raw.decode("utf-8") or "{}") + except Exception: + payload = {} + + print( + "POST %s auth=%s stream=%s tools=%s tool_results=%s model=%s" + % ( + self.path, + "ok" if self._auth_ok() else "missing", + bool(payload.get("stream")), + bool(payload.get("tools")), + any(m.get("role") == "tool" for m in payload.get("messages", []) if isinstance(m, dict)), + payload.get("model"), + ), + flush=True, + ) + + if self.path != "/v1/chat/completions": + self._send_json(404, {"error": {"message": "not found"}}) + return + if not self._auth_ok(): + self._send_json(401, {"error": {"message": "missing bearer credential"}}) + return + + request_text = json.dumps(payload) + completion_id = "chatcmpl-kimi-e2e-%d" % int(time.time() * 1000) + if "Reply with exactly: OK" in request_text: + self._send_json( + 200, + { + "id": completion_id, + "object": "chat.completion", + "created": int(time.time()), + "model": model, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "OK"}, + "finish_reason": "stop", + } + ], + }, + ) + return + + has_tools = isinstance(payload.get("tools"), list) and len(payload.get("tools")) > 0 + has_tool_result = any( + m.get("role") == "tool" for m in payload.get("messages", []) if isinstance(m, dict) + ) + if has_tools and not has_tool_result: + tool_call = { + "index": 0, + "id": "call_kimi_exec", + "type": "function", + "function": { + "name": "exec", + "arguments": json.dumps({"command": "hostname; date; uptime"}), + }, + } + if payload.get("stream"): + self._send_sse( + [ + chunk(completion_id, {"role": "assistant"}), + chunk(completion_id, {"tool_calls": [tool_call]}), + chunk(completion_id, {}, "tool_calls"), + ] + ) + else: + self._send_json( + 200, + { + "id": completion_id, + "object": "chat.completion", + "created": int(time.time()), + "model": model, + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": tool_call["id"], + "type": tool_call["type"], + "function": tool_call["function"], + } + ], + }, + "finish_reason": "tool_calls", + } + ], + }, + ) + return + + final_text = "hostname, date, and uptime completed successfully." + if payload.get("stream"): + self._send_sse( + [ + chunk(completion_id, {"role": "assistant"}), + chunk(completion_id, {"content": final_text}), + chunk(completion_id, {}, "stop"), + ] + ) + else: + self._send_json( + 200, + { + "id": completion_id, + "object": "chat.completion", + "created": int(time.time()), + "model": model, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": final_text}, + "finish_reason": "stop", + } + ], + }, + ) + + +ThreadingHTTPServer(("0.0.0.0", port), Handler).serve_forever() +PY + KIMI_MOCK_PID=$! + + for _ in $(seq 1 30); do + if curl -sf "http://127.0.0.1:${KIMI_MOCK_PORT}/v1/models" >/dev/null 2>&1; then + return 0 + fi + sleep 1 + done + return 1 +} + +load_shell_path() { + local local_bin + if [ -f "$HOME/.bashrc" ]; then + # shellcheck source=/dev/null + source "$HOME/.bashrc" 2>/dev/null || true + fi + export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" + if [ -s "$NVM_DIR/nvm.sh" ]; then + # shellcheck source=/dev/null + . "$NVM_DIR/nvm.sh" + fi + local_bin="$HOME/.local/bin" + if [ -d "$local_bin" ]; then + PATH=":${PATH}:" + PATH="${PATH//:${local_bin}:/:}" + PATH="${PATH#:}" + PATH="${PATH%:}" + export PATH="$local_bin:$PATH" + fi +} + +cli_command_available_from_source() { + [ -f "$REPO/dist/nemoclaw.js" ] && command -v node >/dev/null 2>&1 && command -v openshell >/dev/null 2>&1 +} + +prepare_source_cli() { + local rc=0 + : >"$BUILD_LOG" + load_shell_path + + if ! command -v npm >/dev/null 2>&1; then + echo "npm is not available on PATH" >>"$BUILD_LOG" + return 127 + fi + if ! command -v node >/dev/null 2>&1; then + echo "node is not available on PATH" >>"$BUILD_LOG" + return 127 + fi + + info "Installing npm dependencies and building source CLI" + ( + cd "$REPO" \ + && npm ci --ignore-scripts \ + && npm run build:cli + ) >>"$BUILD_LOG" 2>&1 || rc=$? + if [ "$rc" -ne 0 ]; then + return "$rc" + fi + + if ! command -v openshell >/dev/null 2>&1; then + info "Installing OpenShell CLI" + bash "$REPO/scripts/install-openshell.sh" >>"$BUILD_LOG" 2>&1 || rc=$? + load_shell_path + if [ "$rc" -ne 0 ]; then + return "$rc" + fi + fi + + if ! command -v openshell >/dev/null 2>&1; then + echo "openshell is not available on PATH after installation" >>"$BUILD_LOG" + return 127 + fi +} + +destroy_sandbox_best_effort() { + if [ "${NEMOCLAW_E2E_KEEP_SANDBOX:-}" = "1" ]; then + return 0 + fi + set +e + if cli_command_available_from_source; then + run_with_timeout 120 node "$REPO/bin/nemoclaw.js" "$SANDBOX_NAME" destroy --yes >/dev/null 2>&1 + elif command -v nemoclaw >/dev/null 2>&1; then + run_with_timeout 120 nemoclaw "$SANDBOX_NAME" destroy --yes >/dev/null 2>&1 + fi + if command -v openshell >/dev/null 2>&1; then + run_with_timeout 60 openshell sandbox delete "$SANDBOX_NAME" >/dev/null 2>&1 + fi + set -uo pipefail +} + +cleanup() { + stop_kimi_mock + rm -f "$KIMI_MOCK_LOG" 2>/dev/null || true + destroy_sandbox_best_effort +} + +run_kimi_onboard() { + local onboard_exit=0 + local prep_exit=0 + export NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" + export NEMOCLAW_RECREATE_SANDBOX=1 + export NEMOCLAW_NON_INTERACTIVE=1 + export NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 + export NEMOCLAW_YES=1 + export NEMOCLAW_MODEL="$KIMI_MODEL" + export NEMOCLAW_PREFERRED_API=openai-completions + export NEMOCLAW_POLICY_TIER=restricted + export NEMOCLAW_POLICY_MODE=skip + if use_kimi_mock; then + export NEMOCLAW_PROVIDER=custom + export NEMOCLAW_ENDPOINT_URL="$KIMI_ENDPOINT_URL" + export COMPATIBLE_API_KEY="$KIMI_MOCK_API_KEY" + unset NVIDIA_API_KEY NVIDIA_API_KEY OPENAI_API_KEY ANTHROPIC_API_KEY GEMINI_API_KEY + else + export NEMOCLAW_PROVIDER=custom + unset NEMOCLAW_ENDPOINT_URL NEMOCLAW_COMPAT_MODEL NEMOCLAW_E2E_USE_HOSTED_INFERENCE COMPATIBLE_API_KEY + unset OPENAI_API_KEY ANTHROPIC_API_KEY GEMINI_API_KEY + if ! ensure_public_nvidia_api_key; then + fail "K1: NVIDIA_API_KEY must be a public VPN NVIDIA inference nvapi-* key" + summary + fi + fi + unset TELEGRAM_BOT_TOKEN DISCORD_BOT_TOKEN SLACK_BOT_TOKEN SLACK_APP_TOKEN + + prepare_source_cli || prep_exit=$? + if [ "$prep_exit" -ne 0 ]; then + fail "K1: source CLI/OpenShell preparation failed (exit $prep_exit)" + info "Last 100 lines of build/setup log:" + tail -100 "$BUILD_LOG" 2>/dev/null || true + summary + fi + + destroy_sandbox_best_effort + info "Using source-built CLI at $REPO/bin/nemoclaw.js" + run_with_timeout 1500 node "$REPO/bin/nemoclaw.js" onboard --fresh --non-interactive --yes-i-accept-third-party-software \ + >"$ONBOARD_LOG" 2>&1 || onboard_exit=$? + + if [ "$onboard_exit" -eq 0 ]; then + if use_kimi_mock; then + pass "K1: onboard completed for Kimi compatible endpoint sandbox" + else + pass "K1: onboard completed for VPN NVIDIA Kimi sandbox" + fi + else + fail "K1: onboard failed (exit $onboard_exit)" + info "Last 100 lines of onboard log:" + tail -100 "$ONBOARD_LOG" 2>/dev/null || true + summary + fi +} + +check_openclaw_config() { + local output rc=0 script + script=$( + cat <<'SH' +python3 - "$1" <<'PY' +import json +import sys + +model = sys.argv[1] +cfg = json.load(open("/sandbox/.openclaw/openclaw.json", encoding="utf-8")) +errors = [] +providers = cfg.get("models", {}).get("providers", {}) +inference = providers.get("inference") if isinstance(providers, dict) else None +if sorted(providers.keys()) != ["inference"]: + errors.append("provider keys are %r" % sorted(providers.keys())) +if not isinstance(inference, dict): + errors.append("models.providers.inference is missing") +else: + if inference.get("baseUrl") != "https://inference.local/v1": + errors.append("inference baseUrl is %r" % inference.get("baseUrl")) + if inference.get("api") != "openai-completions": + errors.append("inference api is %r" % inference.get("api")) + models = inference.get("models") or [] + selected = next((m for m in models if m.get("id") == model), None) + if not selected: + errors.append("Kimi model entry is missing") + else: + compat = selected.get("compat") or {} + for key, expected in { + "supportsStore": False, + "requiresStringContent": True, + "maxTokensField": "max_tokens", + "requiresToolResultName": True, + }.items(): + if compat.get(key) != expected: + errors.append("compat[%s] is %r" % (key, compat.get(key))) +primary = cfg.get("agents", {}).get("defaults", {}).get("model", {}).get("primary") +if primary != "inference/" + model: + errors.append("primary model is %r" % primary) +plugins = cfg.get("plugins", {}) +paths = plugins.get("load", {}).get("paths", []) +entries = plugins.get("entries", {}) +if "/usr/local/share/nemoclaw/openclaw-plugins/kimi-inference-compat" not in paths: + errors.append("Kimi plugin load path missing") +if not entries.get("nemoclaw-kimi-inference-compat", {}).get("enabled"): + errors.append("Kimi plugin entry is not enabled") +tools = cfg.get("tools", {}) +if tools.get("toolSearch") is not False: + errors.append("tools.toolSearch is %r" % tools.get("toolSearch")) +print(json.dumps({ + "provider_keys": sorted(providers.keys()) if isinstance(providers, dict) else [], + "primary": primary, + "plugin_enabled": entries.get("nemoclaw-kimi-inference-compat", {}).get("enabled"), + "toolSearch": tools.get("toolSearch"), + "errors": errors, +})) +sys.exit(1 if errors else 0) +PY +SH + ) + output=$(sandbox_exec_sh_script "$script" "$KIMI_MODEL" 2>&1) || rc=$? + info "OpenClaw config summary: ${output:0:800}" + if [ "$rc" -eq 0 ]; then + pass "K2: openclaw.json has managed Kimi compat and plugin wiring" + else + fail "K2: openclaw.json Kimi compat/plugin wiring is wrong" + fi +} + +check_inference_route() { + local response rc=0 + response=$(openshell sandbox exec --name "$SANDBOX_NAME" -- curl -sk --connect-timeout 5 --max-time 20 https://inference.local/v1/models 2>&1) || rc=$? + if [ "$rc" -eq 0 ] && echo "$response" | grep -q "$KIMI_MODEL"; then + if use_kimi_mock; then + pass "K3: sandbox inference.local models route reaches Kimi mock" + else + pass "K3: sandbox inference.local models route reaches VPN NVIDIA Kimi" + fi + else + fail "K3: sandbox inference.local models route failed (${response:0:400})" + fi +} + +run_agent_prompt() { + local prompt remote_cmd agent_exit=0 final_text + prompt="Use the exec tool to run hostname, date, and uptime. Run each command and then say exactly: hostname, date, and uptime completed successfully." + remote_cmd="rm -f /sandbox/.openclaw/agents/main/sessions/${SESSION_ID}.jsonl.lock /sandbox/.openclaw/agents/main/sessions/${SESSION_ID}.trajectory.jsonl 2>/dev/null || true; nemoclaw-start openclaw agent --agent main --json --session-id $(quote_for_remote_sh "$SESSION_ID") -m $(quote_for_remote_sh "$prompt")" + run_with_timeout 420 openshell sandbox exec --name "$SANDBOX_NAME" -- sh -lc "$remote_cmd" >"$AGENT_LOG" 2>&1 || agent_exit=$? + final_text="$( + python3 - "$AGENT_LOG" <<'PY' 2>/dev/null || true +import json +import sys + +text = open(sys.argv[1], encoding="utf-8", errors="replace").read() +for idx, ch in enumerate(text): + if ch != "{": + continue + try: + data = json.loads(text[idx:]) + except Exception: + continue + payloads = data.get("payloads") or [] + texts = [p.get("text") for p in payloads if isinstance(p, dict) and isinstance(p.get("text"), str)] + if texts: + print(texts[-1]) + break + meta_text = data.get("meta", {}).get("finalAssistantVisibleText") + if isinstance(meta_text, str): + print(meta_text) + break +PY + )" + if [ "$agent_exit" -ne 0 ]; then + fail "K4: OpenClaw agent command failed (exit $agent_exit)" + info "Parsed final assistant text: ${final_text:-}" + info "Agent log tail:" + tail -120 "$AGENT_LOG" 2>/dev/null || true + return + fi + + if [ "${final_text%.}" = "hostname, date, and uptime completed successfully" ]; then + pass "K4: OpenClaw agent returned the expected final text" + else + pass "K4: OpenClaw agent command completed; trajectory acceptance validates final tool results" + info "Non-canonical visible final text from command output: ${final_text:-}" + fi +} + +extract_runtime_session_id() { + python3 - "$AGENT_LOG" <<'PY' 2>/dev/null || true +import json +import sys + +text = open(sys.argv[1], encoding="utf-8", errors="replace").read() +for idx, ch in enumerate(text): + if ch != "{": + continue + try: + data = json.loads(text[idx:]) + except Exception: + continue + sid = ( + data.get("result", {}) + .get("meta", {}) + .get("agentMeta", {}) + .get("sessionId") + ) + if sid: + print(sid) + break +PY +} + +check_trajectory_acceptance() { + local output rc=0 script runtime_session_id + runtime_session_id="$(extract_runtime_session_id)" + script=$( + cat <<'SH' +python3 - "$1" "$2" <<'PY' +import json +import pathlib +import sys + +explicit_sid = sys.argv[1] +runtime_sid = sys.argv[2] if len(sys.argv) > 2 else "" +candidate_sids = [sid for sid in [runtime_sid, explicit_sid] if sid] +root = pathlib.Path("/sandbox/.openclaw") +base = pathlib.Path("/sandbox/.openclaw/agents/main/sessions") + + +def add_candidate(pairs, session_path, trajectory_path, label): + key = (str(session_path), str(trajectory_path)) + if key not in {item[:2] for item in pairs}: + pairs.append((str(session_path), str(trajectory_path), label)) + + +pairs = [] +for sid in candidate_sids: + add_candidate(pairs, base / (sid + ".jsonl"), base / (sid + ".trajectory.jsonl"), sid) + +for trajectory_path in root.rglob("*.trajectory.jsonl"): + stem = trajectory_path.name[: -len(".trajectory.jsonl")] + add_candidate(pairs, trajectory_path.with_name(stem + ".jsonl"), trajectory_path, "recursive") + +session_path = None +trajectory_path = None +for session_candidate, trajectory_candidate, _label in pairs: + maybe_session = pathlib.Path(session_candidate) + maybe_trajectory = pathlib.Path(trajectory_candidate) + if maybe_session.exists() and maybe_trajectory.exists(): + session_path = maybe_session + trajectory_path = maybe_trajectory + break + +if not session_path or not trajectory_path: + diagnostic = { + "errors": ["missing session/trajectory jsonl pair"], + "explicitSessionId": explicit_sid, + "runtimeSessionId": runtime_sid, + "checkedPairs": pairs[:20], + "sessionFiles": [str(p) for p in root.rglob("*.jsonl")][:40], + "trajectoryFiles": [str(p) for p in root.rglob("*.trajectory.jsonl")][:40], + } + print(json.dumps(diagnostic, indent=2)) + sys.exit(1) + +session = [json.loads(line) for line in session_path.read_text().splitlines() if line.strip()] +trajectory = [json.loads(line) for line in trajectory_path.read_text().splitlines() if line.strip()] +errors = [] +artifacts = [item for item in trajectory if item.get("type") == "trace.artifacts"] +completed = [item for item in trajectory if item.get("type") == "model.completed"] +if len(artifacts) != 1: + errors.append("expected 1 trace.artifacts record, got %d" % len(artifacts)) +artifact_data = artifacts[-1].get("data", {}) if artifacts else {} +completed_data = completed[-1].get("data", {}) if completed else {} +metas = artifact_data.get("toolMetas", []) +assistant_tool_messages = [ + item.get("message", {}) + for item in session + if item.get("type") == "message" + and item.get("message", {}).get("role") == "assistant" + and any(block.get("type") == "toolCall" for block in item.get("message", {}).get("content", [])) +] +source_calls = [] +for message in assistant_tool_messages: + source_calls.extend(message.get("content", [])) +source_commands = [block.get("arguments", {}).get("command") for block in source_calls] +messages = [item.get("message", {}) for item in session if item.get("type") == "message"] +tool_result_indices = [idx for idx, msg in enumerate(messages) if msg.get("role") == "toolResult"] +assistant_indices = [idx for idx, msg in enumerate(messages) if msg.get("role") == "assistant"] +raw = session_path.read_text() + "\n" + trajectory_path.read_text() + +if artifact_data.get("finalStatus") != "success": + errors.append("finalStatus is %r" % artifact_data.get("finalStatus")) +if len(metas) != 3: + errors.append("expected 3 trace.artifacts.toolMetas, got %d" % len(metas)) +if [meta.get("toolName") for meta in metas] != ["exec", "exec", "exec"]: + errors.append("toolMeta tool names are %r" % [meta.get("toolName") for meta in metas]) +if sorted(meta.get("meta") for meta in metas) != ["date", "hostname", "uptime"]: + errors.append("toolMeta command set is %r" % sorted(meta.get("meta") for meta in metas)) +if source_commands != ["hostname", "date", "uptime"]: + errors.append("source assistant command order is %r" % source_commands) +if any(isinstance(command, str) and ";" in command for command in source_commands): + errors.append("source assistant still contains a combined semicolon command") +if artifact_data.get("promptErrorSource") is not None: + errors.append("promptErrorSource is %r" % artifact_data.get("promptErrorSource")) +if completed_data.get("promptErrorSource") is not None: + errors.append("model.completed promptErrorSource is %r" % completed_data.get("promptErrorSource")) +for field in ["aborted", "externalAbort", "timedOut", "idleTimedOut", "timedOutDuringCompaction"]: + if artifact_data.get(field): + errors.append("%s is %r" % (field, artifact_data.get(field))) +if "abandoned" in raw.lower(): + errors.append("trajectory/session contains 'abandoned'") +if "want me to continue" in raw.lower(): + errors.append("trajectory/session contains 'want me to continue'") +def normalize_final_text(value): + if not isinstance(value, str): + return value + return value.strip().removesuffix(".") + +final_texts = artifact_data.get("assistantTexts") or [] +expected_final_text = "hostname, date, and uptime completed successfully" +if not final_texts or normalize_final_text(final_texts[-1]) != expected_final_text: + errors.append("final assistant text is %r" % (final_texts[-1] if final_texts else None)) +if not tool_result_indices or not assistant_indices or max(assistant_indices) <= max(tool_result_indices): + errors.append("final assistant response did not occur after all tool results") + +summary = { + "explicitSessionId": explicit_sid, + "runtimeSessionId": runtime_sid, + "sessionPath": str(session_path), + "trajectoryPath": str(trajectory_path), + "finalStatus": artifact_data.get("finalStatus"), + "toolMetasCount": len(metas), + "toolMetaToolNames": [meta.get("toolName") for meta in metas], + "toolMetaCommandSet": sorted(meta.get("meta") for meta in metas), + "sourceAssistantCommands": source_commands, + "sourceHasCombinedSemicolonCommand": any(isinstance(command, str) and ";" in command for command in source_commands), + "promptErrorSource": artifact_data.get("promptErrorSource"), + "containsAbandoned": "abandoned" in raw.lower(), + "containsWantMeToContinue": "want me to continue" in raw.lower(), + "finalAssistantText": final_texts[-1] if final_texts else None, + "finalAssistantAfterAllToolResults": bool(tool_result_indices and assistant_indices and max(assistant_indices) > max(tool_result_indices)), + "messageRoles": [msg.get("role") for msg in messages], + "errors": errors, +} +print(json.dumps(summary, indent=2)) +sys.exit(1 if errors else 0) +PY +SH + ) + output=$(sandbox_exec_sh_script "$script" "$SESSION_ID" "$runtime_session_id" 2>&1) || rc=$? + info "Trajectory summary:" + printf '%s\n' "$output" | sed 's/^/ /' + if [ "$rc" -eq 0 ]; then + pass "K5: trajectory proves split Kimi exec calls completed cleanly" + else + fail "K5: trajectory acceptance checks failed" + fi +} + +check_upstream_observed_agent_traffic() { + if ! use_kimi_mock; then + local route rc=0 + route=$(openshell inference get -g nemoclaw 2>&1 || openshell inference get 2>&1) || rc=$? + if [ "$rc" -eq 0 ] && echo "$route" | grep -q "compatible-endpoint" && echo "$route" | grep -q "$KIMI_MODEL"; then + pass "K6: OpenShell route is VPN NVIDIA Kimi" + else + fail "K6: OpenShell route is not VPN NVIDIA Kimi (${route:0:400})" + fi + return + fi + + local stream_count + stream_count=$(grep -c "POST /v1/chat/completions auth=ok stream=True" "$KIMI_MOCK_LOG" 2>/dev/null || true) + if [ "$stream_count" -ge 2 ]; then + pass "K6: Kimi mock observed authenticated streamed tool-call and final-answer traffic" + else + fail "K6: Kimi mock did not observe both streamed agent requests" + info "Mock log:" + sed 's/^/ /' "$KIMI_MOCK_LOG" 2>/dev/null || true + fi +} + +# Repo root +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +if [ -d /workspace ] && [ -f /workspace/install.sh ]; then + REPO="/workspace" +elif [ -f "${SCRIPT_DIR}/../../install.sh" ]; then + REPO="$(cd "${SCRIPT_DIR}/../.." && pwd)" +elif [ -f "./install.sh" ]; then + REPO="$(pwd)" +else + echo "ERROR: Cannot find repo root." + exit 1 +fi + +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-kimi-compat}" +KIMI_USE_MOCK="${NEMOCLAW_KIMI_USE_MOCK:-0}" +KIMI_MOCK_PORT="${NEMOCLAW_KIMI_MOCK_PORT:-18146}" +KIMI_MODEL="${NEMOCLAW_KIMI_MODEL:-moonshotai/kimi-k2.6}" +KIMI_MOCK_API_KEY="${NEMOCLAW_KIMI_MOCK_API_KEY:-fake-kimi-compatible-key-e2e}" +KIMI_MOCK_HOST="${NEMOCLAW_KIMI_MOCK_HOST:-host.openshell.internal}" +KIMI_ENDPOINT_URL="${NEMOCLAW_KIMI_MOCK_ENDPOINT_URL:-http://${KIMI_MOCK_HOST}:${KIMI_MOCK_PORT}/v1}" +SESSION_ID="${NEMOCLAW_KIMI_SESSION_ID:-kimi-e2e-$(date +%s)}" +KIMI_MOCK_LOG="$(mktemp)" +ONBOARD_LOG="/tmp/nemoclaw-e2e-kimi-inference-compat-onboard.log" +AGENT_LOG="/tmp/nemoclaw-e2e-kimi-inference-compat-agent.log" +BUILD_LOG="/tmp/nemoclaw-e2e-kimi-inference-compat-build.log" +KIMI_MOCK_PID="" + +trap cleanup EXIT + +echo "" +echo "============================================================" +echo " Kimi Inference Compatibility E2E (#2620 / #3046)" +echo " $(date)" +echo "============================================================" +echo "" + +section "Phase 0: Prerequisites" +if ! docker info >/dev/null 2>&1; then + fail "Docker is not running" + summary +fi +pass "Docker is running" + +if ! command -v python3 >/dev/null 2>&1; then + fail "python3 not found" + summary +fi +pass "python3 is available" + +load_shell_path +info "Repo: $REPO" +info "Sandbox: $SANDBOX_NAME" +info "Model: $KIMI_MODEL" +if use_kimi_mock; then + info "Mode: hermetic mock" + info "Mock endpoint URL for gateway: $KIMI_ENDPOINT_URL" +else + info "Mode: live public VPN NVIDIA inference via compatible-endpoint" +fi + +section "Phase 1: Kimi upstream" +if use_kimi_mock; then + if start_kimi_mock; then + pass "K0: Kimi-compatible mock endpoint started" + else + fail "K0: Kimi-compatible mock endpoint failed to start" + info "Mock log:" + sed 's/^/ /' "$KIMI_MOCK_LOG" 2>/dev/null || true + summary + fi +elif ensure_public_nvidia_api_key; then + pass "K0: public VPN NVIDIA inference key is available for Kimi" +else + fail "K0: NVIDIA_API_KEY must be a public VPN NVIDIA inference nvapi-* key" + summary +fi + +section "Phase 2: Onboard fresh Kimi sandbox" +run_kimi_onboard + +section "Phase 3: Runtime assertions" +check_openclaw_config +check_inference_route +run_agent_prompt +check_trajectory_acceptance +check_upstream_observed_agent_traffic + +trap - EXIT +cleanup +summary diff --git a/test/e2e-vpn/test-launchable-smoke.sh b/test/e2e-vpn/test-launchable-smoke.sh new file mode 100755 index 00000000000..e9fb7dbc5ed --- /dev/null +++ b/test/e2e-vpn/test-launchable-smoke.sh @@ -0,0 +1,599 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Launchable Install-Flow Smoke Test +# +# Validates the Brev launchable install path (scripts/brev-launchable-ci-cpu.sh) +# end-to-end: bootstrap → artifact verification → onboard → sandbox health → +# live inference → cleanup. +# +# This is the long-living safety net for the community install path. If any +# regression breaks brev-launchable-ci-cpu.sh (e.g., the Apr 20-25 Brev outage +# from issues #2472/#2482, or the container reachability fallback from #2425), +# this smoke test catches it before community users are affected. +# +# Key insight: brev-launchable-ci-cpu.sh has ZERO Brev dependencies — it's a +# generic Ubuntu bootstrap script. It runs on ubuntu-latest GitHub runners +# with no BREV_API_TOKEN needed. +# +# What this tests: +# 1. Run brev-launchable-ci-cpu.sh with NEMOCLAW_REF=current branch +# 2. Verify installation artifacts (nemoclaw, openshell, Node.js ≥22, Docker, sentinel) +# 3. nemoclaw onboard --non-interactive with hosted inference +# 4. Sandbox health: nemoclaw list, status, gateway running +# 5. Live inference through the sandbox (same pattern as test-full-e2e.sh Phase 4) +# 6. Destroy + cleanup +# +# Prerequisites: +# - Ubuntu runner (ubuntu-latest) +# - Docker running +# - NVIDIA_API_KEY set for hosted inference +# - Network access to inference.nvidia.com +# - NEMOCLAW_NON_INTERACTIVE=1 +# - NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 +# +# Environment variables: +# NEMOCLAW_REF — git ref for brev-launchable-ci-cpu.sh (default: current branch) +# NEMOCLAW_SANDBOX_NAME — sandbox name (default: e2e-launchable) +# NEMOCLAW_RECREATE_SANDBOX — set to 1 to recreate if exists +# NVIDIA_API_KEY — required for hosted inference +# SKIP_DOCKER_PULL — set to 1 to skip Docker image pre-pulls (speeds up CI) +# +# Usage: +# NEMOCLAW_NON_INTERACTIVE=1 NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ +# NVIDIA_API_KEY=... bash test/e2e-vpn/test-launchable-smoke.sh +# +# See: https://github.com/NVIDIA/NemoClaw/issues/2599 + +set -uo pipefail + +export NEMOCLAW_E2E_DEFAULT_TIMEOUT=1800 +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +# shellcheck source=test/e2e-vpn/e2e-timeout.sh +source "${SCRIPT_DIR}/e2e-timeout.sh" +# shellcheck source=test/e2e-vpn/lib/openclaw-json.sh +source "${SCRIPT_DIR}/lib/openclaw-json.sh" + +PASS=0 +FAIL=0 +SKIP=0 +TOTAL=0 + +pass() { + ((PASS++)) + ((TOTAL++)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} +fail() { + ((FAIL++)) + ((TOTAL++)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} +# shellcheck disable=SC2329 +skip() { + ((SKIP++)) + ((TOTAL++)) + printf '\033[33m SKIP: %s\033[0m\n' "$1" +} +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } + +# Parse chat completion response — handles both content and reasoning_content +parse_chat_content() { + python3 -c " +import json, sys +try: + r = json.load(sys.stdin) + c = r['choices'][0]['message'] + content = c.get('content') or c.get('reasoning_content') or c.get('reasoning') or '' + print(content.strip()) +except Exception as e: + print(f'PARSE_ERROR: {e}', file=sys.stderr) + sys.exit(1) +" +} + +# shellcheck source=test/e2e-vpn/lib/ci-compatible-inference.sh +. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/lib/ci-compatible-inference.sh" + +# Determine repo root +if [ -f "$(cd "$(dirname "$0")/../.." && pwd)/scripts/brev-launchable-ci-cpu.sh" ]; then + REPO="$(cd "$(dirname "$0")/../.." && pwd)" +else + echo "ERROR: Cannot find repo root (expected scripts/brev-launchable-ci-cpu.sh)." + exit 1 +fi + +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-launchable}" +INSTALL_LOG="/tmp/nemoclaw-launchable-install.log" +TEST_LOG="/tmp/nemoclaw-launchable-test.log" + +# The launchable script clones into ~/NemoClaw by default. For CI, use +# a unique directory so we don't collide with the checkout. +NEMOCLAW_CLONE_DIR="${NEMOCLAW_CLONE_DIR:-${HOME}/NemoClaw-launchable}" +export NEMOCLAW_CLONE_DIR + +# The launchable script clones from github.com/NVIDIA/NemoClaw using +# NEMOCLAW_REF as the branch. To test the CURRENT code (not main HEAD), +# we pre-seed the clone directory from the checkout (see Phase 0) and +# create a branch named "main" at the current commit. The script detects +# an existing .git dir, does fetch+checkout (which is a no-op since we're +# already on the right commit), then proceeds to npm install + build. +# This lets us test on forks where the branch name doesn't exist upstream. +NEMOCLAW_REF="${NEMOCLAW_REF:-main}" +export NEMOCLAW_REF + +# Skip Docker image pre-pulls by default in CI — the images will be pulled +# at onboard time and this avoids flaky pulls blocking the install step. +export SKIP_DOCKER_PULL="${SKIP_DOCKER_PULL:-1}" + +exec > >(tee -a "$TEST_LOG") 2>&1 + +# shellcheck source=test/e2e-vpn/lib/sandbox-teardown.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib/sandbox-teardown.sh" +register_sandbox_for_teardown "$SANDBOX_NAME" +nemoclaw_e2e_configure_compatible_inference || exit 1 +HOSTED_INFERENCE_BASE_URL="$(nemoclaw_e2e_hosted_inference_base_url)" +HOSTED_INFERENCE_MODEL="$(nemoclaw_e2e_hosted_inference_model)" +HOSTED_INFERENCE_KEY="$(nemoclaw_e2e_hosted_inference_key)" + +# ══════════════════════════════════════════════════════════════════ +# Phase 0: Pre-cleanup +# ══════════════════════════════════════════════════════════════════ +section "Phase 0: Pre-cleanup" +info "Destroying any leftover sandbox/gateway from previous runs..." +if command -v nemoclaw >/dev/null 2>&1; then + nemoclaw "$SANDBOX_NAME" destroy --yes 2>/dev/null || true +fi +if command -v openshell >/dev/null 2>&1; then + openshell sandbox delete "$SANDBOX_NAME" 2>/dev/null || true + openshell gateway destroy -g nemoclaw 2>/dev/null || true +fi +# Clean up any previous launchable clone (sudo because launchable may have +# created root-owned files on a previous run) +sudo rm -rf "$NEMOCLAW_CLONE_DIR" 2>/dev/null || rm -rf "$NEMOCLAW_CLONE_DIR" || true + +# Pre-seed the clone directory from the checked-out repo so the launchable +# script tests THIS code (not main HEAD). The script's step 5 detects +# $NEMOCLAW_CLONE_DIR/.git and runs the refresh path (fetch+checkout) +# instead of a fresh clone from NVIDIA/NemoClaw. We create a "main" branch +# at the current commit so NEMOCLAW_REF=main resolves locally. +info "Pre-seeding $NEMOCLAW_CLONE_DIR from checkout at $REPO..." +git clone --local --no-hardlinks "$REPO" "$NEMOCLAW_CLONE_DIR" +# Ensure a "main" branch exists at the current commit for the script's +# `git fetch origin main && git checkout main` to succeed. Point origin +# at the clone itself so fetch resolves locally (the CI checkout may be +# in detached HEAD and lack a "main" branch). +git -C "$NEMOCLAW_CLONE_DIR" checkout -B main HEAD 2>/dev/null || true +git -C "$NEMOCLAW_CLONE_DIR" remote set-url origin "$NEMOCLAW_CLONE_DIR" +pass "Pre-cleanup complete (clone dir pre-seeded)" + +# ══════════════════════════════════════════════════════════════════ +# Phase 1: Prerequisites +# ══════════════════════════════════════════════════════════════════ +section "Phase 1: Prerequisites" + +if docker info >/dev/null 2>&1; then + pass "Docker is running" +else + fail "Docker is not running — cannot continue" + exit 1 +fi + +if ! nemoclaw_e2e_require_hosted_inference_key; then + exit 1 +fi + +if nemoclaw_e2e_probe_hosted_inference; then + pass "Network access to ${HOSTED_INFERENCE_BASE_URL}" +else + fail "Cannot reach ${HOSTED_INFERENCE_BASE_URL}" + exit 1 +fi + +if [ "${NEMOCLAW_NON_INTERACTIVE:-}" != "1" ]; then + fail "NEMOCLAW_NON_INTERACTIVE=1 is required" + exit 1 +fi + +if [ "${NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE:-}" != "1" ]; then + fail "NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 is required for non-interactive install" + exit 1 +fi + +if [ -f "$REPO/scripts/brev-launchable-ci-cpu.sh" ]; then + pass "brev-launchable-ci-cpu.sh found at $REPO/scripts/" +else + fail "brev-launchable-ci-cpu.sh not found" + exit 1 +fi + +info "NEMOCLAW_REF=$NEMOCLAW_REF" +info "NEMOCLAW_CLONE_DIR=$NEMOCLAW_CLONE_DIR" + +# ══════════════════════════════════════════════════════════════════ +# Phase 2: Run brev-launchable-ci-cpu.sh +# ══════════════════════════════════════════════════════════════════ +section "Phase 2: Run brev-launchable-ci-cpu.sh (launchable install path)" + +info "Running the launchable bootstrap script..." +info "This installs Docker, Node.js 22, OpenShell, clones NemoClaw, builds CLI+plugin." +info "Expected duration: 3-8 minutes." + +# The launchable script expects to run as root (it uses sudo internally). +# On GitHub runners, we already have passwordless sudo. +# Redirect is intentional — log file stays runner-owned, not root-owned. +# shellcheck disable=SC2024 +sudo -E bash "$REPO/scripts/brev-launchable-ci-cpu.sh" >"$INSTALL_LOG" 2>&1 & +install_pid=$! +tail -f "$INSTALL_LOG" --pid=$install_pid 2>/dev/null & +tail_pid=$! +wait $install_pid +install_exit=$? +kill $tail_pid 2>/dev/null || true +wait $tail_pid 2>/dev/null || true + +if [ $install_exit -eq 0 ]; then + pass "brev-launchable-ci-cpu.sh completed (exit 0)" +else + fail "brev-launchable-ci-cpu.sh failed (exit $install_exit)" + info "Last 30 lines of install log:" + tail -30 "$INSTALL_LOG" + exit 1 +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 3: Verify installation artifacts +# ══════════════════════════════════════════════════════════════════ +section "Phase 3: Verify installation artifacts" + +# Refresh PATH — the launchable script installs binaries to /usr/local/bin +# and Node.js via nodesource. On the GH runner the shell may not have +# picked up the new PATH entries yet. +export PATH="/usr/local/bin:$PATH" +if [ "${GITHUB_ACTIONS:-}" = "true" ] \ + && [ "${GITHUB_REPOSITORY:-}" = "NVIDIA/NemoClaw" ] \ + && [ "${GITHUB_REF:-}" = "refs/heads/fix/native-messaging-websocket" ] \ + && [ -n "${NEMOCLAW_OPENSHELL_BIN:-}" ]; then + main_openshell_dir="$(dirname "$NEMOCLAW_OPENSHELL_BIN")" + export PATH="$main_openshell_dir:$PATH" +fi +hash -r 2>/dev/null || true + +# 3a: nemoclaw on PATH and --help works +if command -v nemoclaw >/dev/null 2>&1; then + pass "nemoclaw on PATH: $(command -v nemoclaw)" +else + fail "nemoclaw not found on PATH after launchable install" +fi + +if nemoclaw --help >/dev/null 2>&1; then + pass "nemoclaw --help exits 0" +else + fail "nemoclaw --help failed" +fi + +# 3b: openshell on PATH and --version works +if command -v openshell >/dev/null 2>&1; then + os_version="$(openshell --version 2>&1 || echo unknown)" + pass "openshell on PATH: $(command -v openshell) (${os_version})" +else + fail "openshell not found on PATH after launchable install" +fi + +# 3c: Node.js >= 22 +# The launchable script installs Node.js via nodesource as root. On GH runners, +# a pre-installed Node may shadow the new one in PATH. Refresh the hash table +# and check the version that the launchable script's npm actually uses. +hash -r 2>/dev/null || true +if command -v node >/dev/null 2>&1; then + node_version="$(node --version 2>/dev/null)" + node_major="$(node -p 'process.versions.node.split(".")[0]' 2>/dev/null || echo 0)" + if [ "$node_major" -ge 22 ]; then + pass "Node.js >= 22 installed: ${node_version}" + else + # On ubuntu-latest GH runners, nodesource may not override the pre-installed + # Node 20. This is a known issue with the launchable script (#TBD). Log it + # as a warning but don't block the test — the CLI still works with Node 20. + info "Node.js ${node_version} found (< 22). Checking if onboard can proceed..." + if [ "$node_major" -ge 20 ]; then + skip "Node.js ${node_version} — launchable installed Node < 22 but >= 20 (usable)" + else + fail "Node.js version too old: ${node_version} (need >= 20)" + fi + fi +else + fail "Node.js not found on PATH after launchable install" +fi + +# 3d: Docker running +if docker info >/dev/null 2>&1; then + pass "Docker running after launchable install" +else + fail "Docker not running after launchable install" +fi + +# 3e: Sentinel file +SENTINEL="/var/run/nemoclaw-launchable-ready" +if [ -f "$SENTINEL" ]; then + pass "Sentinel file exists: $SENTINEL" +else + fail "Sentinel file missing: $SENTINEL" +fi + +# 3f: Clone directory exists with built artifacts +if [ -d "$NEMOCLAW_CLONE_DIR/.git" ]; then + pass "NemoClaw cloned at $NEMOCLAW_CLONE_DIR" +else + fail "NemoClaw clone directory missing: $NEMOCLAW_CLONE_DIR" +fi + +if [ -d "$NEMOCLAW_CLONE_DIR/dist" ]; then + pass "CLI built (dist/ exists)" +else + fail "CLI not built (dist/ missing)" +fi + +if [ -d "$NEMOCLAW_CLONE_DIR/nemoclaw/dist" ]; then + pass "Plugin built (nemoclaw/dist/ exists)" +else + fail "Plugin not built (nemoclaw/dist/ missing)" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 4: Onboard (non-interactive, hosted inference) +# ══════════════════════════════════════════════════════════════════ +section "Phase 4: Onboard (non-interactive, hosted inference)" + +# Run onboard from the launchable clone directory — this is the real +# community path: the user's NemoClaw is in ~/NemoClaw, not a CI checkout. +cd "$NEMOCLAW_CLONE_DIR" || { + fail "Could not cd to $NEMOCLAW_CLONE_DIR" + exit 1 +} + +info "Running nemoclaw onboard --non-interactive..." +info "Provider: ${NEMOCLAW_PROVIDER:-configured hosted inference}" +info "Sandbox name: $SANDBOX_NAME" + +ONBOARD_LOG="/tmp/nemoclaw-launchable-onboard.log" +export NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" +export NEMOCLAW_RECREATE_SANDBOX="${NEMOCLAW_RECREATE_SANDBOX:-1}" + +nemoclaw onboard --non-interactive >"$ONBOARD_LOG" 2>&1 & +onboard_pid=$! +tail -f "$ONBOARD_LOG" --pid=$onboard_pid 2>/dev/null & +tail_pid=$! +wait $onboard_pid +onboard_exit=$? +kill $tail_pid 2>/dev/null || true +wait $tail_pid 2>/dev/null || true + +if [ $onboard_exit -eq 0 ]; then + pass "nemoclaw onboard completed (exit 0)" +else + fail "nemoclaw onboard failed (exit $onboard_exit)" + info "Last 30 lines of onboard log:" + tail -30 "$ONBOARD_LOG" + exit 1 +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 5: Sandbox health verification +# ══════════════════════════════════════════════════════════════════ +section "Phase 5: Sandbox health verification" + +# 5a: nemoclaw list +if list_output=$(nemoclaw list 2>&1); then + if grep -Fq -- "$SANDBOX_NAME" <<<"$list_output"; then + pass "nemoclaw list contains '${SANDBOX_NAME}'" + else + fail "nemoclaw list does not contain '${SANDBOX_NAME}'" + fi +else + fail "nemoclaw list failed: ${list_output:0:200}" +fi + +# 5b: nemoclaw status +if status_output=$(nemoclaw "$SANDBOX_NAME" status 2>&1); then + pass "nemoclaw ${SANDBOX_NAME} status exits 0" +else + fail "nemoclaw ${SANDBOX_NAME} status failed: ${status_output:0:200}" +fi + +# 5c: Inference configured by onboard +if inf_check=$(openshell inference get 2>&1); then + expected_provider="$(nemoclaw_e2e_expected_route_provider)" + expected_model="" + if nemoclaw_e2e_using_compatible_inference; then + expected_model="$HOSTED_INFERENCE_MODEL" + fi + if nemoclaw_e2e_inference_output_matches "$inf_check" "$expected_provider" "$expected_model"; then + pass "Inference configured via onboard (${expected_provider})" + else + inf_check_plain="$(printf '%s' "$inf_check" | nemoclaw_e2e_strip_ansi)" + fail "Inference not configured - onboard did not set up ${expected_provider}: ${inf_check_plain:0:200}" + fi +else + fail "openshell inference get failed: ${inf_check:0:200}" +fi + +# 5d: Gateway running +if docker ps --format '{{.Names}}' 2>/dev/null | grep -q "nemoclaw\|openshell"; then + pass "Gateway container running" +else + skip "Could not confirm gateway container (may have different naming)" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 6: Live inference through the sandbox +# ══════════════════════════════════════════════════════════════════ +section "Phase 6: Live inference" + +# ── Test 6a: Direct hosted inference endpoint (sanity check) ── +info "[LIVE] Direct API test → ${HOSTED_INFERENCE_BASE_URL}..." +api_response=$(curl -s --max-time 30 \ + -X POST "${HOSTED_INFERENCE_BASE_URL}/chat/completions" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $HOSTED_INFERENCE_KEY" \ + -d "$(printf '{"model":"%s","messages":[{"role":"user","content":"Reply with exactly one word: PONG"}],"max_tokens":100}' "$HOSTED_INFERENCE_MODEL")" 2>/dev/null) || true + +if [ -n "$api_response" ]; then + api_content=$(echo "$api_response" | parse_chat_content 2>/dev/null) || true + if grep -qi "PONG" <<<"$api_content"; then + pass "[LIVE] Direct API: model responded with PONG" + else + fail "[LIVE] Direct API: expected PONG, got: ${api_content:0:200}" + fi +else + fail "[LIVE] Direct API: empty response from curl" +fi + +# ── Test 6b: Inference through sandbox (routing check) ── +info "[ROUTING] inference.local DNS + OpenShell proxy reachable from sandbox..." +ssh_config="$(mktemp)" +sandbox_response="" + +if openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config" 2>/dev/null; then + sandbox_response=$(run_with_timeout 90 ssh -F "$ssh_config" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" \ + "curl -s --max-time 60 https://inference.local/v1/chat/completions \ + -H 'Content-Type: application/json' \ + -d '{\"model\":\"$HOSTED_INFERENCE_MODEL\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with exactly one word: PONG\"}],\"max_tokens\":100}'" \ + 2>&1) || true +fi +rm -f "$ssh_config" + +# Retry sandbox inference up to 3 times — live models are not deterministic +# and the gateway proxy can return unexpected responses on first attempt. +sandbox_content="" +pong_ok=false +for pong_attempt in 1 2 3; do + if [ -n "$sandbox_response" ]; then + sandbox_content=$(echo "$sandbox_response" | parse_chat_content 2>/dev/null) || true + if grep -qi "PONG" <<<"$sandbox_content"; then + pong_ok=true + break + fi + info "Sandbox inference attempt ${pong_attempt}/3: got '${sandbox_content:0:80}', retrying in 5s..." + else + info "Sandbox inference attempt ${pong_attempt}/3: empty response, retrying in 5s..." + fi + [ "$pong_attempt" -lt 3 ] || break + sleep 5 + ssh_config="$(mktemp)" + sandbox_response="" + if openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config" 2>/dev/null; then + sandbox_response=$(run_with_timeout 90 ssh -F "$ssh_config" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" \ + "curl -s --max-time 60 https://inference.local/v1/chat/completions \ + -H 'Content-Type: application/json' \ + -d '{\"model\":\"$HOSTED_INFERENCE_MODEL\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with exactly one word: PONG\"}],\"max_tokens\":100}'" \ + 2>&1) || true + fi + rm -f "$ssh_config" +done + +if $pong_ok; then + pass "[ROUTING] inference.local: OpenShell routed curl to the hosted inference endpoint and returned PONG" +else + fail "[ROUTING] inference.local: expected PONG after 3 attempts, got: ${sandbox_content:0:200}" +fi + +# ── Test 6c: openclaw-mediated turn (the real proof) ── +info "[LIVE] openclaw agent → openclaw HTTP client → inference.local..." +ssh_config="$(mktemp)" +agent_response="" +agent_stderr="" +agent_rc=0 +agent_stderr_file="$(mktemp)" + +if openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config" 2>/dev/null; then + agent_session_id="e2e-launchable-$(date +%s)-$$" + agent_response=$(run_with_timeout 120 ssh -F "$ssh_config" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" \ + "openclaw agent --agent main --json --thinking off --session-id '${agent_session_id}' -m 'What is 6 multiplied by 7? Reply with only the integer, no extra words.'" \ + 2>"$agent_stderr_file") || agent_rc=$? + agent_stderr="$(<"$agent_stderr_file")" +else + agent_rc=255 + agent_stderr="failed to get SSH config for ${SANDBOX_NAME}" +fi +rm -f "$ssh_config" "$agent_stderr_file" + +agent_reply=$(printf '%s' "$agent_response" | parse_openclaw_agent_text 2>/dev/null) || true + +if grep -qE "(^|[^0-9])42([^0-9]|$)" <<<"$agent_reply"; then + pass "[LIVE] openclaw agent: model answered 6×7=42 through openclaw → inference.local" +else + fail "[LIVE] openclaw agent: expected '42' in agent reply; rc=${agent_rc}; reply='${agent_reply:0:200}'; stdout='${agent_response:0:300}'; stderr='${agent_stderr:0:300}'" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 7: Cleanup +# ══════════════════════════════════════════════════════════════════ +section "Phase 7: Cleanup" + +[[ "${NEMOCLAW_E2E_KEEP_SANDBOX:-}" = "1" ]] || nemoclaw "$SANDBOX_NAME" destroy --yes 2>&1 | tail -3 || true +openshell gateway destroy -g nemoclaw 2>/dev/null || true + +# Verify against the registry file directly. `nemoclaw list` triggers +# gateway recovery which can restart a destroyed gateway — avoid it here. +registry_file="${HOME}/.nemoclaw/sandboxes.json" +if [ -f "$registry_file" ] && grep -Fq "\"${SANDBOX_NAME}\"" "$registry_file"; then + fail "Sandbox ${SANDBOX_NAME} still in registry after destroy" +else + pass "Sandbox ${SANDBOX_NAME} removed" +fi + +# Clean up the launchable clone directory (sudo because launchable ran as root +# and npm install creates root-owned files in node_modules/) +sudo rm -rf "$NEMOCLAW_CLONE_DIR" 2>/dev/null || rm -rf "$NEMOCLAW_CLONE_DIR" || true +pass "Launchable clone directory cleaned up" + +# ══════════════════════════════════════════════════════════════════ +# Summary +# ══════════════════════════════════════════════════════════════════ +echo "" +echo "========================================" +echo " Launchable Install-Flow Smoke Test Results:" +echo " Passed: $PASS" +echo " Failed: $FAIL" +echo " Skipped: $SKIP" +echo " Total: $TOTAL" +echo "========================================" +echo "" +echo " What this tested (issue #2599):" +echo " - brev-launchable-ci-cpu.sh bootstrap (Docker, Node.js, OpenShell, NemoClaw)" +echo " - Installation artifacts (binaries on PATH, sentinel file, built outputs)" +echo " - Onboard via launchable-installed NemoClaw (hosted inference)" +echo " - Sandbox health (list, status, inference config, gateway)" +echo " - Direct hosted inference" +echo " - Sandbox inference routing (curl → inference.local)" +echo " - openclaw agent mediated inference (the full stack)" +echo " - Destroy + cleanup" +echo "" + +if [ "$FAIL" -eq 0 ]; then + printf '\n\033[1;32m LAUNCHABLE SMOKE TEST PASSED — community install path verified end-to-end.\033[0m\n' + exit 0 +else + printf '\n\033[1;31m %d test(s) failed.\033[0m\n' "$FAIL" + exit 1 +fi diff --git a/test/e2e-vpn/test-messaging-compatible-endpoint.sh b/test/e2e-vpn/test-messaging-compatible-endpoint.sh new file mode 100755 index 00000000000..bcbe3a15b86 --- /dev/null +++ b/test/e2e-vpn/test-messaging-compatible-endpoint.sh @@ -0,0 +1,679 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Telegram + OpenAI-compatible endpoint regression E2E (#2766, #2572) +# +# Hermetic path: +# - starts a local OpenAI-compatible mock endpoint +# - onboards with NEMOCLAW_PROVIDER=custom and Telegram enabled +# - verifies OpenClaw keeps the managed inference.local provider shape +# - verifies a sandbox-side chat completion reaches the mock with auth +# - verifies openclaw's HTTP client completes a turn through the custom +# endpoint (exercises the FORWARD-mode rewrite in http-proxy-fix.js, +# the path that caused "LLM request failed: network connection error" +# for deepinfra/together.ai users on NemoClaw 0.0.24 — see #2572) +# - verifies no RFC 7230 hop-by-hop proxy headers leak to the upstream +# +# Prerequisites: +# - Docker running +# - NemoClaw installed or a source checkout that install.sh can install +# +# Environment: +# NEMOCLAW_SANDBOX_NAME — sandbox name (default: e2e-msg-compat) +# NEMOCLAW_COMPAT_MOCK_PORT — mock endpoint port (default: 18089) +# NEMOCLAW_COMPAT_MODEL — model id for the compatible endpoint mock +# NEMOCLAW_COMPAT_MOCK_API_KEY — optional; defaults to a fake hermetic key +# TELEGRAM_BOT_TOKEN — optional; defaults to a fake Telegram token +# TELEGRAM_ALLOWED_IDS — optional; defaults to a fake allowlist +# +# Usage: +# NEMOCLAW_NON_INTERACTIVE=1 NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ +# bash test/e2e-vpn/test-messaging-compatible-endpoint.sh + +set -uo pipefail + +export NEMOCLAW_E2E_DEFAULT_TIMEOUT=1800 +SCRIPT_DIR_TIMEOUT="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +# shellcheck source=test/e2e-vpn/e2e-timeout.sh +. "${SCRIPT_DIR_TIMEOUT}/e2e-timeout.sh" +# shellcheck source=test/e2e-vpn/lib/openclaw-json.sh +. "${SCRIPT_DIR_TIMEOUT}/lib/openclaw-json.sh" + +PASS=0 +FAIL=0 +SKIP=0 +TOTAL=0 + +pass() { + ((PASS++)) + ((TOTAL++)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} +fail() { + ((FAIL++)) + ((TOTAL++)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} +skip() { + ((SKIP++)) + ((TOTAL++)) + printf '\033[33m SKIP: %s\033[0m\n' "$1" +} +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } + +summary() { + echo "" + echo "============================================================" + echo " Messaging Compatible Endpoint E2E Results" + echo "============================================================" + echo " PASS: $PASS" + echo " FAIL: $FAIL" + echo " SKIP: $SKIP" + echo " TOTAL: $TOTAL" + echo "============================================================" + if [ "$FAIL" -gt 0 ]; then + exit 1 + fi +} + +host_ip_for_sandbox() { + local ip_addr + ip_addr="$(ip route get 1.1.1.1 2>/dev/null | awk '{for (i=1;i<=NF;i++) if ($i=="src") {print $(i+1); exit}}')" + if [ -n "$ip_addr" ]; then + echo "$ip_addr" + return + fi + ip_addr="$(hostname -I 2>/dev/null | awk '{print $1}')" + if [ -n "$ip_addr" ]; then + echo "$ip_addr" + return + fi + if [ "$(uname -s 2>/dev/null)" = "Darwin" ]; then + for iface in en0 en1 bridge100; do + ip_addr="$(ipconfig getifaddr "$iface" 2>/dev/null || true)" + if [ -n "$ip_addr" ]; then + echo "$ip_addr" + return + fi + done + ip_addr="$(ifconfig 2>/dev/null | awk '/inet / && $2 !~ /^127\./ {print $2; exit}')" + if [ -n "$ip_addr" ]; then + echo "$ip_addr" + return + fi + fi + echo "127.0.0.1" +} + +quote_for_remote_sh() { + local value="${1:-}" + printf "'%s'" "$(printf '%s' "$value" | sed "s/'/'\\\\''/g")" +} + +sandbox_exec_sh_script() { + local script="$1" + shift + local encoded remote_cmd arg + encoded="$(printf '%s' "$script" | base64 | tr -d '\n')" + remote_cmd="tmp=\$(mktemp); trap 'rm -f \"\$tmp\"' EXIT; printf %s $(quote_for_remote_sh "$encoded") | base64 -d > \"\$tmp\"; sh \"\$tmp\"" + for arg in "$@"; do + remote_cmd+=" $(quote_for_remote_sh "$arg")" + done + openshell sandbox exec --name "$SANDBOX_NAME" -- sh -lc "$remote_cmd" +} + +stop_compat_mock() { + if [ -n "${COMPAT_MOCK_PID:-}" ] && kill -0 "$COMPAT_MOCK_PID" 2>/dev/null; then + kill "$COMPAT_MOCK_PID" 2>/dev/null || true + wait "$COMPAT_MOCK_PID" 2>/dev/null || true + fi + COMPAT_MOCK_PID="" +} + +start_compat_mock() { + : >"$COMPAT_MOCK_LOG" + python3 - "$COMPAT_MOCK_PORT" "$COMPAT_MODEL" "$COMPATIBLE_KEY" >"$COMPAT_MOCK_LOG" 2>&1 <<'PY' & +import json +import sys +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +port = int(sys.argv[1]) +model = sys.argv[2] +api_key = sys.argv[3] + +# RFC 7230 §6.1 hop-by-hop headers that http-proxy-fix.js must strip before +# the request reaches the upstream. If any of these arrive at the mock it +# means the FORWARD-mode rewrite leaked proxy-hop fields — the bug class +# that hit deepinfra users on NemoClaw 0.0.24 (issue #2490). +HOP_BY_HOP = { + "proxy-authorization", "proxy-connection", "proxy-authenticate", + "connection", "keep-alive", "te", "trailer", "transfer-encoding", "upgrade", +} + + +class Handler(BaseHTTPRequestHandler): + def log_message(self, fmt, *args): + return + + def _log_proxy_hop_headers(self): + leaked = [k for k in self.headers if k.lower() in HOP_BY_HOP] + print("proxy_hop_headers=%s" % ("none" if not leaked else ",".join(leaked)), flush=True) + + def _send(self, status, payload): + body = json.dumps(payload).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def _send_sse(self): + body = ( + "event: response.output_text.delta\n" + "data: {\"delta\":\"OK\"}\n\n" + "event: response.completed\n" + "data: {}\n\n" + ).encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def _send_chat_sse(self, content): + chunk = json.dumps({ + "id": "chatcmpl-mock", + "object": "chat.completion.chunk", + "choices": [{"index": 0, "delta": {"role": "assistant", "content": content}, "finish_reason": None}], + }) + done_chunk = json.dumps({ + "id": "chatcmpl-mock", + "object": "chat.completion.chunk", + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + }) + body = ( + "data: %s\n\ndata: %s\n\ndata: [DONE]\n\n" % (chunk, done_chunk) + ).encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def _auth_ok(self): + return self.headers.get("Authorization", "") == "Bearer " + api_key + + def do_GET(self): + if self.path == "/v1/models": + print("GET /v1/models", flush=True) + self._send(200, {"object": "list", "data": [{"id": model, "object": "model"}]}) + return + self._send(404, {"error": {"message": "not found"}}) + + def do_POST(self): + length = int(self.headers.get("Content-Length", "0") or "0") + raw = self.rfile.read(length) if length else b"" + try: + payload = json.loads(raw.decode("utf-8") or "{}") + except Exception: + payload = {} + + if self.path == "/v1/responses": + print("POST /v1/responses auth=%s stream=%s" % ("ok" if self._auth_ok() else "missing", payload.get("stream")), flush=True) + if not self._auth_ok(): + self._send(401, {"error": {"message": "missing bearer credential"}}) + return + if payload.get("stream"): + self._send_sse() + return + self._send(200, { + "id": "resp-mock", + "object": "response", + "output": [{ + "type": "function_call", + "name": "emit_ok", + "arguments": "{\"value\":\"OK\"}" + }], + }) + return + + if self.path == "/v1/chat/completions": + self._log_proxy_hop_headers() + print("POST /v1/chat/completions auth=%s model=%s stream=%s" % ("ok" if self._auth_ok() else "missing", payload.get("model"), payload.get("stream")), flush=True) + if not self._auth_ok(): + self._send(401, {"error": {"message": "missing bearer credential"}}) + return + if payload.get("stream"): + self._send_chat_sse("PONG from compatible endpoint mock") + return + self._send(200, { + "id": "chatcmpl-mock", + "object": "chat.completion", + "choices": [{ + "index": 0, + "message": { + "role": "assistant", + "content": "PONG from compatible endpoint mock" + }, + "finish_reason": "stop" + }], + }) + return + + self._send(404, {"error": {"message": "not found"}}) + + +ThreadingHTTPServer(("0.0.0.0", port), Handler).serve_forever() +PY + COMPAT_MOCK_PID=$! + + for _ in $(seq 1 30); do + if curl -sf "http://127.0.0.1:${COMPAT_MOCK_PORT}/v1/models" >/dev/null 2>&1; then + return 0 + fi + sleep 1 + done + return 1 +} + +load_shell_path() { + local local_bin + if [ -f "$HOME/.bashrc" ]; then + # shellcheck source=/dev/null + source "$HOME/.bashrc" 2>/dev/null || true + fi + export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" + if [ -s "$NVM_DIR/nvm.sh" ]; then + # shellcheck source=/dev/null + . "$NVM_DIR/nvm.sh" + fi + local_bin="$HOME/.local/bin" + if [ -d "$local_bin" ]; then + PATH=":${PATH}:" + PATH="${PATH//:${local_bin}:/:}" + PATH="${PATH#:}" + PATH="${PATH%:}" + export PATH="$local_bin:$PATH" + fi +} + +cli_command_available_from_source() { + [ -f "$REPO/dist/nemoclaw.js" ] && command -v node >/dev/null 2>&1 && command -v openshell >/dev/null 2>&1 +} + +run_cli() { + if cli_command_available_from_source; then + node "$REPO/bin/nemoclaw.js" "$@" + else + nemoclaw "$@" + fi +} + +destroy_sandbox_best_effort() { + if [ "${NEMOCLAW_E2E_KEEP_SANDBOX:-}" = "1" ]; then + return 0 + fi + set +e + if cli_command_available_from_source; then + run_with_timeout 120 node "$REPO/bin/nemoclaw.js" "$SANDBOX_NAME" destroy --yes >/dev/null 2>&1 + elif command -v nemoclaw >/dev/null 2>&1; then + run_with_timeout 120 nemoclaw "$SANDBOX_NAME" destroy --yes >/dev/null 2>&1 + fi + if command -v openshell >/dev/null 2>&1; then + run_with_timeout 60 openshell sandbox delete "$SANDBOX_NAME" >/dev/null 2>&1 + fi + set -uo pipefail +} + +run_compatible_onboard() { + local onboard_exit=0 + local onboard_cmd_desc + export NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" + export NEMOCLAW_RECREATE_SANDBOX=1 + export NEMOCLAW_NON_INTERACTIVE=1 + export NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 + export NEMOCLAW_SKIP_TELEGRAM_REACHABILITY=1 + export NEMOCLAW_PROVIDER=custom + export NEMOCLAW_ENDPOINT_URL="$COMPAT_ENDPOINT_URL" + export NEMOCLAW_MODEL="$COMPAT_MODEL" + export NEMOCLAW_PREFERRED_API=openai-completions + export NEMOCLAW_POLICY_MODE=custom + export NEMOCLAW_POLICY_PRESETS=telegram + export COMPATIBLE_API_KEY="$COMPATIBLE_KEY" + export TELEGRAM_BOT_TOKEN="$TELEGRAM_TOKEN" + export TELEGRAM_ALLOWED_IDS="$TELEGRAM_IDS" + unset DISCORD_BOT_TOKEN SLACK_BOT_TOKEN SLACK_APP_TOKEN + + if cli_command_available_from_source; then + onboard_cmd_desc="source CLI onboard" + info "Using source-built CLI at $REPO/bin/nemoclaw.js" + destroy_sandbox_best_effort + run_with_timeout 1200 node "$REPO/bin/nemoclaw.js" onboard --fresh --non-interactive --yes-i-accept-third-party-software \ + >"$ONBOARD_LOG" 2>&1 || onboard_exit=$? + else + onboard_cmd_desc="install.sh" + info "Source CLI is not built yet; running install.sh from this checkout." + bash "$REPO/install.sh" --non-interactive --yes-i-accept-third-party-software --fresh \ + >"$ONBOARD_LOG" 2>&1 || onboard_exit=$? + load_shell_path + fi + + if [ "$onboard_exit" -eq 0 ]; then + pass "C1: ${onboard_cmd_desc} completed for compatible endpoint + Telegram" + else + fail "C1: ${onboard_cmd_desc} failed (exit $onboard_exit)" + info "Last 80 lines of onboard log:" + tail -80 "$ONBOARD_LOG" 2>/dev/null || true + summary + fi +} + +check_openclaw_config() { + local output rc=0 script + script=$( + cat <<'SH' +python3 - "$1" <<'PY' +import json +import sys + +model = sys.argv[1] +cfg = json.load(open("/sandbox/.openclaw/openclaw.json", encoding="utf-8")) +providers = cfg.get("models", {}).get("providers", {}) +errors = [] +if "deepinfra" in providers: + errors.append("direct deepinfra provider is present") +if sorted(providers.keys()) != ["inference"]: + errors.append("provider keys are %r" % sorted(providers.keys())) +inference = providers.get("inference") if isinstance(providers, dict) else None +if not isinstance(inference, dict): + errors.append("models.providers.inference is missing") +else: + if inference.get("baseUrl") != "https://inference.local/v1": + errors.append("inference baseUrl is %r" % inference.get("baseUrl")) + if inference.get("apiKey") != "unused": + errors.append("inference apiKey is not the non-secret placeholder") +primary = cfg.get("agents", {}).get("defaults", {}).get("model", {}).get("primary") +if primary != "inference/" + model: + errors.append("primary model is %r" % primary) +if not cfg.get("channels", {}).get("telegram"): + errors.append("telegram channel config missing") +print(json.dumps({ + "provider_keys": sorted(providers.keys()) if isinstance(providers, dict) else [], + "inference_base": inference.get("baseUrl") if isinstance(inference, dict) else None, + "inference_api_key": inference.get("apiKey") if isinstance(inference, dict) else None, + "primary": primary, + "telegram_present": bool(cfg.get("channels", {}).get("telegram")), + "errors": errors, +})) +sys.exit(1 if errors else 0) +PY +SH + ) + output=$(sandbox_exec_sh_script "$script" "$COMPAT_MODEL" 2>&1) || rc=$? + info "OpenClaw config summary: ${output:0:500}" + if [ "$rc" -eq 0 ]; then + pass "C3: openclaw.json uses managed inference.local provider and Telegram config" + else + fail "C3: openclaw.json compatible endpoint shape is wrong" + fi +} + +check_gateway_ready() { + local result script + script=$( + cat <<'SH' +last="" +for _attempt in $(seq 1 30); do + result=$(node <<'NODE' 2>&1 || true +const net = require("net"); +let done = false; +const sock = net.connect(18789, "127.0.0.1"); +function finish(line) { + if (done) return; + done = true; + console.log(line); + sock.destroy(); +} +sock.on("connect", () => finish("OPEN")); +sock.on("error", (err) => finish("ERROR " + err.message)); +sock.setTimeout(1000, () => finish("TIMEOUT")); +NODE + ) + if echo "$result" | grep -q "OPEN"; then + echo "$result" + exit 0 + fi + last="$result" + sleep 1 +done +echo "$last" +exit 1 +SH + ) + result=$(sandbox_exec_sh_script "$script" 2>&1 || true) + if echo "$result" | grep -q "OPEN"; then + pass "C4: Gateway stayed up after Telegram provider initialization" + else + fail "C4: Gateway is not serving after Telegram-compatible onboard (${result:0:200})" + info "Gateway log tail:" + openshell sandbox exec --name "$SANDBOX_NAME" -- cat /tmp/gateway.log 2>/dev/null | tail -60 || true + fi +} + +check_sandbox_inference() { + local payload payload_arg response rc=0 content + payload=$(COMPAT_MODEL="$COMPAT_MODEL" python3 -c ' +import json +import os +print(json.dumps({ + "model": os.environ["COMPAT_MODEL"], + "messages": [{"role": "user", "content": "Reply with exactly: PONG"}], + "max_tokens": 32, +})) +') + payload_arg="$(printf '%q' "$payload")" + response=$(openshell sandbox exec --name "$SANDBOX_NAME" -- sh -lc "curl -sS --max-time 60 https://inference.local/v1/chat/completions -H 'Content-Type: application/json' -d $payload_arg" 2>&1) || rc=$? + content=$(printf '%s' "$response" | python3 -c 'import json,sys; d=json.load(sys.stdin); print(d["choices"][0]["message"]["content"])' 2>/dev/null) || true + if [ "$rc" -eq 0 ] && echo "$content" | grep -q "PONG"; then + pass "C5: Sandbox inference.local chat completion returned mock content" + else + fail "C5: Sandbox inference.local chat completion failed (${response:0:400})" + fi +} + +# C8 + C9: Run openclaw agent --json inside the sandbox and verify the +# openclaw HTTP client (axios/follow-redirects) completes a turn through +# the custom compatible endpoint. This exercises the FORWARD-mode rewrite +# branch of nemoclaw-blueprint/scripts/http-proxy-fix.js — the path that +# caused "LLM request failed: network connection error" for deepinfra users +# on NemoClaw 0.0.24 (issue #2490). curl (used in C5) bypasses Node's +# http.request entirely and cannot catch this class of regression. +check_openclaw_agent_turn() { + local session_id raw ssh_cfg reply rc=0 + session_id="e2e-compat-agent-$(date +%s)-$$" + ssh_cfg="$(mktemp)" + + if ! openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_cfg" 2>/dev/null; then + rm -f "$ssh_cfg" + fail "C8: openclaw agent turn — could not get SSH config" + return + fi + + # Snapshot hop-header log count before the agent turn so C9 can prove a + # *new* line was written by this request and not reused from the C5 curl hit. + local hop_count_before + hop_count_before=$(grep -c "proxy_hop_headers=" "$COMPAT_MOCK_LOG" 2>/dev/null) || hop_count_before=0 + + # 2>/dev/null drops openclaw progress/log lines so stdout is JSON-only. + raw=$(run_with_timeout 90 ssh -F "$ssh_cfg" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" \ + "openclaw agent --agent main --json --session-id '${session_id}' -m 'Reply with only: PONG'" \ + 2>/dev/null) || rc=$? + rm -f "$ssh_cfg" + + # Fail closed on provider/transport errors so a coincidental PONG in a + # stack trace or error message cannot mask an SSRF block or gateway failure. + if printf '%s' "$raw" | grep -qiE "SsrFBlockedError|Blocked hostname|transport error|ECONNREFUSED|EAI_AGAIN|gateway unavailable|network connection error"; then + fail "C8: openclaw agent turn failed with provider/transport error (exit ${rc}): ${raw:0:300}" + return + fi + + reply=$(printf '%s' "$raw" | parse_openclaw_agent_text 2>/dev/null) || true + + if [ "$rc" -eq 0 ] && printf '%s' "$reply" | grep -qi "PONG"; then + pass "C8: openclaw agent completed turn via compatible endpoint (http-proxy-fix.js FORWARD-mode path exercised)" + else + fail "C8: openclaw agent turn failed (exit ${rc}); reply='${reply:0:200}', raw='${raw:0:200}'" + fi + + # C9: Verify http-proxy-fix.js stripped proxy hop headers — they must not + # reach the upstream mock. The mock logs "proxy_hop_headers=none" when + # clean, or "proxy_hop_headers=" when the strip failed. + # Read every line appended after the SSH command so C5's earlier + # /v1/chat/completions entry cannot satisfy this check, and so a retry + # or follow-up call can't slip a leaked-header request past us. + local new_hop_lines leaked + new_hop_lines=$(grep "proxy_hop_headers=" "$COMPAT_MOCK_LOG" 2>/dev/null \ + | tail -n +"$((hop_count_before + 1))") || true + if [ -z "$new_hop_lines" ]; then + fail "C9: Mock logged no proxy_hop_headers line for the agent turn — agent did not reach /v1/chat/completions" + else + leaked=$(printf '%s\n' "$new_hop_lines" \ + | sed 's/.*proxy_hop_headers=//' \ + | grep -v '^none$' \ + | paste -sd',' -) || true + if [ -z "$leaked" ]; then + pass "C9: No proxy hop headers leaked to the compatible endpoint upstream (http-proxy-fix.js strip verified)" + else + fail "C9: Proxy hop headers leaked to upstream — http-proxy-fix.js strip broken: ${leaked}" + fi + fi +} + +cleanup() { + stop_compat_mock + rm -f "$COMPAT_MOCK_LOG" 2>/dev/null || true + destroy_sandbox_best_effort +} + +# ── Repo root ───────────────────────────────────────────────────── +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +if [ -d /workspace ] && [ -f /workspace/install.sh ]; then + REPO="/workspace" +elif [ -f "${SCRIPT_DIR}/../../install.sh" ]; then + REPO="$(cd "${SCRIPT_DIR}/../.." && pwd)" +elif [ -f "./install.sh" ]; then + REPO="$(pwd)" +else + echo "ERROR: Cannot find repo root." + exit 1 +fi + +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-msg-compat}" +COMPAT_MOCK_PORT="${NEMOCLAW_COMPAT_MOCK_PORT:-18089}" +COMPAT_MODEL="${NEMOCLAW_COMPAT_MODEL:-mock/deepseek-compatible}" +COMPATIBLE_KEY="${NEMOCLAW_COMPAT_MOCK_API_KEY:-fake-compatible-key-e2e}" +TELEGRAM_TOKEN="${TELEGRAM_BOT_TOKEN:-test-fake-telegram-token-e2e}" +TELEGRAM_IDS="${TELEGRAM_ALLOWED_IDS:-123456789}" +COMPAT_MOCK_LOG="$(mktemp)" +COMPAT_MOCK_PID="" +ONBOARD_LOG="/tmp/nemoclaw-e2e-messaging-compatible-endpoint-install.log" + +trap cleanup EXIT + +echo "" +echo "============================================================" +echo " Telegram + Compatible Endpoint E2E (#2766, #2572)" +echo " $(date)" +echo "============================================================" +echo "" + +section "Phase 0: Prerequisites" +if ! docker info >/dev/null 2>&1; then + fail "Docker is not running" + summary +fi +pass "Docker is running" + +if ! command -v python3 >/dev/null 2>&1; then + fail "python3 not found" + summary +fi +pass "python3 is available" + +load_shell_path +info "Repo: $REPO" +info "Sandbox: $SANDBOX_NAME" +info "Model: $COMPAT_MODEL" + +section "Phase 1: Local compatible endpoint mock" +COMPAT_HOST="$(host_ip_for_sandbox)" +COMPAT_ENDPOINT_URL="http://${COMPAT_HOST}:${COMPAT_MOCK_PORT}/v1" +info "Starting mock endpoint at ${COMPAT_ENDPOINT_URL}" +if start_compat_mock; then + pass "C0: Compatible endpoint mock started" +else + fail "C0: Compatible endpoint mock failed to start" + info "Mock log:" + sed 's/^/ /' "$COMPAT_MOCK_LOG" || true + summary +fi + +if curl -sf "${COMPAT_ENDPOINT_URL}/models" >/dev/null 2>&1; then + pass "C0b: Compatible endpoint mock is reachable through host address" +else + fail "C0b: Compatible endpoint mock is not reachable at ${COMPAT_ENDPOINT_URL}" + summary +fi + +section "Phase 2: Onboard custom provider with Telegram" +run_compatible_onboard + +if grep -q "Compatible endpoint responds through inference.local" "$ONBOARD_LOG" 2>/dev/null; then + pass "C2: Onboard ran the compatible endpoint sandbox smoke check" +else + fail "C2: Onboard log does not show the compatible endpoint sandbox smoke check" +fi + +section "Phase 3: Runtime assertions" +if openshell provider get compatible-endpoint >/dev/null 2>&1; then + pass "C2b: Gateway has the compatible-endpoint provider" +else + fail "C2b: Gateway is missing the compatible-endpoint provider" +fi + +check_openclaw_config +check_gateway_ready +check_sandbox_inference +check_openclaw_agent_turn + +if grep -q "POST /v1/chat/completions auth=ok" "$COMPAT_MOCK_LOG" 2>/dev/null; then + pass "C6: Compatible mock received authenticated chat traffic" +else + fail "C6: Compatible mock did not record authenticated chat traffic" + info "Mock log:" + sed 's/^/ /' "$COMPAT_MOCK_LOG" || true +fi + +if [ -n "${TELEGRAM_BOT_TOKEN_REAL:-}" ] \ + && [ -n "${TELEGRAM_CHAT_ID_E2E:-}" ] \ + && [ -n "${COMPATIBLE_API_KEY:-}" ] \ + && [ -n "${NEMOCLAW_ENDPOINT_URL:-}" ] \ + && [ -n "${NEMOCLAW_COMPAT_MODEL:-}" ]; then + skip "C7: Live Telegram reply requires an inbound user-message driver; hermetic route passed" +else + skip "C7: Live Telegram-compatible round trip secrets not fully set" +fi + +trap - EXIT +cleanup +summary diff --git a/test/e2e-vpn/test-messaging-providers.sh b/test/e2e-vpn/test-messaging-providers.sh new file mode 100755 index 00000000000..8d4c6012842 --- /dev/null +++ b/test/e2e-vpn/test-messaging-providers.sh @@ -0,0 +1,3235 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# shellcheck disable=SC2016,SC2034 +# SC2016: Single-quoted strings are intentional — Node.js code passed via SSH. +# SC2034: Some variables are used indirectly or reserved for later phases. + +# Messaging Credential Provider E2E Tests +# +# Validates that messaging credentials (Telegram, Discord, Slack, WeChat) +# flow correctly through the OpenShell provider/placeholder/L7-proxy pipeline, +# and holds WhatsApp's QR-only channel to the same config/policy/no-secret +# standard even though it has no host-side token provider. Tests every +# layer of the chain introduced in PR #1081: +# +# 1. Provider creation — openshell stores the real token +# 2. Sandbox attachment — --provider flags wire providers to the sandbox +# 3. Credential isolation — real tokens never appear in sandbox env, +# process list, or filesystem +# 4. Config patching — openclaw.json channels use placeholder values +# 5. OpenClaw runtime discovery — channels list as installed/configured +# 6. Telegram diagnostics — startup/credential breadcrumbs stay sanitized +# 7. Network reachability — Node.js can reach messaging APIs through proxy +# 8. Native Discord gateway path — WebSocket L7 path is tested hermetically +# 9. L7 proxy rewriting — placeholder is rewritten to real token at egress +# 10. WhatsApp QR-only parity — channel add/rebuild applies policy, bakes +# openclaw.json, creates no providers, and leaks no token placeholders +# +# Uses fake tokens by default (no external accounts needed). With fake tokens, +# the live API probes return 401/404 — proving the full chain worked (request +# reached the real API with the token rewritten). The OpenClaw plugin-send phase +# then sends messages to host-side fake provider APIs when complete real +# credentials/targets are not configured. +# +# Prerequisites: +# - Docker running +# - NemoClaw installed (install.sh or brev-setup.sh already ran) +# - NVIDIA_API_KEY set +# - openshell on PATH +# +# Environment variables: +# NVIDIA_API_KEY — required +# NEMOCLAW_NON_INTERACTIVE=1 — required +# NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 — required +# NEMOCLAW_SANDBOX_NAME — sandbox name (default: e2e-msg-provider) +# TELEGRAM_BOT_TOKEN — defaults to fake token +# DISCORD_BOT_TOKEN — defaults to fake token +# TELEGRAM_ALLOWED_IDS — comma-separated Telegram user IDs for DM allowlisting +# TELEGRAM_AUTHORIZED_CHAT_IDS — compatibility alias for TELEGRAM_ALLOWED_IDS +# TELEGRAM_CHAT_ID — compatibility alias for TELEGRAM_ALLOWED_IDS +# TELEGRAM_BOT_TOKEN_REAL — optional: enables Phase 6 real OpenClaw send +# DISCORD_BOT_TOKEN_REAL — optional: enables Phase 6 real OpenClaw send +# SLACK_BOT_TOKEN_REAL — optional: enables Phase 6 real OpenClaw send +# SLACK_APP_TOKEN_REAL — optional paired Slack app token for real Slack run +# SLACK_BOT_TOKEN — defaults to fake token (xoxb-fake-...) +# SLACK_APP_TOKEN — defaults to fake token (xapp-fake-...) +# SLACK_ALLOWED_USERS — comma-separated Slack user IDs for DM and channel @mention allowlisting +# SLACK_BOT_TOKEN_REVOKED — optional: revoked xoxb- token to test auth pre-validation (#2340) +# SLACK_APP_TOKEN_REVOKED — optional: paired xapp- token for the revoked bot token +# WECHAT_BOT_TOKEN — defaults to fake token; presence skips host-side QR login +# WECHAT_ACCOUNT_ID — defaults to fake iLink account ID (manifest hook account key) +# WECHAT_BASE_URL — defaults to fake iLink baseUrl (per-account API host) +# WECHAT_USER_ID — defaults to fake operator wechat user ID (seeds DM allowlist) +# WECHAT_ALLOWED_IDS — optional: comma-separated DM allowlist for wechat +# WhatsApp — QR-only; the test enables it via `channels add whatsapp` +# WHATSAPP_TOKEN / WHATSAPP_BOT_TOKEN / WHATSAPP_SESSION_SECRET +# — overwritten with fake decoys to prove NemoClaw ignores host-side +# WhatsApp credential-shaped env vars +# TELEGRAM_CHAT_ID_E2E — optional: target for real Telegram send +# DISCORD_CHANNEL_ID_E2E — optional: target for real Discord send +# SLACK_CHANNEL_ID_E2E — optional: target for real Slack send +# NEMOCLAW_TELEGRAM_INBOUND_REPLY_E2E=1 — optional: wait for a real Telegram-client DM +# from an allowed user and verify inbound + +# outbound gateway breadcrumbs +# NEMOCLAW_TELEGRAM_INBOUND_WAIT_SECONDS — optional: wait time for the live inbound +# proof (default: 90) +# NEMOCLAW_OPENSHELL_BIN — optional OpenShell binary under test +# NEMOCLAW_FRESH=1 — auto-set to discard interrupted onboard sessions +# +# Usage: +# NEMOCLAW_NON_INTERACTIVE=1 NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ +# NVIDIA_API_KEY=nvapi-... bash test/e2e-vpn/test-messaging-providers.sh +# +# See: https://github.com/NVIDIA/NemoClaw/pull/1081 + +set -uo pipefail + +PASS=0 +FAIL=0 +SKIP=0 +TOTAL=0 + +pass() { + ((PASS++)) + ((TOTAL++)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} +fail() { + ((FAIL++)) + ((TOTAL++)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} +skip() { + ((SKIP++)) + ((TOTAL++)) + printf '\033[33m SKIP: %s\033[0m\n' "$1" +} +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } +is_fake_slack_token() { + case "${1:-}" in + xoxb-fake-* | xoxb-test-* | xapp-fake-* | xapp-test-*) return 0 ;; + *) return 1 ;; + esac +} +is_unresolved_placeholder_rejection() { + printf '%s\n' "$1" | grep -qiE 'credential_injection_failed|unresolved credential placeholder' +} + +text_contains_all() { + local haystack="$1" + shift + + local needle + for needle in "$@"; do + case "$haystack" in + *"$needle"*) ;; + *) return 1 ;; + esac + done + + return 0 +} + +text_contains_any() { + local haystack="$1" + shift + + local needle + for needle in "$@"; do + case "$haystack" in + *"$needle"*) return 0 ;; + esac + done + + return 1 +} + +# Determine repo root +if [ -d /workspace ] && [ -f /workspace/install.sh ]; then + REPO="/workspace" +elif [ -f "$(cd "$(dirname "$0")/../.." && pwd)/install.sh" ]; then + REPO="$(cd "$(dirname "$0")/../.." && pwd)" +else + echo "ERROR: Cannot find repo root." + exit 1 +fi + +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-msg-provider}" +OPENSHELL_BIN="${NEMOCLAW_OPENSHELL_BIN:-openshell}" +REGISTRY="$HOME/.nemoclaw/sandboxes.json" + +openshell() { + if [ "$OPENSHELL_BIN" = "openshell" ]; then + command openshell "$@" + else + "$OPENSHELL_BIN" "$@" + fi +} + +registry_plan_channel_contains() { + local item="$1" + node -e ' +const fs = require("fs"); +const [registryPath, sandboxName, channelId] = process.argv.slice(1); +if (!fs.existsSync(registryPath)) process.exit(1); +const registry = JSON.parse(fs.readFileSync(registryPath, "utf8")); +const channels = registry.sandboxes?.[sandboxName]?.messaging?.plan?.channels; +process.exit(Array.isArray(channels) && channels.some((channel) => channel?.channelId === channelId) ? 0 : 1); +' "$REGISTRY" "$SANDBOX_NAME" "$item" +} + +assert_openclaw_config_activation() { + local assertion_id="$1" + local channel="$2" + local label="$3" + local channel_present channel_enabled plugin_enabled + + channel_present=$(printf '%s\n' "$channel_json" | CHANNEL="$channel" python3 -c ' +import json +import os +import sys +try: + channels = json.load(sys.stdin) + print("true" if isinstance(channels.get(os.environ["CHANNEL"]), dict) else "false") +except Exception: + print("error") +' 2>/dev/null || true) + channel_enabled=$(printf '%s\n' "$channel_json" | CHANNEL="$channel" python3 -c ' +import json +import os +import sys +try: + channels = json.load(sys.stdin) + entry = channels.get(os.environ["CHANNEL"], {}) + print("true" if isinstance(entry, dict) and entry.get("enabled") is True else "false") +except Exception: + print("error") +' 2>/dev/null || true) + plugin_enabled=$(printf '%s\n' "$plugin_entries_json" | CHANNEL="$channel" python3 -c ' +import json +import os +import sys +try: + entries = json.load(sys.stdin) + entry = entries.get(os.environ["CHANNEL"], {}) + print("true" if isinstance(entry, dict) and entry.get("enabled") is True else "false") +except Exception: + print("error") +' 2>/dev/null || true) + + if [ "$channel_present" != "true" ]; then + skip "${assertion_id}: ${label} channel block not in openclaw.json (expected in non-root sandbox)" + return + fi + + if [ "$channel_enabled" = "true" ] && [ "$plugin_enabled" = "true" ]; then + pass "${assertion_id}: ${label} channel and plugin are explicitly enabled in openclaw.json" + else + fail "${assertion_id}: ${label} OpenClaw activation missing (channels.${channel}.enabled=${channel_enabled}, plugins.entries.${channel}.enabled=${plugin_enabled})" + fi +} + +summarize_openclaw_config_activation() { + CHANNEL_JSON="$channel_json" PLUGIN_ENTRIES_JSON="$plugin_entries_json" python3 -c ' +import json +import os + +channels_to_check = ("telegram", "discord", "slack", "whatsapp") +try: + channels = json.loads(os.environ.get("CHANNEL_JSON", "{}")) + entries = json.loads(os.environ.get("PLUGIN_ENTRIES_JSON", "{}")) +except json.JSONDecodeError as exc: + print("parse_error=%s" % exc.msg) + raise SystemExit(0) + +summary = [] +for channel in channels_to_check: + channel_entry = channels.get(channel, {}) + plugin_entry = entries.get(channel, {}) + summary.append( + "%s:channel=%s,plugin=%s" + % ( + channel, + isinstance(channel_entry, dict) and channel_entry.get("enabled") is True, + isinstance(plugin_entry, dict) and plugin_entry.get("enabled") is True, + ) + ) +print("; ".join(summary)) +' 2>/dev/null || printf 'unavailable' +} + +summarize_openclaw_runtime_channels() { + printf '%s\n' "$openclaw_channels_list_json" | python3 -c ' +import json +import sys + +channels_to_check = ("telegram", "discord", "slack", "whatsapp") +try: + data = json.load(sys.stdin) +except json.JSONDecodeError as exc: + print("parse_error=%s" % exc.msg) + raise SystemExit(0) + +chat = data.get("chat") if isinstance(data, dict) else None +if not isinstance(chat, dict): + print("missing_chat") + raise SystemExit(0) + +summary = [] +for channel in channels_to_check: + entry = chat.get(channel) + if not isinstance(entry, dict): + summary.append("%s:missing" % channel) + continue + accounts = entry.get("accounts") + if isinstance(accounts, list): + account_ids = [str(item) for item in accounts if isinstance(item, str)] + else: + account_ids = ["<%s>" % type(accounts).__name__] + summary.append( + "%s:installed=%s,origin=%s,accounts=%s" + % (channel, entry.get("installed"), entry.get("origin"), ",".join(account_ids)) + ) +print("; ".join(summary)) +' 2>/dev/null || printf 'unavailable' +} + +assert_openclaw_runtime_channel() { + local assertion_id="$1" + local channel="$2" + local label="$3" + local expected_account="${4:-default}" + local runtime_state + + runtime_state=$(printf '%s\n' "$openclaw_channels_list_json" | CHANNEL="$channel" ACCOUNT="$expected_account" python3 -c ' +import json +import os +import sys + +channel = os.environ["CHANNEL"] +expected = os.environ.get("ACCOUNT", "") +try: + data = json.load(sys.stdin) +except json.JSONDecodeError as exc: + print("error invalid_json=%s" % exc.msg) + raise SystemExit(0) + +if not isinstance(data, dict): + print("no top_level_type=%s" % type(data).__name__) + raise SystemExit(0) + +chat = data.get("chat") +if not isinstance(chat, dict): + print("no missing_chat") + raise SystemExit(0) + +entry = chat.get(channel) +if not isinstance(entry, dict): + print("no missing_channel") + raise SystemExit(0) + +# OpenClaw `channels list --all --json` currently reports configured account +# ids as a list of strings, for example: {"chat":{"slack":{"accounts":["default"]}}}. +# Keep this strict so schema drift fails loudly instead of hiding a discovery +# regression behind a permissive compatibility parser. +accounts = entry.get("accounts") +if not isinstance(accounts, list) or any(not isinstance(item, str) for item in accounts): + print( + "no installed=%s origin=%s accounts_shape=%s" + % (entry.get("installed"), entry.get("origin"), type(accounts).__name__) + ) + raise SystemExit(0) + +installed = entry.get("installed") is True +configured = entry.get("origin") == "configured" +account_ok = not expected or expected in accounts +if installed and configured and account_ok: + print("yes") +else: + print( + "no installed=%s origin=%s accounts=%s" + % (entry.get("installed"), entry.get("origin"), accounts) + ) +' 2>/dev/null || true) + + if [ "$runtime_state" = "yes" ]; then + pass "${assertion_id}: OpenClaw channels list reports ${label} installed and configured" + else + fail "${assertion_id}: OpenClaw channels list did not report ${label} installed/configured (${runtime_state}; summary=${openclaw_channels_summary:-unavailable})" + fi +} + +assert_openclaw_runtime_channel_installed() { + local assertion_id="$1" + local channel="$2" + local label="$3" + local runtime_state + + runtime_state=$(printf '%s\n' "$openclaw_channels_list_json" | CHANNEL="$channel" python3 -c ' +import json +import os +import sys + +channel = os.environ["CHANNEL"] +try: + data = json.load(sys.stdin) +except json.JSONDecodeError as exc: + print("error invalid_json=%s" % exc.msg) + raise SystemExit(0) + +chat = data.get("chat") if isinstance(data, dict) else None +if not isinstance(chat, dict): + print("no missing_chat") + raise SystemExit(0) + +entry = chat.get(channel) +if not isinstance(entry, dict): + print("no missing_channel") + raise SystemExit(0) + +accounts = entry.get("accounts") +if not isinstance(accounts, list) or any(not isinstance(item, str) for item in accounts): + print( + "no installed=%s origin=%s accounts_shape=%s" + % (entry.get("installed"), entry.get("origin"), type(accounts).__name__) + ) + raise SystemExit(0) + +installed = entry.get("installed") is True +origin_ok = entry.get("origin") in ("available", "configured") +if installed and origin_ok: + print("yes") +else: + print( + "no installed=%s origin=%s accounts=%s" + % (entry.get("installed"), entry.get("origin"), accounts) + ) +' 2>/dev/null || true) + + if [ "$runtime_state" = "yes" ]; then + pass "${assertion_id}: OpenClaw channels list reports ${label} plugin installed" + else + fail "${assertion_id}: OpenClaw channels list did not report ${label} plugin installed (${runtime_state}; summary=${openclaw_channels_summary:-unavailable})" + fi +} + +# shellcheck source=test/e2e-vpn/lib/sandbox-teardown.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib/sandbox-teardown.sh" +register_sandbox_for_teardown "$SANDBOX_NAME" + +# Default to hermetic fake tokens, but let repository live-message secrets win +# when they are available. The workflow always provides fake env_json values so +# the _REAL variables must take precedence here. +TELEGRAM_TOKEN="${TELEGRAM_BOT_TOKEN_REAL:-${TELEGRAM_BOT_TOKEN:-test-fake-telegram-token-e2e}}" +DISCORD_TOKEN="${DISCORD_BOT_TOKEN_REAL:-${DISCORD_BOT_TOKEN:-test-fake-discord-token-e2e}}" +SLACK_TOKEN="${SLACK_BOT_TOKEN_REAL:-${SLACK_BOT_TOKEN:-xoxb-fake-slack-token-e2e}}" +SLACK_APP="${SLACK_APP_TOKEN_REAL:-${SLACK_APP_TOKEN:-xapp-fake-slack-app-token-e2e}}" +if [ -n "${TELEGRAM_ALLOWED_IDS:-}" ]; then + TELEGRAM_IDS="$TELEGRAM_ALLOWED_IDS" + TELEGRAM_ALLOWLIST_ENV_KEY="TELEGRAM_ALLOWED_IDS" +elif [ -n "${TELEGRAM_AUTHORIZED_CHAT_IDS:-}" ]; then + TELEGRAM_IDS="$TELEGRAM_AUTHORIZED_CHAT_IDS" + TELEGRAM_ALLOWLIST_ENV_KEY="TELEGRAM_AUTHORIZED_CHAT_IDS" +elif [ -n "${TELEGRAM_CHAT_ID:-}" ]; then + TELEGRAM_IDS="$TELEGRAM_CHAT_ID" + TELEGRAM_ALLOWLIST_ENV_KEY="TELEGRAM_CHAT_ID" +else + TELEGRAM_IDS="123456789,987654321" + TELEGRAM_ALLOWLIST_ENV_KEY="TELEGRAM_AUTHORIZED_CHAT_IDS" +fi +SLACK_IDS="${SLACK_ALLOWED_USERS-U0AR85ATALW,U09E2ESLACK}" +# WeChat: pre-seeding WECHAT_BOT_TOKEN + the per-account metadata env vars lets +# the non-interactive onboard path (src/lib/onboard.ts:8433) treat wechat as +# "already configured" and skip the host-qr handler entirely. Fake values are +# enough — Phase 1-3 verify placeholders/isolation; no live iLink contact is +# made because no token exchange happens at build time. +WECHAT_TOKEN="${WECHAT_BOT_TOKEN:-test-fake-wechat-token-e2e}" +WECHAT_ACCOUNT="${WECHAT_ACCOUNT_ID:-e2e-fake-account-12345}" +WECHAT_BASE="${WECHAT_BASE_URL:-https://ilinkai.wechat.com}" +WECHAT_USER="${WECHAT_USER_ID:-wxid_e2efakeoperator}" +WECHAT_IDS="${WECHAT_ALLOWED_IDS:-${WECHAT_USER}}" +# WhatsApp is QR-only, but seed host-side decoys to prove they are ignored. +WHATSAPP_TOKEN_DECOY="test-fake-whatsapp-token-e2e" +WHATSAPP_BOT_TOKEN_DECOY="test-fake-whatsapp-bot-token-e2e" +WHATSAPP_SESSION_SECRET_DECOY="test-fake-whatsapp-session-secret-e2e" +export TELEGRAM_BOT_TOKEN="$TELEGRAM_TOKEN" +export DISCORD_BOT_TOKEN="$DISCORD_TOKEN" +export SLACK_BOT_TOKEN="$SLACK_TOKEN" +export SLACK_APP_TOKEN="$SLACK_APP" +case "$TELEGRAM_ALLOWLIST_ENV_KEY" in + TELEGRAM_ALLOWED_IDS) + export TELEGRAM_ALLOWED_IDS="$TELEGRAM_IDS" + ;; + TELEGRAM_AUTHORIZED_CHAT_IDS) + unset TELEGRAM_ALLOWED_IDS + export TELEGRAM_AUTHORIZED_CHAT_IDS="$TELEGRAM_IDS" + ;; + TELEGRAM_CHAT_ID) + unset TELEGRAM_ALLOWED_IDS TELEGRAM_AUTHORIZED_CHAT_IDS + export TELEGRAM_CHAT_ID="$TELEGRAM_IDS" + ;; +esac +export SLACK_ALLOWED_USERS="$SLACK_IDS" +export WECHAT_BOT_TOKEN="$WECHAT_TOKEN" +export WECHAT_ACCOUNT_ID="$WECHAT_ACCOUNT" +export WECHAT_BASE_URL="$WECHAT_BASE" +export WECHAT_USER_ID="$WECHAT_USER" +export WECHAT_ALLOWED_IDS="$WECHAT_IDS" +export WHATSAPP_TOKEN="$WHATSAPP_TOKEN_DECOY" +export WHATSAPP_BOT_TOKEN="$WHATSAPP_BOT_TOKEN_DECOY" +export WHATSAPP_SESSION_SECRET="$WHATSAPP_SESSION_SECRET_DECOY" + +# NEMOCLAW_EXTRA_PLACEHOLDER_KEYS — operator-supplied per-profile credentials. +# The host-side parser at src/lib/onboard/extra-placeholder-keys.ts accepts +# only entries that extend a canonical channel envKey with a non-empty +# `_`, rejects bare canonical keys, the control env, and arbitrary +# host secret names. The fixtures below cover three observable outcomes +# Phase 2c asserts on: +# +# 1. TELEGRAM_BOT_TOKEN_AGENT_A — extension + token exported -> provider +# row registered, placeholder injected into the sandbox env. +# 2. TELEGRAM_BOT_TOKEN_AGENT_MISSING — extension + token NOT exported -> +# registerExtraPlaceholderProviders pushes a token=null row that +# upsertMessagingProviders skips at the gateway; no placeholder is +# injected for that key. +# 3. GITHUB_TOKEN — host secret shape -> rejected at the parser layer +# before any provider row is built; the raw value must never reach +# the sandbox provider gateway. +EXTRAS_TELEGRAM_AGENT_A_TOKEN="test-fake-telegram-token-agent-a-e2e" +EXTRAS_TELEGRAM_AGENT_B_TOKEN="test-fake-telegram-token-agent-b-e2e" +EXTRAS_GITHUB_DECOY="test-fake-host-secret-that-must-not-leak" +export NEMOCLAW_EXTRA_PLACEHOLDER_KEYS="TELEGRAM_BOT_TOKEN_AGENT_A TELEGRAM_BOT_TOKEN_AGENT_B TELEGRAM_BOT_TOKEN_AGENT_MISSING GITHUB_TOKEN" +export TELEGRAM_BOT_TOKEN_AGENT_A="$EXTRAS_TELEGRAM_AGENT_A_TOKEN" +export TELEGRAM_BOT_TOKEN_AGENT_B="$EXTRAS_TELEGRAM_AGENT_B_TOKEN" +unset TELEGRAM_BOT_TOKEN_AGENT_MISSING +export GITHUB_TOKEN="$EXTRAS_GITHUB_DECOY" + +# Run a command inside the sandbox via stdin (avoids exposing sensitive args in process list) +sandbox_exec_stdin() { + local cmd="$1" + local ssh_config + ssh_config="$(mktemp)" + openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config" 2>/dev/null + + local result + result=$(timeout 60 ssh -F "$ssh_config" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" \ + "$cmd" \ + 2>/dev/null) || true + + rm -f "$ssh_config" + echo "$result" +} + +# Run a command inside the sandbox and capture output +sandbox_exec() { + local cmd="$1" + local ssh_config + ssh_config="$(mktemp)" + openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config" 2>/dev/null + + local result + result=$(timeout 60 ssh -F "$ssh_config" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" \ + "$cmd" \ + 2>&1) || true + + rm -f "$ssh_config" + echo "$result" +} + +read_gateway_log() { + openshell sandbox exec --name "$SANDBOX_NAME" -- cat /tmp/gateway.log 2>/dev/null || true +} + +run_telegram_inbound_reply_probe() { + if [ "${NEMOCLAW_TELEGRAM_INBOUND_REPLY_E2E:-}" != "1" ]; then + return + fi + + section "Phase 6a: Live Telegram Inbound Reply Proof" + + local wait_seconds="${NEMOCLAW_TELEGRAM_INBOUND_WAIT_SECONDS:-90}" + if ! [[ "$wait_seconds" =~ ^[0-9]+$ ]]; then + wait_seconds=90 + fi + if [ "$wait_seconds" -lt 1 ]; then + wait_seconds=90 + fi + + if [ -z "${TELEGRAM_BOT_TOKEN_REAL:-}" ]; then + fail "M19b: Live Telegram inbound proof requires TELEGRAM_BOT_TOKEN_REAL" + return + fi + if [ "$TELEGRAM_ALLOWLIST_ENV_KEY" = "TELEGRAM_ALLOWED_IDS" ]; then + fail "M19b: Live Telegram inbound proof must be run with TELEGRAM_AUTHORIZED_CHAT_IDS or TELEGRAM_CHAT_ID to exercise alias compatibility" + return + fi + if [ -z "$TELEGRAM_IDS" ]; then + fail "M19b: Live Telegram inbound proof requires a non-empty Telegram allowlist alias" + return + fi + + local log_before_lines + log_before_lines=$(read_gateway_log | wc -l | tr -d ' ') + if [ -z "$log_before_lines" ]; then + log_before_lines=0 + fi + + info "Live Telegram inbound proof is using ${TELEGRAM_ALLOWLIST_ENV_KEY}; send a fresh direct message from an allowed Telegram client to the bot now." + info "Waiting up to ${wait_seconds}s for inbound getUpdates and outbound sendMessage breadcrumbs in /tmp/gateway.log..." + + local deadline now delta_log saw_inbound saw_outbound + deadline=$(($(date +%s) + wait_seconds)) + saw_inbound=0 + saw_outbound=0 + while true; do + delta_log=$(read_gateway_log | awk -v start="$log_before_lines" 'NR > start') + if echo "$delta_log" | grep -qF "[telegram] [default] inbound update received"; then + saw_inbound=1 + fi + if echo "$delta_log" | grep -qF "[telegram] [default] outbound sendMessage attempted"; then + saw_outbound=1 + fi + if [ "$saw_inbound" = "1" ] && [ "$saw_outbound" = "1" ]; then + pass "M19b: Telegram client DM produced inbound getUpdates and outbound reply breadcrumbs" + return + fi + now=$(date +%s) + if [ "$now" -ge "$deadline" ]; then + break + fi + sleep 5 + done + + fail "M19b: Timed out waiting for Telegram inbound/reply breadcrumbs (inbound=${saw_inbound}, outbound=${saw_outbound})" +} + +run_openclaw_message_send() { + local channel="$1" + local target="$2" + local message="$3" + local channel_b64 target_b64 message_b64 + channel_b64=$(printf '%s' "$channel" | base64 | tr -d '\n') + target_b64=$(printf '%s' "$target" | base64 | tr -d '\n') + message_b64=$(printf '%s' "$message" | base64 | tr -d '\n') + + sandbox_exec_stdin "OPENCLAW_MESSAGE_CHANNEL_B64='$channel_b64' OPENCLAW_MESSAGE_TARGET_B64='$target_b64' OPENCLAW_MESSAGE_TEXT_B64='$message_b64' bash -s" <<'SH' +decode_b64() { + printf '%s' "$1" | base64 -d +} + +channel="$(decode_b64 "$OPENCLAW_MESSAGE_CHANNEL_B64")" +target="$(decode_b64 "$OPENCLAW_MESSAGE_TARGET_B64")" +message="$(decode_b64 "$OPENCLAW_MESSAGE_TEXT_B64")" + +set +e +OPENCLAW_NO_COLOR=1 openclaw message send --channel "$channel" --target "$target" --message "$message" --json +rc=$? +echo "__OPENCLAW_MESSAGE_SEND_EXIT__:$rc" +SH +} + +openclaw_message_send_exit_code() { + awk -F: '/^__OPENCLAW_MESSAGE_SEND_EXIT__:/ { code = $2 } END { if (code != "") print code }' +} + +# shellcheck source=test/e2e-vpn/lib/discord-gateway-proof.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib/discord-gateway-proof.sh" +# shellcheck source=test/e2e-vpn/lib/discord-rest-policy-proof.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib/discord-rest-policy-proof.sh" +# shellcheck source=test/e2e-vpn/lib/slack-api-proof.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib/slack-api-proof.sh" +# shellcheck source=test/e2e-vpn/lib/telegram-api-proof.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib/telegram-api-proof.sh" + +# ══════════════════════════════════════════════════════════════════ +# Phase 0: Prerequisites +# ══════════════════════════════════════════════════════════════════ +section "Phase 0: Prerequisites" + +if [ -z "${NVIDIA_API_KEY:-}" ] && [ -n "${NVIDIA_API_KEY:-}" ]; then + export NVIDIA_API_KEY="${NVIDIA_API_KEY}" + info "Using legacy NVIDIA_API_KEY as fallback for NVIDIA_API_KEY" +fi +if [ -z "${NVIDIA_API_KEY:-}" ]; then + fail "NVIDIA_API_KEY not set" + exit 1 +fi +pass "NVIDIA_API_KEY is set" + +if ! docker info >/dev/null 2>&1; then + fail "Docker is not running" + exit 1 +fi +pass "Docker is running" + +info "Telegram token: configured (${#TELEGRAM_TOKEN} chars)" +telegram_allowed_id_count=0 +if [ -n "$TELEGRAM_IDS" ]; then + IFS=',' read -ra _telegram_allowed_ids <<<"$TELEGRAM_IDS" + for _tid in "${_telegram_allowed_ids[@]}"; do + _tid="${_tid//[[:space:]]/}" + [ -n "$_tid" ] && ((telegram_allowed_id_count++)) + done +fi +info "Telegram allowlist source: ${TELEGRAM_ALLOWLIST_ENV_KEY} (${telegram_allowed_id_count} ID(s))" +info "Discord token: configured (${#DISCORD_TOKEN} chars)" +info "Slack bot token: configured (${#SLACK_TOKEN} chars)" +info "Slack app token: configured (${#SLACK_APP} chars)" +slack_allowed_user_count=0 +if [ -n "$SLACK_IDS" ]; then + IFS=',' read -ra _slack_allowed_ids <<<"$SLACK_IDS" + for _sid in "${_slack_allowed_ids[@]}"; do + _sid="${_sid//[[:space:]]/}" + [ -n "$_sid" ] && ((slack_allowed_user_count++)) + done +fi +info "Slack allowed users configured: ${slack_allowed_user_count} ID(s)" +info "WeChat token: configured (${#WECHAT_TOKEN} chars), account=${WECHAT_ACCOUNT}" +info "Sandbox name: $SANDBOX_NAME" + +# ══════════════════════════════════════════════════════════════════ +# Phase 1: Install NemoClaw (non-interactive mode) +# ══════════════════════════════════════════════════════════════════ +section "Phase 1: Install NemoClaw with messaging tokens" + +cd "$REPO" || exit 1 + +# Pre-cleanup: destroy any leftover sandbox from previous runs +info "Pre-cleanup..." +if command -v nemoclaw >/dev/null 2>&1; then + nemoclaw "$SANDBOX_NAME" destroy --yes 2>/dev/null || true +fi +if openshell --version >/dev/null 2>&1; then + openshell sandbox delete "$SANDBOX_NAME" 2>/dev/null || true + openshell gateway destroy -g nemoclaw 2>/dev/null || true +fi +pass "Pre-cleanup complete" + +if [ -z "${NEMOCLAW_SKIP_TELEGRAM_REACHABILITY:-}" ]; then + if [ -z "${TELEGRAM_BOT_TOKEN_REAL:-}" ] && [[ "$TELEGRAM_TOKEN" == *fake* ]]; then + export NEMOCLAW_SKIP_TELEGRAM_REACHABILITY=1 + info "Skipping onboarding Telegram reachability probe for fake-token E2E" + elif [ -z "${TELEGRAM_BOT_TOKEN_REAL:-}" ] \ + && ! curl -fsS --max-time 10 https://api.telegram.org/ >/dev/null 2>&1; then + export NEMOCLAW_SKIP_TELEGRAM_REACHABILITY=1 + info "Host cannot reach api.telegram.org; skipping manifest Telegram reachability check" + fi +fi +if [ -z "${NEMOCLAW_SKIP_SLACK_AUTH_VALIDATION:-}" ] \ + && [ -z "${SLACK_BOT_TOKEN_REAL:-}" ] \ + && [ -z "${SLACK_APP_TOKEN_REAL:-}" ] \ + && { is_fake_slack_token "$SLACK_TOKEN" || is_fake_slack_token "$SLACK_APP"; }; then + # This E2E uses fake Slack tokens to prove placeholder/proxy behavior against + # the hermetic fake Slack API. Keep real-token runs on the live validation path. + export NEMOCLAW_SKIP_SLACK_AUTH_VALIDATION=1 + info "Skipping onboarding Slack auth validation for fake-token E2E" +fi + +# Pre-merge Slack policy into the base sandbox policy. +# +# The base policy (openclaw-sandbox.yaml) includes Telegram and Discord +# network rules but NOT Slack — Slack access normally comes from the +# slack.yaml preset, applied in onboard Step 8. However, the sandbox +# container starts in Step 6, so the gateway boots without Slack access. +# The Slack SDK's connection attempt hangs or gets a CONNECT 403 before +# the preset is applied, preventing the gateway from serving on 18789. +# +# By appending the Slack rules to the base policy BEFORE install.sh, the +# sandbox is created with Slack access from the start. The Slack SDK gets +# a fast "invalid_auth" response, the channel guard catches it, and the +# gateway continues serving. +# Ref: #2340 +BASE_POLICY="$REPO/nemoclaw-blueprint/policies/openclaw-sandbox.yaml" +SLACK_PRESET="$REPO/nemoclaw-blueprint/policies/presets/slack.yaml" +if [ -f "$BASE_POLICY" ] && [ -f "$SLACK_PRESET" ] && ! grep -q "api.slack.com" "$BASE_POLICY"; then + BASE_POLICY_BAK="$(mktemp)" + cp "$BASE_POLICY" "$BASE_POLICY_BAK" + _previous_exit_trap=$(trap -p EXIT | sed "s/^trap -- '//;s/' EXIT$//") + trap ''"${_previous_exit_trap:+$_previous_exit_trap;}"' cp "$BASE_POLICY_BAK" "$BASE_POLICY" 2>/dev/null || true; rm -f "$BASE_POLICY_BAK"' EXIT + info "Pre-merging Slack network policy into base sandbox policy..." + cat >>"$BASE_POLICY" <<'SLACK_POLICY_EOF' + + # ── Slack — pre-merged for messaging E2E (#2340) ────────────── + # Normally applied as a preset in onboard Step 8, but the sandbox + # container starts before presets are applied. Inline here so the + # gateway has Slack access from first boot. + slack: + name: slack + endpoints: + - host: slack.com + port: 443 + protocol: rest + enforcement: enforce + rules: + - allow: { method: GET, path: "/**" } + - allow: { method: POST, path: "/**" } + - host: api.slack.com + port: 443 + protocol: rest + enforcement: enforce + rules: + - allow: { method: GET, path: "/**" } + - allow: { method: POST, path: "/**" } + - host: hooks.slack.com + port: 443 + protocol: rest + enforcement: enforce + rules: + - allow: { method: GET, path: "/**" } + - allow: { method: POST, path: "/**" } + - host: wss-primary.slack.com + port: 443 + protocol: websocket + enforcement: enforce + rules: + - allow: { method: GET, path: "/**" } + - allow: { method: WEBSOCKET_TEXT, path: "/**" } + - host: wss-backup.slack.com + port: 443 + protocol: websocket + enforcement: enforce + rules: + - allow: { method: GET, path: "/**" } + - allow: { method: WEBSOCKET_TEXT, path: "/**" } + binaries: + - { path: /usr/local/bin/node } + - { path: /usr/bin/node } +SLACK_POLICY_EOF + if ! grep -q "api.slack.com" "$BASE_POLICY"; then + fail "Failed to append Slack policy to base sandbox policy" + exit 1 + fi + pass "Slack network policy pre-merged into base policy" +else + if grep -q "api.slack.com" "$BASE_POLICY" 2>/dev/null; then + info "Slack policy already present in base policy — skipping pre-merge" + else + fail "Cannot pre-merge Slack policy: missing base policy or preset file" + exit 1 + fi +fi + +# Run install.sh --non-interactive which installs Node.js, openshell, +# NemoClaw, and runs onboard. Messaging tokens are already exported so +# the onboard step creates providers and attaches them to the sandbox. +info "Running install.sh --non-interactive..." +info "This installs Node.js, openshell, NemoClaw, and runs onboard with messaging providers." +info "Expected duration: 5-10 minutes on first run." + +export NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" +export NEMOCLAW_RECREATE_SANDBOX=1 +export NEMOCLAW_FRESH=1 + +INSTALL_LOG="/tmp/nemoclaw-e2e-install.log" +bash install.sh --non-interactive >"$INSTALL_LOG" 2>&1 & +install_pid=$! +tail -f "$INSTALL_LOG" --pid=$install_pid 2>/dev/null & +tail_pid=$! +wait $install_pid +install_exit=$? +kill $tail_pid 2>/dev/null || true +wait $tail_pid 2>/dev/null || true + +# Source shell profile to pick up nvm/PATH changes from install.sh +if [ -f "$HOME/.bashrc" ]; then + # shellcheck source=/dev/null + source "$HOME/.bashrc" 2>/dev/null || true +fi +export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" +if [ -s "$NVM_DIR/nvm.sh" ]; then + # shellcheck source=/dev/null + . "$NVM_DIR/nvm.sh" +fi +if [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]]; then + export PATH="$HOME/.local/bin:$PATH" +fi + +if [ $install_exit -eq 0 ]; then + pass "M0: install.sh completed (exit 0)" +else + fail "M0: install.sh failed (exit $install_exit)" + info "Last 30 lines of install log:" + tail -30 "$INSTALL_LOG" 2>/dev/null || true + exit 1 +fi + +# Verify tools are on PATH +if ! openshell --version >/dev/null 2>&1; then + fail "openshell not found on PATH after install" + exit 1 +fi +pass "openshell installed ($(openshell --version 2>&1 || echo unknown))" + +if ! command -v nemoclaw >/dev/null 2>&1; then + fail "nemoclaw not found on PATH after install" + exit 1 +fi +pass "nemoclaw installed at $(command -v nemoclaw)" + +# Verify sandbox is ready +sandbox_list=$(openshell sandbox list 2>&1 || true) +if echo "$sandbox_list" | grep -q "$SANDBOX_NAME.*Ready"; then + pass "M0b: Sandbox '$SANDBOX_NAME' is Ready" +else + fail "M0b: Sandbox '$SANDBOX_NAME' not Ready (list: ${sandbox_list:0:200})" + exit 1 +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 1b: Enable WhatsApp QR-only channel +# ══════════════════════════════════════════════════════════════════ +section "Phase 1b: Enable WhatsApp QR-only channel" + +WHATSAPP_ADD_LOG="/tmp/nemoclaw-e2e-whatsapp-add.log" +if nemoclaw "$SANDBOX_NAME" channels add whatsapp >"$WHATSAPP_ADD_LOG" 2>&1; then + whatsapp_add_exit=0 +else + whatsapp_add_exit=$? +fi +cat "$WHATSAPP_ADD_LOG" + +if [ "$whatsapp_add_exit" -eq 0 ] && grep -q "Enabled whatsapp channel" "$WHATSAPP_ADD_LOG"; then + pass "M-WA0: channels add whatsapp registered QR-only channel" +else + fail "M-WA0: channels add whatsapp failed or did not register channel" + tail -30 "$WHATSAPP_ADD_LOG" 2>/dev/null || true + exit 1 +fi + +if openshell provider get "${SANDBOX_NAME}-whatsapp-bridge" >/dev/null 2>&1; then + fail "M-WA1: Unexpected WhatsApp bridge provider exists in gateway" +else + pass "M-WA1: WhatsApp QR-only channel creates no bridge provider" +fi + +if registry_plan_channel_contains "whatsapp"; then + pass "M-WA2: registry.messaging.plan.channels contains whatsapp after channel add" +else + fail "M-WA2: registry.messaging.plan.channels missing whatsapp after channel add" +fi + +whatsapp_policy_pre=$(openshell policy get --full "$SANDBOX_NAME" 2>/dev/null || true) +if text_contains_all "$whatsapp_policy_pre" \ + "web.whatsapp.com" \ + "whatsapp.net" \ + "raw.githubusercontent.com"; then + pass "M-WA3: WhatsApp policy preset applied before rebuild" +else + fail "M-WA3: WhatsApp policy preset missing expected endpoints before rebuild" +fi + +WHATSAPP_REBUILD_LOG="/tmp/nemoclaw-e2e-whatsapp-rebuild.log" +info "Rebuilding sandbox so WhatsApp is baked into openclaw.json..." +if nemoclaw "$SANDBOX_NAME" rebuild --yes >"$WHATSAPP_REBUILD_LOG" 2>&1; then + pass "M-WA4: Rebuild completed after WhatsApp channel add" +else + fail "M-WA4: Rebuild failed after WhatsApp channel add" + tail -50 "$WHATSAPP_REBUILD_LOG" 2>/dev/null || true + exit 1 +fi + +whatsapp_policy_post=$(openshell policy get --full "$SANDBOX_NAME" 2>/dev/null || true) +if text_contains_all "$whatsapp_policy_post" \ + "web.whatsapp.com" \ + "whatsapp.net" \ + "raw.githubusercontent.com" \ + && text_contains_any "$whatsapp_policy_post" \ + "/usr/local/bin/node" \ + "/usr/bin/node"; then + pass "M-WA5: WhatsApp policy preset survived rebuild with Node binary scope" +else + fail "M-WA5: WhatsApp policy preset missing expected endpoints/binaries after rebuild" +fi + +sandbox_list=$(openshell sandbox list 2>&1 || true) +if echo "$sandbox_list" | grep -q "$SANDBOX_NAME.*Ready"; then + pass "M-WA6: Sandbox '$SANDBOX_NAME' is Ready after WhatsApp rebuild" +else + fail "M-WA6: Sandbox '$SANDBOX_NAME' not Ready after WhatsApp rebuild (list: ${sandbox_list:0:200})" + exit 1 +fi + +# M-WA6b: WhatsApp compact-QR pairing wiring (NemoClaw#4522). The entrypoint +# installs a NemoClaw-owned preload that forces the `qrcode` package (which +# OpenClaw's renderQrTerminal uses to render the pairing QR) into +# `{ small: true }` half-block rendering so the in-sandbox pairing QR fits a +# phone-camera frame. The preload is wired into the connect-session NODE_OPTIONS +# and the openclaw() guard injects it for the `channels login --channel whatsapp` +# invocation. Verify the preload file (root-owned/read-only in root mode; +# read-only in non-root mode) and the guard wiring are present in the sandbox. +whatsapp_qr_preload_stat=$(sandbox_exec "stat -c '%U:%a' /tmp/nemoclaw-whatsapp-qr-compact.js 2>/dev/null || echo missing") +entrypoint_start_log_stat=$(sandbox_exec "stat -c '%U:%a' /tmp/nemoclaw-start.log 2>/dev/null || echo missing") +if [ "$whatsapp_qr_preload_stat" = "root:444" ]; then + pass "M-WA6b: WhatsApp compact-QR preload installed root:444 (#4522)" +elif [ "$whatsapp_qr_preload_stat" = "sandbox:444" ] && [ "$entrypoint_start_log_stat" = "sandbox:600" ]; then + # /tmp/nemoclaw-start.log is written before sandbox-init.sh is sourced: + # root mode creates root:600, while non-root mode creates sandbox:600. + # Only accept sandbox-owned sourced files when that independent init-time + # signal proves privilege separation was already disabled. + pass "M-WA6b: WhatsApp compact-QR preload installed sandbox:444 (non-root mode) (#4522)" +elif [ "$whatsapp_qr_preload_stat" = "missing" ]; then + fail "M-WA6b: WhatsApp compact-QR preload not installed in sandbox (#4522)" +else + fail "M-WA6b: WhatsApp compact-QR preload has unexpected owner/mode: ${whatsapp_qr_preload_stat} (entrypoint start log: ${entrypoint_start_log_stat}) (#4522)" +fi + +# Assert on the generic manifest-runtime wiring, not just the filename: the +# filename also appears in install banners and path assignments. After the +# messaging manifest migration, WhatsApp contributes a connect preload entry +# and the shared openclaw() guard reads that list for WhatsApp login. +whatsapp_qr_connect_list=$(sandbox_exec "grep -cFx -- '/tmp/nemoclaw-whatsapp-qr-compact.js' /tmp/nemoclaw-messaging-connect-preloads.list 2>/dev/null || echo 0") +whatsapp_qr_connect_export=$(sandbox_exec "grep -cF -- '--require \$_nemoclaw_preload' /tmp/nemoclaw-proxy-env.sh 2>/dev/null || echo 0") +whatsapp_qr_guard_wiring=$(sandbox_exec "grep -cF -- '_nemoclaw_messaging_connect_node_options' /tmp/nemoclaw-proxy-env.sh 2>/dev/null || echo 0") +if [ "${whatsapp_qr_connect_list:-0}" -ge 1 ] 2>/dev/null \ + && [ "${whatsapp_qr_connect_export:-0}" -ge 1 ] 2>/dev/null \ + && [ "${whatsapp_qr_guard_wiring:-0}" -ge 1 ] 2>/dev/null; then + pass "M-WA6c: openclaw() guard injects manifest connect preloads for WhatsApp login (#4522)" +else + fail "M-WA6c: openclaw() guard missing manifest connect preload injection for WhatsApp login (#4522)" +fi + +# M-WA6d: Prove the rendered QR SIZE in the real sandbox, not just that the +# preload file/wiring exist (NemoClaw#4522). Render a representative WhatsApp +# pairing payload through the EXACT renderer the channel-login onQr callback +# uses — `renderQrTerminal` from the baked OpenClaw's plugin-sdk/media-runtime — +# once with the connect-session NODE_OPTIONS sourced (the preload active, as in +# the reporter workflow) and once with NODE_OPTIONS cleared. Assert the sourced +# render is compact and strictly smaller than the cleared baseline. +# +# The probe runs from the global node_modules parent so the bare +# `openclaw/...` specifier resolves against the globally-installed CLI. If the +# renderer cannot be resolved/executed at all (an infra/resolution issue, not a +# size regression) the sub-check SKIPs rather than failing the suite — an actual +# oversized render still yields a number above the ceiling and fails. The +# hard-gated, version-pinned size proof is intentionally not part of this +# VPN-only shell tree. +WHATSAPP_QR_RENDER_PROBE=$( + cat <<'PROBE' +import { renderQrTerminal } from "openclaw/plugin-sdk/media-runtime"; +const strip = (s) => s.replace(/\x1b\[[0-9;]*m/g, ""); +const qr = "2@" + "ABcd12".repeat(8) + "," + "a8K3".repeat(11) + "=," + + "Xy90".repeat(11) + "=," + "Qr5T".repeat(9) + "="; +const out = strip(await renderQrTerminal(qr)); +process.stdout.write(String(out.split("\n").length)); +PROBE +) +whatsapp_qr_render_b64=$(printf '%s' "$WHATSAPP_QR_RENDER_PROBE" | base64 | tr -d '\n') +# Build a remote command that writes the probe to the global lib dir and runs +# it twice (preload sourced vs NODE_OPTIONS cleared), printing both row counts. +whatsapp_qr_render_remote=$( + cat < "\$PROBE_FILE" 2>/dev/null || { echo "RENDER_PROBE_UNAVAILABLE: write failed"; exit 0; } +cd "\$LIBDIR" || { echo "RENDER_PROBE_UNAVAILABLE: cd failed"; exit 0; } +# Compact render: source the connect-session env so the preload is on NODE_OPTIONS. +COMPACT="\$( [ -f /tmp/nemoclaw-proxy-env.sh ] && . /tmp/nemoclaw-proxy-env.sh 2>/dev/null; node "\$PROBE_FILE" 2>/dev/null )" || COMPACT="" +# Baseline render: explicitly clear NODE_OPTIONS so the preload is absent. +BASELINE="\$( NODE_OPTIONS="" node "\$PROBE_FILE" 2>/dev/null )" || BASELINE="" +rm -f "\$PROBE_FILE" 2>/dev/null || true +echo "RENDER_COMPACT=\${COMPACT:-NA} RENDER_BASELINE=\${BASELINE:-NA}" +REMOTE +) +whatsapp_qr_render_out=$(sandbox_exec "$whatsapp_qr_render_remote") +whatsapp_qr_compact_rows=$(printf '%s' "$whatsapp_qr_render_out" | sed -n 's/.*RENDER_COMPACT=\([0-9]*\).*/\1/p') +whatsapp_qr_baseline_rows=$(printf '%s' "$whatsapp_qr_render_out" | sed -n 's/.*RENDER_BASELINE=\([0-9]*\).*/\1/p') +if [ -n "$whatsapp_qr_compact_rows" ] && [ -n "$whatsapp_qr_baseline_rows" ]; then + if [ "$whatsapp_qr_compact_rows" -le 40 ] && [ "$whatsapp_qr_compact_rows" -lt "$whatsapp_qr_baseline_rows" ]; then + pass "M-WA6d: in-sandbox pairing QR renders compact (${whatsapp_qr_compact_rows} rows, baseline ${whatsapp_qr_baseline_rows}) (#4522)" + else + fail "M-WA6d: in-sandbox pairing QR not compact (compact=${whatsapp_qr_compact_rows} rows, baseline=${whatsapp_qr_baseline_rows}) (#4522)" + fi +else + skip "M-WA6d: in-sandbox QR render probe unavailable (${whatsapp_qr_render_out:0:160}) (#4522)" +fi + +# M1: Verify Telegram provider exists in gateway +if openshell provider get "${SANDBOX_NAME}-telegram-bridge" >/dev/null 2>&1; then + pass "M1: Provider '${SANDBOX_NAME}-telegram-bridge' exists in gateway" +else + fail "M1: Provider '${SANDBOX_NAME}-telegram-bridge' not found in gateway" +fi + +# M2: Verify Discord provider exists in gateway +if openshell provider get "${SANDBOX_NAME}-discord-bridge" >/dev/null 2>&1; then + pass "M2: Provider '${SANDBOX_NAME}-discord-bridge' exists in gateway" +else + fail "M2: Provider '${SANDBOX_NAME}-discord-bridge' not found in gateway" +fi + +# M-W1: Verify WeChat provider exists in gateway. Non-interactive onboard +# saw WECHAT_BOT_TOKEN in env (skipping host-qr login) and registered the +# bridge provider just like the other channels. +if openshell provider get "${SANDBOX_NAME}-wechat-bridge" >/dev/null 2>&1; then + pass "M-W1: Provider '${SANDBOX_NAME}-wechat-bridge' exists in gateway" +else + fail "M-W1: Provider '${SANDBOX_NAME}-wechat-bridge' not found in gateway (non-interactive QR-skip path may be broken)" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 2: Credential Isolation — env vars inside sandbox +# ══════════════════════════════════════════════════════════════════ +section "Phase 2: Credential Isolation" + +# M3: TELEGRAM_BOT_TOKEN inside sandbox must NOT contain the host-side token +sandbox_telegram=$(sandbox_exec "printenv TELEGRAM_BOT_TOKEN" 2>/dev/null || true) +if [ -z "$sandbox_telegram" ]; then + info "TELEGRAM_BOT_TOKEN not set inside sandbox (provider-only mode)" + TELEGRAM_PLACEHOLDER="" +elif echo "$sandbox_telegram" | grep -qF "$TELEGRAM_TOKEN"; then + fail "M3: Real Telegram token leaked into sandbox env" +else + pass "M3: Sandbox TELEGRAM_BOT_TOKEN is a placeholder (not the real token)" + TELEGRAM_PLACEHOLDER="$sandbox_telegram" + info "Telegram placeholder: ${TELEGRAM_PLACEHOLDER:0:30}..." +fi + +# M4: DISCORD_BOT_TOKEN inside sandbox must NOT contain the host-side token +sandbox_discord=$(sandbox_exec "printenv DISCORD_BOT_TOKEN" 2>/dev/null || true) +if [ -z "$sandbox_discord" ]; then + info "DISCORD_BOT_TOKEN not set inside sandbox (provider-only mode)" + DISCORD_PLACEHOLDER="" +elif echo "$sandbox_discord" | grep -qF "$DISCORD_TOKEN"; then + fail "M4: Real Discord token leaked into sandbox env" +else + pass "M4: Sandbox DISCORD_BOT_TOKEN is a placeholder (not the real token)" + DISCORD_PLACEHOLDER="$sandbox_discord" + info "Discord placeholder: ${DISCORD_PLACEHOLDER:0:30}..." +fi + +# M5: At least one placeholder should be present for subsequent phases +if [ -n "$TELEGRAM_PLACEHOLDER" ] || [ -n "$DISCORD_PLACEHOLDER" ]; then + pass "M5: At least one messaging placeholder detected in sandbox" +else + skip "M5: No messaging placeholders found — OpenShell may not inject them as env vars" + info "Subsequent phases that depend on placeholders will adapt" +fi + +# M3/M4 verify the specific TELEGRAM_BOT_TOKEN / DISCORD_BOT_TOKEN +# env vars hold placeholders. The checks below verify the real +# host-side tokens do not appear on ANY observable surface inside +# the sandbox: full environment, process list, or filesystem. + +sandbox_env_all=$(sandbox_exec "env 2>/dev/null" 2>/dev/null || true) +sandbox_ps=$(openshell sandbox exec -n "$SANDBOX_NAME" -- \ + sh -c 'cat /proc/[0-9]*/cmdline 2>/dev/null | tr "\0" "\n"' 2>/dev/null || true) + +if [ -n "$sandbox_ps" ]; then + info "Process cmdlines captured ($(echo "$sandbox_ps" | wc -l | tr -d ' ') lines)" +else + info "Process cmdline capture returned empty — M5b/M5f will skip" +fi + +# M5a: Full environment dump must not contain the real Telegram token +if [ -z "$sandbox_env_all" ]; then + skip "M5a: Environment variable list is empty" +elif echo "$sandbox_env_all" | grep -qF "$TELEGRAM_TOKEN"; then + fail "M5a: Real Telegram token found in full sandbox environment dump" +else + pass "M5a: Real Telegram token absent from full sandbox environment" +fi + +# M5b: Process list must not contain the real Telegram token +if [ -z "$sandbox_ps" ]; then + skip "M5b: Process list is empty" +elif echo "$sandbox_ps" | grep -qF "$TELEGRAM_TOKEN"; then + fail "M5b: Real Telegram token found in sandbox process list" +else + pass "M5b: Real Telegram token absent from sandbox process list" +fi + +# M5c: Recursive filesystem search for the real Telegram token. +# Covers /sandbox (workspace), /home, /etc, /tmp, /var. +sandbox_fs_tg=$(printf '%s' "$TELEGRAM_TOKEN" | sandbox_exec_stdin "grep -rFlm1 -f - /sandbox /home /etc /tmp /var 2>/dev/null || true") +if [ -n "$sandbox_fs_tg" ]; then + fail "M5c: Real Telegram token found on sandbox filesystem: ${sandbox_fs_tg}" +else + pass "M5c: Real Telegram token absent from sandbox filesystem" +fi + +# M5d: Placeholder string must be present in the sandbox environment +if [ -n "$TELEGRAM_PLACEHOLDER" ]; then + if echo "$sandbox_env_all" | grep -qF "$TELEGRAM_PLACEHOLDER"; then + pass "M5d: Telegram placeholder confirmed present in sandbox environment" + else + fail "M5d: Telegram placeholder not found in sandbox environment" + fi +else + skip "M5d: No Telegram placeholder to verify (provider-only mode)" +fi + +# M5e: Full environment dump must not contain the real Discord token +if [ -z "$sandbox_env_all" ]; then + skip "M5e: Environment variable list is empty" +elif echo "$sandbox_env_all" | grep -qF "$DISCORD_TOKEN"; then + fail "M5e: Real Discord token found in full sandbox environment dump" +else + pass "M5e: Real Discord token absent from full sandbox environment" +fi + +# M5f: Process list must not contain the real Discord token +if [ -z "$sandbox_ps" ]; then + skip "M5f: Process list is empty" +elif echo "$sandbox_ps" | grep -qF "$DISCORD_TOKEN"; then + fail "M5f: Real Discord token found in sandbox process list" +else + pass "M5f: Real Discord token absent from sandbox process list" +fi + +# M5g: Recursive filesystem search for the real Discord token +sandbox_fs_dc=$(printf '%s' "$DISCORD_TOKEN" | sandbox_exec_stdin "grep -rFlm1 -f - /sandbox /home /etc /tmp /var 2>/dev/null || true") +if [ -n "$sandbox_fs_dc" ]; then + fail "M5g: Real Discord token found on sandbox filesystem: ${sandbox_fs_dc}" +else + pass "M5g: Real Discord token absent from sandbox filesystem" +fi + +# M5h: Discord placeholder must be present in the sandbox environment +if [ -n "$DISCORD_PLACEHOLDER" ]; then + if echo "$sandbox_env_all" | grep -qF "$DISCORD_PLACEHOLDER"; then + pass "M5h: Discord placeholder confirmed present in sandbox environment" + else + fail "M5h: Discord placeholder not found in sandbox environment" + fi +else + skip "M5h: No Discord placeholder to verify (provider-only mode)" +fi + +# ── Slack credential isolation (#2085) ──────────────────────────── +# Mirrors M5a/M5e/M5g for Slack now that provider-shaped aliases are resolved +# directly by OpenShell. The host-side fake token must never appear on any +# observable surface inside the sandbox. + +# M-S5a: Full environment dump must not contain the real Slack bot token. +if [ -z "$sandbox_env_all" ]; then + skip "M-S5a: Environment variable list is empty" +elif echo "$sandbox_env_all" | grep -qF "$SLACK_TOKEN"; then + fail "M-S5a: Real Slack bot token found in full sandbox environment dump" +else + pass "M-S5a: Real Slack bot token absent from full sandbox environment" +fi + +# M-S5b: Process list must not contain the real Slack bot token. +if [ -z "$sandbox_ps" ]; then + skip "M-S5b: Process list is empty" +elif echo "$sandbox_ps" | grep -qF "$SLACK_TOKEN"; then + fail "M-S5b: Real Slack bot token found in sandbox process list" +else + pass "M-S5b: Real Slack bot token absent from sandbox process list" +fi + +# M-S5c: Recursive filesystem search for the real Slack bot token. +sandbox_fs_sl=$(printf '%s' "$SLACK_TOKEN" | sandbox_exec_stdin "grep -rFlm1 -f - /sandbox /home /etc /tmp /var 2>/dev/null || true") +if [ -n "$sandbox_fs_sl" ]; then + fail "M-S5c: Real Slack bot token found on sandbox filesystem: ${sandbox_fs_sl}" +else + pass "M-S5c: Real Slack bot token absent from sandbox filesystem" +fi + +# M-S5d: Same checks for the xapp- Socket Mode token. +if [ -n "$SLACK_APP" ]; then + if [ -z "$sandbox_env_all" ]; then + skip "M-S5d: Environment variable list is empty" + elif echo "$sandbox_env_all" | grep -qF "$SLACK_APP"; then + fail "M-S5d: Real Slack app token found in full sandbox environment dump" + else + pass "M-S5d: Real Slack app token absent from sandbox environment" + fi + if [ -z "$sandbox_ps" ]; then + skip "M-S5d2: Process list is empty" + elif echo "$sandbox_ps" | grep -qF "$SLACK_APP"; then + fail "M-S5d2: Real Slack app token found in sandbox process list" + else + pass "M-S5d2: Real Slack app token absent from sandbox process list" + fi + sandbox_fs_sapp=$(printf '%s' "$SLACK_APP" | sandbox_exec_stdin "grep -rFlm1 -f - /sandbox /home /etc /tmp /var 2>/dev/null || true") + if [ -n "$sandbox_fs_sapp" ]; then + fail "M-S5e: Real Slack app token found on sandbox filesystem: ${sandbox_fs_sapp}" + else + pass "M-S5e: Real Slack app token absent from sandbox filesystem" + fi +fi + +# M-S5f: openclaw.json must contain the Bolt-shape placeholder, not the +# real token. OpenShell resolves the provider-shaped alias directly on egress. +config_slack=$(sandbox_exec "cat /sandbox/.openclaw/openclaw.json 2>/dev/null | grep -E '\"(bot|app)Token\"'" 2>/dev/null || true) +if [ -n "$config_slack" ] && { + echo "$config_slack" | grep -qF "$SLACK_TOKEN" \ + || echo "$config_slack" | grep -qF "$SLACK_APP" +}; then + fail "M-S5f: Real Slack bot/app token spliced into openclaw.json — apply_slack_token_override regression?" +elif [ -n "$config_slack" ] \ + && echo "$config_slack" | grep -q 'xoxb-OPENSHELL-RESOLVE-ENV-SLACK_BOT_TOKEN' \ + && echo "$config_slack" | grep -q 'xapp-OPENSHELL-RESOLVE-ENV-SLACK_APP_TOKEN'; then + pass "M-S5f: openclaw.json holds both Bolt-shape Slack placeholders (no real token on disk)" +else + skip "M-S5f: Could not extract Slack token fields from openclaw.json" +fi + +# M-S5g: No Slack transport bridge should be installed. NODE_OPTIONS may still +# include non-transport resilience guards, but not the removed token rewriter. +sandbox_node_opts=$(openshell sandbox exec --name "$SANDBOX_NAME" -- bash -lc 'echo "$NODE_OPTIONS"' 2>/dev/null || echo "") +if echo "$sandbox_node_opts" | grep -q "nemoclaw-slack-token-rewriter.js"; then + fail "M-S5g: removed Slack token rewriter preload still present in NODE_OPTIONS" +else + pass "M-S5g: Slack token rewriter preload absent from NODE_OPTIONS" +fi + +# ── WeChat credential isolation ─────────────────────────────────── +# Mirrors M5a/M5b/M5c for WeChat. The host-side WECHAT_BOT_TOKEN must +# never appear on any observable surface inside the sandbox — the +# upstream @tencent-weixin/openclaw-weixin plugin reads it via the +# placeholder in /openclaw-weixin/accounts/.json and the +# L7 proxy rewrites at egress. + +# M-W3: WECHAT_BOT_TOKEN inside the sandbox must NOT contain the host token. +sandbox_wechat=$(sandbox_exec "printenv WECHAT_BOT_TOKEN" 2>/dev/null || true) +if [ -z "$sandbox_wechat" ]; then + info "WECHAT_BOT_TOKEN not set inside sandbox (provider-only mode)" + WECHAT_PLACEHOLDER="" +elif echo "$sandbox_wechat" | grep -qF "$WECHAT_TOKEN"; then + fail "M-W3: Real WeChat token leaked into sandbox env" +else + pass "M-W3: Sandbox WECHAT_BOT_TOKEN is a placeholder (not the real token)" + WECHAT_PLACEHOLDER="$sandbox_wechat" + info "WeChat placeholder: ${WECHAT_PLACEHOLDER:0:30}..." +fi + +# M-W3a: Full environment dump must not contain the real WeChat token. +if [ -z "$sandbox_env_all" ]; then + skip "M-W3a: Environment variable list is empty" +elif echo "$sandbox_env_all" | grep -qF "$WECHAT_TOKEN"; then + fail "M-W3a: Real WeChat token found in full sandbox environment dump" +else + pass "M-W3a: Real WeChat token absent from full sandbox environment" +fi + +# M-W3b: Process list must not contain the real WeChat token. +if [ -z "$sandbox_ps" ]; then + skip "M-W3b: Process list is empty" +elif echo "$sandbox_ps" | grep -qF "$WECHAT_TOKEN"; then + fail "M-W3b: Real WeChat token found in sandbox process list" +else + pass "M-W3b: Real WeChat token absent from sandbox process list" +fi + +# M-W3c: Recursive filesystem search for the real WeChat token. The seed +# script writes the placeholder, not the token — a hit here would mean +# something upstream is splicing the real value into account state files. +sandbox_fs_wc=$(printf '%s' "$WECHAT_TOKEN" | sandbox_exec_stdin "grep -rFlm1 -f - /sandbox /home /etc /tmp /var 2>/dev/null || true") +if [ -n "$sandbox_fs_wc" ]; then + fail "M-W3c: Real WeChat token found on sandbox filesystem: ${sandbox_fs_wc}" +else + pass "M-W3c: Real WeChat token absent from sandbox filesystem" +fi + +# M-W3d: WeChat placeholder must be present in the sandbox environment. +if [ -n "$WECHAT_PLACEHOLDER" ]; then + if echo "$sandbox_env_all" | grep -qF "$WECHAT_PLACEHOLDER"; then + pass "M-W3d: WeChat placeholder confirmed present in sandbox environment" + else + fail "M-W3d: WeChat placeholder not found in sandbox environment" + fi +else + skip "M-W3d: No WeChat placeholder to verify (provider-only mode)" +fi + +# ── WhatsApp QR-only isolation ──────────────────────────────────── +# WhatsApp is deliberately tokenless from NemoClaw's perspective. The operator +# pairs inside the sandbox, and mutable QR session state is allowed in durable +# agent state. There must be no host-side WhatsApp credential provider, +# placeholder, or token env for OpenShell to rewrite. + +if [ -z "$sandbox_env_all" ]; then + skip "M-WA7a: Environment variable list is empty" +elif echo "$sandbox_env_all" | grep -qE '(^|[[:space:]])WHATSAPP_.*(TOKEN|SECRET|AUTH|SESSION)='; then + fail "M-WA7a: WhatsApp credential-like env var found in sandbox environment" +else + pass "M-WA7a: No WhatsApp credential-like env var present in sandbox environment" +fi + +if [ -z "$sandbox_ps" ]; then + skip "M-WA7b: Process list is empty" +elif echo "$sandbox_ps" | grep -qE 'WHATSAPP_.*(TOKEN|SECRET|AUTH|SESSION)|openshell:resolve:env:WHATSAPP'; then + fail "M-WA7b: WhatsApp credential placeholder found in sandbox process list" +else + pass "M-WA7b: No WhatsApp credential placeholder present in sandbox process list" +fi + +sandbox_fs_wa=$(sandbox_exec " + { + grep -rIlm1 -E '(^|[^A-Z0-9_])WHATSAPP_[A-Z0-9_]*(TOKEN|SECRET|AUTH|SESSION)[A-Z0-9_]*=' /sandbox /home /etc /tmp /var 2>/dev/null || true + grep -rIlm1 -F 'openshell:resolve:env:WHATSAPP' /sandbox /home /etc /tmp /var 2>/dev/null || true + grep -rIlm1 -F '$WHATSAPP_TOKEN_DECOY' /sandbox /home /etc /tmp /var 2>/dev/null || true + grep -rIlm1 -F '$WHATSAPP_BOT_TOKEN_DECOY' /sandbox /home /etc /tmp /var 2>/dev/null || true + grep -rIlm1 -F '$WHATSAPP_SESSION_SECRET_DECOY' /sandbox /home /etc /tmp /var 2>/dev/null || true + } | sort -u +") +if [ -n "$sandbox_fs_wa" ]; then + fail "M-WA7c: WhatsApp host credential material found on sandbox filesystem: ${sandbox_fs_wa}" +else + pass "M-WA7c: No WhatsApp host credential material found on sandbox filesystem" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 2c: NEMOCLAW_EXTRA_PLACEHOLDER_KEYS — per-profile credential injection +# +# Validates the operator-supplied extra-placeholder-keys hook end-to-end: +# - the registered provider row exists in the OpenShell gateway under the +# deterministic `${sandbox}-extra-` name +# - the sandbox env exposes the canonical resolve placeholder for an +# extension key, never the raw token value +# - a listed-but-unset key produces no gateway provider row +# - a non-extending host secret name (GITHUB_TOKEN) is refused at the +# parser layer, never registered, never present in sandbox env/fs/log +# - the NEMOCLAW_EXTRA_PLACEHOLDER_KEYS env arg itself reaches the +# container so the in-container revision-collapse refresh sees the +# same list the host-side parser produced +# - two independent extension keys resolve to two distinct placeholders +# (the per-Hermes-profile property the feature exists to enable; the +# Hermes-side `.env` substitution is operator-driven and therefore not +# observable from an OpenClaw E2E) +# ══════════════════════════════════════════════════════════════════ +section "Phase 2c: Extra placeholder keys — per-profile credential injection" + +# X1: Provider list shows the extension-key row with the slugged sandbox name. +provider_list=$(openshell provider list 2>/dev/null || true) +EXTRA_PROVIDER_NAME="${SANDBOX_NAME}-extra-telegram-bot-token-agent-a" +if echo "$provider_list" | grep -qF "$EXTRA_PROVIDER_NAME"; then + pass "X1: Provider '$EXTRA_PROVIDER_NAME' registered for the operator-supplied extension key" +else + fail "X1: Provider '$EXTRA_PROVIDER_NAME' missing from openshell provider list" +fi + +# X2: Listed-but-unset extension key must not produce a provider row. +MISSING_PROVIDER_NAME="${SANDBOX_NAME}-extra-telegram-bot-token-agent-missing" +if echo "$provider_list" | grep -qF "$MISSING_PROVIDER_NAME"; then + fail "X2: Provider '$MISSING_PROVIDER_NAME' was registered despite the operator never exporting the credential" +else + pass "X2: Missing-credential extension key produced no provider row (upsert skipped null token)" +fi + +# X3: Non-extending host secret name must be refused at the parser layer. +HOST_SECRET_PROVIDER_NAME="${SANDBOX_NAME}-extra-github-token" +if echo "$provider_list" | grep -qF "$HOST_SECRET_PROVIDER_NAME"; then + fail "X3: Provider '$HOST_SECRET_PROVIDER_NAME' was registered — host secret name leaked past the parser allowlist" +else + pass "X3: GITHUB_TOKEN refused by the parser; no provider row registered" +fi + +# X4a: Sandbox env exposes the canonical resolve placeholder for the +# first extension key, never the raw operator-supplied token value. +sandbox_extra_env=$(sandbox_exec "printenv TELEGRAM_BOT_TOKEN_AGENT_A" 2>/dev/null || true) +if [ -z "$sandbox_extra_env" ]; then + fail "X4a: TELEGRAM_BOT_TOKEN_AGENT_A is unset inside the sandbox; placeholder injection failed" +elif echo "$sandbox_extra_env" | grep -qF "$EXTRAS_TELEGRAM_AGENT_A_TOKEN"; then + fail "X4a: Raw operator-supplied token leaked into the sandbox TELEGRAM_BOT_TOKEN_AGENT_A env" +elif echo "$sandbox_extra_env" | grep -q "^openshell:resolve:env:"; then + pass "X4a: Sandbox TELEGRAM_BOT_TOKEN_AGENT_A is the canonical resolve placeholder" + info " placeholder: ${sandbox_extra_env:0:40}..." +else + fail "X4a: Sandbox TELEGRAM_BOT_TOKEN_AGENT_A is neither the placeholder nor empty: ${sandbox_extra_env:0:80}" +fi + +# X4b: A second extension key resolves to its own distinct placeholder, so +# two Hermes profiles consuming `${TELEGRAM_BOT_TOKEN_AGENT_A}` and +# `${TELEGRAM_BOT_TOKEN_AGENT_B}` get isolated credentials at L7 egress. +sandbox_extra_env_b=$(sandbox_exec "printenv TELEGRAM_BOT_TOKEN_AGENT_B" 2>/dev/null || true) +if [ -z "$sandbox_extra_env_b" ]; then + fail "X4b: TELEGRAM_BOT_TOKEN_AGENT_B is unset inside the sandbox; placeholder injection failed for the second extension key" +elif echo "$sandbox_extra_env_b" | grep -qF "$EXTRAS_TELEGRAM_AGENT_B_TOKEN"; then + fail "X4b: Raw operator-supplied token leaked into the sandbox TELEGRAM_BOT_TOKEN_AGENT_B env" +elif [ "$sandbox_extra_env_b" = "$sandbox_extra_env" ]; then + fail "X4b: TELEGRAM_BOT_TOKEN_AGENT_A and TELEGRAM_BOT_TOKEN_AGENT_B resolve to the same placeholder; per-key isolation broken" +elif echo "$sandbox_extra_env_b" | grep -q "^openshell:resolve:env:"; then + pass "X4b: Two extension keys resolve to distinct canonical placeholders" +else + fail "X4b: Sandbox TELEGRAM_BOT_TOKEN_AGENT_B is neither the placeholder nor empty: ${sandbox_extra_env_b:0:80}" +fi + +# X5: The control env NEMOCLAW_EXTRA_PLACEHOLDER_KEYS must reach the +# nemoclaw-start.sh process inside the container so the +# refresh_openclaw_provider_placeholders helper sees the per-profile keys +# at boot. Grep the entrypoint log for the deterministic breadcrumb the +# refresh helper emits whenever at least one extension key survives the +# in-container parser — that line only fires after the env arg propagated +# AND the canonical-prefix mirror accepted the entry. +start_log=$(openshell sandbox exec --name "$SANDBOX_NAME" -- cat /tmp/nemoclaw-start.log 2>/dev/null || true) +if [ -z "$start_log" ]; then + fail "X5: /tmp/nemoclaw-start.log unavailable; cannot prove extras reached the in-container refresh helper" +else + extras_breadcrumb=$(echo "$start_log" | grep -E "^\[config\] NEMOCLAW_EXTRA_PLACEHOLDER_KEYS accepted [0-9]+ entry\(ies\):" | tail -1 || true) + if [ -z "$extras_breadcrumb" ]; then + fail "X5: nemoclaw-start did not log an accepted-extras breadcrumb; env arg did not propagate or canonical-prefix mirror rejected it" + info " Last 40 lines of /tmp/nemoclaw-start.log:" + echo "$start_log" | tail -40 | while IFS= read -r line; do info " $line"; done + elif ! echo "$extras_breadcrumb" | grep -qw TELEGRAM_BOT_TOKEN_AGENT_A; then + fail "X5: accepted-extras breadcrumb missing TELEGRAM_BOT_TOKEN_AGENT_A: $extras_breadcrumb" + elif echo "$extras_breadcrumb" | grep -qw GITHUB_TOKEN; then + fail "X5: accepted-extras breadcrumb contains GITHUB_TOKEN — host filter bypass" + else + pass "X5: nemoclaw-start accepted-extras breadcrumb proves NEMOCLAW_EXTRA_PLACEHOLDER_KEYS reached the in-container parser" + info " ${extras_breadcrumb:0:160}" + fi +fi + +# X6: The raw operator-supplied token value must not appear on any +# observable sandbox surface (env dump, process list, filesystem). +sandbox_env_extras_dump=$(sandbox_exec "env 2>/dev/null" 2>/dev/null || true) +if [ -z "$sandbox_env_extras_dump" ]; then + skip "X6a: Sandbox environment dump is empty" +elif echo "$sandbox_env_extras_dump" | grep -qF "$EXTRAS_TELEGRAM_AGENT_A_TOKEN"; then + fail "X6a: Raw extension-key token found in sandbox environment dump" +else + pass "X6a: Raw extension-key token absent from sandbox environment dump" +fi + +sandbox_ps_extras=$(openshell sandbox exec -n "$SANDBOX_NAME" -- \ + sh -c 'cat /proc/[0-9]*/cmdline 2>/dev/null | tr "\0" "\n"' 2>/dev/null || true) +if [ -z "$sandbox_ps_extras" ]; then + skip "X6b: Sandbox process list is empty" +elif echo "$sandbox_ps_extras" | grep -qF "$EXTRAS_TELEGRAM_AGENT_A_TOKEN"; then + fail "X6b: Raw extension-key token found in sandbox process list" +else + pass "X6b: Raw extension-key token absent from sandbox process list" +fi + +sandbox_fs_extras=$(sandbox_exec " + grep -rIlm1 -F '$EXTRAS_TELEGRAM_AGENT_A_TOKEN' /sandbox /home /etc /tmp /var 2>/dev/null || true +") +if [ -n "$sandbox_fs_extras" ]; then + fail "X6c: Raw extension-key token found on sandbox filesystem: ${sandbox_fs_extras}" +else + pass "X6c: Raw extension-key token absent from sandbox filesystem" +fi + +# X7: The refused GITHUB_TOKEN value must not reach the sandbox at all — +# neither as an env var, nor on the filesystem. (The host process exports +# it for the parser-rejection test; the sandbox-create env allowlist must +# drop it.) +sandbox_github_env=$(sandbox_exec "printenv GITHUB_TOKEN" 2>/dev/null || true) +if echo "$sandbox_github_env" | grep -qF "$EXTRAS_GITHUB_DECOY"; then + fail "X7a: Refused GITHUB_TOKEN value reached the sandbox env" +else + pass "X7a: Refused GITHUB_TOKEN value never reached the sandbox env" +fi + +sandbox_fs_github=$(sandbox_exec " + grep -rIlm1 -F '$EXTRAS_GITHUB_DECOY' /sandbox /home /etc /tmp /var 2>/dev/null || true +") +if [ -n "$sandbox_fs_github" ]; then + fail "X7b: Refused GITHUB_TOKEN value found on sandbox filesystem: ${sandbox_fs_github}" +else + pass "X7b: Refused GITHUB_TOKEN value absent from sandbox filesystem" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 3: Config Patching — openclaw.json channels +# ══════════════════════════════════════════════════════════════════ +section "Phase 3: Config Patching Verification" + +# Read openclaw.json and extract channel config +managed_proxy_url="" +channel_json=$(sandbox_exec "python3 -c \" +import json, sys +try: + cfg = json.load(open('/sandbox/.openclaw/openclaw.json')) + channels = cfg.get('channels', {}) + print(json.dumps(channels)) +except Exception as e: + print(json.dumps({'error': str(e)})) +\"" 2>/dev/null || true) +plugin_entries_json=$(sandbox_exec "python3 -c \" +import json +try: + cfg = json.load(open('/sandbox/.openclaw/openclaw.json')) + entries = cfg.get('plugins', {}).get('entries', {}) + print(json.dumps(entries)) +except Exception as e: + print(json.dumps({'error': str(e)})) +\"" 2>/dev/null || true) + +if [ -z "$channel_json" ] || echo "$channel_json" | grep -q '"error"'; then + fail "M6: Could not read openclaw.json channels (${channel_json:0:200})" +else + info "OpenClaw channel activation summary: $(summarize_openclaw_config_activation)" + + assert_openclaw_config_activation "M6a" "telegram" "Telegram" + assert_openclaw_config_activation "M6b" "discord" "Discord" + assert_openclaw_config_activation "M6c" "slack" "Slack" + assert_openclaw_config_activation "M6d" "whatsapp" "WhatsApp" + + # This live nightly check intentionally uses OpenClaw's real runtime surface: + # config activation alone is not enough if the CLI still treats the channel as + # unavailable. Log only derived state; raw channel JSON may grow token/session + # fields in future OpenClaw releases. + openclaw_channels_list_json=$(sandbox_exec "timeout 45 openclaw channels list --all --json --no-color 2>/dev/null" 2>/dev/null || true) + openclaw_channels_summary="$(summarize_openclaw_runtime_channels)" + info "OpenClaw channels list summary: ${openclaw_channels_summary}" + assert_openclaw_runtime_channel "M6e" "telegram" "Telegram" "default" + assert_openclaw_runtime_channel "M6f" "discord" "Discord" "default" + assert_openclaw_runtime_channel "M6g" "slack" "Slack" "default" + # WhatsApp has no host-side token provider; before QR pairing OpenClaw can + # prove only that the external plugin is installed and loadable. + assert_openclaw_runtime_channel_installed "M6h" "whatsapp" "WhatsApp" + + # M6: Telegram channel exists with a bot token + # Note: non-root sandboxes cannot patch openclaw.json (chmod 444, root-owned). + # Channels still work via L7 proxy token rewriting without config patching. + # SKIP (not FAIL) when channels are absent — this is the expected non-root path. + tg_token=$(echo "$channel_json" | python3 -c " +import json, sys +d = json.load(sys.stdin) +accounts = d.get('telegram', {}).get('accounts', {}) +account = accounts.get('default') or accounts.get('main') or {} +print(account.get('botToken', '')) +" 2>/dev/null || true) + + if [ -n "$tg_token" ]; then + pass "M6: Telegram channel botToken present in openclaw.json" + else + skip "M6: Telegram channel not in openclaw.json (expected in non-root sandbox)" + fi + + # M6a/M6b: When the channel block is present in openclaw.json, the + # generated config must mark it enabled at the top level so OpenClaw + # 2026.5.22+ actually loads the bridge. NemoClaw#4314 / #4390 reproduced + # as silent "no bridge / no logs"; the symptom matched the Slack + # regression fixed in #4222. Mirror M6's skip-on-absent pattern — the + # non-root sandbox path cannot patch openclaw.json and the block may be + # missing entirely; we only assert behavior when the block is present. + tg_state=$(echo "$channel_json" | python3 -c " +import json, sys +d = json.load(sys.stdin) +block = d.get('telegram') +if not isinstance(block, dict): + print('absent') +elif block.get('enabled') is True: + print('enabled') +else: + print('missing') +" 2>/dev/null || echo "absent") + case "$tg_state" in + enabled) pass "M6a: channels.telegram.enabled is true (bridge loadable per #4314/#4390)" ;; + missing) fail "M6a: channels.telegram present but enabled flag missing — bridge will silently no-op" ;; + *) skip "M6a: Telegram channel block not in openclaw.json (expected in non-root sandbox)" ;; + esac + + dc_state=$(echo "$channel_json" | python3 -c " +import json, sys +d = json.load(sys.stdin) +block = d.get('discord') +if not isinstance(block, dict): + print('absent') +elif block.get('enabled') is True: + print('enabled') +else: + print('missing') +" 2>/dev/null || echo "absent") + case "$dc_state" in + enabled) pass "M6b: channels.discord.enabled is true (bridge loadable)" ;; + missing) fail "M6b: channels.discord present but enabled flag missing — bridge will silently no-op" ;; + *) skip "M6b: Discord channel block not in openclaw.json (expected in non-root sandbox)" ;; + esac + + # M7: Telegram token is NOT the real/fake host token + if [ -n "$tg_token" ] && [ "$tg_token" != "$TELEGRAM_TOKEN" ]; then + pass "M7: Telegram botToken is not the host-side token (placeholder confirmed)" + elif [ -n "$tg_token" ]; then + fail "M7: Telegram botToken matches host-side token — credential leaked into config!" + else + skip "M7: No Telegram botToken to check" + fi + + # M7b: OpenShell can scope provider placeholders by credential revision + # (openshell:resolve:env:v*_TELEGRAM_BOT_TOKEN). OpenClaw must receive that + # runtime-scoped placeholder in openclaw.json; leaving the canonical + # openshell:resolve:env:TELEGRAM_BOT_TOKEN value in the account config makes + # the Telegram bridge start with an unresolved/invalid token. + if [ -n "$tg_token" ] && [ -n "$TELEGRAM_PLACEHOLDER" ]; then + if [ "$tg_token" = "$TELEGRAM_PLACEHOLDER" ]; then + pass "M7b: Telegram botToken matches the OpenShell runtime placeholder" + elif [ "$tg_token" = "openshell:resolve:env:TELEGRAM_BOT_TOKEN" ]; then + fail "M7b: Telegram botToken stayed canonical instead of using runtime placeholder" + else + fail "M7b: Telegram botToken placeholder mismatch (config='${tg_token:0:40}...', env='${TELEGRAM_PLACEHOLDER:0:40}...')" + fi + elif [ -n "$tg_token" ]; then + skip "M7b: No Telegram runtime placeholder env to compare" + else + skip "M7b: No Telegram botToken to compare" + fi + + # M7c-M7f: The Telegram preload diagnostics are installed by nemoclaw-start.sh. + # Exercise them in-process with a mocked Bot API response so the assertions + # are hermetic while still covering the real sandbox-side preload script. + if [ -n "$tg_token" ]; then + telegram_diag_output=$(sandbox_exec "cat > /tmp/nemoclaw-telegram-diagnostics-e2e.js <<'NODE' +const fs = require('fs'); +const http = require('http'); +const https = require('https'); +const { EventEmitter } = require('events'); + +const diagnosticsPath = '/tmp/nemoclaw-telegram-diagnostics.js'; +const sourceConfigPath = '/sandbox/.openclaw/openclaw.json'; +const prefix = 'openshell:resolve:env:'; +const canonicalPlaceholder = prefix + 'TELEGRAM_BOT_TOKEN'; +const diagnosticPlaceholder = prefix + 'vdiagnostic_TELEGRAM_BOT_TOKEN'; +const invalidProbeToken = '000000:telegram-diagnostics-invalid-e2e'; + +function readTelegramBotToken(config) { + const telegram = config?.channels?.telegram; + const accounts = telegram?.accounts || {}; + const account = accounts.default || accounts.main || accounts[Object.keys(accounts)[0]]; + return typeof account?.botToken === 'string' ? account.botToken : ''; +} + +function writeScenarioConfig(token, scenario) { + const config = JSON.parse(fs.readFileSync(sourceConfigPath, 'utf8')); + const accounts = config.channels.telegram.accounts; + const accountName = accounts.default ? 'default' : accounts.main ? 'main' : Object.keys(accounts)[0]; + accounts[accountName].botToken = token; + const configPath = '/tmp/nemoclaw-telegram-diagnostics-' + scenario + '.json'; + fs.writeFileSync(configPath, JSON.stringify(config)); + return configPath; +} + +function installFakeTelegramHttp(statusCode) { + function makeFakeRequest(callback) { + const req = new EventEmitter(); + req.end = () => { + setImmediate(() => { + const res = new EventEmitter(); + res.statusCode = statusCode; + if (typeof callback === 'function') callback(res); + req.emit('response', res); + res.emit('data', Buffer.from('{}')); + res.emit('end'); + }); + return req; + }; + req.write = () => true; + req.setTimeout = () => req; + req.abort = () => {}; + req.destroy = () => {}; + return req; + } + + for (const mod of [http, https]) { + mod.request = function request(...args) { + const callback = args.find((arg) => typeof arg === 'function'); + return makeFakeRequest(callback); + }; + mod.get = function get(...args) { + const callback = args.find((arg) => typeof arg === 'function'); + const req = makeFakeRequest(callback); + req.end(); + return req; + }; + } +} + +async function main() { + const scenario = process.argv[2] || ''; + if (!fs.existsSync(diagnosticsPath)) { + console.log('E2E_FAIL_MISSING_DIAGNOSTICS_PRELOAD'); + process.exit(2); + } + + const currentConfig = JSON.parse(fs.readFileSync(sourceConfigPath, 'utf8')); + const currentToken = readTelegramBotToken(currentConfig); + if (!currentToken) { + console.log('E2E_SKIP_NO_TELEGRAM_BOTTOKEN'); + return; + } + + if (scenario === 'missing-env') { + process.env.OPENCLAW_CONFIG_PATH = writeScenarioConfig(canonicalPlaceholder, scenario); + delete process.env.TELEGRAM_BOT_TOKEN; + } else if (scenario === 'placeholder-mismatch') { + process.env.OPENCLAW_CONFIG_PATH = writeScenarioConfig(canonicalPlaceholder, scenario); + process.env.TELEGRAM_BOT_TOKEN = currentToken.startsWith(prefix) && currentToken !== canonicalPlaceholder + ? currentToken + : diagnosticPlaceholder; + } else if (scenario === 'startup-401') { + const runtimeToken = currentToken.startsWith(prefix) ? currentToken : diagnosticPlaceholder; + process.env.OPENCLAW_CONFIG_PATH = writeScenarioConfig(runtimeToken, scenario); + process.env.TELEGRAM_BOT_TOKEN = runtimeToken; + installFakeTelegramHttp(401); + } else { + console.log('E2E_FAIL_UNKNOWN_SCENARIO'); + process.exit(2); + } + + require(diagnosticsPath); + + if (scenario === 'startup-401') { + https.request('https://api.telegram.org/bot' + invalidProbeToken + '/getMe').end(); + } + + await new Promise((resolve) => setTimeout(resolve, 50)); +} + +main().catch((error) => { + console.log('E2E_FAIL_DIAGNOSTICS_EXCEPTION: ' + (error && error.stack ? error.stack : error)); + process.exit(2); +}); +NODE +NODE_OPTIONS= node /tmp/nemoclaw-telegram-diagnostics-e2e.js missing-env 2>&1 +NODE_OPTIONS= node /tmp/nemoclaw-telegram-diagnostics-e2e.js placeholder-mismatch 2>&1 +NODE_OPTIONS= node /tmp/nemoclaw-telegram-diagnostics-e2e.js startup-401 2>&1 +") + + if echo "$telegram_diag_output" | grep -q 'E2E_FAIL_'; then + diag_fail_codes=$(printf '%s\n' "$telegram_diag_output" | grep -o 'E2E_FAIL_[A-Z0-9_]*' | sort -u | tr '\n' ' ') + fail "M7c: Telegram diagnostics E2E probe failed (${diag_fail_codes:-E2E_FAIL})" + elif echo "$telegram_diag_output" | grep -q 'E2E_SKIP_NO_TELEGRAM_BOTTOKEN'; then + skip "M7c: Telegram diagnostics skipped because openclaw.json has no botToken" + skip "M7d: Telegram diagnostics skipped because openclaw.json has no botToken" + skip "M7e: Telegram diagnostics skipped because openclaw.json has no botToken" + skip "M7f: Telegram diagnostics skipped because openclaw.json has no botToken" + else + if echo "$telegram_diag_output" | grep -qF '[telegram] [default] credential placeholder configured but TELEGRAM_BOT_TOKEN is missing from runtime env'; then + pass "M7c: Telegram diagnostics report missing runtime placeholder env" + else + fail "M7c: Telegram diagnostics missing-env breadcrumb absent" + fi + + if echo "$telegram_diag_output" | grep -qF '[telegram] [default] credential placeholder mismatch: openclaw.json botToken does not match runtime TELEGRAM_BOT_TOKEN placeholder'; then + pass "M7d: Telegram diagnostics report scoped placeholder mismatch" + else + fail "M7d: Telegram diagnostics placeholder-mismatch breadcrumb absent" + fi + + if echo "$telegram_diag_output" | grep -qF '[telegram] [default] Bot API rejected startup probe with HTTP 401; token invalid or credential placeholder unresolved'; then + pass "M7e: Telegram diagnostics report sanitized startup probe rejection" + else + fail "M7e: Telegram diagnostics startup-probe breadcrumb absent" + fi + + if echo "$telegram_diag_output" | grep -qE 'telegram-diagnostics-invalid-e2e|openshell:resolve:env:'; then + fail "M7f: Telegram diagnostics leaked raw token or credential placeholder" + else + pass "M7f: Telegram diagnostics breadcrumbs are sanitized" + fi + fi + else + skip "M7c: No Telegram botToken for diagnostics probe" + skip "M7d: No Telegram botToken for diagnostics probe" + skip "M7e: No Telegram botToken for diagnostics probe" + skip "M7f: No Telegram botToken for diagnostics probe" + fi + + # M8: Discord channel exists with a token + dc_token=$(echo "$channel_json" | python3 -c " +import json, sys +d = json.load(sys.stdin) +accounts = d.get('discord', {}).get('accounts', {}) +account = accounts.get('default') or accounts.get('main') or {} +print(account.get('token', '')) +" 2>/dev/null || true) + + if [ -n "$dc_token" ]; then + pass "M8: Discord channel token present in openclaw.json" + else + skip "M8: Discord channel not in openclaw.json (expected in non-root sandbox)" + fi + + # M9: Discord token is NOT the real/fake host token + if [ -n "$dc_token" ] && [ "$dc_token" != "$DISCORD_TOKEN" ]; then + pass "M9: Discord token is not the host-side token (placeholder confirmed)" + elif [ -n "$dc_token" ]; then + fail "M9: Discord token matches host-side token — credential leaked into config!" + else + skip "M9: No Discord token to check" + fi + + # M9b: Discord Gateway WebSocket routing uses OpenClaw's managed proxy. + # OpenClaw's Discord plugin validates the per-account proxy and rejects any + # non-loopback host, so NemoClaw must not bake a Discord-only account.proxy + # (the sandbox egress proxy 10.200.0.1:3128 is not loopback). Discord + # gateway/REST egress is carried by the top-level managed proxy + # (proxy.loopbackMode "gateway-only"). The fake Gateway proof in M13b-M13g + # exercises the same OpenShell relay path using that managed proxy config. + dc_proxy=$(echo "$channel_json" | python3 -c " +import json, sys +d = json.load(sys.stdin) +accounts = d.get('discord', {}).get('accounts', {}) +account = accounts.get('default') or accounts.get('main') or {} +print(account.get('proxy', '')) +" 2>/dev/null || true) + + managed_proxy_url=$(sandbox_exec "python3 -c \" +import json +cfg = json.load(open('/sandbox/.openclaw/openclaw.json')) +proxy = cfg.get('proxy') or {} +if proxy.get('enabled') is True: + print(proxy.get('proxyUrl') or '') +\"" 2>/dev/null || true) + expected_managed_proxy="http://${NEMOCLAW_PROXY_HOST:-10.200.0.1}:${NEMOCLAW_PROXY_PORT:-3128}" + if [ -n "$dc_token" ] && [ -z "$dc_proxy" ] && [ "$managed_proxy_url" = "$expected_managed_proxy" ]; then + pass "M9b: Discord relies on OpenClaw managed proxy config, with no per-account loopback proxy" + elif [ -n "$dc_token" ]; then + fail "M9b: Discord proxy wiring wrong; expected account.proxy='' and proxy.proxyUrl='${expected_managed_proxy}' (account.proxy='${dc_proxy}', proxy.proxyUrl='${managed_proxy_url}')" + else + skip "M9b: No Discord channel config to check" + fi + + # M10: Telegram enabled + tg_enabled=$(echo "$channel_json" | python3 -c " +import json, sys +d = json.load(sys.stdin) +accounts = d.get('telegram', {}).get('accounts', {}) +account = accounts.get('default') or accounts.get('main') or {} +print(account.get('enabled', False)) +" 2>/dev/null || true) + + if [ "$tg_enabled" = "True" ]; then + pass "M10: Telegram channel is enabled" + else + skip "M10: Telegram channel not enabled (expected in non-root sandbox)" + fi + + # M11: Discord enabled + dc_enabled=$(echo "$channel_json" | python3 -c " +import json, sys +d = json.load(sys.stdin) +accounts = d.get('discord', {}).get('accounts', {}) +account = accounts.get('default') or accounts.get('main') or {} +print(account.get('enabled', False)) +" 2>/dev/null || true) + + if [ "$dc_enabled" = "True" ]; then + pass "M11: Discord channel is enabled" + else + skip "M11: Discord channel not enabled (expected in non-root sandbox)" + fi + + # M11b: Telegram dmPolicy is allowlist (not pairing) + tg_dm_policy=$(echo "$channel_json" | python3 -c " +import json, sys +d = json.load(sys.stdin) +accounts = d.get('telegram', {}).get('accounts', {}) +account = accounts.get('default') or accounts.get('main') or {} +print(account.get('dmPolicy', '')) +" 2>/dev/null || true) + + if [ "$tg_dm_policy" = "allowlist" ]; then + pass "M11b: Telegram dmPolicy is 'allowlist'" + elif [ -n "$tg_dm_policy" ]; then + fail "M11b: Telegram dmPolicy is '$tg_dm_policy' (expected 'allowlist')" + else + skip "M11b: Telegram dmPolicy not set (channel may not be configured)" + fi + + # M11c: Telegram allowFrom contains the expected user IDs + tg_allow_from=$(echo "$channel_json" | python3 -c " +import json, sys +d = json.load(sys.stdin) +accounts = d.get('telegram', {}).get('accounts', {}) +account = accounts.get('default') or accounts.get('main') or {} +ids = account.get('allowFrom', []) +print(','.join(str(i) for i in ids)) +" 2>/dev/null || true) + + if [ -n "$tg_allow_from" ]; then + # Check that all configured IDs are present + IFS=',' read -ra expected_ids <<<"$TELEGRAM_IDS" + missing_ids=() + tg_allow_from_csv=",${tg_allow_from//[[:space:]]/}," + for eid in "${expected_ids[@]}"; do + eid="${eid//[[:space:]]/}" + [ -z "$eid" ] && continue + if [[ "$tg_allow_from_csv" != *",$eid,"* ]]; then + missing_ids+=("$eid") + fi + done + if [ ${#missing_ids[@]} -eq 0 ]; then + pass "M11c: Telegram allowFrom contains all expected user IDs: $tg_allow_from" + if [ "$TELEGRAM_ALLOWLIST_ENV_KEY" != "TELEGRAM_ALLOWED_IDS" ]; then + pass "M11c-alias: Telegram allowFrom honored ${TELEGRAM_ALLOWLIST_ENV_KEY} alias" + fi + else + fail "M11c: Telegram allowFrom ($tg_allow_from) is missing IDs: ${missing_ids[*]} (expected all of: $TELEGRAM_IDS)" + fi + else + skip "M11c: Telegram allowFrom not set (channel may not be configured)" + fi + + # M11d: Telegram groupPolicy defaults to open so group chats are not silently dropped + tg_group_policy=$(echo "$channel_json" | python3 -c " +import json, sys +d = json.load(sys.stdin) +accounts = d.get('telegram', {}).get('accounts', {}) +account = accounts.get('default') or accounts.get('main') or {} +print(account.get('groupPolicy', '')) +" 2>/dev/null || true) + + if [ "$tg_group_policy" = "open" ]; then + pass "M11d: Telegram groupPolicy is 'open'" + elif [ -n "$tg_group_policy" ]; then + fail "M11d: Telegram groupPolicy is '$tg_group_policy' (expected 'open')" + else + skip "M11d: Telegram groupPolicy not set (channel may not be configured)" + fi + + # M11e: Slack channel configured — gateway must survive auth failure (#2340) + # The Slack channel has placeholder tokens that will fail auth. The channel + # guard preload (NODE_OPTIONS --require) should catch the error. We can't + # verify the guard file via SSH (different container), but we CAN check the + # gateway port from here. This is tested more thoroughly in Phase 7. + slack_configured=$(echo "$channel_json" | python3 -c " +import json, sys +d = json.load(sys.stdin) +print('yes' if 'slack' in d else 'no') +" 2>/dev/null || true) + if [ "$slack_configured" = "yes" ]; then + pass "M11e: Slack channel configured with placeholder tokens (guard needed)" + + # M11f/M11g/M11h: SLACK_ALLOWED_USERS should authorize both DMs and + # channel @mentions from the same users. Config lives on the Slack account + # because OpenClaw supports multi-account Slack channel policy. + sl_dm_policy=$(echo "$channel_json" | python3 -c " +import json, sys +d = json.load(sys.stdin) +account = d.get('slack', {}).get('accounts', {}).get('default', {}) +print(account.get('dmPolicy', '')) +" 2>/dev/null || true) + if [ "$sl_dm_policy" = "allowlist" ]; then + pass "M11f: Slack dmPolicy is 'allowlist'" + elif [ -n "$sl_dm_policy" ]; then + fail "M11f: Slack dmPolicy is '$sl_dm_policy' (expected 'allowlist')" + else + skip "M11f: Slack dmPolicy not set" + fi + + sl_group_policy=$(echo "$channel_json" | python3 -c " +import json, sys +d = json.load(sys.stdin) +account = d.get('slack', {}).get('accounts', {}).get('default', {}) +print(account.get('groupPolicy', '')) +" 2>/dev/null || true) + if [ "$sl_group_policy" = "allowlist" ]; then + pass "M11g: Slack groupPolicy is 'allowlist'" + elif [ -n "$sl_group_policy" ]; then + fail "M11g: Slack groupPolicy is '$sl_group_policy' (expected 'allowlist')" + else + skip "M11g: Slack groupPolicy not set" + fi + + sl_channel_users=$(echo "$channel_json" | python3 -c " +import json, sys +d = json.load(sys.stdin) +account = d.get('slack', {}).get('accounts', {}).get('default', {}) +wildcard = account.get('channels', {}).get('*', {}) +if wildcard.get('enabled') is not True: + print('BAD_ENABLED') +elif wildcard.get('requireMention') is not True: + print('BAD_REQUIRE_MENTION') +else: + users = wildcard.get('users', []) + if not isinstance(users, list): + print('BAD_USERS_TYPE') + elif len(users) == 0: + print('EMPTY_USERS') + else: + print(','.join(str(i) for i in users)) +" 2>/dev/null || true) + if [ "$sl_channel_users" = "BAD_ENABLED" ]; then + fail "M11h: Slack wildcard channel config is not enabled" + elif [ "$sl_channel_users" = "BAD_REQUIRE_MENTION" ]; then + fail "M11h: Slack wildcard channel config does not require mention" + elif [ "$sl_channel_users" = "BAD_USERS_TYPE" ]; then + fail "M11h: Slack wildcard channel users is not a list" + elif [ "$sl_channel_users" = "EMPTY_USERS" ]; then + fail "M11h: Slack wildcard channel users is empty" + elif [ -n "$sl_channel_users" ]; then + IFS=',' read -ra expected_slack_ids <<<"$SLACK_IDS" + missing_slack_ids=() + expected_slack_id_count=0 + sl_channel_users_csv=",${sl_channel_users//[[:space:]]/}," + for sid in "${expected_slack_ids[@]}"; do + sid="${sid//[[:space:]]/}" + [ -z "$sid" ] && continue + ((expected_slack_id_count++)) + if [[ "$sl_channel_users_csv" != *",$sid,"* ]]; then + missing_slack_ids+=("$sid") + fi + done + if [ ${#missing_slack_ids[@]} -eq 0 ]; then + pass "M11h: Slack wildcard channel @mention allowlist contains expected user count (${expected_slack_id_count})" + else + fail "M11h: Slack wildcard channel users missing ${#missing_slack_ids[@]} expected ID(s)" + fi + else + skip "M11h: Slack wildcard channel users not set" + fi + + # Diagnostics: check if the guard was installed and what NODE_OPTIONS looks like + info "Checking guard installation diagnostics:" + guard_exists=$(openshell sandbox exec --name "$SANDBOX_NAME" -- ls -la /tmp/nemoclaw-slack-channel-guard.js 2>/dev/null || echo "EXEC_FAILED") + info " Guard file: $guard_exists" + node_opts=$(openshell sandbox exec --name "$SANDBOX_NAME" -- bash -c 'echo "$NODE_OPTIONS"' 2>/dev/null || echo "EXEC_FAILED") + info " NODE_OPTIONS: $node_opts" + else + skip "M11e: No Slack channel in config" + fi + + # M-WA8/M-WA9: WhatsApp is QR-only, but it still needs a real channel block + # baked into openclaw.json after `channels add whatsapp` + rebuild. There + # should be no token, auth, or OpenShell placeholder field in that account. + whatsapp_account_json=$(echo "$channel_json" | python3 -c " +import json, sys +d = json.load(sys.stdin) +account = d.get('whatsapp', {}).get('accounts', {}).get('default', {}) +print(json.dumps(account, sort_keys=True)) +" 2>/dev/null || true) + whatsapp_enabled=$(echo "$whatsapp_account_json" | python3 -c " +import json, sys +try: + account = json.load(sys.stdin) + print(account.get('enabled', False)) +except Exception: + print(False) +" 2>/dev/null || true) + whatsapp_health_monitor=$(echo "$whatsapp_account_json" | python3 -c " +import json, sys +try: + account = json.load(sys.stdin) + print(account.get('healthMonitor', {}).get('enabled', None)) +except Exception: + print(None) +" 2>/dev/null || true) + + if [ "$whatsapp_enabled" = "True" ]; then + pass "M-WA8: WhatsApp account is enabled in openclaw.json" + else + fail "M-WA8: WhatsApp account missing or disabled in openclaw.json (${whatsapp_account_json:0:200})" + fi + + if [ "$whatsapp_health_monitor" = "False" ]; then + pass "M-WA8a: WhatsApp health monitor is disabled for unpaired QR session" + else + fail "M-WA8a: WhatsApp health monitor is not disabled (${whatsapp_account_json:0:200})" + fi + + whatsapp_secret_fields=$(echo "$whatsapp_account_json" | python3 -c " +import json, sys +try: + account = json.load(sys.stdin) +except Exception: + print('BAD_JSON') + sys.exit(0) +bad = [] +def walk(value, path=''): + if isinstance(value, dict): + for key, child in value.items(): + next_path = f'{path}.{key}' if path else key + if any(word in key.lower() for word in ('token', 'secret', 'auth', 'session')): + bad.append(next_path) + walk(child, next_path) + elif isinstance(value, list): + for idx, child in enumerate(value): + walk(child, f'{path}[{idx}]') + elif isinstance(value, str) and 'openshell:resolve:env:WHATSAPP' in value: + bad.append(path) +walk(account) +print(','.join(bad)) +" 2>/dev/null || true) + if [ -z "$whatsapp_secret_fields" ]; then + pass "M-WA9: WhatsApp config has no token/auth/session provider placeholders" + else + fail "M-WA9: WhatsApp config contains secret-like fields: ${whatsapp_secret_fields}" + fi + + # M-W7: WeChat plugin install registry is restored alongside the channel + # block, the plugin entry is enabled, and the install spec is pinned to a + # concrete semver. The upstream plugin loader needs this install metadata + # after OpenClaw config rewrites (plugins.entries alone is not enough), + # and a floating spec (e.g. "@latest") would silently bypass the + # installer-trust pinning enforced by the WeChat package-install allowlist and + # wechat.seedOpenClawAccount manifest hook (WECHAT_PLUGIN_SPEC=@2.4.3). + wechat_plugins_json=$(sandbox_exec "python3 -c \" +import json +cfg = json.load(open('/sandbox/.openclaw/openclaw.json')) +plugins = cfg.get('plugins', {}) or {} +print(json.dumps({ + 'install': plugins.get('installs', {}).get('openclaw-weixin', {}), + 'entry': plugins.get('entries', {}).get('openclaw-weixin', {}), +})) +\"" 2>/dev/null || true) + if echo "$wechat_plugins_json" | python3 -c " +import json, re, sys +try: + data = json.load(sys.stdin) +except Exception: + sys.exit(2) +inst = data.get(\"install\") if isinstance(data, dict) else None +entry = data.get(\"entry\") if isinstance(data, dict) else None +spec = inst.get(\"spec\") if isinstance(inst, dict) else None +install_path = inst.get(\"installPath\") if isinstance(inst, dict) else None +ok = ( + isinstance(inst, dict) + and inst.get(\"source\") == \"npm\" + and isinstance(spec, str) + and bool(re.fullmatch(r\"@tencent-weixin/openclaw-weixin@\d+\.\d+\.\d+\", spec)) + and isinstance(install_path, str) + and bool(install_path.strip()) + and isinstance(entry, dict) + and entry.get(\"enabled\") is True +) +sys.exit(0 if ok else 1) +" 2>/dev/null; then + pass "M-W7: WeChat plugin install registry restored, entry enabled, spec pinned in openclaw.json" + else + fail "M-W7: WeChat plugin install registry missing/invalid, entry not enabled, or spec not pinned to a concrete semver" + fi + + # M-W8: WeChat channel registered under channels.openclaw-weixin with the + # configured accountId enabled. Written by the manifest post-agent-install + # hook during image build. Absence here means WeChat metadata was empty or + # the manifest build-file output was skipped — both regressions on the + # non-interactive QR-skip path. + wechat_enabled=$(echo "$channel_json" | python3 -c " +import json, sys +d = json.load(sys.stdin) +accounts = d.get('openclaw-weixin', {}).get('accounts', {}) +account = accounts.get('$WECHAT_ACCOUNT', {}) +print(account.get('enabled', False)) +" 2>/dev/null || true) + if [ "$wechat_enabled" = "True" ]; then + pass "M-W8: WeChat account '$WECHAT_ACCOUNT' is enabled in openclaw.json (channels.openclaw-weixin)" + else + fail "M-W8: WeChat account not enabled in openclaw.json (channels.openclaw-weixin missing or disabled)" + fi +fi + +# M-W9: Per-account credential file holds the WECHAT_BOT_TOKEN placeholder, +# not the real token. The manifest post-agent-install hook writes +# /openclaw-weixin/accounts/.json with +# token = "openshell:resolve:env:WECHAT_BOT_TOKEN". A real-token hit +# would mean someone bypassed the placeholder constant. +wechat_account_json=$(sandbox_exec "cat /sandbox/.openclaw/openclaw-weixin/accounts/${WECHAT_ACCOUNT}.json 2>/dev/null || true" 2>/dev/null || true) +if [ -z "$wechat_account_json" ] || echo "$wechat_account_json" | grep -qi "no such file"; then + fail "M-W9: WeChat per-account credential file not found (manifest post-agent-install hook may have been skipped)" +else + if echo "$wechat_account_json" | grep -qF "$WECHAT_TOKEN"; then + fail "M-W9: Real WeChat token spliced into accounts/${WECHAT_ACCOUNT}.json — manifest seed placeholder regression" + elif echo "$wechat_account_json" | grep -qF "openshell:resolve:env:WECHAT_BOT_TOKEN"; then + pass "M-W9: WeChat per-account credential file uses the L7-resolved placeholder" + else + fail "M-W9: WeChat per-account credential file has unexpected token shape: $(echo "$wechat_account_json" | tr -d '\n' | cut -c1-200)" + fi +fi + +# M-W10: Accounts index lists the configured accountId. Written by +# the manifest post-agent-install hook before the per-account file; the upstream plugin's +# auth/accounts.ts boots accounts that appear in this index. +wechat_index_json=$(sandbox_exec "cat /sandbox/.openclaw/openclaw-weixin/accounts.json 2>/dev/null || true" 2>/dev/null || true) +if [ -z "$wechat_index_json" ] || echo "$wechat_index_json" | grep -qi "no such file"; then + fail "M-W10: WeChat accounts.json index not found" +else + if echo "$wechat_index_json" | python3 -c " +import json, sys +try: + ids = json.load(sys.stdin) + sys.exit(0 if isinstance(ids, list) and '$WECHAT_ACCOUNT' in ids else 1) +except Exception: + sys.exit(2) +" 2>/dev/null; then + pass "M-W10: WeChat accounts.json index contains '$WECHAT_ACCOUNT'" + else + fail "M-W10: WeChat accounts.json missing '$WECHAT_ACCOUNT' (raw: $(echo "$wechat_index_json" | tr -d '\n' | cut -c1-200))" + fi +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 4: Network Reachability +# ══════════════════════════════════════════════════════════════════ +section "Phase 4: Network Reachability" + +# M12: Node.js can reach api.telegram.org through the proxy +tg_reach=$(sandbox_exec 'node -e " +const https = require(\"https\"); +const req = https.get(\"https://api.telegram.org/\", (res) => { + console.log(\"HTTP_\" + res.statusCode); + res.resume(); +}); +req.on(\"error\", (e) => console.log(\"ERROR: \" + e.message)); +req.setTimeout(15000, () => { req.destroy(); console.log(\"TIMEOUT\"); }); +"' 2>/dev/null || true) + +if echo "$tg_reach" | grep -q "HTTP_"; then + pass "M12: Node.js reached api.telegram.org (${tg_reach})" +elif echo "$tg_reach" | grep -q "TIMEOUT"; then + skip "M12: api.telegram.org timed out (network may be slow)" +elif echo "$tg_reach" | grep -qiE "ERROR:.*(ECONNRESET|reset|socket hang up|ENETUNREACH|EHOSTUNREACH|ETIMEDOUT)"; then + skip "M12: api.telegram.org unreachable from this network (${tg_reach:0:160})" +else + fail "M12: Node.js could not reach api.telegram.org (${tg_reach:0:200})" +fi + +# M13: Node.js can reach Discord API/CDN through the proxy +live_discord_policy=$(openshell policy get --full "$SANDBOX_NAME" 2>/dev/null || true) +if echo "$live_discord_policy" | grep -q "discord.com" \ + && echo "$live_discord_policy" | grep -q "cdn.discordapp.com" \ + && { echo "$live_discord_policy" | grep -q "/usr/local/bin/node" || echo "$live_discord_policy" | grep -q "/usr/bin/node"; }; then + pass "M13-policy: Live policy contains Discord endpoints and Node binaries" +else + fail "M13-policy: Live policy is missing expected Discord preset endpoint/binary entries" +fi + +live_proxy_env=$(sandbox_exec 'printf "HTTPS_PROXY=%s\nhttps_proxy=%s\nNO_PROXY=%s\nno_proxy=%s\n" "$HTTPS_PROXY" "$https_proxy" "$NO_PROXY" "$no_proxy"' 2>/dev/null || true) +info "Sandbox proxy env: ${live_proxy_env//$'\n'/ }" +if echo "$live_proxy_env" | grep -qE "https?_proxy=.*10\.200\.0\.1:3128|HTTPS_PROXY=.*10\.200\.0\.1:3128"; then + pass "M13-proxy: Sandbox uses the OpenShell gateway proxy" +else + fail "M13-proxy: Sandbox proxy env does not point at OpenShell gateway: ${live_proxy_env:0:200}" +fi + +# Regression context for #3477: curl is intentionally not in the Discord +# preset's binary whitelist, but a live curl CONNECT 403 is ambiguous because +# an upstream network policy can produce the same symptom. Treat the live probe +# as diagnostics only; M13-rest-d/e below provide the hermetic whitelist proof. +live_dc_curl=$(sandbox_exec 'set +e +rm -f /tmp/nemoclaw-discord-curl.err /tmp/nemoclaw-discord-curl.body +curl -v --max-time 10 https://discord.com/ \ + -o /tmp/nemoclaw-discord-curl.body \ + 2>/tmp/nemoclaw-discord-curl.err +rc=$? +printf "RC=%s\n" "$rc" +grep -E "Uses proxy|CONNECT discord.com:443|HTTP/1\\.[01] 403|CONNECT tunnel failed|Connection established|policy_denied|Forbidden" /tmp/nemoclaw-discord-curl.err /tmp/nemoclaw-discord-curl.body 2>/dev/null || true +' 2>/dev/null || true) +info "Discord curl probe: ${live_dc_curl:0:500}" +if echo "$live_dc_curl" | grep -qiE "CONNECT tunnel failed.*403|CONNECT discord\.com:443|HTTP/1\.[01] 403|policy_denied|Forbidden" \ + && ! echo "$live_dc_curl" | grep -qiE "Connection established|200 Connection"; then + info "M13-curl: ambiguous live CONNECT 403 may be upstream or local; hermetic M13-rest-d/e prove whitelist behavior; output: ${live_dc_curl:0:300}" +elif echo "$live_dc_curl" | grep -qiE "Connection established|200 Connection"; then + fail "M13-curl: curl unexpectedly established a tunnel to Discord; binary whitelist may be too broad" +else + info "M13-curl: live curl probe inconclusive; hermetic M13-rest-d/e prove whitelist behavior; output: ${live_dc_curl:0:200}" +fi + +dc_reach=$(sandbox_exec 'node - <<'"'"'NODE'"'"' +const https = require("https"); +const targets = [ + ["api", "https://discord.com/api/v10/gateway"], + ["cdn", "https://cdn.discordapp.com/"], +]; +let pending = targets.length; +let failed = false; + +function done() { + pending -= 1; + if (pending === 0) process.exit(failed ? 1 : 0); +} + +for (const [name, url] of targets) { + const req = https.get(url, (res) => { + console.log(`${name}:HTTP_${res.statusCode}`); + res.resume(); + done(); + }); + req.on("error", (error) => { + failed = true; + console.log(`${name}:ERROR_${error.message}`); + done(); + }); + req.setTimeout(15000, () => { + failed = true; + req.destroy(); + console.log(`${name}:TIMEOUT`); + done(); + }); +} +NODE +' 2>/dev/null || true) + +info "Discord Node probe: ${dc_reach:0:500}" +if echo "$dc_reach" | grep -q "api:HTTP_" \ + && echo "$dc_reach" | grep -q "cdn:HTTP_"; then + pass "M13: Node.js reached Discord API and CDN through the same proxy (${dc_reach//$'\n'/ })" +elif echo "$dc_reach" | grep -qiE "CONNECT.*403|policy_denied|forbidden"; then + fail "M13: Node.js was denied by the proxy despite the Discord preset being applied: ${dc_reach:0:300}" +elif echo "$dc_reach" | grep -qiE "TIMEOUT|ENETUNREACH|EHOSTUNREACH|ETIMEDOUT|ECONNRESET|socket hang up|network"; then + skip "M13: Live Discord unreachable from this network (${dc_reach:0:200})" +else + fail "M13: Node.js could not reach Discord API/CDN (${dc_reach:0:200})" +fi + +# M13-rest-a-M13-rest-e: Hermetic Discord-shaped HTTPS REST binary whitelist proof. +fake_rest_ready=0 +if start_fake_discord_rest_api; then + fake_rest_ready=1 + pass "M13-rest-a: Hermetic fake Discord REST API started on host port ${FAKE_DISCORD_REST_PORT}" +else + skip "M13-rest-a: Could not start hermetic fake Discord REST API" +fi + +fake_rest_policy_ready=0 +if [ "$fake_rest_ready" = "1" ]; then + if apply_fake_discord_rest_policy "$SANDBOX_NAME" "$FAKE_DISCORD_REST_PORT" >/tmp/nemoclaw-fake-discord-rest-policy.log 2>&1; then + fake_rest_policy_ready=1 + pass "M13-rest-b: Applied Node-only HTTPS policy for fake Discord REST API" + else + fail "M13-rest-b: Failed to apply fake Discord REST policy: $(tail -20 /tmp/nemoclaw-fake-discord-rest-policy.log 2>/dev/null | tr '\n' ' ' | cut -c1-300)" + fi +else + skip "M13-rest-b: Fake Discord REST API unavailable; skipping policy apply" +fi + +fake_rest_node="" +if [ "$fake_rest_policy_ready" = "1" ]; then + fake_rest_node=$(run_fake_discord_rest_node_request "$FAKE_DISCORD_REST_PORT" "/api/v10/gateway" || true) +fi +info "Fake Discord REST Node probe: ${fake_rest_node:0:300}" +if [ "$fake_rest_policy_ready" != "1" ]; then + skip "M13-rest-c: Fake Discord REST policy unavailable; skipping Node proof" +elif echo "$fake_rest_node" | grep -q "^200 "; then + pass "M13-rest-c: Node reached the fake Discord REST API through OpenShell" +else + fail "M13-rest-c: Node failed to reach fake Discord REST API: ${fake_rest_node:0:300}" +fi + +fake_rest_curl="" +if [ "$fake_rest_policy_ready" = "1" ]; then + fake_rest_curl=$(run_fake_discord_rest_curl_request "$FAKE_DISCORD_REST_PORT" || true) +fi +info "Fake Discord REST curl probe: ${fake_rest_curl:0:500}" +if [ "$fake_rest_policy_ready" != "1" ]; then + skip "M13-rest-d: Fake Discord REST policy unavailable; skipping curl denial proof" +elif echo "$fake_rest_curl" | grep -qiE "CONNECT tunnel failed.*403|HTTP/1\.[01] 403|policy_denied|Forbidden" \ + && ! echo "$fake_rest_curl" | grep -qiE "Connection established|200 Connection"; then + pass "M13-rest-d: curl was denied before reaching the fake Discord REST API" +elif echo "$fake_rest_curl" | grep -qiE "Connection established|200 Connection"; then + fail "M13-rest-d: curl unexpectedly established a tunnel to the fake Discord REST API" +else + fail "M13-rest-d: Fake Discord REST curl denial had unexpected shape: ${fake_rest_curl:0:300}" +fi + +fake_rest_capture="" +if [ "$fake_rest_policy_ready" = "1" ]; then + fake_rest_capture=$(fake_discord_rest_capture_counts || true) +fi +info "Fake Discord REST capture counts: ${fake_rest_capture}" +if [ "$fake_rest_policy_ready" != "1" ]; then + skip "M13-rest-e: Fake Discord REST policy unavailable; skipping capture proof" +elif echo "$fake_rest_capture" | grep -q "node=1" \ + && echo "$fake_rest_capture" | grep -q "curl=0"; then + pass "M13-rest-e: Fake server saw Node but no curl request" +else + fail "M13-rest-e: Unexpected fake Discord REST capture counts: ${fake_rest_capture}" +fi + +# M13b-M13g: Hermetic Discord Gateway over OpenShell's native WebSocket L7 path. +# M13d-config drives the fake Gateway using the generated OpenClaw managed +# proxy URL. With current OpenClaw, Discord should rely on this top-level proxy +# config instead of a NemoClaw-owned per-account loopback proxy. +fake_gateway_ready=0 +if start_fake_discord_gateway "$DISCORD_TOKEN"; then + fake_gateway_ready=1 + pass "M13b: Hermetic fake Discord Gateway started on host port ${FAKE_DISCORD_GATEWAY_PORT}" +else + fail "M13b: Failed to start hermetic fake Discord Gateway" +fi + +if [ "$fake_gateway_ready" = "1" ] \ + && apply_fake_discord_gateway_policy "$SANDBOX_NAME" "$FAKE_DISCORD_GATEWAY_PORT" >/tmp/nemoclaw-fake-discord-policy.log 2>&1; then + pass "M13c: Applied native WebSocket policy with credential rewrite for fake Discord Gateway" +else + fail "M13c: Failed to apply fake Discord Gateway policy: $(tail -20 /tmp/nemoclaw-fake-discord-policy.log 2>/dev/null | tr '\n' ' ' | cut -c1-300)" +fi + +dc_ws_config_proxy="" +managed_proxy_safe="${managed_proxy_url:-}" +if [ "$fake_gateway_ready" = "1" ] && [ -n "$managed_proxy_safe" ]; then + dc_ws_config_proxy=$(run_fake_discord_gateway_node_client "$FAKE_DISCORD_GATEWAY_PORT" "openshell:resolve:env:DISCORD_BOT_TOKEN" "$managed_proxy_safe" || true) +fi +info "OpenClaw-managed-proxy fake Discord Gateway probe: ${dc_ws_config_proxy:0:500}" + +if [ "$fake_gateway_ready" != "1" ]; then + skip "M13d-config: Fake Discord Gateway unavailable; skipping OpenClaw managed proxy proof" +elif [ -z "$managed_proxy_safe" ]; then + fail "M13d-config: No OpenClaw managed proxy URL in openclaw.json to exercise against fake Gateway" +elif echo "$dc_ws_config_proxy" | grep -q "^UPGRADE$" \ + && echo "$dc_ws_config_proxy" | grep -q "^HELLO$" \ + && echo "$dc_ws_config_proxy" | grep -q "^IDENTIFY_SENT_PLACEHOLDER$" \ + && echo "$dc_ws_config_proxy" | grep -q "^READY$" \ + && echo "$dc_ws_config_proxy" | grep -q "^HEARTBEAT_ACK$"; then + pass "M13d-config: OpenClaw managed proxy URL from openclaw.json reaches fake Gateway through OpenShell" +else + fail "M13d-config: OpenClaw managed proxy URL from openclaw.json failed against fake Gateway: ${dc_ws_config_proxy:0:400}" +fi + +dc_ws_native="" +if [ "$fake_gateway_ready" = "1" ]; then + dc_ws_native=$(run_fake_discord_gateway_node_client "$FAKE_DISCORD_GATEWAY_PORT" "openshell:resolve:env:DISCORD_BOT_TOKEN" || true) +fi +info "Native fake Discord Gateway probe: ${dc_ws_native:0:500}" + +if echo "$dc_ws_native" | grep -q "^UPGRADE$"; then + pass "M13d: Native WebSocket upgrade reached fake Discord Gateway through OpenShell" +else + fail "M13d: Native WebSocket upgrade failed: ${dc_ws_native:0:300}" +fi + +if echo "$dc_ws_native" | grep -q "^HELLO$" \ + && echo "$dc_ws_native" | grep -q "^IDENTIFY_SENT_PLACEHOLDER$" \ + && echo "$dc_ws_native" | grep -q "^READY$" \ + && echo "$dc_ws_native" | grep -q "^HEARTBEAT_ACK$"; then + pass "M13e: Discord HELLO, placeholder IDENTIFY, READY, and heartbeat ACK completed" +else + fail "M13e: Discord Gateway protocol proof incomplete: ${dc_ws_native:0:400}" +fi + +fake_gateway_capture_check="" +if [ "$fake_gateway_ready" = "1" ]; then + fake_gateway_capture_check=$(check_fake_discord_gateway_rewrite_capture "$FAKE_DISCORD_GATEWAY_CAPTURE_FILE" "$DISCORD_TOKEN" 2>&1 || true) +fi + +if [ "$fake_gateway_ready" = "1" ] && [ "$fake_gateway_capture_check" = "OK" ]; then + pass "M13f: Fake Gateway proved placeholder-to-token rewrite without logging the raw token" +else + if [ "$fake_gateway_ready" = "1" ]; then + info "Fake Discord Gateway capture check: ${fake_gateway_capture_check:0:300}" + fi + fail "M13f: Fake Gateway did not prove placeholder-to-token rewrite at the relay boundary" +fi + +capture_before_negative=0 +capture_after_negative=0 +dc_ws_negative="" +if [ "$fake_gateway_ready" = "1" ]; then + capture_before_negative=$(wc -l <"$FAKE_DISCORD_GATEWAY_CAPTURE_FILE" 2>/dev/null || echo 0) + dc_ws_negative=$(run_fake_discord_gateway_node_client "$FAKE_DISCORD_GATEWAY_PORT" "openshell:resolve:env:DEFINITELY_NOT_REGISTERED" || true) + capture_after_negative=$(wc -l <"$FAKE_DISCORD_GATEWAY_CAPTURE_FILE" 2>/dev/null || echo 0) +fi +info "Native fake Discord Gateway negative probe: ${dc_ws_negative:0:300}" + +if [ "$fake_gateway_ready" = "1" ] \ + && ! echo "$dc_ws_negative" | grep -q "^READY$" \ + && ! tail -n "$((capture_after_negative - capture_before_negative))" "$FAKE_DISCORD_GATEWAY_CAPTURE_FILE" 2>/dev/null | grep -Fq "DEFINITELY_NOT_REGISTERED"; then + pass "M13g: Unregistered Discord WebSocket placeholder is rejected before upstream token exposure" +else + fail "M13g: Unregistered Discord WebSocket placeholder reached READY or leaked upstream" +fi + +# M14 (negative): curl should be blocked by binary restriction +curl_reach=$(sandbox_exec "curl -s --max-time 10 https://api.telegram.org/ 2>&1" 2>/dev/null || true) +if echo "$curl_reach" | grep -qiE "(blocked|denied|forbidden|refused|not found|no such)"; then + pass "M14: curl to api.telegram.org blocked (binary restriction enforced)" +elif [ -z "$curl_reach" ]; then + pass "M14: curl returned empty (likely blocked by policy)" +else + # curl may not be installed in the sandbox at all + if echo "$curl_reach" | grep -qiE "(command not found|not installed)"; then + pass "M14: curl not available in sandbox (defense in depth)" + else + info "M14: curl output: ${curl_reach:0:200}" + skip "M14: Could not confirm curl is blocked (may need manual check)" + fi +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 5: L7 Proxy Token Rewriting +# ══════════════════════════════════════════════════════════════════ +section "Phase 5: L7 Proxy Token Rewriting" + +# M15-M16: Telegram getMe with placeholder token +# If proxy rewrites correctly: reaches Telegram → 401 (fake) or 200 (real) +# If proxy is broken: proxy error, timeout, or mangled URL +info "Calling api.telegram.org/bot{placeholder}/getMe from inside sandbox..." +tg_api=$(sandbox_exec 'node -e " +const https = require(\"https\"); +const token = process.env.TELEGRAM_BOT_TOKEN || \"missing\"; +const url = \"https://api.telegram.org/bot\" + token + \"/getMe\"; +const req = https.get(url, (res) => { + let body = \"\"; + res.on(\"data\", (d) => body += d); + res.on(\"end\", () => console.log(res.statusCode + \" \" + body.slice(0, 300))); +}); +req.on(\"error\", (e) => console.log(\"ERROR: \" + e.message)); +req.setTimeout(30000, () => { req.destroy(); console.log(\"TIMEOUT\"); }); +"' 2>/dev/null || true) + +info "Telegram API response: ${tg_api:0:300}" + +# Filter out Node.js warnings (e.g. UNDICI-EHPA) before extracting status code +tg_status=$(echo "$tg_api" | grep -E '^[0-9]' | head -1 | awk '{print $1}') +if [ "$tg_status" = "200" ]; then + pass "M15: Telegram getMe returned 200 — real token verified!" +elif [ "$tg_status" = "401" ] || [ "$tg_status" = "404" ]; then + # Telegram returns 404 (not 401) for invalid bot tokens in the URL path. + # Either status proves the L7 proxy rewrote the placeholder and the request + # reached the real Telegram API. + pass "M15: Telegram getMe returned $tg_status — L7 proxy rewrote placeholder (fake token rejected by API)" + pass "M16: Full chain verified: sandbox → proxy → token rewrite → Telegram API" +elif echo "$tg_api" | grep -q "TIMEOUT"; then + skip "M15: Telegram API timed out (network issue, not a plumbing failure)" +elif echo "$tg_api" | grep -qiE "ERROR:.*(ECONNRESET|reset|socket hang up|ENETUNREACH|EHOSTUNREACH|ETIMEDOUT)"; then + skip "M15: Telegram API unreachable from this network (${tg_api:0:160})" +elif echo "$tg_api" | grep -q "ERROR"; then + fail "M15: Telegram API call failed with error: ${tg_api:0:200}" +else + fail "M15: Unexpected Telegram response (status=$tg_status): ${tg_api:0:200}" +fi + +# M17: Discord users/@me with placeholder token +info "Calling discord.com/api/v10/users/@me from inside sandbox..." +dc_api=$(sandbox_exec 'node -e " +const https = require(\"https\"); +const token = process.env.DISCORD_BOT_TOKEN || \"missing\"; +const options = { + hostname: \"discord.com\", + path: \"/api/v10/users/@me\", + headers: { \"Authorization\": \"Bot \" + token }, +}; +const req = https.get(options, (res) => { + let body = \"\"; + res.on(\"data\", (d) => body += d); + res.on(\"end\", () => console.log(res.statusCode + \" \" + body.slice(0, 300))); +}); +req.on(\"error\", (e) => console.log(\"ERROR: \" + e.message)); +req.setTimeout(30000, () => { req.destroy(); console.log(\"TIMEOUT\"); }); +"' 2>/dev/null || true) + +info "Discord API response: ${dc_api:0:300}" + +# Filter out Node.js warnings (e.g. UNDICI-EHPA) before extracting status code +dc_status=$(echo "$dc_api" | grep -E '^[0-9]' | head -1 | awk '{print $1}') +if [ "$dc_status" = "200" ]; then + pass "M17: Discord users/@me returned 200 — real token verified!" +elif [ "$dc_status" = "401" ]; then + pass "M17: Discord users/@me returned 401 — L7 proxy rewrote placeholder (fake token rejected by API)" +elif echo "$dc_api" | grep -q "TIMEOUT"; then + skip "M17: Discord API timed out (network issue, not a plumbing failure)" +elif echo "$dc_api" | grep -q "ERROR"; then + fail "M17: Discord API call failed with error: ${dc_api:0:200}" +else + fail "M17: Unexpected Discord response (status=$dc_status): ${dc_api:0:200}" +fi + +# ── Slack: OpenShell alias/body rewrite chain (#2085) ───────────── +# Verifies the full chain hermetically: Bolt-shape placeholder in the +# Authorization header → OpenShell resolves the provider-shaped alias and +# substitutes the real env value → a host-side fake Slack API receives the +# resolved token and returns Slack-shaped invalid_auth. + +fake_slack_ready=0 +if start_fake_slack_api "$SLACK_TOKEN" "$SLACK_APP"; then + fake_slack_ready=1 + pass "M-S14a: Hermetic fake Slack API started on host port ${FAKE_SLACK_API_PORT}" +else + fail "M-S14a: Failed to start hermetic fake Slack API" +fi + +if [ "$fake_slack_ready" = "1" ] \ + && apply_fake_slack_api_policy "$SANDBOX_NAME" "$FAKE_SLACK_API_PORT" >/tmp/nemoclaw-fake-slack-policy.log 2>&1; then + pass "M-S14b: Applied REST policy for hermetic fake Slack API" +else + fail "M-S14b: Failed to apply fake Slack API policy: $(tail -20 /tmp/nemoclaw-fake-slack-policy.log 2>/dev/null | tr '\n' ' ' | cut -c1-300)" +fi + +check_fake_slack_capture_token() { + local path="$1" + local expected_token="$2" + node - "$FAKE_SLACK_API_CAPTURE_FILE" "$path" "$expected_token" <<'NODE' +const fs = require("fs"); +const [file, path, expectedToken] = process.argv.slice(2); +const rows = fs + .readFileSync(file, "utf8") + .trim() + .split(/\n+/) + .filter(Boolean) + .map((line) => JSON.parse(line)) + .filter((row) => row.event === "request" && row.path === path); +const last = rows.at(-1); +if (!last) { + console.log(`NO_REQUEST ${path}`); + process.exit(2); +} +if (last.authorization !== undefined || last.body !== undefined) { + console.log("RAW_CAPTURE_LEAK"); + process.exit(6); +} +if (last.tokenMatchesExpected !== true) { + console.log("BAD_AUTH_REWRITE"); + process.exit(3); +} +if (last.bodyMatchesExpected !== true) { + console.log("BAD_BODY_REWRITE"); + process.exit(4); +} +if (last.tokenLooksPlaceholder) { + console.log("PLACEHOLDER_LEAK"); + process.exit(5); +} +console.log("OK"); +NODE +} + +check_fake_slack_capture_message() { + local path="$1" + local expected_channel="$2" + local expected_text="$3" + node - "$FAKE_SLACK_API_CAPTURE_FILE" "$path" "$expected_channel" "$expected_text" <<'NODE' +const fs = require("fs"); +const [file, path, expectedChannel, expectedText] = process.argv.slice(2); +const rows = fs + .readFileSync(file, "utf8") + .trim() + .split(/\n+/) + .filter(Boolean) + .map((line) => JSON.parse(line)) + .filter((row) => row.event === "request" && row.path === path); +const last = rows.at(-1); +if (!last) { + console.log(`NO_REQUEST ${path}`); + process.exit(2); +} +if (last.channel !== expectedChannel) { + console.log(`BAD_CHANNEL ${last.channel}`); + process.exit(3); +} +if (last.text !== expectedText) { + console.log(`BAD_TEXT ${last.text}`); + process.exit(4); +} +console.log("OK"); +NODE +} + +info "Calling fake Slack /api/auth.test from inside sandbox with Bolt-shape placeholder..." +sl_api="" +if [ "$fake_slack_ready" = "1" ]; then + sl_api=$(run_fake_slack_api_node_request "$FAKE_SLACK_API_PORT" "/api/auth.test" "Bearer xoxb-OPENSHELL-RESOLVE-ENV-SLACK_BOT_TOKEN" || true) +fi + +info "Slack auth.test response: ${sl_api:0:300}" +sl_status=$(echo "$sl_api" | grep -E '^[0-9]' | head -1 | awk '{print $1}') + +if [ "$sl_status" = "200" ] && echo "$sl_api" | grep -q '"ok":true'; then + pass "M-S15: Slack auth.test returned ok:true — real token round-trip verified!" +elif [ "$sl_status" = "200" ] && echo "$sl_api" | grep -qE 'invalid_auth|not_authed'; then + pass "M-S15: Slack auth.test returned invalid_auth — full chain verified (OpenShell alias rewrite → fake Slack)" + sl_capture=$(check_fake_slack_capture_token "/api/auth.test" "$SLACK_TOKEN" || true) + if [ "$sl_capture" = "OK" ]; then + pass "M-S15a: fake Slack saw host-side bot token in header and urlencoded body" + else + fail "M-S15a: fake Slack capture did not prove bot header/body rewrite: ${sl_capture:0:300}" + fi +elif echo "$sl_api" | grep -q "TIMEOUT"; then + skip "M-S15: fake Slack API timed out" +elif echo "$sl_api" | grep -q "ERROR"; then + fail "M-S15: Slack API call failed with error: ${sl_api:0:200}" +elif echo "$sl_api" | grep -qF 'OPENSHELL-RESOLVE-ENV-'; then + fail "M-S15: OpenShell did not resolve the Bolt-shape alias" +elif echo "$sl_api" | grep -qF 'openshell:resolve:env:'; then + fail "M-S15: L7 proxy did not substitute the canonical placeholder — substitution chain broken" +else + fail "M-S15: Unexpected Slack response (status=$sl_status): ${sl_api:0:200}" +fi + +# M-S15b: L7 proxy substitution for SLACK_BOT_TOKEN, isolated from the +# alias path. Sends the canonical openshell:resolve:env:SLACK_BOT_TOKEN +# placeholder directly. If the L7 proxy substitutes correctly, the fake Slack API +# receives the host-side xoxb token and returns invalid_auth. +# +# Mirrors the proof technique already used by Telegram M15 and Discord +# M17 (they get 401/404 from the real APIs because the L7 proxy +# substituted the canonical form into a real fake-token-shape value). +info "Probing L7 proxy substitution for SLACK_BOT_TOKEN (canonical placeholder, bypasses rewriter)..." +sl_canonical="" +if [ "$fake_slack_ready" = "1" ]; then + sl_canonical=$(run_fake_slack_api_node_request "$FAKE_SLACK_API_PORT" "/api/auth.test" "Bearer openshell:resolve:env:SLACK_BOT_TOKEN" || true) +fi + +info "Slack auth.test (canonical) response: ${sl_canonical:0:300}" +sl_canon_status=$(echo "$sl_canonical" | grep -E '^[0-9]' | head -1 | awk '{print $1}') + +if [ "$sl_canon_status" = "200" ] && echo "$sl_canonical" | grep -qE 'invalid_auth|not_authed'; then + pass "M-S15b: L7 proxy substitutes openshell:resolve:env:SLACK_BOT_TOKEN at egress (parallels Telegram M15 / Discord M17)" +elif echo "$sl_canonical" | grep -q "TIMEOUT"; then + skip "M-S15b: canonical-placeholder probe timed out" +elif echo "$sl_canonical" | grep -qF 'openshell:resolve:env:' || echo "$sl_canonical" | grep -qiF 'invalid token'; then + fail "M-S15b: L7 proxy passed canonical placeholder through unchanged — substitution not happening for SLACK_BOT_TOKEN" +else + fail "M-S15b: Unexpected response (status=$sl_canon_status): ${sl_canonical:0:200}" +fi + +# M-S15c: Negative control — the env-var name in the canonical +# placeholder is not registered as a provider. The L7 proxy's response +# differs from M-S15b's "successful substitution" path, which gives us +# a positive signal that substitution happens at all. If M-S15b and +# M-S15c return identical responses, the proxy isn't substituting; if +# they differ, the proxy distinguishes set vs unset env vars (i.e., +# substitution is actually running on the substring it recognizes). +info "Probing L7 proxy substitution with an unset env var (negative control)..." +sl_unset="" +if [ "$fake_slack_ready" = "1" ]; then + sl_unset=$(run_fake_slack_api_node_request "$FAKE_SLACK_API_PORT" "/api/auth.test" "Bearer openshell:resolve:env:DEFINITELY_NOT_SET_XYZ" || true) +fi + +info "Slack auth.test (unset env) response: ${sl_unset:0:300}" +# OpenShell may reject the unresolved placeholder with an explicit +# credential_injection_failed response or a connection-level failure. +# Either shape proves the unresolved placeholder did not reach upstream. +if is_unresolved_placeholder_rejection "$sl_unset"; then + pass "M-S15c: unset-var failed closed before upstream exposure" +elif echo "$sl_unset" | grep -qE 'ERROR:.*(socket hang up|ECONNRESET|EPIPE|hang up|reset)'; then + pass "M-S15c: unset-var triggered connection-level failure — proxy refuses to forward unsubstituted placeholder" +elif echo "$sl_unset" | grep -qE '^200\b'; then + fail "M-S15c: unset-var returned HTTP 200 — proxy passed canonical placeholder through unchanged for unset env (substitution may be a no-op)" +elif echo "$sl_unset" | grep -qE '^401\b|bad_auth|DEFINITELY_NOT_SET_XYZ'; then + fail "M-S15c: unset-var request reached fake Slack — unresolved placeholder escaped the proxy boundary" +elif [ -z "$sl_unset" ] || echo "$sl_unset" | grep -q "TIMEOUT"; then + skip "M-S15c: unset-var probe timed out or returned no output" +else + skip "M-S15c: unset-var produced an unclassified result: ${sl_unset:0:200}" +fi + +# M-S16: Socket Mode HTTPS leg (apps.connections.open). Bolt's Socket +# Mode opens a websocket only after this POST succeeds, so this is the +# call that the xapp- token actually authenticates. We don't bother +# upgrading WSS in the test — the auth check is on the HTTPS POST. +info "Calling fake Slack /api/apps.connections.open with Bolt-shape xapp- placeholder..." +sl_app_api="" +if [ "$fake_slack_ready" = "1" ]; then + sl_app_api=$(run_fake_slack_api_node_request "$FAKE_SLACK_API_PORT" "/api/apps.connections.open" "Bearer xapp-OPENSHELL-RESOLVE-ENV-SLACK_APP_TOKEN" || true) +fi + +info "Slack apps.connections.open response: ${sl_app_api:0:300}" +sl_app_status=$(echo "$sl_app_api" | grep -E '^[0-9]' | head -1 | awk '{print $1}') + +if [ "$sl_app_status" = "200" ] && echo "$sl_app_api" | grep -q '"ok":true'; then + pass "M-S16: apps.connections.open returned ok:true — real xapp token round-trip verified!" +elif [ "$sl_app_status" = "200" ] && echo "$sl_app_api" | grep -qE 'invalid_auth|not_authed|not_allowed_token_type'; then + pass "M-S16: apps.connections.open auth-rejected — Socket Mode HTTPS leg verified (OpenShell alias rewrite → fake Slack)" + sl_app_capture=$(check_fake_slack_capture_token "/api/apps.connections.open" "$SLACK_APP" || true) + if [ "$sl_app_capture" = "OK" ]; then + pass "M-S16a: fake Slack saw host-side app token in header and urlencoded body" + else + fail "M-S16a: fake Slack capture did not prove app header/body rewrite: ${sl_app_capture:0:300}" + fi +elif echo "$sl_app_api" | grep -q "TIMEOUT"; then + skip "M-S16: apps.connections.open timed out" +elif echo "$sl_app_api" | grep -qF 'OPENSHELL-RESOLVE-ENV-'; then + fail "M-S16: OpenShell did not resolve the xapp- alias for Socket Mode path" +else + fail "M-S16: Unexpected apps.connections.open response (status=$sl_app_status): ${sl_app_api:0:200}" +fi + +# M-S16b: L7 proxy substitution for SLACK_APP_TOKEN, isolated. Same +# rationale as M-S15b — sends the canonical placeholder directly so only +# the L7 proxy substitution is exercised. +info "Probing L7 proxy substitution for SLACK_APP_TOKEN (canonical placeholder)..." +sl_app_canonical="" +if [ "$fake_slack_ready" = "1" ]; then + sl_app_canonical=$(run_fake_slack_api_node_request "$FAKE_SLACK_API_PORT" "/api/apps.connections.open" "Bearer openshell:resolve:env:SLACK_APP_TOKEN" || true) +fi + +info "Slack apps.connections.open (canonical) response: ${sl_app_canonical:0:300}" +sl_app_canon_status=$(echo "$sl_app_canonical" | grep -E '^[0-9]' | head -1 | awk '{print $1}') + +info "Probing L7 proxy substitution for an unset app-token env var (negative control)..." +sl_app_unset="" +if [ "$fake_slack_ready" = "1" ]; then + sl_app_unset=$(run_fake_slack_api_node_request "$FAKE_SLACK_API_PORT" "/api/apps.connections.open" "Bearer openshell:resolve:env:DEFINITELY_NOT_SET_SLACK_APP_TOKEN" || true) +fi + +info "Slack apps.connections.open (unset env) response: ${sl_app_unset:0:300}" +if [ "$sl_app_canon_status" = "200" ] && echo "$sl_app_canonical" | grep -qE 'invalid_auth|not_authed|not_allowed_token_type'; then + if is_unresolved_placeholder_rejection "$sl_app_unset"; then + pass "M-S16b: unset app-token failed closed before upstream exposure" + elif echo "$sl_app_unset" | grep -qE 'ERROR:.*(socket hang up|ECONNRESET|EPIPE|hang up|reset)'; then + pass "M-S16b: L7 proxy substitutes openshell:resolve:env:SLACK_APP_TOKEN at egress (unset-var control diverged)" + elif echo "$sl_app_unset" | grep -qE '^200\b'; then + fail "M-S16b: unset app-token env returned HTTP 200 — proxy may be passing canonical placeholders through unchanged" + elif echo "$sl_app_unset" | grep -qE '^401\b|bad_auth|DEFINITELY_NOT_SET_SLACK_APP_TOKEN'; then + fail "M-S16b: unset app-token request reached fake Slack — unresolved placeholder escaped the proxy boundary" + elif [ -z "$sl_app_unset" ] || echo "$sl_app_unset" | grep -q "TIMEOUT"; then + skip "M-S16b: unset app-token control timed out or returned no output" + else + skip "M-S16b: unset app-token control produced an unclassified result: ${sl_app_unset:0:200}" + fi +elif echo "$sl_app_canonical" | grep -q "TIMEOUT"; then + skip "M-S16b: canonical-placeholder probe timed out" +elif echo "$sl_app_canonical" | grep -qF 'openshell:resolve:env:'; then + fail "M-S16b: L7 proxy passed canonical placeholder through unchanged for SLACK_APP_TOKEN" +else + fail "M-S16b: Unexpected response (status=$sl_app_canon_status): ${sl_app_canonical:0:200}" +fi + +# M-S17: Slack channel @mention allowlist proof (#3729). This runs inside the +# sandbox, imports OpenClaw's installed Slack test API, and verifies: +# - the configured Slack user can prepare a channel app_mention +# - another user is denied by channels.*.users +# - sendMessageSlack posts back to the channel through the hermetic fake API +info "Running Slack channel @mention allowlist proof through installed OpenClaw..." +sl_channel_proof="" +sl_allowed_user="${SLACK_IDS%%,*}" +sl_allowed_user="${sl_allowed_user//[[:space:]]/}" +slack_openclaw_plugin_mock_send_ok=0 +if [ "$fake_slack_ready" = "1" ] && [ -n "$sl_allowed_user" ]; then + sl_channel_proof=$(run_fake_slack_channel_mention_proof "$FAKE_SLACK_API_PORT" "$sl_allowed_user" "U999DENIED" || true) +fi + +info "Slack channel @mention proof response: ${sl_channel_proof:0:500}" +if echo "$sl_channel_proof" | grep -q '"ok":true' \ + && echo "$sl_channel_proof" | grep -q '"deniedPrepared":true'; then + pass "M-S17: Slack channel @mention allowlist accepts configured user and denies another user" + sl_post_capture=$(check_fake_slack_capture_token "/api/chat.postMessage" "$SLACK_TOKEN" || true) + if [ "$sl_post_capture" = "OK" ]; then + pass "M-S17a: fake Slack saw host-side bot token for channel reply" + else + fail "M-S17a: fake Slack capture did not prove channel reply token rewrite: ${sl_post_capture:0:300}" + fi + sl_message_capture=$(check_fake_slack_capture_message "/api/chat.postMessage" "C0E2ESLACK" "NemoClaw Slack channel mention proof" || true) + if [ "$sl_message_capture" = "OK" ]; then + pass "M-S17b: fake Slack captured non-secret channel/text metadata for channel reply" + else + fail "M-S17b: fake Slack did not capture expected channel reply metadata: ${sl_message_capture:0:300}" + fi + sl_proof_kind=$(printf '%s\n' "$sl_channel_proof" | python3 -c ' +import json +import sys +for line in sys.stdin: + line = line.strip() + if not line.startswith("{"): + continue + try: + value = json.loads(line) + except Exception: + continue + print(value.get("proof", "")) + break +' 2>/dev/null || true) + if [ "$sl_proof_kind" = "openclaw-private-helper" ] && [ "$sl_message_capture" = "OK" ]; then + slack_openclaw_plugin_mock_send_ok=1 + pass "M-S17c: installed OpenClaw Slack send helper drove the host-side fake Slack message" + else + fail "M-S17c: Slack proof did not use the installed OpenClaw Slack send helper (proof=${sl_proof_kind:-missing})" + fi + # M-S17d (#4752): a denied explicit @-mention prepares no command but must + # still emit exactly one bounded sender-facing feedback action. + if echo "$sl_channel_proof" | grep -q '"deniedFeedbackCount":1' \ + && echo "$sl_channel_proof" | grep -q '"deniedFeedbackMethod":"chat.postEphemeral"'; then + pass "M-S17d: denied Slack @mention sent exactly one bounded sender feedback action" + else + fail "M-S17d: denied Slack @mention did not send bounded sender feedback: ${sl_channel_proof:0:500}" + fi +elif [ "$fake_slack_ready" != "1" ]; then + skip "M-S17: fake Slack API was not ready" +elif [ -z "$sl_allowed_user" ]; then + skip "M-S17: SLACK_ALLOWED_USERS is empty" +else + fail "M-S17: Slack channel @mention proof failed: ${sl_channel_proof:0:500}" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 6: OpenClaw Plugin Sends +# ══════════════════════════════════════════════════════════════════ +section "Phase 6: OpenClaw Plugin Sends" + +if [ -n "${TELEGRAM_BOT_TOKEN_REAL:-}" ] && [ -n "${TELEGRAM_CHAT_ID_E2E:-}" ]; then + info "Real Telegram token available — testing live round-trip" + + # M18: Telegram getMe with real token should return 200 + bot info + # Note: the real token must be set up as the provider credential, not as env + # For this to work, the sandbox must have been created with the real token + if [ "$tg_status" = "200" ]; then + pass "M18: Telegram getMe returned 200 with real token" + if echo "$tg_api" | grep -q '"ok":true'; then + pass "M18b: Telegram response contains ok:true" + fi + else + fail "M18: Expected Telegram getMe 200 with real token, got: $tg_status" + fi + + # M19: real send through OpenClaw's message CLI/plugin path. + info "Sending Telegram test message through OpenClaw plugin to chat ${TELEGRAM_CHAT_ID_E2E}..." + send_result=$(run_openclaw_message_send \ + "telegram" \ + "${TELEGRAM_CHAT_ID_E2E}" \ + "NemoClaw OpenClaw Telegram plugin E2E $(date -u +%Y-%m-%dT%H:%M:%SZ)" || true) + send_exit=$(printf '%s\n' "$send_result" | openclaw_message_send_exit_code) + + if [ "$send_exit" = "0" ]; then + pass "M19: Telegram openclaw message send succeeded through plugin" + else + fail "M19: Telegram openclaw message send failed: ${send_result:0:300}" + fi +else + telegram_mock_chat_id="${TELEGRAM_CHAT_ID_E2E:-42424242}" + telegram_mock_text="NemoClaw OpenClaw Telegram plugin mock E2E" + info "Complete real Telegram credentials are not available — using host-side fake Telegram Bot API" + if start_fake_telegram_api "$TELEGRAM_TOKEN"; then + pass "M18: Host-side fake Telegram Bot API started for OpenClaw plugin send" + if apply_fake_telegram_api_policy "$SANDBOX_NAME" "$FAKE_TELEGRAM_API_PORT" >/tmp/nemoclaw-fake-telegram-policy.log 2>&1; then + pass "M18a: Applied REST policy for host-side fake Telegram Bot API" + tg_mock_send_result=$(run_openclaw_telegram_mock_send "$FAKE_TELEGRAM_API_PORT" "$telegram_mock_chat_id" "$telegram_mock_text" || true) + tg_mock_send_exit=$(printf '%s\n' "$tg_mock_send_result" | openclaw_message_send_exit_code) + tg_mock_capture=$(check_fake_telegram_capture_send "$TELEGRAM_TOKEN" "$telegram_mock_chat_id" "$telegram_mock_text" || true) + + if [ "$tg_mock_send_exit" = "0" ] && [ "$tg_mock_capture" = "OK" ]; then + pass "M19: Telegram installed OpenClaw send helper posted through host mock" + elif [ "$tg_mock_send_exit" != "0" ]; then + fail "M19: Telegram OpenClaw mock helper send failed: ${tg_mock_send_result:0:300}" + else + fail "M19: Fake Telegram did not capture the expected rewritten message: ${tg_mock_capture:0:300}" + fi + else + fail "M18a: Failed to apply fake Telegram policy: $(tail -20 /tmp/nemoclaw-fake-telegram-policy.log 2>/dev/null | tr '\n' ' ' | cut -c1-300)" + fail "M19: Telegram OpenClaw mock message send could not run without fake Telegram policy" + fi + else + fail "M18: Could not start host-side fake Telegram Bot API" + fail "M19: Telegram OpenClaw mock message send could not run without fake Telegram" + fi +fi + +run_telegram_inbound_reply_probe + +if [ -n "${DISCORD_BOT_TOKEN_REAL:-}" ] && [ -n "${DISCORD_CHANNEL_ID_E2E:-}" ]; then + if [ "$dc_status" = "200" ]; then + pass "M20: Discord users/@me returned 200 with real token" + else + fail "M20: Expected Discord users/@me 200 with real token, got: $dc_status" + fi + + info "Sending Discord test message through OpenClaw plugin to channel ${DISCORD_CHANNEL_ID_E2E}..." + dc_send_result=$(run_openclaw_message_send \ + "discord" \ + "channel:${DISCORD_CHANNEL_ID_E2E}" \ + "NemoClaw OpenClaw Discord plugin E2E $(date -u +%Y-%m-%dT%H:%M:%SZ)" || true) + dc_send_exit=$(printf '%s\n' "$dc_send_result" | openclaw_message_send_exit_code) + + if [ "$dc_send_exit" = "0" ]; then + pass "M21: Discord openclaw message send succeeded through plugin" + else + fail "M21: Discord openclaw message send failed: ${dc_send_result:0:300}" + fi +else + discord_mock_channel_id="${DISCORD_CHANNEL_ID_E2E:-420000000000000123}" + discord_mock_text="NemoClaw OpenClaw Discord plugin mock E2E" + info "Complete real Discord credentials are not available — using host-side fake Discord message API" + if start_fake_discord_message_api "$DISCORD_TOKEN"; then + pass "M20: Host-side fake Discord message API started for OpenClaw plugin send" + if apply_fake_discord_message_api_policy "$SANDBOX_NAME" "$FAKE_DISCORD_MESSAGE_API_PORT" >/tmp/nemoclaw-fake-discord-message-policy.log 2>&1; then + pass "M20a: Applied REST policy for host-side fake Discord message API" + dc_mock_send_result=$(run_fake_discord_plugin_send_proof "$FAKE_DISCORD_MESSAGE_API_PORT" "$discord_mock_channel_id" "$discord_mock_text" || true) + dc_mock_capture=$(check_fake_discord_message_capture "$discord_mock_channel_id" "$discord_mock_text" || true) + + if echo "$dc_mock_send_result" | grep -q '"ok":true' && [ "$dc_mock_capture" = "OK" ]; then + pass "M21: Discord installed OpenClaw send helper posted through host mock" + elif ! echo "$dc_mock_send_result" | grep -q '"ok":true'; then + fail "M21: Discord OpenClaw mock message send failed: ${dc_mock_send_result:0:500}" + else + fail "M21: Fake Discord did not capture the expected rewritten message: ${dc_mock_capture:0:300}" + fi + else + fail "M20a: Failed to apply fake Discord message policy: $(tail -20 /tmp/nemoclaw-fake-discord-message-policy.log 2>/dev/null | tr '\n' ' ' | cut -c1-300)" + fail "M21: Discord OpenClaw mock message send could not run without fake Discord policy" + fi + else + fail "M20: Could not start host-side fake Discord message API" + fail "M21: Discord OpenClaw mock message send could not run without fake Discord" + fi +fi + +if [ -n "${SLACK_BOT_TOKEN_REAL:-}" ] && [ -n "${SLACK_CHANNEL_ID_E2E:-}" ]; then + pass "M22: Complete real Slack credentials are available for live OpenClaw send" + info "Sending Slack test message through OpenClaw plugin to channel ${SLACK_CHANNEL_ID_E2E}..." + sl_send_result=$(run_openclaw_message_send \ + "slack" \ + "channel:${SLACK_CHANNEL_ID_E2E}" \ + "NemoClaw OpenClaw Slack plugin E2E $(date -u +%Y-%m-%dT%H:%M:%SZ)" || true) + sl_send_exit=$(printf '%s\n' "$sl_send_result" | openclaw_message_send_exit_code) + + if [ "$sl_send_exit" = "0" ]; then + pass "M23: Slack openclaw message send succeeded through plugin" + else + fail "M23: Slack openclaw message send failed: ${sl_send_result:0:300}" + fi +else + info "Complete real Slack credentials are not available — requiring installed OpenClaw Slack helper proof against host fake Slack" + if [ "$slack_openclaw_plugin_mock_send_ok" = "1" ]; then + pass "M22: Slack host mock accepted the OpenShell-rewritten bot token" + pass "M23: Slack installed OpenClaw send helper posted through host mock" + else + fail "M22: Slack host mock did not prove OpenShell-rewritten bot token through installed OpenClaw helper" + fail "M23: Slack installed OpenClaw send helper did not post through host mock" + fi +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 7: Slack channel guard (#2340) +# +# The sandbox was installed with fake Slack tokens. After the +# OpenShell alias rewrite change (#2085 follow-up) the failure mode is: +# 1. Bolt accepts the xoxb-OPENSHELL-RESOLVE-ENV-… placeholder +# (matches its prefix regex). +# 2. OpenShell resolves the alias at egress. +# 3. The L7 proxy substitutes the fake xoxb-fake-… token from env. +# 4. The Slack API rejects the fake token. +# 5. @slack/web-api emits an unhandled rejection — the guard catches it. +# Pre-refactor the catch happened earlier (Bolt's in-process xapp- prefix +# check), but the observable here is the same: gateway stays up, log shows +# the guard caught a Slack rejection. +# ══════════════════════════════════════════════════════════════════ +section "Phase 7: Slack channel guard (#2340)" + +# S1: Gateway is serving on port 18789 — the guard caught the Slack rejection +gw_port=$(sandbox_exec 'node -e " +const net = require(\"net\"); +const sock = net.connect(18789, \"127.0.0.1\"); +sock.on(\"connect\", () => { console.log(\"OPEN\"); sock.end(); }); +sock.on(\"error\", () => console.log(\"CLOSED\")); +setTimeout(() => { console.log(\"TIMEOUT\"); sock.destroy(); }, 5000); +"' 2>/dev/null || true) +if echo "$gw_port" | grep -q "OPEN"; then + pass "S1: Gateway is serving on port 18789 — Slack auth failure did not crash it" +else + fail "S1: Gateway is not serving on port 18789 (${gw_port:0:200})" + # Dump early entrypoint log — captures crashes that happen before + # touch /tmp/gateway.log (e.g., Landlock read failures, seccomp blocks). + start_log=$(openshell sandbox exec --name "$SANDBOX_NAME" -- cat /tmp/nemoclaw-start.log 2>/dev/null || true) + if [ -n "$start_log" ]; then + info "Entrypoint log (last 40 lines of /tmp/nemoclaw-start.log):" + echo "$start_log" | tail -40 | while IFS= read -r line; do + info " $line" + done + fi +fi + +# S2: Dump gateway.log for diagnostics (must use openshell exec — SSH user +# cannot read the file because it's 600 gateway:gateway). +gw_log=$(openshell sandbox exec --name "$SANDBOX_NAME" -- cat /tmp/gateway.log 2>/dev/null || true) +if [ -z "$gw_log" ]; then + # Container may have already exited + gw_log=$(nemoclaw "$SANDBOX_NAME" logs 2>&1 | tail -200 || true) +fi + +info "Gateway log (last 30 lines):" +echo "$gw_log" | tail -30 | while IFS= read -r line; do + info " $line" +done + +if echo "$gw_log" | grep -q "provider failed to start:.*gateway continues"; then + pass "S2: Gateway log shows Slack rejection was caught by channel guard" +elif echo "$gw_log" | grep -qi "slack"; then + info "Slack-related lines: $(echo "$gw_log" | grep -i slack | head -5)" + skip "S2: Gateway log has Slack output but not the guard catch message" +elif [ -z "$gw_log" ]; then + skip "S2: Could not read gateway log (container may have exited)" +else + skip "S2: No Slack-related output in gateway log" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 7b: Channel runtime registry verification (#4156) +# ══════════════════════════════════════════════════════════════════ +# Asserts that the new runtime-channel diagnostic (`nemoclaw +# doctor --json` → Messaging → "Runtime channel registry") fires after +# rebuild. If the docker image was baked correctly, the diagnostic +# reports each configured channel as visible to the OpenClaw runtime; +# if the bake failed (the gap behind #4156), it reports the missing set +# instead of silently passing. +section "Phase 7b: Channel runtime registry verification (#4156)" + +doctor_json=$(nemoclaw "$SANDBOX_NAME" doctor --json 2>/dev/null || true) +if [ -z "$doctor_json" ]; then + skip "RT0: Could not collect doctor --json output" +else + runtime_check=$(echo "$doctor_json" | python3 -c " +import json, sys +try: + report = json.load(sys.stdin) +except Exception as e: + print(json.dumps({'error': str(e)})); sys.exit(0) +match = next( + (c for c in report.get('checks', []) if c.get('label') == 'Runtime channel registry'), + None, +) +print(json.dumps(match or {'missing': True})) +" 2>/dev/null || echo '{"error":"parse"}') + + if echo "$runtime_check" | grep -q '"missing"'; then + skip "RT1: doctor --json had no Runtime channel registry check (no configured channels)" + else + info "Runtime channel registry check: ${runtime_check:0:300}" + rt_status=$(echo "$runtime_check" | python3 -c "import json,sys; print(json.load(sys.stdin).get('status',''))" 2>/dev/null || echo "") + if [ "$rt_status" = "ok" ]; then + pass "RT1: doctor reports configured channels are visible to OpenClaw runtime registry" + elif [ "$rt_status" = "warn" ]; then + # A warn is still a pass for this E2E: it means the diagnostic detected + # the very gap #4156 closes (e.g. a channel configured but absent from + # /sandbox/.openclaw/openclaw.json after rebuild). The detail field + # surfaces which channels are missing so the suite output stays useful. + pass "RT1: doctor surfaced runtime channel registry warning (detail: $(echo "$runtime_check" | python3 -c "import json,sys; print(json.load(sys.stdin).get('detail',''))"))" + else + fail "RT1: Unexpected Runtime channel registry status '$rt_status' (raw: ${runtime_check:0:300})" + fi + fi +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 8: Cleanup +# ══════════════════════════════════════════════════════════════════ +section "Phase 8: Cleanup" + +info "Destroying sandbox '$SANDBOX_NAME'..." +if [[ "${NEMOCLAW_E2E_KEEP_SANDBOX:-}" = "1" ]]; then + skip "Cleanup: NEMOCLAW_E2E_KEEP_SANDBOX=1 — leaving sandbox '$SANDBOX_NAME' for inspection" +else + nemoclaw "$SANDBOX_NAME" destroy --yes 2>/dev/null || true + openshell sandbox delete "$SANDBOX_NAME" 2>/dev/null || true +fi + +# Verify cleanup +if [[ "${NEMOCLAW_E2E_KEEP_SANDBOX:-}" = "1" ]]; then + pass "Cleanup: Sandbox '$SANDBOX_NAME' intentionally kept" +elif openshell sandbox list 2>&1 | grep -q "$SANDBOX_NAME"; then + fail "Cleanup: Sandbox '$SANDBOX_NAME' still present after cleanup" +else + pass "Cleanup: Sandbox '$SANDBOX_NAME' removed" +fi + +# ══════════════════════════════════════════════════════════════════ +# Summary +# ══════════════════════════════════════════════════════════════════ +echo "" +echo "========================================" +echo " Messaging Provider Test Results:" +echo " Passed: $PASS" +echo " Failed: $FAIL" +echo " Skipped: $SKIP" +echo " Total: $TOTAL" +echo "========================================" + +if [ "$FAIL" -eq 0 ]; then + printf '\n\033[1;32m Messaging provider tests PASSED.\033[0m\n' + exit 0 +else + printf '\n\033[1;31m %d test(s) FAILED.\033[0m\n' "$FAIL" + exit 1 +fi diff --git a/test/e2e-vpn/test-model-router-provider-routed-inference.sh b/test/e2e-vpn/test-model-router-provider-routed-inference.sh new file mode 100755 index 00000000000..ee068600ce4 --- /dev/null +++ b/test/e2e-vpn/test-model-router-provider-routed-inference.sh @@ -0,0 +1,196 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Coverage guard for #3255 — Model Router (Provider Routed) onboard must +# produce a working inference.local route instead of HTTP 503. + +set -uo pipefail + +PASS=0 +FAIL=0 +TOTAL=0 + +pass() { + ((PASS++)) + ((TOTAL++)) + echo " OK: $1" +} +fail() { + ((FAIL++)) + ((TOTAL++)) + echo " ERROR: $1" +} +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } + +is_routed_pong_response() { + local raw="$1" + python3 - "$raw" <<'PY' +import json, re, sys +raw = sys.argv[1] +try: + data = json.loads(raw) +except Exception: + raise SystemExit(1) +model = str(data.get("model", "")) +choices = data.get("choices") or [] +content = "" +if choices and isinstance(choices[0], dict): + message = choices[0].get("message") or {} + content = str(message.get("content", "")) +ok_model = model == "nvidia-routed" or model.startswith("nvidia-routed") +ok_content = re.search(r"\bPONG\b", content, re.IGNORECASE) is not None +raise SystemExit(0 if ok_model and ok_content else 1) +PY +} + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO="$(cd "${SCRIPT_DIR}/../.." && pwd)" +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-model-router}" +ONBOARD_LOG="${E2E_MODEL_ROUTER_ONBOARD_LOG:-/tmp/nemoclaw-e2e-model-router-onboard.log}" +RESPONSE_LOG="${E2E_MODEL_ROUTER_RESPONSE_LOG:-/tmp/nemoclaw-e2e-model-router-response.log}" +HEALTH_LOG="${E2E_MODEL_ROUTER_HEALTH_LOG:-/tmp/nemoclaw-e2e-model-router-health.log}" +TIMEOUT_CMD="${TIMEOUT_CMD:-timeout}" + +# shellcheck source=test/e2e-vpn/lib/install-path-refresh.sh +. "${SCRIPT_DIR}/lib/install-path-refresh.sh" +# shellcheck source=test/e2e-vpn/lib/sandbox-teardown.sh +. "${SCRIPT_DIR}/lib/sandbox-teardown.sh" +register_sandbox_for_teardown "$SANDBOX_NAME" + +redact_file() { + local file="$1" + [ -f "$file" ] || return 0 + python3 - "$file" <<'PY' +import os, sys +path = sys.argv[1] +secrets = [os.environ.get("NVIDIA_API_KEY", ""), os.environ.get("NEMOCLAW_PROVIDER_KEY", "")] +text = open(path, "r", errors="replace").read() +for secret in filter(None, secrets): + text = text.replace(secret, "") +open(path, "w").write(text) +PY +} + +# shellcheck disable=SC2317,SC2329 # Invoked indirectly by the EXIT trap. +cleanup() { + local rc=$? + redact_file "$ONBOARD_LOG" + redact_file "$RESPONSE_LOG" + redact_file "$HEALTH_LOG" + if [ "${NEMOCLAW_E2E_KEEP_SANDBOX:-0}" != "1" ]; then + nemoclaw "$SANDBOX_NAME" destroy --yes >/dev/null 2>&1 || true + fi + exit "$rc" +} +trap cleanup EXIT # invoked by EXIT trap + +section "Prerequisites" +if docker info >/dev/null 2>&1; then + pass "Docker is running" +else + fail "Docker is not running" + exit 1 +fi + +if [ -n "${NVIDIA_API_KEY:-}" ] && [[ "${NVIDIA_API_KEY}" == nvapi-* ]]; then + pass "NVIDIA_API_KEY is set" +else + fail "NVIDIA_API_KEY is required and must start with nvapi-" + exit 1 +fi + +section "Install NemoClaw from checkout" +if ! command -v nemoclaw >/dev/null 2>&1; then + NEMOCLAW_NON_INTERACTIVE=1 \ + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ + bash "${REPO}/install.sh" --non-interactive --yes-i-accept-third-party-software >"$ONBOARD_LOG" 2>&1 || true + nemoclaw_refresh_install_env +fi + +if command -v nemoclaw >/dev/null 2>&1; then + pass "nemoclaw is available: $(nemoclaw --version 2>/dev/null || echo unknown)" +else + fail "nemoclaw not found after install" + exit 1 +fi + +section "Onboard with Model Router provider" +rm -f "$HOME/.nemoclaw/onboard.lock" 2>/dev/null || true +nemoclaw "$SANDBOX_NAME" destroy --yes >/dev/null 2>&1 || true + +env \ + NEMOCLAW_PROVIDER_KEY="$NVIDIA_API_KEY" \ + NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" \ + NEMOCLAW_NON_INTERACTIVE=1 \ + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ + NEMOCLAW_POLICY_TIER="open" \ + NEMOCLAW_PROVIDER="routed" \ + NVIDIA_API_KEY="$NVIDIA_API_KEY" \ + "$TIMEOUT_CMD" 1500 nemoclaw onboard --fresh --non-interactive --yes-i-accept-third-party-software \ + >"$ONBOARD_LOG" 2>&1 +onboard_rc=$? +redact_file "$ONBOARD_LOG" +if [ "$onboard_rc" -eq 0 ]; then + pass "Model Router onboard completed" +else + fail "Model Router onboard failed (exit ${onboard_rc}); see ${ONBOARD_LOG}" + exit 1 +fi + +section "Host model-router health" +health="" +for _ in $(seq 1 20); do + health="$(curl -s --max-time 10 http://127.0.0.1:4000/health 2>&1 || true)" + printf '%s\n' "$health" >"$HEALTH_LOG" + redact_file "$HEALTH_LOG" + if echo "$health" | grep -Eq '"healthy_count"[[:space:]]*:[[:space:]]*[1-9]'; then + pass "model-router reports at least one healthy endpoint" + break + fi + sleep 3 +done +if ! echo "$health" | grep -Eq '"healthy_count"[[:space:]]*:[[:space:]]*[1-9]'; then + fail "model-router has no healthy endpoints; expected #3255 main-equivalent failure" + info "Health excerpt: $(head -c 500 "$HEALTH_LOG")" + exit 1 +fi + +section "Sandbox inference.local routed completion" +response="" +for _ in $(seq 1 3); do + response="$(openshell sandbox exec --name "$SANDBOX_NAME" -- \ + curl -sk --max-time 90 https://inference.local/v1/chat/completions \ + -H 'Content-Type: application/json' \ + -d '{"model":"nvidia-routed","messages":[{"role":"user","content":"Reply with exactly one word: PONG"}],"max_tokens":50}' \ + 2>&1 || true)" + printf '%s\n' "$response" >"$RESPONSE_LOG" + redact_file "$RESPONSE_LOG" + if is_routed_pong_response "$response"; then + pass "inference.local returned a routed Model Router completion" + break + fi + if echo "$response" | grep -qi 'inference service unavailable\|HTTP 503\|healthy_count.*0'; then + break + fi + sleep 5 +done + +if is_routed_pong_response "$response"; then + : +else + fail "Model Router inference.local did not return a routed completion; expected #3255 main-equivalent failure" + info "Response excerpt: $(head -c 500 "$RESPONSE_LOG")" + exit 1 +fi + +section "Summary" +if [ "$FAIL" -eq 0 ]; then + pass "Model Router provider-routed inference guard passed" + exit 0 +fi +exit 1 diff --git a/test/e2e-vpn/test-network-policy.sh b/test/e2e-vpn/test-network-policy.sh new file mode 100755 index 00000000000..3ccc474e86b --- /dev/null +++ b/test/e2e-vpn/test-network-policy.sh @@ -0,0 +1,1136 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# ============================================================================= +# test-network-policy.sh +# NemoClaw Network Policy E2E Tests +# +# Covers: +# TC-NET-01: Deny-by-default egress (blocked URL returns 403) +# TC-NET-02: Whitelisted endpoint access (PyPI reachable via curl GET; POST blocked) +# TC-NET-03: Live policy-add without restart (slack preset) +# TC-NET-04: policy-add --dry-run (no changes applied) +# TC-NET-05: Hot-reload (policy change without sandbox restart) +# TC-NET-06: Permissive policy mode (open all egress) +# TC-NET-07: Inference exemption + direct provider blocked +# TC-NET-08: Jira per-binary policy enforcement +# TC-NET-09: SSRF validation (dangerous IPs rejected) +# TC-NET-10: OpenClaw web_fetch can reach approved host gateway target, +# while OpenShell still denies unapproved host gateway ports +# TC-NET-11: Homebrew preset installs and runs a formula end-to-end +# +# Prerequisites: +# - Docker running +# - NemoClaw installed (or install.sh available) +# - NVIDIA_API_KEY for sandbox onboard +# ============================================================================= + +set -euo pipefail + +# ── Overall timeout ────────────────────────────────────────────────────────── +export NEMOCLAW_E2E_DEFAULT_TIMEOUT=3600 +SCRIPT_DIR_TIMEOUT="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +# shellcheck source=test/e2e-vpn/e2e-timeout.sh +source "${SCRIPT_DIR_TIMEOUT}/e2e-timeout.sh" +# shellcheck source=test/e2e-vpn/lib/install-path-refresh.sh +source "${SCRIPT_DIR_TIMEOUT}/lib/install-path-refresh.sh" +# ── Config ─────────────────────────────────────────────────────────────────── +SANDBOX_NAME="e2e-net-policy" +LOG_FILE="test-network-policy-$(date +%Y%m%d-%H%M%S).log" +SANDBOX_EXEC_TIMEOUT_SECONDS=120 +PACKAGE_MANAGER_SANDBOX_TIMEOUT_SECONDS=300 + +# ── Colors ─────────────────────────────────────────────────────────────────── +GREEN='\033[0;32m' +RED='\033[0;31m' +YELLOW='\033[1;33m' +CYAN='\033[0;36m' +NC='\033[0m' + +PASS=0 +FAIL=0 +SKIP=0 +TOTAL=0 + +# Log a timestamped message to stdout and the log file. +log() { echo -e "${CYAN}[$(date +%H:%M:%S)]${NC} $*" | tee -a "$LOG_FILE"; } +# Record a passing test assertion. +pass() { + ((PASS += 1)) + ((TOTAL += 1)) + echo -e "${GREEN} PASS${NC} $1" | tee -a "$LOG_FILE" +} +# Record a failing test assertion with a reason. +fail() { + ((FAIL += 1)) + ((TOTAL += 1)) + echo -e "${RED} FAIL${NC} $1 — $2" | tee -a "$LOG_FILE" +} +# Record a skipped test with a reason. +skip() { + ((SKIP += 1)) + ((TOTAL += 1)) + echo -e "${YELLOW} SKIP${NC} $1 — $2" | tee -a "$LOG_FILE" +} + +# ── Resolve repo root ──────────────────────────────────────────────────────── +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" + +# ── Install NemoClaw if not present ────────────────────────────────────────── +install_nemoclaw() { + export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" + if [ -s "$NVM_DIR/nvm.sh" ]; then + # shellcheck source=/dev/null + . "$NVM_DIR/nvm.sh" + fi + nemoclaw_ensure_local_bin_on_path + + if command -v nemoclaw >/dev/null 2>&1; then + log "nemoclaw already installed: $(nemoclaw --version 2>/dev/null || echo unknown)" + return + fi + log "=== Installing NemoClaw via install.sh ===" + NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" \ + NVIDIA_API_KEY="${NVIDIA_API_KEY:-nvapi-DUMMY-FOR-INSTALL}" \ + NEMOCLAW_NON_INTERACTIVE=1 \ + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ + NEMOCLAW_POLICY_TIER="restricted" \ + bash "$REPO_ROOT/install.sh" --non-interactive --yes-i-accept-third-party-software \ + 2>&1 | tee -a "$LOG_FILE" + nemoclaw_refresh_install_env + if ! command -v nemoclaw >/dev/null 2>&1; then + log "ERROR: install.sh failed — nemoclaw not found" + exit 1 + fi +} + +# ── Pre-flight ─────────────────────────────────────────────────────────────── +preflight() { + log "=== Pre-flight checks ===" + if ! docker info >/dev/null 2>&1; then + log "ERROR: Docker is not running." + exit 1 + fi + log "Docker is running" + install_nemoclaw + if ! command -v expect >/dev/null 2>&1; then + log "Installing expect..." + if ! (sudo apt-get update -qq && sudo apt-get install -y -qq expect >/dev/null 2>&1); then + log "WARNING: failed to install expect — interactive tests will skip" + fi + if ! command -v expect >/dev/null 2>&1; then + log "WARNING: expect not available — interactive tests will skip" + fi + fi + if ! command -v python3 >/dev/null 2>&1; then + log "ERROR: python3 is required for JSON parsing" + exit 1 + fi + log "nemoclaw: $(nemoclaw --version 2>/dev/null || echo unknown)" + log "python3: $(python3 --version 2>/dev/null || echo unknown)" + log "Pre-flight complete" +} + +# Apply a network policy preset by name (non-interactive). +apply_preset() { + local preset_name="$1" + log " Applying preset '$preset_name' (non-interactive)..." + local exit_code=0 + nemoclaw "$SANDBOX_NAME" policy-add "$preset_name" --yes 2>&1 | tee -a "$LOG_FILE" || exit_code=$? + sleep 3 + return "$exit_code" +} + +# Apply a network policy preset via interactive prompts using expect. +apply_preset_interactive() { + local preset_name="$1" + if ! command -v expect >/dev/null 2>&1; then + log " expect not available — cannot test interactive mode" + return 2 + fi + local preset_list preset_num + preset_list=$(NEMOCLAW_NON_INTERACTIVE='' nemoclaw "$SANDBOX_NAME" policy-add &1) || true + preset_num=$(echo "$preset_list" | grep -oE '[0-9]+\).*'"$preset_name" | grep -oE '^[0-9]+') || true + if [[ -z "$preset_num" ]]; then + log " Could not find '$preset_name' in interactive preset list" + return 1 + fi + log " Applying preset '$preset_name' (#$preset_num) via interactive expect..." + local exit_code=0 + set +e + NEMOCLAW_NON_INTERACTIVE='' expect <&1 | tee -a "$LOG_FILE" +set timeout 30 +spawn env NEMOCLAW_NON_INTERACTIVE= nemoclaw $SANDBOX_NAME policy-add +expect "Choose preset*" +send "$preset_num\r" +expect "*Y/n*" +send "Y\r" +expect eof +EOF + exit_code=${PIPESTATUS[0]} + set -e + sleep 3 + return "$exit_code" +} + +# Execute a command inside the sandbox via SSH. +sandbox_exec() { + local cmd="$1" + local timeout_seconds="${2:-$SANDBOX_EXEC_TIMEOUT_SECONDS}" + local ssh_cfg + ssh_cfg="$(mktemp)" + if ! openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_cfg" 2>/dev/null; then + log " [sandbox_exec] Failed to get SSH config" + rm -f "$ssh_cfg" + echo "" + return 1 + fi + local result ssh_exit=0 + result=$(run_with_timeout "$timeout_seconds" ssh -F "$ssh_cfg" \ + -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" "$cmd" 2>&1) || ssh_exit=$? + rm -f "$ssh_cfg" + echo "$result" + return $ssh_exit +} + +start_e2e_http_server() { + local docroot="$1" + local port_file="$2" + local log_file="$3" + python3 - "$docroot" "$port_file" >"$log_file" 2>&1 <<'PYHTTP' & +import functools +import http.server +import socketserver +import sys + +docroot = sys.argv[1] +port_file = sys.argv[2] + +class ReusableTCPServer(socketserver.TCPServer): + allow_reuse_address = True + +handler = functools.partial(http.server.SimpleHTTPRequestHandler, directory=docroot) +with ReusableTCPServer(("0.0.0.0", 0), handler) as server: + with open(port_file, "w", encoding="utf-8") as handle: + handle.write(str(server.server_address[1])) + handle.flush() + print(f"serving {docroot} on port {server.server_address[1]}", flush=True) + server.serve_forever() +PYHTTP + echo "$!" +} + +wait_for_e2e_http_port() { + local port_file="$1" + local pid="$2" + local _ + for _ in {1..50}; do + if [ -s "$port_file" ]; then + tr -d '[:space:]' <"$port_file" + return 0 + fi + if ! kill -0 "$pid" 2>/dev/null; then + return 1 + fi + sleep 0.1 + done + return 1 +} + +# ── Onboard sandbox ───────────────────────────────────────────────────────── +setup_sandbox() { + local api_key="${NVIDIA_API_KEY:-}" + if [[ -z "$api_key" ]]; then + log "ERROR: NVIDIA_API_KEY not set" + exit 1 + fi + + # Unconditional destroy — `nemoclaw list` does not always surface sandboxes + # stuck in a not-ready state, and a not-ready sandbox blocks onboard with + # "already exists but is not ready" before NEMOCLAW_RECREATE_SANDBOX=1 kicks in. + log "Preflight: destroying any existing '$SANDBOX_NAME' sandbox..." + nemoclaw "$SANDBOX_NAME" destroy --yes 2>/dev/null || true + + log "=== Onboarding sandbox '$SANDBOX_NAME' with restricted policy ===" + rm -f "$HOME/.nemoclaw/onboard.lock" 2>/dev/null || true + NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" \ + NEMOCLAW_NON_INTERACTIVE=1 \ + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ + NEMOCLAW_POLICY_TIER="restricted" \ + NEMOCLAW_WEB_SEARCH_ENABLED=1 \ + NEMOCLAW_RECREATE_SANDBOX=1 \ + run_with_timeout 600 nemoclaw onboard --non-interactive --yes-i-accept-third-party-software \ + 2>&1 | tee -a "$LOG_FILE" || { + log "FATAL: Onboard failed" + exit 1 + } + log "Sandbox '$SANDBOX_NAME' onboarded with restricted policy" +} + +# ============================================================================= +# TC-NET-01: Deny-by-default egress +# ============================================================================= +test_net_01_deny_default() { + log "=== TC-NET-01: Deny-by-Default Egress ===" + + local blocked_url="https://example.com/" + log " Probing blocked URL from inside sandbox: $blocked_url" + + local response + response=$(sandbox_exec "node -e \" +fetch('$blocked_url', {signal: AbortSignal.timeout(15000)}) + .then(r => console.log('STATUS_' + r.status)) + .catch(e => console.log('ERROR_' + (e.cause?.code || e.code || e.message))) +\"" 2>&1) || true + + log " Response: $response" + + if echo "$response" | grep -qE "STATUS_403|ERROR_"; then + pass "TC-NET-01: Non-whitelisted URL blocked ($response)" + elif echo "$response" | grep -qE "STATUS_2"; then + fail "TC-NET-01: Deny default" "Non-whitelisted URL returned success ($response)" + else + fail "TC-NET-01: Deny default" "Unexpected response ($response)" + fi +} + +# ============================================================================= +# TC-NET-02: Whitelisted endpoint access +# ============================================================================= +test_net_02_whitelist_access() { + log "=== TC-NET-02: Whitelisted Endpoint Access ===" + + log " Adding pypi preset for whitelist test..." + if ! apply_preset "pypi"; then + fail "TC-NET-02: Setup" "Could not apply pypi preset" + return + fi + + log " Probing PyPI read-only access from inside sandbox using curl..." + + local pypi_code + pypi_code=$(sandbox_exec "curl -sS -o /dev/null -w '%{http_code}' --max-time 20 https://pypi.org/simple/requests/ 2>&1" 2>&1) || true + log " pypi.org GET status: $pypi_code" + + if [ "$pypi_code" = "200" ]; then + pass "TC-NET-02: pypi.org reachable via curl GET after preset applied" + else + fail "TC-NET-02: Whitelist" "curl GET to pypi.org did not return 200: ${pypi_code:0:200}" + fi + + # Use a real PyPI artifact instead of a placeholder path. The placeholder + # can legitimately return 404, which proves egress but is easy to misread as + # a failed GET probe when QA verifies the case manually. + local files_code + files_code=$(sandbox_exec "curl -LsS -o /dev/null -w '%{http_code}' --max-time 20 https://files.pythonhosted.org/packages/source/r/requests/requests-2.32.5.tar.gz 2>&1" 2>&1) || true + log " files.pythonhosted.org GET status: $files_code" + + if echo "$files_code" | grep -qE "^[23][0-9][0-9]$"; then + pass "TC-NET-02: files.pythonhosted.org artifact reachable via curl GET" + else + fail "TC-NET-02: Whitelist" "curl GET to files.pythonhosted.org artifact did not return 2xx/3xx: ${files_code:0:200}" + fi + + local post_code + post_code=$(sandbox_exec "curl -sS -o /dev/null -w '%{http_code}' -X POST --max-time 20 https://pypi.org/simple/le/ 2>&1" 2>&1) || true + log " pypi.org POST status: $post_code" + + if [ "$post_code" = "403" ]; then + pass "TC-NET-02: PyPI POST remains blocked under read-only preset" + else + fail "TC-NET-02: Whitelist" "curl POST to pypi.org should remain blocked with 403: ${post_code:0:200}" + fi + + # #4014 validates network-policy egress only. Keep pip as a log-only + # diagnostic so package-manager behavior cannot fail this regression. + log " Optional diagnostic: probing PyPI from inside sandbox using pip..." + + local response + response=$(sandbox_exec "rm -rf /tmp/pip-test && pip download --no-deps --no-cache-dir --dest /tmp/pip-test requests 2>&1 && echo PIP_OK || echo PIP_FAIL" 2>&1) || true + + log " pip diagnostic response: ${response:0:300}" + + if echo "$response" | grep -q "PIP_OK"; then + log " pip diagnostic succeeded after pypi preset was applied" + elif echo "$response" | grep -qiE "Downloading|Successfully"; then + log " pip diagnostic reached PyPI after pypi preset was applied" + else + log " pip diagnostic did not succeed; ignoring for #4014 because curl egress checks are authoritative: ${response:0:200}" + fi +} + +# ============================================================================= +# TC-NET-11: Homebrew preset install/use path +# ============================================================================= +test_net_11_brew_install_hello() { + log "=== TC-NET-11: Homebrew Preset Installs and Runs hello ===" + + log " Adding brew preset for Homebrew formula install test..." + if ! apply_preset "brew"; then + fail "TC-NET-11: Setup" "Could not apply brew preset" + return + fi + + local policy_list + if ! policy_list=$(nemoclaw "$SANDBOX_NAME" policy-list 2>&1); then + fail "TC-NET-11: policy-list" "policy-list failed after brew preset: ${policy_list:0:500}" + return + fi + log " policy-list: ${policy_list:0:600}" + if printf '%s\n' "$policy_list" | grep -E "^[[:space:]]*●[[:space:]]+brew[[:space:]]" >/dev/null; then + pass "TC-NET-11: policy-list shows brew applied" + else + fail "TC-NET-11: policy-list" "brew preset not marked applied: ${policy_list:0:500}" + return + fi + + local connect_probe connect_rc=0 + connect_probe=$(run_with_timeout 60 nemoclaw "$SANDBOX_NAME" connect --probe-only 2>&1) || connect_rc=$? + log " connect --probe-only: ${connect_probe:0:500}" + if [[ $connect_rc -eq 0 ]]; then + pass "TC-NET-11: nemoclaw connect --probe-only reaches sandbox" + else + fail "TC-NET-11: connect --probe-only" "connect probe failed: ${connect_probe:0:500}" + return + fi + + log " Probing Homebrew policy endpoints and installing hello through the wrapper..." + local brew_probe_script brew_probe_b64 response + brew_probe_script="$( + cat <<'BREW_PROBE' +set -euo pipefail +export HOMEBREW_NO_AUTO_UPDATE=1 +export HOMEBREW_NO_ENV_HINTS=1 + +check_status() { + local name="$1" + local url="$2" + local status + status=$(curl -sS -o /dev/null -w "%{http_code}" --connect-timeout 10 --max-time 30 "$url") || { + echo "BREW_ENDPOINT_${name}_CURL_FAILED" + return 1 + } + case "$status" in + 2??|3??|401) + echo "BREW_ENDPOINT_${name}_OK_${status}" + ;; + *) + echo "BREW_ENDPOINT_${name}_BAD_${status}" + return 1 + ;; + esac +} + +check_status formulae https://formulae.brew.sh +check_status raw https://raw.githubusercontent.com/Homebrew/brew/HEAD/README.md +git ls-remote https://github.com/Homebrew/brew.git HEAD >/dev/null +echo "BREW_ENDPOINT_github_OK" +check_status ghcr https://ghcr.io/v2/ + +command -v brew +brew --prefix +brew install --quiet hello +command -v hello +hello +BREW_PROBE + )" + brew_probe_b64="$(printf '%s' "$brew_probe_script" | base64 | tr -d '\n')" + response=$(sandbox_exec "printf '%s' '${brew_probe_b64}' | base64 -d > /tmp/nemoclaw-brew-e2e.sh +bash /tmp/nemoclaw-brew-e2e.sh" "$PACKAGE_MANAGER_SANDBOX_TIMEOUT_SECONDS" 2>&1) || true + + log " Response: ${response:0:1000}" + + if echo "$response" | grep -q "BREW_ENDPOINT_formulae_OK_" \ + && echo "$response" | grep -q "BREW_ENDPOINT_raw_OK_" \ + && echo "$response" | grep -q "BREW_ENDPOINT_github_OK" \ + && echo "$response" | grep -q "BREW_ENDPOINT_ghcr_OK_" \ + && echo "$response" | grep -q "/usr/local/bin/brew" \ + && echo "$response" | grep -q "/home/linuxbrew/.linuxbrew" \ + && echo "$response" | grep -q "/home/linuxbrew/.linuxbrew/bin/hello" \ + && echo "$response" | grep -q "Hello, world!"; then + pass "TC-NET-11: brew preset installed hello and ran the formula command" + else + fail "TC-NET-11: Homebrew install" "brew install/use path failed: ${response:0:500}" + fi +} + +# ============================================================================= +# TC-NET-03: Live policy-add without restart +# ============================================================================= +test_net_03_live_policy_add() { + log "=== TC-NET-03: Live Policy-Add Without Restart ===" + + local target_url="https://slack.com/" + + log " Step 1: Verify slack.com is blocked before policy-add..." + local before + before=$(sandbox_exec "node -e \" +fetch('$target_url', {signal: AbortSignal.timeout(15000)}) + .then(r => console.log('STATUS_' + r.status)) + .catch(e => console.log('ERROR_' + (e.cause?.code || e.code || e.message))) +\"" 2>&1) || true + log " Before policy-add: $before" + + if echo "$before" | grep -qE "STATUS_[23][0-9][0-9]"; then + skip "TC-NET-03" "slack.com already reachable before policy-add (preset may be pre-applied)" + return + fi + + log " Step 2: Adding slack preset (interactive mode)..." + local interactive_rc=0 + apply_preset_interactive "slack" || interactive_rc=$? + if [[ $interactive_rc -eq 2 ]]; then + log " Interactive mode unavailable (expect missing) — falling back to non-interactive..." + if ! apply_preset "slack"; then + fail "TC-NET-03: Setup" "Could not apply slack preset" + return + fi + elif [[ $interactive_rc -ne 0 ]]; then + fail "TC-NET-03: Interactive policy-add" "interactive flow failed (exit $interactive_rc)" + return + fi + + sleep 5 + + log " Step 3: Verify slack.com is reachable after policy-add..." + local after + after=$(sandbox_exec "node -e \" +fetch('$target_url', {signal: AbortSignal.timeout(30000)}) + .then(r => console.log('STATUS_' + r.status)) + .catch(e => console.log('ERROR_' + (e.cause?.code || e.code || e.message))) +\"" 2>&1) || true + log " After policy-add: $after" + + if echo "$after" | grep -qE "STATUS_[2-4][0-9][0-9]"; then + pass "TC-NET-03: Endpoint reachable after live policy-add ($after)" + elif echo "$after" | grep -qE "ERROR_"; then + fail "TC-NET-03: Live policy-add" "slack.com still proxy-blocked after policy-add ($after)" + else + fail "TC-NET-03: Live policy-add" "Unexpected response after policy-add ($after)" + fi +} + +# ============================================================================= +# TC-NET-04: policy-add --dry-run +# ============================================================================= +test_net_04_dry_run() { + log "=== TC-NET-04: Policy-Add --dry-run ===" + + local target_url="https://api.atlassian.com/" + + log " Step 1: Verify api.atlassian.com is blocked..." + local before + before=$(sandbox_exec "node -e \" +fetch('$target_url', {signal: AbortSignal.timeout(15000)}) + .then(r => console.log('STATUS_' + r.status)) + .catch(e => console.log('ERROR_' + (e.cause?.code || e.code || e.message))) +\"" 2>&1) || true + log " Before dry-run: $before" + + log " Step 2: Running policy-add --dry-run jira..." + local dry_output dry_rc=0 + dry_output=$(nemoclaw "$SANDBOX_NAME" policy-add jira --dry-run 2>&1) || dry_rc=$? + log " Dry-run output (exit $dry_rc): ${dry_output:0:300}" + + if [[ $dry_rc -eq 0 ]] && echo "$dry_output" | grep -qiE "atlassian|would be opened"; then + pass "TC-NET-04: Dry-run printed endpoint info" + else + fail "TC-NET-04: Dry-run output" "Expected endpoint info in output: ${dry_output:0:200}" + fi + + log " Step 3: Verify api.atlassian.com is still blocked after dry-run..." + local after + after=$(sandbox_exec "node -e \" +fetch('$target_url', {signal: AbortSignal.timeout(15000)}) + .then(r => console.log('STATUS_' + r.status)) + .catch(e => console.log('ERROR_' + (e.cause?.code || e.code || e.message))) +\"" 2>&1) || true + log " After dry-run: $after" + + if echo "$after" | grep -qE "STATUS_403|ERROR_"; then + pass "TC-NET-04: Policy unchanged after dry-run (blocked: $after)" + elif echo "$after" | grep -qE "STATUS_[23]"; then + fail "TC-NET-04: Dry-run side effect" "api.atlassian.com reachable after dry-run (policy was modified)" + else + fail "TC-NET-04: Dry-run verification" "Unexpected response ($after)" + fi +} + +# ============================================================================= +# TC-NET-08: Jira per-binary policy enforcement +# ============================================================================= +test_net_08_jira_per_binary_enforcement() { + log "=== TC-NET-08: Jira Per-Binary Policy Enforcement ===" + local curl_probe_url="https://api.atlassian.com/oauth/token/accessible-resources" + + log " Step 1: Applying jira preset..." + if ! apply_preset "jira"; then + fail "TC-NET-08: Setup" "Could not apply jira preset" + return + fi + + log " Step 2: Verify Node HTTPS can reach Atlassian API..." + local node_response + node_response=$(sandbox_exec "node -e \" +const https = require('https'); +const req = https.get('https://api.atlassian.com', (res) => { + console.log('NODE_STATUS_' + res.statusCode); + res.resume(); +}); +req.setTimeout(30000, () => { + console.log('NODE_ERROR_TIMEOUT'); + req.destroy(); +}); +req.on('error', (error) => console.log('NODE_ERROR_' + (error.code || error.message))); +\"" 2>&1) || true + log " Node response: $node_response" + + if echo "$node_response" | grep -qE "NODE_STATUS_[23][0-9][0-9]"; then + pass "TC-NET-08: Node reaches Atlassian API after jira preset ($node_response)" + elif echo "$node_response" | grep -qE "NODE_STATUS_403|NODE_ERROR_"; then + fail "TC-NET-08: Node policy" "Node did not reach Atlassian API after jira preset ($node_response)" + return + else + fail "TC-NET-08: Node policy" "Unexpected Node response ($node_response)" + return + fi + + log " Step 3: Verify curl remains blocked by the Jira preset..." + local curl_before + curl_before=$(sandbox_exec "set +e +OUT=\$(curl -sS -o /dev/null -w 'CURL_STATUS_%{http_code} CURL_APPCONNECT_%{time_appconnect}' --max-time 10 ${curl_probe_url} 2>&1) +RC=\$? +echo \"\$OUT CURL_RC_\$RC\" +" 2>&1) || true + log " Curl before explicit approval: $curl_before" + + if echo "$curl_before" | grep -qE "CURL_STATUS_[1-9][0-9][0-9]" \ + && ! echo "$curl_before" | grep -qE "CURL_STATUS_403.*CURL_APPCONNECT_0(\.0+)?( |$)"; then + fail "TC-NET-08: Curl pre-approval" "curl reached Atlassian without explicit approval ($curl_before)" + return + elif echo "$curl_before" | grep -qE "CURL_STATUS_000|CURL_STATUS_403|CURL_RC_[1-9]|denied|policy|forbidden"; then + if echo "$curl_before" | grep -qE "CURL_APPCONNECT_0(\.0+)?( |$)"; then + pass "TC-NET-08: curl blocked before explicit approval and before outbound TLS ($curl_before)" + else + fail "TC-NET-08: Curl pre-approval" "curl was denied but appeared to establish outbound TLS ($curl_before)" + return + fi + else + fail "TC-NET-08: Curl pre-approval" "Unexpected curl denial signal ($curl_before)" + return + fi + + log " Step 4: Explicitly allow curl to api.atlassian.com via OpenShell policy update..." + if ! openshell policy update "$SANDBOX_NAME" \ + --add-endpoint api.atlassian.com:443:read-only:rest:enforce \ + --binary /usr/bin/curl \ + --binary /usr/local/bin/curl \ + --wait 2>&1 | tee -a "$LOG_FILE"; then + fail "TC-NET-08: Curl approval" "Could not apply explicit curl approval" + return + fi + sleep 5 + + log " Step 5: Verify curl reaches Atlassian after explicit approval..." + local curl_after + curl_after=$(sandbox_exec "set +e +rm -f /tmp/nemoclaw-jira-curl-body +OUT=\$(curl -sS -o /tmp/nemoclaw-jira-curl-body -w 'CURL_STATUS_%{http_code}' --max-time 10 ${curl_probe_url} 2>&1) +RC=\$? +printf '%s CURL_RC_%s CURL_BODY_' \"\$OUT\" \"\$RC\" +head -c 120 /tmp/nemoclaw-jira-curl-body 2>/dev/null || true +printf '\n' +" 2>&1) || true + log " Curl after explicit approval: $curl_after" + + if echo "$curl_after" | grep -qE "CURL_STATUS_401" \ + && echo "$curl_after" | grep -qE "Unauthorized|unauthorized"; then + pass "TC-NET-08: curl reaches Atlassian after explicit approval ($curl_after)" + else + fail "TC-NET-08: Curl post-approval" "curl did not reach Atlassian after explicit approval ($curl_after)" + fi +} + +# ============================================================================= +# TC-NET-07: Inference exemption + direct provider blocked +# ============================================================================= +test_net_07_inference_exemption() { + log "=== TC-NET-07: Inference Exemption + Direct Provider Blocked ===" + + log " Step 1: Send prompt via inference.local (should succeed)..." + local inference_response + inference_response=$(sandbox_exec "curl -s --max-time 60 https://inference.local/v1/chat/completions \ + -H 'Content-Type: application/json' \ + -d '{\"model\":\"nvidia/nemotron-3-super-120b-a12b\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with exactly one word: PONG\"}],\"max_tokens\":50}'" 2>&1) || true + + log " Inference response: ${inference_response:0:200}" + + local content + content=$(echo "$inference_response" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d['choices'][0]['message']['content'])" 2>/dev/null) || true + + if [[ -n "$content" ]]; then + pass "TC-NET-07: Inference via inference.local succeeded" + else + fail "TC-NET-07: Inference" "No response from inference.local: ${inference_response:0:200}" + return + fi + + log " Step 2: Attempt direct connection to provider (should be blocked)..." + local direct_response + direct_response=$(sandbox_exec "node -e \" +fetch('https://inference.nvidia.com/v1/models', {signal: AbortSignal.timeout(15000)}) + .then(r => console.log('STATUS_' + r.status)) + .catch(e => console.log('ERROR_' + (e.cause?.code || e.code || e.message))) +\"" 2>&1) || true + + log " Direct provider response: $direct_response" + + if echo "$direct_response" | grep -qE "STATUS_403|ERROR_"; then + pass "TC-NET-07: Direct provider access blocked ($direct_response)" + elif echo "$direct_response" | grep -qE "STATUS_[23]"; then + fail "TC-NET-07: Direct provider" "Direct access to provider succeeded ($direct_response)" + else + fail "TC-NET-07: Direct provider" "Unexpected response ($direct_response)" + fi +} + +# ============================================================================= +# TC-NET-05: Hot-reload — policy takes effect without sandbox restart +# ============================================================================= +test_net_05_hot_reload() { + log "=== TC-NET-05: Hot-Reload (no sandbox restart) ===" + + log " Capturing sandbox start time before policy change..." + local starttime_before + starttime_before=$(sandbox_exec "cat /proc/1/stat 2>/dev/null | awk '{print \$22}'" 2>&1) || true + log " Start time before: $starttime_before" + + log " Adding npm preset..." + if ! apply_preset "npm"; then + fail "TC-NET-05: Setup" "Could not apply npm preset" + return + fi + + log " Capturing sandbox start time after policy change..." + local starttime_after + starttime_after=$(sandbox_exec "cat /proc/1/stat 2>/dev/null | awk '{print \$22}'" 2>&1) || true + log " Start time after: $starttime_after" + + if [[ -n "$starttime_before" && -n "$starttime_after" && "$starttime_before" == "$starttime_after" ]]; then + pass "TC-NET-05: Sandbox start time unchanged after policy-add (no restart)" + elif [[ -z "$starttime_before" || -z "$starttime_after" ]]; then + skip "TC-NET-05" "Could not capture sandbox start time" + else + fail "TC-NET-05: Hot-reload" "Sandbox start time changed ($starttime_before → $starttime_after) — sandbox was restarted" + fi +} + +# ============================================================================= +# TC-NET-06: Permissive policy mode +# ============================================================================= +test_net_06_permissive_mode() { + log "=== TC-NET-06: Permissive Policy Mode ===" + + log " Step 1: Verify npm registry is blocked under restricted policy..." + local before + before=$(sandbox_exec "npm ping 2>&1 && echo NPM_OK || echo NPM_FAIL" 2>&1) || true + log " Before permissive: ${before:0:200}" + + if echo "$before" | grep -q "NPM_OK"; then + log " npm already reachable (preset may be applied from earlier test)" + fi + + log " Step 2: Applying permissive policy via openshell..." + local permissive_path="$REPO_ROOT/nemoclaw-blueprint/policies/openclaw-sandbox-permissive.yaml" + if ! openshell policy set --policy "$permissive_path" --wait "$SANDBOX_NAME" 2>&1 | tee -a "$LOG_FILE"; then + fail "TC-NET-06: Setup" "Could not apply permissive policy ($permissive_path)" + return + fi + sleep 5 + + log " Step 3: Verify npm registry is reachable under permissive policy..." + local during + during=$(sandbox_exec "npm ping 2>&1 && echo NPM_OK || echo NPM_FAIL" 2>&1) || true + log " During permissive: ${during:0:200}" + + if echo "$during" | grep -q "NPM_OK"; then + pass "TC-NET-06: npm reachable under permissive policy" + else + fail "TC-NET-06: Permissive" "npm still blocked under permissive policy (${during:0:200})" + fi +} + +# ============================================================================= +# TC-NET-09: SSRF validation +# ============================================================================= +test_net_09_ssrf_validation() { + log "=== TC-NET-09: SSRF Validation ===" + + log " Testing SSRF validation via Node.js..." + local result + result=$(node -e " +const { isPrivateIp } = require('$REPO_ROOT/nemoclaw/dist/blueprint/ssrf'); +const dangerous = ['169.254.169.254', '127.0.0.1', '10.0.0.1', '192.168.1.1', '0.0.0.0']; +const safe = ['8.8.8.8', '142.250.80.46']; +let pass = true; +for (const ip of dangerous) { + if (!isPrivateIp(ip)) { console.log('FAIL: ' + ip + ' not blocked'); pass = false; } +} +for (const ip of safe) { + if (isPrivateIp(ip)) { console.log('FAIL: ' + ip + ' incorrectly blocked'); pass = false; } +} +console.log(pass ? 'SSRF_PASS' : 'SSRF_FAIL'); +" 2>&1) || true + + log " Result: $result" + + if echo "$result" | grep -q "SSRF_PASS"; then + pass "TC-NET-09: SSRF validation correctly blocks dangerous IPs" + else + fail "TC-NET-09: SSRF" "Validation failed: $result" + fi +} + +# ============================================================================= +# TC-NET-10: OpenClaw web_fetch host gateway compatibility +# ============================================================================= +test_net_10_openclaw_web_fetch_host_gateway() { + log "=== TC-NET-10: OpenClaw web_fetch Host Gateway ===" + + local host_dir server_log port port_file server_pid marker + local deny_host_dir deny_server_log deny_port deny_port_file deny_server_pid deny_marker + marker="NEMOCLAW_HOST_GATEWAY_WEB_FETCH_OK" + deny_marker="NEMOCLAW_HOST_GATEWAY_WEB_FETCH_DENIED_PORT_SHOULD_NOT_LEAK" + host_dir="$(mktemp -d)" + deny_host_dir="$(mktemp -d)" + server_log="$host_dir/http.log" + deny_server_log="$deny_host_dir/http.log" + port_file="$host_dir/port" + deny_port_file="$deny_host_dir/port" + printf '%s\n' "$marker" >"$host_dir/index.html" + printf '%s\n' "$deny_marker" >"$deny_host_dir/index.html" + + server_pid="$(start_e2e_http_server "$host_dir" "$port_file" "$server_log")" + deny_server_pid="$(start_e2e_http_server "$deny_host_dir" "$deny_port_file" "$deny_server_log")" + if ! port="$(wait_for_e2e_http_port "$port_file" "$server_pid")"; then + fail "TC-NET-10: Setup" "host HTTP server failed to publish a port ($(cat "$server_log" 2>/dev/null))" + kill "$server_pid" "$deny_server_pid" 2>/dev/null || true + wait "$server_pid" "$deny_server_pid" 2>/dev/null || true + rm -rf "$host_dir" "$deny_host_dir" + return + fi + if ! deny_port="$(wait_for_e2e_http_port "$deny_port_file" "$deny_server_pid")"; then + fail "TC-NET-10: Setup" "deny host HTTP server failed to publish a port ($(cat "$deny_server_log" 2>/dev/null))" + kill "$server_pid" "$deny_server_pid" 2>/dev/null || true + wait "$server_pid" "$deny_server_pid" 2>/dev/null || true + rm -rf "$host_dir" "$deny_host_dir" + return + fi + if ! kill -0 "$server_pid" 2>/dev/null; then + fail "TC-NET-10: Setup" "host HTTP server failed to start ($(cat "$server_log" 2>/dev/null))" + rm -rf "$host_dir" + kill "$deny_server_pid" 2>/dev/null || true + wait "$deny_server_pid" 2>/dev/null || true + rm -rf "$deny_host_dir" + return + fi + if ! kill -0 "$deny_server_pid" 2>/dev/null; then + fail "TC-NET-10: Setup" "deny host HTTP server failed to start ($(cat "$deny_server_log" 2>/dev/null))" + kill "$server_pid" 2>/dev/null || true + wait "$server_pid" 2>/dev/null || true + rm -rf "$host_dir" "$deny_host_dir" + return + fi + + cleanup_host_server() { + kill "$server_pid" 2>/dev/null || true + wait "$server_pid" 2>/dev/null || true + kill "$deny_server_pid" 2>/dev/null || true + wait "$deny_server_pid" 2>/dev/null || true + rm -rf "$host_dir" "$deny_host_dir" + } + + log " Allowing node/openclaw access to host.openshell.internal:${port}..." + local host_gateway_policy + host_gateway_policy="$(mktemp "${TMPDIR:-/tmp}/nemoclaw-host-gateway-policy.XXXXXX.yaml")" + cat >"$host_gateway_policy" <&1 | tee -a "$LOG_FILE"; then + rm -f "$host_gateway_policy" + fail "TC-NET-10: Setup" "Could not allow host.openshell.internal:${port}" + cleanup_host_server + return + fi + rm -f "$host_gateway_policy" + sleep 5 + + local direct + direct=$(sandbox_exec "node -e \" +fetch('http://host.openshell.internal:${port}/', {signal: AbortSignal.timeout(15000)}) + .then(async r => console.log('STATUS_' + r.status + ' ' + (await r.text()).slice(0, 120))) + .catch(e => console.log('ERROR_' + (e.cause?.code || e.code || e.message))) +\"" 2>&1) || true + log " Direct Node host-gateway fetch: $direct" + if ! echo "$direct" | grep -q "$marker"; then + fail "TC-NET-10: Setup" "host gateway policy/proxy probe failed before OpenClaw web_fetch ($direct)" + cleanup_host_server + return + fi + + log " Verifying unapproved host.openshell.internal:${deny_port} remains denied..." + local denied_direct + denied_direct=$(sandbox_exec "node -e \" +fetch('http://host.openshell.internal:${deny_port}/', {signal: AbortSignal.timeout(15000)}) + .then(async r => console.log('STATUS_' + r.status + ' ' + (await r.text()).slice(0, 120))) + .catch(e => console.log('ERROR_' + (e.cause?.code || e.code || e.message))) +\"" 2>&1) || true + log " Direct Node denied-port probe: $denied_direct" + if echo "$denied_direct" | grep -q "$deny_marker"; then + fail "TC-NET-10: OpenShell policy" "unapproved host gateway port was reachable before OpenClaw web_fetch deny-case ($denied_direct)" + cleanup_host_server + return + fi + if echo "$denied_direct" | grep -qiE "STATUS_403|ERROR_|denied|policy|forbidden|not allowed|not permitted"; then + pass "TC-NET-10: OpenShell policy denies unapproved host gateway port" + else + fail "TC-NET-10: OpenShell policy" "unexpected denied-port response before OpenClaw web_fetch deny-case ($denied_direct)" + cleanup_host_server + return + fi + + local web_fetch_probe_script web_fetch_probe_b64 web_fetch_output web_fetch_rc=0 + web_fetch_probe_script="$( + cat <<'NODE' +import fs from "node:fs"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +const [approvedUrl, deniedUrl, marker, denyMarker] = process.argv.slice(2); +const distDir = "/usr/local/lib/node_modules/openclaw/dist"; + +function fail(code, detail) { + console.log(`E2E_FAIL_${code}: ${String(detail || "").slice(0, 1200)}`); + process.exitCode = 1; +} + +function findDistFile(prefix) { + const candidates = fs + .readdirSync(distDir) + .filter((name) => name.startsWith(prefix) && name.endsWith(".js")) + .sort(); + if (candidates.length !== 1) { + throw new Error(`expected one ${prefix}*.js file, found ${candidates.length}: ${candidates.join(", ")}`); + } + return path.join(distDir, candidates[0]); +} + +function summarize(value) { + return JSON.stringify(value, (_key, inner) => { + if (typeof inner === "string" && inner.length > 1200) return `${inner.slice(0, 1200)}...`; + return inner; + }); +} + +async function main() { + const configPath = process.env.OPENCLAW_CONFIG_PATH || "/sandbox/.openclaw/openclaw.json"; + const config = JSON.parse(fs.readFileSync(configPath, "utf8")); + const fetchConfig = config?.tools?.web?.fetch; + if (fetchConfig?.useTrustedEnvProxy !== true) { + fail("CONFIG_MISSING_TRUSTED_ENV_PROXY", `tools.web.fetch.useTrustedEnvProxy=${fetchConfig?.useTrustedEnvProxy}`); + return; + } + + const mod = await import(pathToFileURL(findDistFile("openclaw-tools-")).href); + const createOpenClawTools = mod.t || mod.createOpenClawTools; + if (typeof createOpenClawTools !== "function") { + fail("OPENCLAW_TOOLS_EXPORT_MISSING", Object.keys(mod).join(",")); + return; + } + + const tools = createOpenClawTools({ + config, + sandboxed: true, + workspaceDir: "/sandbox/.openclaw/workspace-main", + wrapBeforeToolCallHook: false, + disablePluginTools: true, + disableMessageTool: true, + }); + const webFetch = tools.find((tool) => tool?.name === "web_fetch"); + if (!webFetch || typeof webFetch.execute !== "function") { + fail("WEB_FETCH_TOOL_MISSING", tools.map((tool) => tool?.name).filter(Boolean).join(",")); + return; + } + + let approvedRaw = ""; + try { + const approved = await webFetch.execute("e2e-approved-host-gateway", { + url: approvedUrl, + extractMode: "text", + maxChars: 2000, + }); + approvedRaw = summarize(approved); + } catch (error) { + const detail = error && (error.stack || error.message) ? error.stack || error.message : error; + if (/SsrFBlockedError|Blocked hostname|private\/internal\/special-use/i.test(String(detail))) { + fail("SSRF_BLOCKED_HOST_GATEWAY_APPROVED", detail); + return; + } + fail("APPROVED_FETCH_ERROR", detail); + return; + } + if (!approvedRaw.includes(marker)) { + fail("APPROVED_MARKER_MISSING", approvedRaw); + return; + } + console.log("E2E_WEB_FETCH_APPROVED_OK"); + + try { + const denied = await webFetch.execute("e2e-denied-host-gateway", { + url: deniedUrl, + extractMode: "text", + maxChars: 2000, + }); + const deniedRaw = summarize(denied); + if (deniedRaw.includes(denyMarker)) { + fail("DENIED_PORT_REACHED", deniedRaw); + return; + } + fail("DENIED_PORT_UNEXPECTED_SUCCESS", deniedRaw); + } catch (error) { + const detail = String(error && (error.stack || error.message) ? error.stack || error.message : error); + if (/SsrFBlockedError|Blocked hostname|private\/internal\/special-use/i.test(detail)) { + fail("SSRF_BLOCKED_HOST_GATEWAY_DENIED", detail); + return; + } + if (/Web fetch failed \\(403\\)|\\b403\\b|policy|denied|forbidden|fetch failed|ECONN|UND_ERR|proxy/i.test(detail)) { + console.log(`E2E_WEB_FETCH_DENIED_OK ${detail.split("\n")[0].slice(0, 300)}`); + return; + } + fail("DENIED_PORT_UNEXPECTED_ERROR", detail); + } +} + +main().catch((error) => { + fail("UNCAUGHT", error && (error.stack || error.message) ? error.stack || error.message : error); +}); +NODE + )" + web_fetch_probe_b64="$(printf '%s' "$web_fetch_probe_script" | base64 | tr -d '\n')" + web_fetch_output=$(sandbox_exec "printf '%s' '${web_fetch_probe_b64}' | base64 -d > /tmp/nemoclaw-web-fetch-e2e.mjs +nemoclaw-start node /tmp/nemoclaw-web-fetch-e2e.mjs 'http://host.openshell.internal:${port}/' 'http://host.openshell.internal:${deny_port}/' '${marker}' '${deny_marker}'" 2>&1) || web_fetch_rc=$? + cleanup_host_server + + log " OpenClaw web_fetch probe: ${web_fetch_output:0:1000}" + if printf '%s' "$web_fetch_output" | grep -q "E2E_FAIL_SSRF_BLOCKED_HOST_GATEWAY"; then + fail "TC-NET-10: OpenClaw web_fetch" "OpenClaw SSRF guard blocked host gateway before OpenShell policy (${web_fetch_output:0:500})" + return + fi + + if printf '%s' "$web_fetch_output" | grep -q "E2E_FAIL_DENIED_PORT_REACHED"; then + fail "TC-NET-10: OpenClaw web_fetch policy" "web_fetch reached unapproved host gateway port (${web_fetch_output:0:500})" + return + fi + + if printf '%s' "$web_fetch_output" | grep -q "E2E_WEB_FETCH_APPROVED_OK"; then + pass "TC-NET-10: OpenClaw web_fetch reached approved host.openshell.internal target" + else + fail "TC-NET-10: OpenClaw web_fetch" "approved marker not returned (exit ${web_fetch_rc}, output='${web_fetch_output:0:500}')" + return + fi + + if printf '%s' "$web_fetch_output" | grep -q "E2E_WEB_FETCH_DENIED_OK"; then + pass "TC-NET-10: OpenClaw web_fetch cannot reach unapproved host gateway port" + else + fail "TC-NET-10: OpenClaw web_fetch policy" "unapproved host gateway port did not produce a policy denial signal (exit ${web_fetch_rc}, output='${web_fetch_output:0:500}')" + fi +} + +# ── Teardown ───────────────────────────────────────────────────────────────── +teardown() { + # Do not unlink ~/.nemoclaw/onboard.lock: that lock is global and PID- + # ownership-aware in src/lib/onboard-session.ts (acquireOnboardLock + # verifies the holder's PID liveness and inode), so an unconditional rm + # here could yank a concurrent run's live lock. A crashed process leaves + # a stale lock that the next onboard cleans up automatically. + set +e + nemoclaw "$SANDBOX_NAME" destroy --yes 2>/dev/null || true + set -e +} + +# ── Summary ────────────────────────────────────────────────────────────────── +summary() { + echo "" + echo "============================================================" + echo " Network Policy E2E Results" + echo "============================================================" + echo -e " ${GREEN}PASS: $PASS${NC}" + echo -e " ${RED}FAIL: $FAIL${NC}" + echo -e " ${YELLOW}SKIP: $SKIP${NC}" + echo " TOTAL: $TOTAL" + echo "============================================================" + echo " Log: $LOG_FILE" + echo "============================================================" + echo "" + + if [[ $FAIL -gt 0 ]]; then + exit 1 + fi + exit 0 +} + +# ── Main ───────────────────────────────────────────────────────────────────── +main() { + echo "" + echo "============================================================" + echo " NemoClaw Network Policy E2E Tests" + echo " $(date)" + echo "============================================================" + echo "" + + preflight + setup_sandbox + + test_net_01_deny_default + test_net_11_brew_install_hello + test_net_02_whitelist_access + test_net_03_live_policy_add + test_net_04_dry_run + test_net_08_jira_per_binary_enforcement + test_net_05_hot_reload + test_net_07_inference_exemption + test_net_09_ssrf_validation + test_net_10_openclaw_web_fetch_host_gateway + test_net_06_permissive_mode # last — opens all egress, affects subsequent tests + + trap - EXIT + teardown + summary +} + +trap teardown EXIT +main "$@" diff --git a/test/e2e-vpn/test-ollama-auth-proxy-e2e.sh b/test/e2e-vpn/test-ollama-auth-proxy-e2e.sh new file mode 100755 index 00000000000..ca50a72e8b3 --- /dev/null +++ b/test/e2e-vpn/test-ollama-auth-proxy-e2e.sh @@ -0,0 +1,568 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Ollama Auth Proxy E2E — real Ollama, real inference, real proxy. +# +# Validates the full proxy chain introduced in PR #1922: +# 1. Install Ollama + pull a small model +# 2. Start Ollama on 127.0.0.1 (localhost only) +# 3. Start the auth proxy on 0.0.0.0:11435 +# 4. Verify proxy auth (reject bad tokens, accept good tokens) +# 5. Verify real inference through the proxy +# 6. Verify proxy recovery (kill + restart from persisted token) +# 7. Verify token persistence (file exists, permissions, content) +# 8. Verify container reachability check works against the proxy +# +# Does NOT require GPU — runs CPU inference with a small model. +# Does NOT require OpenShell/sandbox — tests the host-side proxy chain only. +# +# Usage: +# bash test/e2e-vpn/test-ollama-auth-proxy-e2e.sh +# +# Triggered via workflow_dispatch (manual) or as part of nightly. + +# ShellCheck cannot see EXIT trap invocations of cleanup helpers in this E2E script. +# shellcheck disable=SC2317 +set -uo pipefail + +PASS=0 +FAIL=0 +TOTAL=0 +PROXY_PID="" +OLLAMA_PID="" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +PROXY_SCRIPT="$SCRIPT_DIR/scripts/ollama-auth-proxy.js" +TOKEN_DIR="$(mktemp -d)" +TOKEN_FILE="$TOKEN_DIR/.nemoclaw/ollama-proxy-token" +OLLAMA_PORT=11434 +PROXY_PORT=11435 +MODEL="qwen2.5:0.5b" + +pass() { + ((PASS++)) + ((TOTAL++)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} +fail() { + ((FAIL++)) + ((TOTAL++)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} + +# shellcheck disable=SC2329 # invoked via trap +cleanup() { + if [ -n "${PROXY_PID:-}" ]; then + kill "$PROXY_PID" 2>/dev/null || true + fi + # Don't kill system Ollama — only kill if we started it + if [ -n "${OLLAMA_PID:-}" ]; then + kill "$OLLAMA_PID" 2>/dev/null || true + fi + rm -rf "$TOKEN_DIR" +} +trap cleanup EXIT + +# ══════════════════════════════════════════════════════════════════ +# Phase 1: Prerequisites +# ══════════════════════════════════════════════════════════════════ +section "Phase 1: Prerequisites" + +if ! command -v node >/dev/null 2>&1; then + fail "Node.js not found" + exit 1 +fi +pass "Node.js available: $(node --version)" + +if ! command -v curl >/dev/null 2>&1; then + fail "curl not found" + exit 1 +fi +pass "curl available" + +if [ ! -f "$PROXY_SCRIPT" ]; then + fail "Proxy script not found at $PROXY_SCRIPT" + exit 1 +fi +pass "Proxy script exists" + +# ══════════════════════════════════════════════════════════════════ +# Phase 2: Install Ollama + pull model +# ══════════════════════════════════════════════════════════════════ +section "Phase 2: Install Ollama and pull model" + +if command -v ollama >/dev/null 2>&1; then + pass "Ollama already installed: $(ollama --version 2>/dev/null || echo unknown)" +else + info "Installing Ollama..." + if curl -fsSL https://ollama.com/install.sh | sh 2>&1; then + pass "Ollama installed" + else + fail "Ollama install failed" + exit 1 + fi +fi + +# Stop any existing Ollama so we control the binding +pkill -f "ollama serve" 2>/dev/null || true +systemctl --user stop ollama 2>/dev/null || true +systemctl stop ollama 2>/dev/null || true +sleep 2 + +# Start Ollama on localhost only (mirrors what onboard does with the proxy) +info "Starting Ollama on 127.0.0.1:${OLLAMA_PORT}..." +OLLAMA_HOST="127.0.0.1:${OLLAMA_PORT}" ollama serve >/dev/null 2>&1 & +OLLAMA_PID=$! +sleep 3 + +if curl -sf "http://127.0.0.1:${OLLAMA_PORT}/api/tags" >/dev/null 2>&1; then + pass "Ollama running on 127.0.0.1:${OLLAMA_PORT}" +else + fail "Ollama failed to start on 127.0.0.1:${OLLAMA_PORT}" + exit 1 +fi + +# Pull the small model +info "Pulling model ${MODEL} (this may take a few minutes on first run)..." +if ollama pull "$MODEL" 2>&1; then + pass "Model $MODEL pulled" +else + fail "Failed to pull $MODEL" + exit 1 +fi + +# Verify model is available +if curl -sf "http://127.0.0.1:${OLLAMA_PORT}/api/tags" | grep -q "$MODEL"; then + pass "Model $MODEL available in Ollama" +else + fail "Model $MODEL not found in /api/tags" + exit 1 +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 3: Start auth proxy +# ══════════════════════════════════════════════════════════════════ +section "Phase 3: Start auth proxy" + +TOKEN=$(node -e "console.log(require('crypto').randomBytes(24).toString('hex'))") +info "Generated proxy token: ${TOKEN:0:8}..." + +# Persist token (mirrors onboard behavior) +mkdir -p "$TOKEN_DIR/.nemoclaw" +echo "$TOKEN" >"$TOKEN_FILE" +chmod 600 "$TOKEN_FILE" + +OLLAMA_PROXY_TOKEN="$TOKEN" \ + OLLAMA_PROXY_PORT="$PROXY_PORT" \ + OLLAMA_BACKEND_PORT="$OLLAMA_PORT" \ + node "$PROXY_SCRIPT" & +PROXY_PID=$! +sleep 2 + +# Liveness probe: any response means the proxy is up. After #3338 unauth +# requests to /api/tags get 401, so we just verify a real HTTP status was +# returned (any 3-digit code, not 000 = no response). +STATUS=$(curl -s -o /dev/null -w "%{http_code}" "http://127.0.0.1:${PROXY_PORT}/api/tags") +if [[ "$STATUS" =~ ^[1-9][0-9]{2}$ ]]; then + pass "Auth proxy running on 0.0.0.0:${PROXY_PORT} (HTTP $STATUS)" +else + fail "Auth proxy failed to start (no HTTP response: '$STATUS')" + exit 1 +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 4: Auth verification +# ══════════════════════════════════════════════════════════════════ +section "Phase 4: Auth verification" + +# 4a: Unauthenticated request to protected endpoint → 401 +STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST \ + "http://127.0.0.1:${PROXY_PORT}/api/generate" -d '{}') +if [ "$STATUS" = "401" ]; then + pass "Unauthenticated POST /api/generate → 401" +else + fail "Expected 401 for unauthenticated POST, got $STATUS" +fi + +# 4b: Wrong token → 401 +WRONG_AUTH="Bearer wrong-token-$(date +%s)" +STATUS=$(curl -s -o /dev/null -w "%{http_code}" \ + -H "Authorization: $WRONG_AUTH" \ + -X POST "http://127.0.0.1:${PROXY_PORT}/api/generate" -d '{}') +if [ "$STATUS" = "401" ]; then + pass "Wrong token POST /api/generate → 401" +else + fail "Expected 401 for wrong token, got $STATUS" +fi + +# 4c: Correct token → 200 (forwarded to Ollama) +CORRECT_AUTH="Bearer $TOKEN" +STATUS=$(curl -s -o /dev/null -w "%{http_code}" \ + -H "Authorization: $CORRECT_AUTH" \ + "http://127.0.0.1:${PROXY_PORT}/api/tags") +if [ "$STATUS" = "200" ]; then + pass "Correct token GET /api/tags → 200" +else + fail "Expected 200 for correct token, got $STATUS" +fi + +# 4d: GET /api/tags without auth → 401 (no health-check bypass — #3338) +STATUS=$(curl -s -o /dev/null -w "%{http_code}" \ + "http://127.0.0.1:${PROXY_PORT}/api/tags") +if [ "$STATUS" = "401" ]; then + pass "Unauthenticated GET /api/tags → 401" +else + fail "Expected 401 for unauthenticated GET /api/tags, got $STATUS" +fi + +# 4e: POST /api/tags without auth → 401 +STATUS=$(curl -s -o /dev/null -w "%{http_code}" \ + -X POST "http://127.0.0.1:${PROXY_PORT}/api/tags" -d '{}') +if [ "$STATUS" = "401" ]; then + pass "Unauthenticated POST /api/tags → 401" +else + fail "Expected 401 for unauthenticated POST /api/tags, got $STATUS" +fi + +# 4f: Authorization header stripped before forwarding (Ollama doesn't see it) +# Verify by checking that Ollama gets a clean request +BODY=$(curl -sf -H "Authorization: $CORRECT_AUTH" \ + "http://127.0.0.1:${PROXY_PORT}/api/tags" 2>/dev/null) +if echo "$BODY" | grep -q "$MODEL"; then + pass "Proxy strips auth header — Ollama responds normally" +else + fail "Proxy may not be stripping auth header correctly" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 5: Real inference through proxy +# ══════════════════════════════════════════════════════════════════ +section "Phase 5: Real inference through proxy" + +# 5a: OpenAI-compatible chat completions through proxy +info "Testing inference: POST /v1/chat/completions through proxy..." +INFERENCE_RESPONSE=$(curl -s --max-time 120 \ + -H "Authorization: $CORRECT_AUTH" \ + -H "Content-Type: application/json" \ + -X POST "http://127.0.0.1:${PROXY_PORT}/v1/chat/completions" \ + -d "{ + \"model\": \"$MODEL\", + \"messages\": [{\"role\": \"user\", \"content\": \"Reply with exactly one word: PONG\"}], + \"max_tokens\": 50 + }" 2>/dev/null) || true + +if [ -n "$INFERENCE_RESPONSE" ]; then + # Check for a valid response structure + if echo "$INFERENCE_RESPONSE" | python3 -c " +import json, sys +r = json.load(sys.stdin) +c = r.get('choices', [{}])[0].get('message', {}).get('content', '') +print(c.strip()) +sys.exit(0 if c.strip() else 1) +" 2>/dev/null; then + pass "Inference through proxy: got chat completion response" + else + fail "Inference through proxy: invalid response structure" + info "Response: ${INFERENCE_RESPONSE:0:300}" + fi +else + fail "Inference through proxy: empty response" +fi + +# 5b: Ollama native /api/generate through proxy +info "Testing inference: POST /api/generate through proxy..." +GENERATE_RESPONSE=$(curl -s --max-time 120 \ + -H "Authorization: $CORRECT_AUTH" \ + -H "Content-Type: application/json" \ + -X POST "http://127.0.0.1:${PROXY_PORT}/api/generate" \ + -d "{ + \"model\": \"$MODEL\", + \"prompt\": \"Reply with one word: PONG\", + \"stream\": false + }" 2>/dev/null) || true + +if [ -n "$GENERATE_RESPONSE" ]; then + if echo "$GENERATE_RESPONSE" | python3 -c " +import json, sys +r = json.load(sys.stdin) +print(r.get('response', '').strip()) +sys.exit(0 if r.get('response', '').strip() else 1) +" 2>/dev/null; then + pass "Inference through proxy: got /api/generate response" + else + fail "Inference through proxy: invalid /api/generate response" + info "Response: ${GENERATE_RESPONSE:0:300}" + fi +else + fail "Inference through proxy: empty /api/generate response" +fi + +# 5c: Inference WITHOUT token → 401 (not forwarded) +STATUS=$(curl -s -o /dev/null -w "%{http_code}" --max-time 10 \ + -H "Content-Type: application/json" \ + -X POST "http://127.0.0.1:${PROXY_PORT}/v1/chat/completions" \ + -d "{ + \"model\": \"$MODEL\", + \"messages\": [{\"role\": \"user\", \"content\": \"test\"}] + }" 2>/dev/null) +if [ "$STATUS" = "401" ]; then + pass "Inference without token → 401 (not forwarded to Ollama)" +else + fail "Expected 401 for unauthenticated inference, got $STATUS" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 6: Token persistence +# ══════════════════════════════════════════════════════════════════ +section "Phase 6: Token persistence" + +# 6a: Token file exists +if [ -f "$TOKEN_FILE" ]; then + pass "Token file exists at $TOKEN_FILE" +else + fail "Token file missing" +fi + +# 6b: Token file has correct permissions +PERMS=$(stat -c "%a" "$TOKEN_FILE" 2>/dev/null || stat -f "%Lp" "$TOKEN_FILE" 2>/dev/null) +if [ "$PERMS" = "600" ]; then + pass "Token file permissions: 600" +else + fail "Token file permissions: expected 600, got $PERMS" +fi + +# 6c: Token file content matches +FILE_TOKEN=$(tr -d '[:space:]' <"$TOKEN_FILE") +if [ "$FILE_TOKEN" = "$TOKEN" ]; then + pass "Token file content matches generated token" +else + fail "Token file content mismatch" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 7: Proxy recovery (kill + restart) +# ══════════════════════════════════════════════════════════════════ +section "Phase 7: Proxy recovery" + +# 7a: Kill the proxy +info "Killing proxy (PID: $PROXY_PID)..." +kill "$PROXY_PID" 2>/dev/null || true +PROXY_PID="" +sleep 2 + +# Verify it's dead +STATUS=$(curl -s -o /dev/null -w "%{http_code}" --connect-timeout 2 \ + "http://127.0.0.1:${PROXY_PORT}/api/tags" 2>/dev/null) || STATUS="000" +if [ "$STATUS" = "000" ] || [ "$STATUS" = "" ]; then + pass "Proxy confirmed dead after kill" +else + fail "Proxy still responding after kill (status: $STATUS)" +fi + +# 7b: Restart proxy with persisted token (simulates reboot recovery) +info "Restarting proxy from persisted token..." +PERSISTED_TOKEN=$(tr -d '[:space:]' <"$TOKEN_FILE") +OLLAMA_PROXY_TOKEN="$PERSISTED_TOKEN" \ + OLLAMA_PROXY_PORT="$PROXY_PORT" \ + OLLAMA_BACKEND_PORT="$OLLAMA_PORT" \ + node "$PROXY_SCRIPT" & +PROXY_PID=$! +sleep 2 + +# Liveness probe: 401 proves the restarted proxy is alive (the token check +# is exercised in the 7c inference call below). +STATUS=$(curl -s -o /dev/null -w "%{http_code}" "http://127.0.0.1:${PROXY_PORT}/api/tags") +if [[ "$STATUS" =~ ^[1-9][0-9]{2}$ ]]; then + pass "Proxy restarted from persisted token (HTTP $STATUS)" +else + fail "Proxy failed to restart (no HTTP response: '$STATUS')" +fi + +# 7c: Verify inference still works with the same token after restart +RECOVER_AUTH="Bearer $PERSISTED_TOKEN" +RECOVER_RESPONSE=$(curl -s --max-time 60 \ + -H "Authorization: $RECOVER_AUTH" \ + -H "Content-Type: application/json" \ + -X POST "http://127.0.0.1:${PROXY_PORT}/v1/chat/completions" \ + -d "{ + \"model\": \"$MODEL\", + \"messages\": [{\"role\": \"user\", \"content\": \"Say OK\"}], + \"max_tokens\": 10 + }" 2>/dev/null) || true + +if [ -n "$RECOVER_RESPONSE" ] && echo "$RECOVER_RESPONSE" | python3 -c " +import json, sys +r = json.load(sys.stdin) +sys.exit(0 if r.get('choices') else 1) +" 2>/dev/null; then + pass "Inference works after proxy restart with persisted token" +else + fail "Inference failed after proxy restart" +fi + +# 7d: Verify old token still works (same token persisted) +if [ "$TOKEN" = "$PERSISTED_TOKEN" ]; then + pass "Persisted token matches original — no token rotation on restart" +else + fail "Token changed on restart (should be the same persisted token)" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 8: Container reachability check (Docker, if available) +# ══════════════════════════════════════════════════════════════════ +section "Phase 8: Container reachability (Docker)" + +if command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then + info "Docker available — testing container-to-proxy reachability..." + + # Reachability only — the probe container doesn't carry the proxy token, + # so we accept any 3-digit HTTP code (the expected response after #3338 is + # 401). Mirrors how validateLocalProvider checks reachability. + # Drop Docker's own stderr (image-pull progress on cold runners) so it can't + # pollute the captured HTTP code. curl with -s -o /dev/null -w "%{http_code}" + # emits only the 3-digit code on stdout. + CONTAINER_STATUS=$(docker run --rm \ + --add-host "host.openshell.internal:host-gateway" \ + curlimages/curl:8.10.1 \ + -s -o /dev/null -w "%{http_code}" --connect-timeout 5 --max-time 10 \ + "http://host.openshell.internal:${PROXY_PORT}/api/tags" 2>/dev/null) || CONTAINER_STATUS="000" + + if [[ "$CONTAINER_STATUS" =~ ^[1-9][0-9]{2}$ ]]; then + pass "Container can reach proxy at host.openshell.internal:${PROXY_PORT} (HTTP $CONTAINER_STATUS)" + else + fail "Container cannot reach proxy — reachability check would fail during onboard" + info "Result: ${CONTAINER_STATUS:0:200}" + fi + + # Verify container CANNOT reach Ollama directly on localhost + DIRECT_RESULT=$(docker run --rm \ + --add-host "host.openshell.internal:host-gateway" \ + curlimages/curl:8.10.1 \ + -sf --connect-timeout 3 "http://host.openshell.internal:${OLLAMA_PORT}/api/tags" 2>&1) || DIRECT_RESULT="" + + if [ -z "$DIRECT_RESULT" ]; then + pass "Container CANNOT reach Ollama directly on ${OLLAMA_PORT} (localhost-only binding works)" + else + fail "Container CAN reach Ollama on ${OLLAMA_PORT} — Ollama may be on 0.0.0.0" + fi +else + info "Docker not available — skipping container reachability tests" + pass "Container reachability: skipped (no Docker)" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase: Token divergence after simulated re-onboard (issue #2553) +# ══════════════════════════════════════════════════════════════════ +section "Token Divergence Regression (issue #2553)" + +# Proxy should be running from earlier phases with the original token. +# Simulate a re-onboard that writes a NEW token to the file but +# leaves the proxy running with the OLD token. +ORIGINAL_TOKEN=$(cat "$TOKEN_FILE" 2>/dev/null || echo "") +DIVERGENT_TOKEN="divergent-$(date +%s)-$(node -e 'console.log(require("node:crypto").randomBytes(16).toString("hex"))')" + +if [ -n "$ORIGINAL_TOKEN" ]; then + info "Original token: ${ORIGINAL_TOKEN:0:16}..." + info "Writing divergent token to file: ${DIVERGENT_TOKEN:0:16}..." + echo "$DIVERGENT_TOKEN" >"$TOKEN_FILE" + + # Verify proxy still runs with OLD token (divergence exists) + OLD_TOKEN_OK=false + curl -sf --max-time 3 \ + -H "Authorization: Bearer $ORIGINAL_TOKEN" \ + "http://localhost:${PROXY_PORT}/v1/models" >/dev/null 2>&1 && OLD_TOKEN_OK=true + + NEW_TOKEN_OK=false + curl -sf --max-time 3 \ + -H "Authorization: Bearer $DIVERGENT_TOKEN" \ + "http://localhost:${PROXY_PORT}/v1/models" >/dev/null 2>&1 && NEW_TOKEN_OK=true + + if [ "$OLD_TOKEN_OK" = true ] && [ "$NEW_TOKEN_OK" = false ]; then + pass "Confirmed: proxy running with old token, rejects new token (divergence exists)" + else + fail "Divergence not reproduced (old=$OLD_TOKEN_OK new=$NEW_TOKEN_OK) — aborting test" + echo "$ORIGINAL_TOKEN" >"$TOKEN_FILE" + exit 1 + fi + + # Simulate what the fixed ensureOllamaAuthProxy() does: + # 1. Read token from file + # 2. Probe running proxy with that token + # 3. If rejected, kill proxy and restart with file token + info "Simulating ensureOllamaAuthProxy() fix logic..." + FILE_TOKEN=$(cat "$TOKEN_FILE" 2>/dev/null) + PROBE_RC=0 + curl -sf --max-time 3 -H "Authorization: Bearer $FILE_TOKEN" \ + "http://localhost:${PROXY_PORT}/v1/models" >/dev/null 2>&1 || PROBE_RC=$? + + if [ "$PROBE_RC" -ne 0 ]; then + info "Proxy rejects file token (expected) — killing and restarting with correct token..." + kill "$PROXY_PID" 2>/dev/null || true + sleep 1 + OLLAMA_PROXY_TOKEN="$FILE_TOKEN" \ + OLLAMA_PROXY_PORT="$PROXY_PORT" \ + OLLAMA_BACKEND_PORT="$OLLAMA_PORT" \ + node "$PROXY_SCRIPT" & + PROXY_PID=$! + sleep 2 + info "Restarted proxy (PID $PROXY_PID) with file token" + else + info "Proxy already accepts file token — no restart needed" + fi + + # After the fix, the proxy should accept the divergent (file) token + sleep 2 + FIXED_OK=false + curl -sf --max-time 3 \ + -H "Authorization: Bearer $DIVERGENT_TOKEN" \ + "http://localhost:${PROXY_PORT}/v1/models" >/dev/null 2>&1 && FIXED_OK=true + + if [ "$FIXED_OK" = true ]; then + pass "After ensureOllamaAuthProxy: proxy accepts the file token (divergence fixed)" + else + fail "After ensureOllamaAuthProxy: proxy still rejects file token (divergence NOT fixed)" + fi + + # Restore original token for cleanup + echo "$ORIGINAL_TOKEN" >"$TOKEN_FILE" +else + info "No token file found — skipping divergence test" + pass "Token divergence: skipped (no prior token)" +fi + +# ══════════════════════════════════════════════════════════════════ +# Summary +# ══════════════════════════════════════════════════════════════════ +echo "" +echo "========================================" +echo " Ollama Auth Proxy E2E Results:" +echo " Passed: $PASS" +echo " Failed: $FAIL" +echo " Total: $TOTAL" +echo "========================================" +echo "" +echo " What this tested:" +echo " - Ollama on localhost (127.0.0.1 only)" +echo " - Auth proxy token validation (accept/reject)" +echo " - Real inference through proxy (chat + generate)" +echo " - Token file persistence (exists, permissions, content)" +echo " - Proxy kill + restart from persisted token" +echo " - Inference after proxy recovery" +echo " - Container-to-proxy reachability (if Docker available)" +echo " - Container cannot reach Ollama directly (localhost binding)" +echo " - Token divergence detection + auto-fix (issue #2553)" +echo "" + +if [ "$FAIL" -eq 0 ]; then + printf '\n\033[1;32m OLLAMA AUTH PROXY E2E PASSED\033[0m\n' + exit 0 +else + printf '\n\033[1;31m %d test(s) failed.\033[0m\n' "$FAIL" + exit 1 +fi diff --git a/test/e2e-vpn/test-onboard-negative-paths.sh b/test/e2e-vpn/test-onboard-negative-paths.sh new file mode 100755 index 00000000000..af035e7cabd --- /dev/null +++ b/test/e2e-vpn/test-onboard-negative-paths.sh @@ -0,0 +1,610 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# E2E: onboard negative and edge-case paths. +# +# Regression coverage for issue #2573. The nightly happy-path onboard test +# should not be the only place that exercises non-interactive validation. +# +# Scenarios: +# 1. NEMOCLAW_POLICY_MODE=restricted falls back to tier suggestions. +# 2. NEMOCLAW_POLICY_MODE=nonexistent falls back to tier suggestions. +# 3. Invalid NVIDIA API key format is rejected without a stack trace. +# 4. Non-NVIDIA provider keys are not forced to use nvapi-. +# 5. A host listener on the configured gateway port produces a friendly conflict. +# 6. Custom non-interactive policy presets are applied. +# 7. NEMOCLAW_PROVIDER=custom and NEMOCLAW_MODEL are honored. +# 8. --from without --name/NEMOCLAW_SANDBOX_NAME fails before defaulting. +# 9. --from with NEMOCLAW_SANDBOX_NAME proceeds past entry validation. + +set -uo pipefail + +export NEMOCLAW_E2E_DEFAULT_TIMEOUT=1800 +SCRIPT_DIR_TIMEOUT="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +# shellcheck source=test/e2e-vpn/e2e-timeout.sh +source "${SCRIPT_DIR_TIMEOUT}/e2e-timeout.sh" +# shellcheck source=test/e2e-vpn/lib/ci-compatible-inference.sh +. "${SCRIPT_DIR_TIMEOUT}/lib/ci-compatible-inference.sh" + +LOG_FILE="${NEMOCLAW_E2E_LOG:-/tmp/nemoclaw-e2e-onboard-negative-paths.log}" +exec > >(tee "$LOG_FILE") 2>&1 + +PASS=0 +FAIL=0 +SKIP=0 +TOTAL=0 +PORT_HOLDER_PID="" + +pass() { + ((PASS++)) + ((TOTAL++)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} +fail() { + ((FAIL++)) + ((TOTAL++)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} +skip() { + ((SKIP++)) + ((TOTAL++)) + printf '\033[33m SKIP: %s\033[0m\n' "$1" +} +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } + +if [ -d /workspace ] && [ -f /workspace/install.sh ]; then + REPO="/workspace" +elif [ -f "$(cd "$(dirname "$0")/../.." && pwd)/install.sh" ]; then + REPO="$(cd "$(dirname "$0")/../.." && pwd)" +else + echo "ERROR: Cannot find repo root." + exit 1 +fi + +run_nemoclaw() { + node "$REPO/bin/nemoclaw.js" "$@" +} + +if ! command -v nemoclaw >/dev/null 2>&1; then + nemoclaw() { node "$REPO/bin/nemoclaw.js" "$@"; } +fi + +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-onboard-negative}" +PORT_CONFLICT_PORT="${NEMOCLAW_ONBOARD_NEGATIVE_CONFLICT_PORT:-18080}" +SESSION_FILE="$HOME/.nemoclaw/onboard-session.json" +REGISTRY_FILE="$HOME/.nemoclaw/sandboxes.json" +RESTORE_API_KEY="${NVIDIA_API_KEY:-}" +if [ -n "$RESTORE_API_KEY" ]; then + export NVIDIA_API_KEY="$RESTORE_API_KEY" +fi +nemoclaw_e2e_configure_compatible_inference || { + fail "Hosted CI inference could not be configured" + exit 1 +} +CLOUD_MODEL="${NEMOCLAW_ONBOARD_NEGATIVE_MODEL:-$(nemoclaw_e2e_hosted_inference_model)}" +HOSTED_INFERENCE_BASE_URL="$(nemoclaw_e2e_hosted_inference_base_url)" +EXPECTED_PROVIDER="$(nemoclaw_e2e_expected_route_provider)" +ONBOARD_INFERENCE_ENV=( + "NEMOCLAW_PROVIDER=custom" + "NEMOCLAW_MODEL=$CLOUD_MODEL" + "NVIDIA_API_KEY=$RESTORE_API_KEY" +) +if nemoclaw_e2e_using_compatible_inference; then + ONBOARD_INFERENCE_ENV=( + "NEMOCLAW_PROVIDER=custom" + "NEMOCLAW_ENDPOINT_URL=$HOSTED_INFERENCE_BASE_URL" + "NEMOCLAW_MODEL=$CLOUD_MODEL" + "NEMOCLAW_COMPAT_MODEL=$CLOUD_MODEL" + "COMPATIBLE_API_KEY=$RESTORE_API_KEY" + "NVIDIA_API_KEY=$RESTORE_API_KEY" + ) +fi + +# shellcheck source=test/e2e-vpn/lib/sandbox-teardown.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib/sandbox-teardown.sh" +register_sandbox_for_teardown "$SANDBOX_NAME" +register_sandbox_for_teardown "${SANDBOX_NAME}-bad-key" +register_sandbox_for_teardown "${SANDBOX_NAME}-port" + +cleanup_extra() { + set +e + if [ -n "$PORT_HOLDER_PID" ]; then + kill "$PORT_HOLDER_PID" >/dev/null 2>&1 || true + wait "$PORT_HOLDER_PID" >/dev/null 2>&1 || true + fi + openshell sandbox delete "$SANDBOX_NAME" >/dev/null 2>&1 || true + openshell sandbox delete "${SANDBOX_NAME}-bad-key" >/dev/null 2>&1 || true + openshell sandbox delete "${SANDBOX_NAME}-port" >/dev/null 2>&1 || true + openshell forward stop 18789 >/dev/null 2>&1 || true + openshell gateway destroy -g nemoclaw >/dev/null 2>&1 || true + rm -f "$SESSION_FILE" +} +trap 'cleanup_extra; _nemoclaw_sandbox_teardown' EXIT + +print_summary() { + echo "" + echo "========================================" + echo " PASS: $PASS" + echo " FAIL: $FAIL" + echo " SKIP: $SKIP" + echo " TOTAL: $TOTAL" + echo "========================================" + echo "" +} + +assert_no_stack_trace() { + local output="$1" + if printf '%s\n' "$output" | grep -Eq '(^|[[:space:]])(TypeError|ReferenceError|SyntaxError):|^[[:space:]]+at '; then + return 1 + fi + return 0 +} + +ensure_cli_build() { + if [ -f "$REPO/dist/lib/onboard.js" ] && [ -f "$REPO/dist/lib/validation.js" ]; then + return 0 + fi + info "dist/ is missing; building CLI..." + (cd "$REPO" && npm run build:cli) +} + +run_policy_fallback_check() { + local mode="$1" + node - "$REPO" "$mode" <<'NODE' +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); + +const repo = process.argv[2]; +const mode = process.argv[3]; +const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-negative-policy-")); +process.env.HOME = home; +process.env.NEMOCLAW_NON_INTERACTIVE = "1"; +process.env.NEMOCLAW_POLICY_TIER = "balanced"; +process.env.NEMOCLAW_POLICY_MODE = mode; +process.env.NEMOCLAW_POLICY_PRESETS = ""; + +try { + Object.defineProperty(process, "platform", { value: "darwin" }); +} catch {} + +const credentials = require(path.join(repo, "dist", "lib", "credentials", "store.js")); +const runner = require(path.join(repo, "dist", "lib", "runner.js")); +const registry = require(path.join(repo, "dist", "lib", "state", "registry.js")); +const policies = require(path.join(repo, "dist", "lib", "policy", "index.js")); +const resolveOpenshell = require(path.join(repo, "dist", "lib", "adapters", "openshell", "resolve.js")); + +credentials.prompt = async (msg) => { throw new Error(`unexpected prompt: ${msg}`); }; +credentials.ensureApiKey = async () => {}; +credentials.getCredential = () => null; +runner.run = () => ({ status: 0, stdout: "", stderr: "" }); +runner.runCapture = (command) => { + const text = Array.isArray(command) ? command.join(" ") : String(command); + if (text.includes("sandbox list")) return "test-sb Ready"; + return ""; +}; +registry.registerSandbox = () => true; +registry.updateSandbox = () => true; +registry.getSandbox = () => ({ name: "test-sb", model: null, provider: null }); +resolveOpenshell.resolveOpenshell = () => "/usr/bin/true"; + +const appliedCalls = []; +policies.applyPreset = (_sandbox, name) => { appliedCalls.push(name); return true; }; +policies.applyPresets = (_sandbox, names) => { + for (const name of names) appliedCalls.push(name); + return true; +}; +policies.getAppliedPresets = () => []; + +const warnings = []; +console.log = () => {}; +console.warn = (msg) => warnings.push(String(msg)); + +(async () => { + const { setupPoliciesWithSelection } = require(path.join(repo, "dist", "lib", "onboard.js")); + const applied = await setupPoliciesWithSelection("test-sb", {}); + if (!Array.isArray(applied) || applied.length === 0) { + throw new Error(`expected fallback presets for ${mode}, got ${JSON.stringify(applied)}`); + } + if (appliedCalls.length === 0) { + throw new Error(`expected preset application calls for ${mode}`); + } + if (!warnings.some((line) => line.includes(`Unsupported NEMOCLAW_POLICY_MODE: ${mode}`))) { + throw new Error(`missing unsupported-mode warning for ${mode}: ${warnings.join(" | ")}`); + } + if (!warnings.some((line) => line.includes("Falling back to suggested presets"))) { + throw new Error(`missing fallback warning for ${mode}: ${warnings.join(" | ")}`); + } + const hasTierHint = warnings.some((line) => line.includes("NEMOCLAW_POLICY_TIER=restricted")); + if (mode === "restricted" && !hasTierHint) { + throw new Error(`missing tier hint for restricted mode: ${warnings.join(" | ")}`); + } + if (mode !== "restricted" && hasTierHint) { + throw new Error(`unexpected tier hint for ${mode}: ${warnings.join(" | ")}`); + } +})() + .then(() => fs.rmSync(home, { recursive: true, force: true })) + .catch((err) => { + fs.rmSync(home, { recursive: true, force: true }); + console.error(err && err.stack ? err.stack : err); + process.exit(1); + }); +NODE +} + +run_validation_check() { + node - "$REPO" <<'NODE' +const path = require("node:path"); +const repo = process.argv[2]; +const { validateNvidiaApiKeyValue } = require(path.join(repo, "dist", "lib", "validation.js")); + +const nvidiaError = validateNvidiaApiKeyValue("not-a-nvidia-key", "NVIDIA_API_KEY"); +if (!nvidiaError || !nvidiaError.includes("Must start with nvapi-")) { + throw new Error(`expected NVIDIA key prefix rejection, got: ${nvidiaError}`); +} + +const anthropicError = validateNvidiaApiKeyValue("sk-ant-test-key-without-nvapi-prefix", "ANTHROPIC_API_KEY"); +if (anthropicError !== null) { + throw new Error(`expected Anthropic key to bypass nvapi- prefix enforcement, got: ${anthropicError}`); +} +NODE +} + +start_port_holder() { + local port="$1" + PORT_HOLDER_PID="" + node - "$port" <<'NODE' >/tmp/nemoclaw-e2e-port-holder.log 2>&1 & +const net = require("node:net"); +const port = Number(process.argv[2]); +const server = net.createServer((socket) => socket.end()); +server.on("error", (err) => { + console.error(err && err.message ? err.message : err); + process.exit(2); +}); +server.listen(port, "127.0.0.1", () => { + console.log("ready"); +}); +setInterval(() => {}, 1000); +NODE + PORT_HOLDER_PID=$! + local _i + for _i in $(seq 1 40); do + if node -e 'const net=require("node:net"); const port=Number(process.argv[1]); const s=net.connect(port,"127.0.0.1"); s.once("connect",()=>{s.destroy(); process.exit(0);}); s.once("error",()=>process.exit(1)); setTimeout(()=>process.exit(1),250);' "$port" >/dev/null 2>&1; then + return 0 + fi + if ! kill -0 "$PORT_HOLDER_PID" >/dev/null 2>&1; then + PORT_HOLDER_PID="" + return 1 + fi + sleep 0.25 + done + return 1 +} + +section "Phase 0: Prerequisites" + +if command -v node >/dev/null 2>&1; then + pass "Node.js available" +else + fail "Node.js not found" + print_summary + exit 1 +fi + +if ensure_cli_build; then + pass "CLI build output available" +else + fail "Could not build CLI" + print_summary + exit 1 +fi + +if docker info >/dev/null 2>&1; then + pass "Docker is running" +else + fail "Docker is not running" + print_summary + exit 1 +fi + +if command -v openshell >/dev/null 2>&1; then + pass "openshell CLI installed" +else + fail "openshell CLI not found" + print_summary + exit 1 +fi + +if ! nemoclaw_e2e_require_hosted_inference_key; then + print_summary + exit 1 +fi + +section "Phase 1: Pre-cleanup" +info "Destroying leftover test sandboxes and gateway state..." +run_nemoclaw "$SANDBOX_NAME" destroy --yes >/dev/null 2>&1 || true +run_nemoclaw "${SANDBOX_NAME}-bad-key" destroy --yes >/dev/null 2>&1 || true +run_nemoclaw "${SANDBOX_NAME}-port" destroy --yes >/dev/null 2>&1 || true +openshell sandbox delete "$SANDBOX_NAME" >/dev/null 2>&1 || true +openshell sandbox delete "${SANDBOX_NAME}-bad-key" >/dev/null 2>&1 || true +openshell sandbox delete "${SANDBOX_NAME}-port" >/dev/null 2>&1 || true +openshell forward stop 18789 >/dev/null 2>&1 || true +openshell gateway destroy -g nemoclaw >/dev/null 2>&1 || true +rm -f "$SESSION_FILE" +pass "Pre-cleanup complete" + +section "Phase 2: Policy-mode fallback validation" + +if run_policy_fallback_check restricted; then + pass "NEMOCLAW_POLICY_MODE=restricted falls back to suggested presets" +else + fail "NEMOCLAW_POLICY_MODE=restricted did not fall back cleanly" +fi + +if run_policy_fallback_check nonexistent; then + pass "NEMOCLAW_POLICY_MODE=nonexistent falls back to suggested presets" +else + fail "NEMOCLAW_POLICY_MODE=nonexistent did not fall back cleanly" +fi + +section "Phase 3: Entry option validation" + +FROM_GUARD_LOG="$(mktemp)" +env -u NEMOCLAW_SANDBOX_NAME \ + "${ONBOARD_INFERENCE_ENV[@]}" \ + NEMOCLAW_NON_INTERACTIVE=1 \ + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ + NEMOCLAW_POLICY_MODE=skip \ + node "$REPO/bin/nemoclaw.js" onboard --non-interactive --from "$REPO/Dockerfile" \ + >"$FROM_GUARD_LOG" 2>&1 +from_guard_exit=$? +from_guard_output="$(cat "$FROM_GUARD_LOG")" +rm -f "$FROM_GUARD_LOG" +rm -f "$SESSION_FILE" + +if [ "$from_guard_exit" -eq 1 ]; then + pass "--from without sandbox name exited 1" +else + fail "--from without sandbox name exited $from_guard_exit (expected 1)" +fi + +if printf '%s\n' "$from_guard_output" | grep -q -- "--from requires --name "; then + pass "--from missing-name guard message is explicit" +else + fail "--from missing-name guard message missing" +fi + +if assert_no_stack_trace "$from_guard_output"; then + pass "--from missing-name guard did not print a stack trace" +else + fail "--from missing-name guard printed a stack trace" +fi + +FROM_ENV_NAME_LOG="$(mktemp)" +env \ + "${ONBOARD_INFERENCE_ENV[@]}" \ + NEMOCLAW_NON_INTERACTIVE=1 \ + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ + NEMOCLAW_SANDBOX_NAME="bad name" \ + NEMOCLAW_POLICY_MODE=skip \ + node "$REPO/bin/nemoclaw.js" onboard --non-interactive --from "$REPO/Dockerfile" \ + >"$FROM_ENV_NAME_LOG" 2>&1 +from_env_name_exit=$? +from_env_name_output="$(cat "$FROM_ENV_NAME_LOG")" +rm -f "$FROM_ENV_NAME_LOG" +rm -f "$SESSION_FILE" + +if [ "$from_env_name_exit" -eq 1 ]; then + pass "--from with NEMOCLAW_SANDBOX_NAME reached name validation" +else + fail "--from with NEMOCLAW_SANDBOX_NAME exited $from_env_name_exit (expected 1)" +fi + +if printf '%s\n' "$from_env_name_output" | grep -q "Invalid sandbox name"; then + pass "--from with env sandbox name used NEMOCLAW_SANDBOX_NAME" +else + fail "--from with env sandbox name did not reach name validation" +fi + +if printf '%s\n' "$from_env_name_output" | grep -q -- "--from requires --name "; then + fail "--from with env sandbox name still printed missing-name guard" +else + pass "--from with env sandbox name did not print missing-name guard" +fi + +section "Phase 4: Provider credential validation" + +INVALID_KEY_LOG="$(mktemp)" +NEMOCLAW_NON_INTERACTIVE=1 \ + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ + NEMOCLAW_SANDBOX_NAME="${SANDBOX_NAME}-bad-key" \ + NEMOCLAW_RECREATE_SANDBOX=1 \ + NEMOCLAW_PROVIDER=custom \ + NEMOCLAW_POLICY_MODE=skip \ + NVIDIA_API_KEY=not-a-nvidia-key \ + node "$REPO/bin/nemoclaw.js" onboard --non-interactive >"$INVALID_KEY_LOG" 2>&1 +invalid_key_exit=$? +invalid_key_output="$(cat "$INVALID_KEY_LOG")" +rm -f "$INVALID_KEY_LOG" +openshell gateway destroy -g nemoclaw >/dev/null 2>&1 || true +rm -f "$SESSION_FILE" + +if [ "$invalid_key_exit" -eq 1 ]; then + pass "Invalid NVIDIA API key exited 1" +else + fail "Invalid NVIDIA API key exited $invalid_key_exit (expected 1)" +fi + +if printf '%s\n' "$invalid_key_output" | grep -q "Invalid NVIDIA API key. Must start with nvapi-"; then + pass "Invalid NVIDIA API key message is explicit" +else + fail "Invalid NVIDIA API key message missing" +fi + +if assert_no_stack_trace "$invalid_key_output"; then + pass "Invalid NVIDIA API key path did not print a stack trace" +else + fail "Invalid NVIDIA API key path printed a stack trace" +fi + +if run_validation_check; then + pass "Provider-aware credential validation accepts non-NVIDIA key prefixes" +else + fail "Provider-aware credential validation rejected a non-NVIDIA key prefix" +fi + +section "Phase 5: Gateway port conflict" + +if start_port_holder "$PORT_CONFLICT_PORT"; then + pass "Held gateway port ${PORT_CONFLICT_PORT} with a host listener" +else + skip "Could not start a local holder on port ${PORT_CONFLICT_PORT}; attempting conflict assertion against any existing listener" +fi + +PORT_CONFLICT_LOG="$(mktemp)" +env \ + "${ONBOARD_INFERENCE_ENV[@]}" \ + NEMOCLAW_NON_INTERACTIVE=1 \ + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ + NEMOCLAW_SANDBOX_NAME="${SANDBOX_NAME}-port" \ + NEMOCLAW_RECREATE_SANDBOX=1 \ + NEMOCLAW_GATEWAY_PORT="$PORT_CONFLICT_PORT" \ + NEMOCLAW_POLICY_MODE=skip \ + node "$REPO/bin/nemoclaw.js" onboard --non-interactive >"$PORT_CONFLICT_LOG" 2>&1 +port_conflict_exit=$? +port_conflict_output="$(cat "$PORT_CONFLICT_LOG")" +rm -f "$PORT_CONFLICT_LOG" + +if [ -n "$PORT_HOLDER_PID" ]; then + kill "$PORT_HOLDER_PID" >/dev/null 2>&1 || true + wait "$PORT_HOLDER_PID" >/dev/null 2>&1 || true + PORT_HOLDER_PID="" +fi +rm -f "$SESSION_FILE" + +if [ "$port_conflict_exit" -eq 1 ]; then + pass "Onboard rejected occupied gateway port" +else + fail "Occupied gateway port exited $port_conflict_exit (expected 1)" +fi + +if printf '%s\n' "$port_conflict_output" | grep -q "Port ${PORT_CONFLICT_PORT} is not available"; then + pass "Port conflict message is user-friendly" +else + fail "Port conflict message missing" +fi + +if assert_no_stack_trace "$port_conflict_output"; then + pass "Port conflict path did not print a stack trace" +else + fail "Port conflict path printed a stack trace" +fi + +section "Phase 6: Live non-interactive onboard honors presets and model" + +LIVE_LOG="$(mktemp)" +env \ + "${ONBOARD_INFERENCE_ENV[@]}" \ + NEMOCLAW_NON_INTERACTIVE=1 \ + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ + NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" \ + NEMOCLAW_RECREATE_SANDBOX=1 \ + NEMOCLAW_POLICY_MODE=custom \ + NEMOCLAW_POLICY_PRESETS=npm,pypi \ + node "$REPO/bin/nemoclaw.js" onboard --non-interactive >"$LIVE_LOG" 2>&1 +live_exit=$? +live_output="$(cat "$LIVE_LOG")" +rm -f "$LIVE_LOG" + +if [ "$live_exit" -eq 0 ]; then + pass "Live non-interactive onboard completed" +else + fail "Live non-interactive onboard exited $live_exit" + printf '%s\n' "$live_output" | tail -120 + print_summary + exit 1 +fi + +if printf '%s\n' "$live_output" | grep -Fq "$CLOUD_MODEL"; then + pass "Live onboard selected requested hosted model" +else + fail "Live onboard output did not confirm requested hosted model" +fi + +if node - "$REGISTRY_FILE" "$SANDBOX_NAME" "$CLOUD_MODEL" "$EXPECTED_PROVIDER" <<'NODE'; then +const fs = require("node:fs"); +const [registryPath, sandboxName, expectedModel, expectedProvider] = process.argv.slice(2); +const registry = JSON.parse(fs.readFileSync(registryPath, "utf8")); +const sandbox = registry.sandboxes && registry.sandboxes[sandboxName]; +if (!sandbox) throw new Error(`missing sandbox registry entry: ${sandboxName}`); +if (sandbox.provider !== expectedProvider) { + throw new Error(`expected provider ${expectedProvider}, got ${sandbox.provider}`); +} +if (sandbox.model !== expectedModel) { + throw new Error(`expected model ${expectedModel}, got ${sandbox.model}`); +} +const policies = Array.isArray(sandbox.policies) ? sandbox.policies : []; +for (const preset of ["npm", "pypi"]) { + if (!policies.includes(preset)) { + throw new Error(`missing policy preset ${preset}; policies=${JSON.stringify(policies)}`); + } +} +NODE + pass "Registry recorded requested provider, model, and policy presets" +else + fail "Registry did not record requested provider, model, and policy presets" +fi + +if node - "$SESSION_FILE" "$SANDBOX_NAME" "$CLOUD_MODEL" "$EXPECTED_PROVIDER" <<'NODE'; then +const fs = require("node:fs"); +const [sessionPath, sandboxName, expectedModel, expectedProvider] = process.argv.slice(2); +const session = JSON.parse(fs.readFileSync(sessionPath, "utf8")); +if (session.status !== "complete") throw new Error(`session status ${session.status}`); +if (session.sandboxName !== sandboxName) throw new Error(`session sandbox ${session.sandboxName}`); +if (session.provider !== expectedProvider) throw new Error(`session provider ${session.provider}`); +if (session.model !== expectedModel) throw new Error(`session model ${session.model}`); +const presets = Array.isArray(session.policyPresets) ? session.policyPresets : []; +for (const preset of ["npm", "pypi"]) { + if (!presets.includes(preset)) { + throw new Error(`missing session policy preset ${preset}; presets=${JSON.stringify(presets)}`); + } +} +NODE + pass "Session recorded requested provider, model, and policy presets" +else + fail "Session did not record requested provider, model, and policy presets" +fi + +section "Phase 7: Final cleanup" + +if [[ "${NEMOCLAW_E2E_KEEP_SANDBOX:-}" != "1" ]]; then + run_nemoclaw "$SANDBOX_NAME" destroy --yes >/dev/null 2>&1 || true +fi +openshell sandbox delete "$SANDBOX_NAME" >/dev/null 2>&1 || true +openshell forward stop 18789 >/dev/null 2>&1 || true +openshell gateway destroy -g nemoclaw >/dev/null 2>&1 || true +rm -f "$SESSION_FILE" + +if openshell sandbox get "$SANDBOX_NAME" >/dev/null 2>&1; then + fail "Sandbox '$SANDBOX_NAME' still exists after cleanup" +else + pass "Sandbox '$SANDBOX_NAME' cleaned up" +fi + +if [ -f "$SESSION_FILE" ]; then + fail "Onboard session file still exists after cleanup" +else + pass "Onboard session file cleaned up" +fi + +pass "Final cleanup complete" +print_summary + +if [ "$FAIL" -ne 0 ]; then + exit 1 +fi diff --git a/test/e2e-vpn/test-onboard-repair.sh b/test/e2e-vpn/test-onboard-repair.sh new file mode 100755 index 00000000000..10f36b476fe --- /dev/null +++ b/test/e2e-vpn/test-onboard-repair.sh @@ -0,0 +1,406 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# E2E: resume repair and invalidation behavior. +# +# Regression coverage for issue #446. +# Validates that: +# 1. Resume recreates a missing recorded sandbox instead of assuming it still exists. +# 2. Resume rejects a different requested sandbox name on the same host. +# 3. Resume rejects explicit provider/model changes that conflict with recorded state. +# +# Prerequisites: +# - Docker running +# - openshell CLI installed +# - Node.js available +# - NVIDIA_API_KEY set before starting the test +# +# Usage: +# NVIDIA_API_KEY=... bash test/e2e-vpn/test-onboard-repair.sh + +set -uo pipefail + +PASS=0 +FAIL=0 +SKIP=0 +TOTAL=0 + +pass() { + ((PASS++)) + ((TOTAL++)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} +fail() { + ((FAIL++)) + ((TOTAL++)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} +skip() { + ((SKIP++)) + ((TOTAL++)) + printf '\033[33m SKIP: %s\033[0m\n' "$1" +} +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } + +if [ -d /workspace ] && [ -f /workspace/install.sh ]; then + REPO="/workspace" +elif [ -f "$(cd "$(dirname "$0")/../.." && pwd)/install.sh" ]; then + REPO="$(cd "$(dirname "$0")/../.." && pwd)" +else + echo "ERROR: Cannot find repo root." + exit 1 +fi + +run_nemoclaw() { + node "$REPO/bin/nemoclaw.js" "$@" +} + +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-repair}" +OTHER_SANDBOX_NAME="${NEMOCLAW_OTHER_SANDBOX_NAME:-e2e-other}" +INSTALL_SANDBOX_NAME="${NEMOCLAW_E2E_INSTALL_SANDBOX_NAME:-}" + +# Shim so the teardown helper's trap can call `nemoclaw destroy` even when +# this repo-local test run has no globally-installed `nemoclaw` on PATH (it +# drives the CLI via `node "$REPO/bin/nemoclaw.js"` via run_nemoclaw). +if ! command -v nemoclaw >/dev/null 2>&1; then + nemoclaw() { node "$REPO/bin/nemoclaw.js" "$@"; } +fi + +# shellcheck source=test/e2e-vpn/lib/sandbox-teardown.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib/sandbox-teardown.sh" +# shellcheck source=test/e2e-vpn/lib/ci-compatible-inference.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib/ci-compatible-inference.sh" +register_sandbox_for_teardown "$SANDBOX_NAME" +register_sandbox_for_teardown "$OTHER_SANDBOX_NAME" +if [ -n "$INSTALL_SANDBOX_NAME" ]; then + register_sandbox_for_teardown "$INSTALL_SANDBOX_NAME" +fi + +SESSION_FILE="$HOME/.nemoclaw/onboard-session.json" +RESTORE_API_KEY="${NVIDIA_API_KEY:-}" + +wait_openshell_sandbox_absent() { + local sandbox_name="$1" + local timeout="${2:-60}" + local deadline=$((SECONDS + timeout)) + local output status + + while [ "$SECONDS" -le "$deadline" ]; do + output="$(openshell sandbox get "$sandbox_name" 2>&1)" + status=$? + if [ "$status" -ne 0 ] && grep -qiE 'NotFound|Not Found|sandbox not found' <<<"$output"; then + return 0 + fi + sleep 1 + done + + info "OpenShell still reports sandbox '$sandbox_name' after ${timeout}s:" + printf '%s\n' "$output" | sed 's/^/ /' + return 1 +} + +# ══════════════════════════════════════════════════════════════════ +# Phase 0: Pre-cleanup +# ══════════════════════════════════════════════════════════════════ +section "Phase 0: Pre-cleanup" +info "Destroying any leftover sandbox/gateway from previous runs..." +if [ -n "$INSTALL_SANDBOX_NAME" ]; then + run_nemoclaw "$INSTALL_SANDBOX_NAME" destroy 2>/dev/null || true +fi +run_nemoclaw "$SANDBOX_NAME" destroy 2>/dev/null || true +run_nemoclaw "$OTHER_SANDBOX_NAME" destroy 2>/dev/null || true +if [ -n "$INSTALL_SANDBOX_NAME" ]; then + openshell sandbox delete "$INSTALL_SANDBOX_NAME" 2>/dev/null || true +fi +openshell sandbox delete "$SANDBOX_NAME" 2>/dev/null || true +openshell sandbox delete "$OTHER_SANDBOX_NAME" 2>/dev/null || true +openshell forward stop 18789 2>/dev/null || true +openshell gateway destroy -g nemoclaw 2>/dev/null || true +rm -f "$SESSION_FILE" +pass "Pre-cleanup complete" + +# ══════════════════════════════════════════════════════════════════ +# Phase 1: Prerequisites +# ══════════════════════════════════════════════════════════════════ +section "Phase 1: Prerequisites" + +if docker info >/dev/null 2>&1; then + pass "Docker is running" +else + fail "Docker is not running — cannot continue" + exit 1 +fi + +if command -v openshell >/dev/null 2>&1; then + pass "openshell CLI installed" +else + fail "openshell CLI not found — cannot continue" + exit 1 +fi + +if command -v node >/dev/null 2>&1; then + pass "Node.js available" +else + fail "Node.js not found — cannot continue" + exit 1 +fi + +if [[ -z "$RESTORE_API_KEY" ]]; then + fail "NVIDIA_API_KEY not set or invalid — required for resume completion" + exit 1 +fi +pass "NVIDIA_API_KEY is set" + +export NVIDIA_API_KEY="$RESTORE_API_KEY" +nemoclaw_e2e_configure_compatible_inference || exit 1 +pass "Exported NVIDIA_API_KEY for the repair run (host writes nothing to disk; OpenShell gateway is the system of record)" + +# ══════════════════════════════════════════════════════════════════ +# Phase 2: Create interrupted resumable state +# ══════════════════════════════════════════════════════════════════ +section "Phase 2: Create interrupted state" +info "Running onboard with E2E failure injection at the policy step..." + +# Force a deterministic interruption after the sandbox and OpenClaw setup +# complete, but before policy setup completes. This keeps repair coverage +# independent of product validation behavior such as policy-mode parsing. +FIRST_LOG="$(mktemp)" +NEMOCLAW_NON_INTERACTIVE=1 \ + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ + NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" \ + NEMOCLAW_RECREATE_SANDBOX=1 \ + NEMOCLAW_POLICY_MODE=suggested \ + NEMOCLAW_E2E_FAILURE_INJECTION=1 \ + NEMOCLAW_E2E_FORCE_FAIL_AT_STEP=policies \ + node "$REPO/bin/nemoclaw.js" onboard --non-interactive >"$FIRST_LOG" 2>&1 +first_exit=$? +first_output="$(cat "$FIRST_LOG")" +rm -f "$FIRST_LOG" + +if [ $first_exit -eq 1 ]; then + pass "First onboard exited 1 (expected interrupted run)" +else + fail "First onboard exited $first_exit (expected 1)" + echo "$first_output" + exit 1 +fi + +if [ -f "$SESSION_FILE" ]; then + pass "Onboard session file created" +else + fail "Onboard session file missing after interrupted run" +fi + +if echo "$first_output" | grep -q "\[e2e\] Forced onboarding failure at step 'policies'."; then + pass "First run failed at policy setup as intended" +else + fail "First run did not fail at the expected policy step" + info "Captured first-onboard stdout/stderr (exit=$first_exit):" + printf '%s\n' "$first_output" | sed 's/^/ /' +fi + +if openshell sandbox get "$SANDBOX_NAME" >/dev/null 2>&1; then + pass "Sandbox '$SANDBOX_NAME' exists after interrupted run" +else + fail "Sandbox '$SANDBOX_NAME' not found after interrupted run" + info "Captured first-onboard stdout/stderr (exit=$first_exit):" + printf '%s\n' "$first_output" | sed 's/^/ /' +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 3: Repair missing sandbox on resume +# ══════════════════════════════════════════════════════════════════ +section "Phase 3: Repair missing sandbox" +info "Deleting the recorded sandbox under the session, then resuming..." + +openshell sandbox delete "$SANDBOX_NAME" >/dev/null 2>&1 || true +openshell forward stop 18789 >/dev/null 2>&1 || true + +if wait_openshell_sandbox_absent "$SANDBOX_NAME" 60; then + pass "Sandbox '$SANDBOX_NAME' removed to simulate stale recorded state" +else + fail "Sandbox '$SANDBOX_NAME' still exists after forced deletion" +fi + +REPAIR_LOG="$(mktemp)" +env -u NVIDIA_API_KEY -u COMPATIBLE_API_KEY \ + NEMOCLAW_NON_INTERACTIVE=1 \ + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ + NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" \ + NEMOCLAW_POLICY_MODE=skip \ + node "$REPO/bin/nemoclaw.js" onboard --resume --non-interactive >"$REPAIR_LOG" 2>&1 +repair_exit=$? +repair_output="$(cat "$REPAIR_LOG")" +rm -f "$REPAIR_LOG" + +if [ $repair_exit -eq 0 ]; then + pass "Resume completed after repairing missing sandbox" +else + fail "Resume exited $repair_exit during missing-sandbox repair" + echo "$repair_output" + exit 1 +fi + +if grep -q "\[resume\] Skipping preflight (cached)" <<<"$repair_output"; then + pass "Repair resume skipped preflight" +else + fail "Repair resume did not skip preflight" +fi + +if grep -q "\[resume\] Skipping gateway (running)" <<<"$repair_output"; then + pass "Repair resume skipped gateway" +else + fail "Repair resume did not skip gateway" +fi + +if grep -q "\[resume\] Recorded sandbox state is unavailable; recreating it." <<<"$repair_output"; then + pass "Repair resume detected missing sandbox" +else + fail "Repair resume did not report missing sandbox recreation" +fi + +# The step numbering is [6/8] in the current onboard flow. +if grep -q "Creating sandbox" <<<"$repair_output"; then + pass "Repair resume recreated sandbox" +else + fail "Repair resume did not rerun sandbox creation" +fi + +if run_nemoclaw "$SANDBOX_NAME" status >/dev/null 2>&1; then + pass "Repaired sandbox '$SANDBOX_NAME' is manageable" +else + fail "Repaired sandbox '$SANDBOX_NAME' status failed" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 4: Reject conflicting sandbox +# ══════════════════════════════════════════════════════════════════ +section "Phase 4: Reject conflicting sandbox" + +# Phase 3 completed the session (resumable=false). Re-create interrupted state +# so the conflict detection path is exercised (it runs before the "no resumable +# session" early-exit). +info "Re-creating interrupted state for conflict testing..." +REINJECT_LOG="$(mktemp)" +NEMOCLAW_NON_INTERACTIVE=1 \ + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ + NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" \ + NEMOCLAW_RECREATE_SANDBOX=1 \ + NEMOCLAW_POLICY_MODE=suggested \ + NEMOCLAW_E2E_FAILURE_INJECTION=1 \ + NEMOCLAW_E2E_FORCE_FAIL_AT_STEP=policies \ + node "$REPO/bin/nemoclaw.js" onboard --non-interactive >"$REINJECT_LOG" 2>&1 || true +rm -f "$REINJECT_LOG" +pass "Re-created interrupted session for conflict tests" + +info "Attempting resume with a different sandbox name..." + +SANDBOX_CONFLICT_LOG="$(mktemp)" +env -u NVIDIA_API_KEY -u COMPATIBLE_API_KEY \ + NEMOCLAW_NON_INTERACTIVE=1 \ + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ + NEMOCLAW_SANDBOX_NAME="$OTHER_SANDBOX_NAME" \ + NEMOCLAW_POLICY_MODE=skip \ + node "$REPO/bin/nemoclaw.js" onboard --resume --non-interactive >"$SANDBOX_CONFLICT_LOG" 2>&1 +sandbox_conflict_exit=$? +sandbox_conflict_output="$(cat "$SANDBOX_CONFLICT_LOG")" +rm -f "$SANDBOX_CONFLICT_LOG" + +if [ $sandbox_conflict_exit -eq 1 ]; then + pass "Resume rejected conflicting sandbox name" +else + fail "Resume exited $sandbox_conflict_exit for conflicting sandbox (expected 1)" +fi + +if echo "$sandbox_conflict_output" | grep -q "Resumable state belongs to sandbox '${SANDBOX_NAME}', not '${OTHER_SANDBOX_NAME}'."; then + pass "Conflicting sandbox message is explicit" +else + fail "Conflicting sandbox message missing or incorrect" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 5: Reject conflicting provider/model +# ══════════════════════════════════════════════════════════════════ +section "Phase 5: Reject conflicting provider and model" +info "Attempting resume with conflicting provider/model inputs..." + +PROVIDER_CONFLICT_LOG="$(mktemp)" +env -u NVIDIA_API_KEY -u COMPATIBLE_API_KEY \ + NEMOCLAW_NON_INTERACTIVE=1 \ + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ + NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" \ + NEMOCLAW_PROVIDER=openai \ + NEMOCLAW_MODEL=gpt-5.4 \ + NEMOCLAW_POLICY_MODE=skip \ + node "$REPO/bin/nemoclaw.js" onboard --resume --non-interactive >"$PROVIDER_CONFLICT_LOG" 2>&1 +provider_conflict_exit=$? +provider_conflict_output="$(cat "$PROVIDER_CONFLICT_LOG")" +rm -f "$PROVIDER_CONFLICT_LOG" + +if [ $provider_conflict_exit -eq 1 ]; then + pass "Resume rejected conflicting provider/model" +else + fail "Resume exited $provider_conflict_exit for conflicting provider/model (expected 1)" +fi + +if echo "$provider_conflict_output" | grep -Eq "Resumable state recorded provider '.*', not '.*'\."; then + pass "Conflicting provider message is explicit" +else + fail "Conflicting provider message missing or incorrect" +fi + +if echo "$provider_conflict_output" | grep -Eq "Resumable state recorded model '.*', not 'gpt-5.4'\."; then + pass "Conflicting model message is explicit" +else + fail "Conflicting model message missing or incorrect" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 6: Final cleanup +# ══════════════════════════════════════════════════════════════════ +section "Phase 6: Final cleanup" + +if [[ "${NEMOCLAW_E2E_KEEP_SANDBOX:-}" != "1" ]]; then + run_nemoclaw "$SANDBOX_NAME" destroy 2>/dev/null || true + run_nemoclaw "$OTHER_SANDBOX_NAME" destroy 2>/dev/null || true +fi +openshell sandbox delete "$SANDBOX_NAME" 2>/dev/null || true +openshell sandbox delete "$OTHER_SANDBOX_NAME" 2>/dev/null || true +openshell forward stop 18789 2>/dev/null || true +openshell gateway destroy -g nemoclaw 2>/dev/null || true +rm -f "$SESSION_FILE" + +if openshell sandbox get "$SANDBOX_NAME" >/dev/null 2>&1; then + fail "Sandbox '$SANDBOX_NAME' still exists after cleanup" +else + pass "Sandbox '$SANDBOX_NAME' cleaned up" +fi + +if [ -f "$SESSION_FILE" ]; then + fail "Onboard session file still exists after cleanup" +else + pass "Onboard session file cleaned up" +fi + +pass "Final cleanup complete" + +# ══════════════════════════════════════════════════════════════════ +# Summary +# ══════════════════════════════════════════════════════════════════ +echo "" +echo "========================================" +echo " PASS: $PASS" +echo " FAIL: $FAIL" +echo " SKIP: $SKIP" +echo " TOTAL: $TOTAL" +echo "========================================" +echo "" + +if [ $FAIL -ne 0 ]; then + exit 1 +fi diff --git a/test/e2e-vpn/test-onboard-resume.sh b/test/e2e-vpn/test-onboard-resume.sh new file mode 100755 index 00000000000..20587374b75 --- /dev/null +++ b/test/e2e-vpn/test-onboard-resume.sh @@ -0,0 +1,448 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# E2E: interrupted onboard -> resume -> verify completion. +# +# Regression test for issue #446. +# Validates that: +# 1. A non-interactive onboard run can fail after sandbox creation while leaving resumable state. +# 2. The onboard session file records the interrupted state safely. +# 3. `nemoclaw onboard --resume --non-interactive` skips cached preflight, +# gateway, and sandbox work, then completes by hydrating the stored credential. +# +# Prerequisites: +# - Docker running +# - openshell CLI installed +# - Node.js available +# - NVIDIA_API_KEY set before starting the test +# +# Usage: +# NVIDIA_API_KEY=... bash test/e2e-vpn/test-onboard-resume.sh + +set -uo pipefail + +export NEMOCLAW_E2E_DEFAULT_TIMEOUT=600 +SCRIPT_DIR_TIMEOUT="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +# shellcheck source=test/e2e-vpn/e2e-timeout.sh +source "${SCRIPT_DIR_TIMEOUT}/e2e-timeout.sh" + +PASS=0 +FAIL=0 +SKIP=0 +TOTAL=0 + +pass() { + ((PASS++)) + ((TOTAL++)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} +fail() { + ((FAIL++)) + ((TOTAL++)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} +skip() { + ((SKIP++)) + ((TOTAL++)) + printf '\033[33m SKIP: %s\033[0m\n' "$1" +} +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } + +if [ -d /workspace ] && [ -f /workspace/install.sh ]; then + REPO="/workspace" +elif [ -f "$(cd "$(dirname "$0")/../.." && pwd)/install.sh" ]; then + REPO="$(cd "$(dirname "$0")/../.." && pwd)" +else + echo "ERROR: Cannot find repo root." + exit 1 +fi + +run_nemoclaw() { + node "$REPO/bin/nemoclaw.js" "$@" +} + +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-resume}" + +# Shim so the teardown helper's trap can call `nemoclaw destroy` even when +# this repo-local test run has no globally-installed `nemoclaw` on PATH (it +# drives the CLI via `node "$REPO/bin/nemoclaw.js"` via run_nemoclaw). +if ! command -v nemoclaw >/dev/null 2>&1; then + nemoclaw() { node "$REPO/bin/nemoclaw.js" "$@"; } +fi + +# shellcheck source=test/e2e-vpn/lib/sandbox-teardown.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib/sandbox-teardown.sh" +# shellcheck source=test/e2e-vpn/lib/ci-compatible-inference.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib/ci-compatible-inference.sh" +register_sandbox_for_teardown "$SANDBOX_NAME" + +SESSION_FILE="$HOME/.nemoclaw/onboard-session.json" +REGISTRY="$HOME/.nemoclaw/sandboxes.json" +RESTORE_API_KEY="${NVIDIA_API_KEY:-}" + +# ══════════════════════════════════════════════════════════════════ +# Phase 0: Pre-cleanup +# ══════════════════════════════════════════════════════════════════ +section "Phase 0: Pre-cleanup" +info "Destroying any leftover sandbox/gateway from previous runs..." +run_nemoclaw "$SANDBOX_NAME" destroy 2>/dev/null || true +openshell sandbox delete "$SANDBOX_NAME" 2>/dev/null || true +openshell forward stop 18789 2>/dev/null || true +openshell gateway destroy -g nemoclaw 2>/dev/null || true +rm -f "$SESSION_FILE" +pass "Pre-cleanup complete" + +# ══════════════════════════════════════════════════════════════════ +# Phase 1: Prerequisites +# ══════════════════════════════════════════════════════════════════ +section "Phase 1: Prerequisites" + +if docker info >/dev/null 2>&1; then + pass "Docker is running" +else + fail "Docker is not running — cannot continue" + exit 1 +fi + +if command -v openshell >/dev/null 2>&1; then + pass "openshell CLI installed" +else + fail "openshell CLI not found — cannot continue" + exit 1 +fi + +if command -v node >/dev/null 2>&1; then + pass "Node.js available" +else + fail "Node.js not found — cannot continue" + exit 1 +fi + +if [[ -z "$RESTORE_API_KEY" ]]; then + fail "NVIDIA_API_KEY not set or invalid — required for resume completion" + exit 1 +fi +pass "NVIDIA_API_KEY is set" + +export NVIDIA_API_KEY="$RESTORE_API_KEY" +nemoclaw_e2e_configure_compatible_inference || exit 1 +HOSTED_INFERENCE_BASE_URL="$(nemoclaw_e2e_hosted_inference_base_url)" +EXPECTED_PROVIDER="$(nemoclaw_e2e_expected_route_provider)" + +if nemoclaw_e2e_probe_hosted_inference; then + pass "Network access to ${HOSTED_INFERENCE_BASE_URL}" +else + fail "Cannot reach ${HOSTED_INFERENCE_BASE_URL}" + exit 1 +fi + +pass "Exported NVIDIA_API_KEY for the resume run (host writes nothing to disk; OpenShell gateway is the system of record)" + +# ══════════════════════════════════════════════════════════════════ +# Phase 2: First onboard (forced failure after sandbox creation) +# ══════════════════════════════════════════════════════════════════ +section "Phase 2: First onboard (interrupted)" +info "Running onboard with E2E failure injection at the policy step..." + +# Force a deterministic interruption after the sandbox and OpenClaw setup +# complete, but before policy setup completes. This keeps resume coverage +# independent of product validation behavior such as policy-mode parsing. +FIRST_LOG="$(mktemp)" +NEMOCLAW_NON_INTERACTIVE=1 \ + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ + NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" \ + NEMOCLAW_RECREATE_SANDBOX=1 \ + NEMOCLAW_POLICY_MODE=suggested \ + NEMOCLAW_E2E_FAILURE_INJECTION=1 \ + NEMOCLAW_E2E_FORCE_FAIL_AT_STEP=policies \ + node "$REPO/bin/nemoclaw.js" onboard --non-interactive >"$FIRST_LOG" 2>&1 +first_exit=$? +first_output="$(cat "$FIRST_LOG")" +rm -f "$FIRST_LOG" + +if [ $first_exit -eq 1 ]; then + pass "First onboard exited 1 (expected interrupted run)" +else + fail "First onboard exited $first_exit (expected 1)" + echo "$first_output" + exit 1 +fi + +if echo "$first_output" | grep -q "Sandbox '${SANDBOX_NAME}' created"; then + pass "Sandbox '$SANDBOX_NAME' created before interruption" +else + fail "Sandbox creation not confirmed in first run output" +fi + +if echo "$first_output" | grep -q "\[e2e\] Forced onboarding failure at step 'policies'."; then + pass "First run failed at policy setup as intended" +else + fail "First run did not fail at the expected policy step" +fi + +if openshell sandbox get "$SANDBOX_NAME" >/dev/null 2>&1; then + pass "Sandbox '$SANDBOX_NAME' exists after interrupted run" +else + fail "Sandbox '$SANDBOX_NAME' not found after interrupted run" +fi + +if [ -f "$SESSION_FILE" ]; then + pass "Onboard session file created" +else + fail "Onboard session file missing after interrupted run" +fi + +node -e ' +const fs = require("fs"); +const file = process.argv[1]; +const data = JSON.parse(fs.readFileSync(file, "utf8")); +if (data.status !== "failed") process.exit(1); +if (data.lastCompletedStep !== "openclaw") process.exit(2); +if (!data.failure || data.failure.step !== "policies") process.exit(3); +' "$SESSION_FILE" +case $? in + 0) pass "Session file recorded openclaw completion and policy failure" ;; + *) fail "Session file did not record the expected interrupted state" ;; +esac + +# ══════════════════════════════════════════════════════════════════ +# Phase 3: Resume and complete +# ══════════════════════════════════════════════════════════════════ +section "Phase 3: Resume" +info "Running onboard --resume with NVIDIA_API_KEY removed from env..." + +RESUME_LOG="$(mktemp)" +env -u NVIDIA_API_KEY -u COMPATIBLE_API_KEY \ + NEMOCLAW_NON_INTERACTIVE=1 \ + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ + NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" \ + NEMOCLAW_POLICY_MODE=skip \ + node "$REPO/bin/nemoclaw.js" onboard --resume --non-interactive >"$RESUME_LOG" 2>&1 +resume_exit=$? +resume_output="$(cat "$RESUME_LOG")" +rm -f "$RESUME_LOG" + +if [ $resume_exit -eq 0 ]; then + pass "Resume completed successfully" +else + fail "Resume exited $resume_exit (expected 0)" + echo "$resume_output" + exit 1 +fi + +if echo "$resume_output" | grep -q "\[resume\] Skipping preflight (cached)"; then + pass "Resume skipped preflight" +else + fail "Resume did not skip preflight" +fi + +if echo "$resume_output" | grep -q "\[resume\] Skipping gateway (running)"; then + pass "Resume skipped gateway" +else + fail "Resume did not skip gateway" +fi + +if echo "$resume_output" | grep -q "\[resume\] Skipping sandbox (${SANDBOX_NAME})"; then + pass "Resume skipped sandbox" +else + fail "Resume did not skip sandbox" +fi + +if echo "$resume_output" | grep -q "\[1/7\] Preflight checks"; then + fail "Resume reran preflight unexpectedly" +else + pass "Resume did not rerun preflight" +fi + +if echo "$resume_output" | grep -q "\[2/7\] Starting OpenShell gateway"; then + fail "Resume reran gateway startup unexpectedly" +else + pass "Resume did not rerun gateway startup" +fi + +if echo "$resume_output" | grep -q "\[5/7\] Creating sandbox"; then + fail "Resume reran sandbox creation unexpectedly" +else + pass "Resume did not rerun sandbox creation" +fi + +# The first onboard completed through openclaw (step 7) before failing at +# policies (step 8). Inference was already configured during that run, so +# the resume path detects it is ready (isInferenceRouteReady) and skips it. +if echo "$resume_output" | grep -q "\[4/7\] Setting up inference provider"; then + pass "Resume re-ran inference setup" +elif echo "$resume_output" | grep -q "\[resume\] Skipping inference\|\[reuse\] Skipping inference"; then + pass "Resume skipped inference (already configured)" +else + fail "Resume neither ran nor skipped inference setup" +fi + +if run_nemoclaw "$SANDBOX_NAME" status >/dev/null 2>&1; then + pass "Sandbox '$SANDBOX_NAME' is manageable after resume" +else + fail "Sandbox '$SANDBOX_NAME' status failed after resume" +fi + +node -e ' +const fs = require("fs"); +const file = process.argv[1]; +const expectedProvider = process.argv[2]; +const data = JSON.parse(fs.readFileSync(file, "utf8")); +if (data.status !== "complete") process.exit(1); +if (data.provider !== expectedProvider) process.exit(2); +if (data.steps.preflight.status !== "complete") process.exit(3); +if (data.steps.gateway.status !== "complete") process.exit(4); +if (data.steps.sandbox.status !== "complete") process.exit(5); +if (data.steps.provider_selection.status !== "complete") process.exit(6); +if (data.steps.inference.status !== "complete") process.exit(7); +if (data.steps.openclaw.status !== "complete") process.exit(8); +if (data.steps.policies.status !== "complete") process.exit(9); +' "$SESSION_FILE" "$EXPECTED_PROVIDER" +case $? in + 0) pass "Session file recorded full completion after resume" ;; + *) fail "Session file did not record the expected completed state after resume" ;; +esac + +if [ -f "$REGISTRY" ] && grep -q "$SANDBOX_NAME" "$REGISTRY"; then + pass "Registry contains resumed sandbox entry" +else + fail "Registry does not contain resumed sandbox entry" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 3.5: Implicit resume (plain `onboard`, no --resume flag) — #5470 +# ══════════════════════════════════════════════════════════════════ +# The fix auto-detects resume from a persisted in_progress session. The +# section above proves explicit `--resume`; this proves a plain `onboard` +# rerun resumes on its own, and that `--fresh` suppresses it. +section "Phase 3.5: Implicit resume from in_progress session" + +# Re-mark the now-complete session as in_progress so a plain `onboard` has +# something to auto-resume. Everything is already provisioned, so the resume +# skips every cached step and finishes fast. +# Mimic an interrupted-but-resumable session: status "in_progress" AND +# resumable !== false. Phase 3 marks the completed session `resumable: false`, +# so flipping status alone would (correctly) be rejected as "no resumable +# session"; resetting resumable reconstructs the interrupted shape the resume +# machine accepts (session-bootstrap.ts:140). +set_session_in_progress() { + node -e ' + const fs = require("fs"); + const file = process.argv[1]; + const data = JSON.parse(fs.readFileSync(file, "utf8")); + data.status = "in_progress"; + data.resumable = true; + fs.writeFileSync(file, JSON.stringify(data, null, 2)); + ' "$SESSION_FILE" +} + +set_session_in_progress +info "Running plain onboard (no --resume) on an in_progress session..." +IMPLICIT_LOG="$(mktemp)" +env -u NVIDIA_API_KEY -u COMPATIBLE_API_KEY \ + NEMOCLAW_NON_INTERACTIVE=1 \ + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ + NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" \ + NEMOCLAW_POLICY_MODE=skip \ + node "$REPO/bin/nemoclaw.js" onboard --non-interactive >"$IMPLICIT_LOG" 2>&1 +implicit_exit=$? +implicit_output="$(cat "$IMPLICIT_LOG")" +rm -f "$IMPLICIT_LOG" + +if [ $implicit_exit -eq 0 ]; then + pass "Implicit resume (plain onboard) completed successfully" +else + fail "Implicit resume exited $implicit_exit (expected 0)" + echo "$implicit_output" +fi + +if echo "$implicit_output" | grep -q "(resume mode)"; then + pass "Plain onboard auto-detected resume mode from in_progress session" +else + fail "Plain onboard did not show '(resume mode)' for an in_progress session" +fi + +if echo "$implicit_output" | grep -q "\[resume\] Skipping\|\[reuse\] Skipping"; then + pass "Implicit resume skipped cached steps" +else + fail "Implicit resume did not skip any cached steps" +fi + +# --fresh must suppress the auto-resume even with an in_progress session. +# Fail-fast at preflight (step 1, before sandbox recreation) so this stays +# cheap and non-destructive; the banner is emitted before that step. +set_session_in_progress +info "Running onboard --fresh on the same in_progress session (fail-fast)..." +FRESH_LOG="$(mktemp)" +NEMOCLAW_NON_INTERACTIVE=1 \ + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ + NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" \ + NEMOCLAW_POLICY_MODE=skip \ + NEMOCLAW_E2E_FAILURE_INJECTION=1 \ + NEMOCLAW_E2E_FORCE_FAIL_AT_STEP=preflight \ + node "$REPO/bin/nemoclaw.js" onboard --fresh --non-interactive >"$FRESH_LOG" 2>&1 +fresh_exit=$? +fresh_output="$(cat "$FRESH_LOG")" +rm -f "$FRESH_LOG" + +# Confirm the run actually executed and aborted at preflight, so the +# banner-absence assertion below is meaningful (not a vacuous pass from an +# unrelated early failure). +if [ $fresh_exit -ne 0 ] && echo "$fresh_output" | grep -q "\[e2e\] Forced onboarding failure at step 'preflight'."; then + pass "--fresh run failed fast at preflight as intended" +else + fail "--fresh run did not fail at preflight as expected (exit $fresh_exit)" + echo "$fresh_output" +fi + +if echo "$fresh_output" | grep -q "(resume mode)"; then + fail "--fresh did not suppress auto-resume (unexpected '(resume mode)')" +else + pass "--fresh suppressed auto-resume despite an in_progress session" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 4: Final cleanup +# ══════════════════════════════════════════════════════════════════ +section "Phase 4: Final cleanup" + +[[ "${NEMOCLAW_E2E_KEEP_SANDBOX:-}" = "1" ]] || run_nemoclaw "$SANDBOX_NAME" destroy 2>/dev/null || true +openshell sandbox delete "$SANDBOX_NAME" 2>/dev/null || true +openshell forward stop 18789 2>/dev/null || true +openshell gateway destroy -g nemoclaw 2>/dev/null || true +rm -f "$SESSION_FILE" + +if openshell sandbox get "$SANDBOX_NAME" >/dev/null 2>&1; then + fail "Sandbox '$SANDBOX_NAME' still exists after cleanup" +else + pass "Sandbox '$SANDBOX_NAME' cleaned up" +fi + +if [ -f "$SESSION_FILE" ]; then + fail "Onboard session file still exists after cleanup" +else + pass "Onboard session file cleaned up" +fi + +pass "Final cleanup complete" + +# ══════════════════════════════════════════════════════════════════ +# Summary +# ══════════════════════════════════════════════════════════════════ +echo "" +echo "========================================" +echo " PASS: $PASS" +echo " FAIL: $FAIL" +echo " SKIP: $SKIP" +echo " TOTAL: $TOTAL" +echo "========================================" +echo "" + +if [ $FAIL -ne 0 ]; then + exit 1 +fi diff --git a/test/e2e-vpn/test-openclaw-discord-pairing.sh b/test/e2e-vpn/test-openclaw-discord-pairing.sh new file mode 100755 index 00000000000..682c37ae84d --- /dev/null +++ b/test/e2e-vpn/test-openclaw-discord-pairing.sh @@ -0,0 +1,645 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# OpenClaw Discord pairing E2E (#4061). +# +# This keeps Discord hermetic while covering the failure boundary from the +# macOS report: +# 1. Discord is configured with a provider-backed token and managed proxy. +# 2. A Discord-shaped gateway probe reaches a fake Gateway through OpenShell +# and proves placeholder-to-token rewrite. +# 3. OpenClaw's runtime writes a Discord pending pairing request into the +# shared state root. +# 4. Connect-shell `openclaw pairing approve discord ` finds and +# approves that request. +# 5. Approval creates the Discord allowFrom store entry where OpenClaw reads it. +# +# Environment variables: +# NEMOCLAW_NON_INTERACTIVE=1 - required +# NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 - required +# NVIDIA_API_KEY - required for onboarding +# NEMOCLAW_SANDBOX_NAME - sandbox name (default: e2e-openclaw-discord-pairing) +# DISCORD_BOT_TOKEN - defaults to a fake token +# +# Usage: +# NEMOCLAW_NON_INTERACTIVE=1 NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ +# NVIDIA_API_KEY=nvapi-... bash test/e2e-vpn/test-openclaw-discord-pairing.sh + +# shellcheck disable=SC2016,SC2329 +# SC2016: Single-quoted strings are intentional for commands evaluated inside +# the sandbox rather than on the host. +# SC2329: sandbox_exec_stdin is used by sourced Discord helper functions. + +set -uo pipefail + +PASS=0 +FAIL=0 +SKIP=0 +TOTAL=0 + +pass() { + ((PASS++)) + ((TOTAL++)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} +fail() { + ((FAIL++)) + ((TOTAL++)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} +skip() { + ((SKIP++)) + ((TOTAL++)) + printf '\033[33m SKIP: %s\033[0m\n' "$1" +} +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } + +run_with_timeout() { + local seconds="$1" + shift + if command -v timeout >/dev/null 2>&1; then + timeout "$seconds" "$@" + elif command -v gtimeout >/dev/null 2>&1; then + gtimeout "$seconds" "$@" + else + "$@" + fi +} + +require_timeout_command() { + if command -v timeout >/dev/null 2>&1 || command -v gtimeout >/dev/null 2>&1; then + return 0 + fi + fail "Neither timeout nor gtimeout is available; cannot enforce INSTALL_TIMEOUT_SECONDS" + exit 1 +} + +if [ -d /workspace ] && [ -f /workspace/install.sh ]; then + REPO="/workspace" +elif [ -f "$(cd "$(dirname "$0")/../.." && pwd)/install.sh" ]; then + REPO="$(cd "$(dirname "$0")/../.." && pwd)" +else + echo "ERROR: Cannot find repo root." + exit 1 +fi + +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-openclaw-discord-pairing}" +OPENSHELL_BIN="${NEMOCLAW_OPENSHELL_BIN:-openshell}" +DISCORD_TOKEN="${DISCORD_BOT_TOKEN:-test-fake-discord-pairing-e2e}" +DISCORD_PAIRING_USER="${NEMOCLAW_DISCORD_PAIRING_USER:-1005536447329222676}" +DISCORD_DM_CHANNEL="${NEMOCLAW_DISCORD_DM_CHANNEL:-1199988877766655554}" + +export NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" +export NEMOCLAW_RECREATE_SANDBOX=1 +export NEMOCLAW_FRESH=1 +export NEMOCLAW_POLICY_TIER="${NEMOCLAW_POLICY_TIER:-open}" +export DISCORD_BOT_TOKEN="$DISCORD_TOKEN" +# The issue path is the pairing flow. Do not seed an allowlist that would bypass +# pairing and hide this regression. +unset DISCORD_ALLOWED_IDS +unset DISCORD_USER_ID + +openshell() { + if [ "$OPENSHELL_BIN" = "openshell" ]; then + command openshell "$@" + else + "$OPENSHELL_BIN" "$@" + fi +} + +sandbox_exec() { + local cmd="$1" + local ssh_config + ssh_config="$(mktemp)" + openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config" 2>/dev/null + + local result status + result=$(run_with_timeout 60 ssh -F "$ssh_config" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" \ + "$cmd" \ + 2>&1) + status=$? + + rm -f "$ssh_config" + printf '%s\n' "$result" + return "$status" +} + +sandbox_exec_stdin() { + local cmd="$1" + local ssh_config + ssh_config="$(mktemp)" + openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config" 2>/dev/null + + local result status + result=$(run_with_timeout 60 ssh -F "$ssh_config" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" \ + "$cmd" \ + 2>/dev/null) + status=$? + + rm -f "$ssh_config" + printf '%s\n' "$result" + return "$status" +} + +quote_for_remote_sh() { + local value="${1:-}" + printf "'%s'" "$(printf '%s' "$value" | sed "s/'/'\\\\''/g")" +} + +sandbox_exec_sh_script() { + local script="$1" + shift + local encoded remote_cmd arg + encoded="$(printf '%s' "$script" | base64 | tr -d '\n')" + remote_cmd="tmp=\$(mktemp); trap 'rm -f \"\$tmp\"' EXIT; printf %s $(quote_for_remote_sh "$encoded") | base64 -d > \"\$tmp\"; sh \"\$tmp\"" + for arg in "$@"; do + remote_cmd+=" $(quote_for_remote_sh "$arg")" + done + run_with_timeout 60 openshell sandbox exec --name "$SANDBOX_NAME" -- sh -lc "$remote_cmd" +} + +# shellcheck source=test/e2e-vpn/lib/sandbox-teardown.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib/sandbox-teardown.sh" +register_sandbox_for_teardown "$SANDBOX_NAME" + +# shellcheck source=test/e2e-vpn/lib/discord-gateway-proof.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib/discord-gateway-proof.sh" + +check_fake_discord_gateway_capture() { + node - "$FAKE_DISCORD_GATEWAY_CAPTURE_FILE" "$DISCORD_TOKEN" <<'NODE' +const fs = require("fs"); +const file = process.argv[2]; +const expected = process.argv[3]; +const serialized = fs.readFileSync(file, "utf8"); +const rows = serialized + .trim() + .split(/\n+/) + .filter(Boolean) + .map((line) => JSON.parse(line)); + +const identify = rows.filter((row) => row.event === "identify").at(-1); +if (!identify) { + console.log("NO_IDENTIFY"); + process.exit(2); +} +if (identify.tokenMatchesExpected !== true) { + console.log("BAD_TOKEN_REWRITE"); + process.exit(3); +} +if (identify.tokenLooksPlaceholder) { + console.log("PLACEHOLDER_LEAK"); + process.exit(4); +} +if (Object.prototype.hasOwnProperty.call(identify, "token")) { + console.log("RAW_TOKEN_CAPTURED"); + process.exit(5); +} +if (serialized.includes(expected)) { + console.log("RAW_TOKEN_LEAK"); + process.exit(6); +} +console.log("OK"); +NODE +} + +section "Phase 0: Prerequisites" + +if [ -z "${NVIDIA_API_KEY:-}" ]; then + fail "NVIDIA_API_KEY not set" + exit 1 +fi +pass "NVIDIA_API_KEY is set" + +if ! docker info >/dev/null 2>&1; then + fail "Docker is not running" + exit 1 +fi +pass "Docker is running" + +info "Sandbox name: $SANDBOX_NAME" +info "Discord token: configured (${#DISCORD_TOKEN} chars)" +info "Discord pairing user: $DISCORD_PAIRING_USER" + +section "Phase 1: Install NemoClaw with Discord enabled" + +cd "$REPO" || exit 1 + +info "Pre-cleanup..." +if command -v nemoclaw >/dev/null 2>&1; then + nemoclaw "$SANDBOX_NAME" destroy --yes 2>/dev/null || true +fi +if openshell --version >/dev/null 2>&1; then + openshell sandbox delete "$SANDBOX_NAME" 2>/dev/null || true + if [[ "${CI:-}" = "true" || "${NEMOCLAW_E2E_DESTROY_GATEWAY:-}" = "1" ]]; then + openshell gateway destroy -g nemoclaw 2>/dev/null || true + fi +fi +pass "Pre-cleanup complete" + +INSTALL_LOG="/tmp/nemoclaw-e2e-openclaw-discord-pairing-install.log" +INSTALL_TIMEOUT_SECONDS="${NEMOCLAW_E2E_INSTALL_TIMEOUT_SECONDS:-1800}" +require_timeout_command +info "Running install.sh --non-interactive..." +run_with_timeout "$INSTALL_TIMEOUT_SECONDS" bash install.sh --non-interactive >"$INSTALL_LOG" 2>&1 & +install_pid=$! +tail -f "$INSTALL_LOG" --pid=$install_pid 2>/dev/null & +tail_pid=$! +wait $install_pid +install_exit=$? +kill $tail_pid 2>/dev/null || true +wait $tail_pid 2>/dev/null || true + +if [ -f "$HOME/.bashrc" ]; then + # shellcheck source=/dev/null + source "$HOME/.bashrc" 2>/dev/null || true +fi +export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" +if [ -s "$NVM_DIR/nvm.sh" ]; then + # shellcheck source=/dev/null + . "$NVM_DIR/nvm.sh" +fi +if [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]]; then + export PATH="$HOME/.local/bin:$PATH" +fi + +if [ $install_exit -eq 0 ]; then + pass "Install completed" +else + fail "install.sh failed (exit $install_exit)" + info "Last 40 lines of install log:" + tail -40 "$INSTALL_LOG" 2>/dev/null || true + exit 1 +fi + +sandbox_list=$(openshell sandbox list 2>&1 || true) +if echo "$sandbox_list" | grep -q "$SANDBOX_NAME.*Ready"; then + pass "Sandbox '$SANDBOX_NAME' is Ready" +else + fail "Sandbox '$SANDBOX_NAME' not Ready (list: ${sandbox_list:0:300})" + exit 1 +fi + +if openshell provider get "${SANDBOX_NAME}-discord-bridge" >/dev/null 2>&1; then + pass "Discord provider exists in OpenShell" +else + fail "Discord provider missing in OpenShell" +fi + +discord_config_check=$(sandbox_exec "python3 - <<'PY' +import json +cfg = json.load(open('/sandbox/.openclaw/openclaw.json')) +account = (cfg.get('channels', {}).get('discord', {}).get('accounts', {}).get('default') or {}) +proxy = cfg.get('proxy') or {} +print(json.dumps({ + 'hasToken': bool(account.get('token')), + 'token': account.get('token', ''), + 'dmPolicy': account.get('dmPolicy', ''), + 'allowFrom': account.get('allowFrom', []), + 'accountProxy': account.get('proxy', ''), + 'managedProxy': proxy.get('proxyUrl', '') if proxy.get('enabled') is True else '', +})) +PY") +info "Discord config summary: ${discord_config_check:0:500}" +if echo "$discord_config_check" | grep -q '"hasToken": true' \ + && echo "$discord_config_check" | grep -Eq 'openshell:resolve:env:[^"]*DISCORD_BOT_TOKEN' \ + && ! echo "$discord_config_check" | grep -q '"dmPolicy": "allowlist"'; then + pass "Discord config uses a placeholder token and remains on pairing policy" +else + fail "Discord config is not set up for pairing: ${discord_config_check:0:500}" +fi + +section "Phase 2: Runtime state root contract" + +state_env=$(sandbox_exec 'printf "OPENCLAW_HOME=%s\nOPENCLAW_STATE_DIR=%s\nOPENCLAW_CONFIG_PATH=%s\nOPENCLAW_OAUTH_DIR=%s\n" "$OPENCLAW_HOME" "$OPENCLAW_STATE_DIR" "$OPENCLAW_CONFIG_PATH" "$OPENCLAW_OAUTH_DIR"') +state_env_status=$? +info "OpenClaw env from connect shell: ${state_env//$'\n'/; }" +if [ $state_env_status -eq 0 ] \ + && echo "$state_env" | grep -q '^OPENCLAW_HOME=/sandbox$' \ + && echo "$state_env" | grep -q '^OPENCLAW_STATE_DIR=/sandbox/.openclaw$' \ + && echo "$state_env" | grep -q '^OPENCLAW_CONFIG_PATH=/sandbox/.openclaw/openclaw.json$' \ + && echo "$state_env" | grep -q '^OPENCLAW_OAUTH_DIR=/sandbox/.openclaw/credentials$'; then + pass "Connect-shell OpenClaw env resolves to /sandbox/.openclaw" +else + fail "Connect-shell OpenClaw env does not resolve to the shared state root" +fi + +pairing_list_empty=$(sandbox_exec 'openclaw pairing list discord --json 2>&1') +pairing_list_empty_status=$? +info "Initial Discord pairing list: ${pairing_list_empty:0:300}" +if [ $pairing_list_empty_status -eq 0 ] \ + && echo "$pairing_list_empty" | grep -q '"channel"[[:space:]]*:[[:space:]]*"discord"'; then + pass "openclaw pairing list discord works in connect shell" +else + fail "openclaw pairing list discord failed before request creation: ${pairing_list_empty:0:300}" +fi + +section "Phase 3: Hermetic Discord gateway proof" + +fake_gateway_ready=0 +if start_fake_discord_gateway "$DISCORD_TOKEN"; then + fake_gateway_ready=1 + pass "Hermetic fake Discord Gateway started on host port ${FAKE_DISCORD_GATEWAY_PORT}" +else + fail "Failed to start hermetic fake Discord Gateway" +fi + +if [ "$fake_gateway_ready" = "1" ] \ + && apply_fake_discord_gateway_policy "$SANDBOX_NAME" "$FAKE_DISCORD_GATEWAY_PORT" >/tmp/nemoclaw-fake-discord-pairing-policy.log 2>&1; then + pass "Applied native WebSocket policy with credential rewrite for fake Discord Gateway" +else + fail "Failed to apply fake Discord Gateway policy: $(tail -20 /tmp/nemoclaw-fake-discord-pairing-policy.log 2>/dev/null | tr '\n' ' ' | cut -c1-300)" +fi + +dc_ws_native="" +if [ "$fake_gateway_ready" = "1" ]; then + dc_ws_native=$(run_fake_discord_gateway_node_client "$FAKE_DISCORD_GATEWAY_PORT" "openshell:resolve:env:DISCORD_BOT_TOKEN" || true) +fi +info "Native fake Discord Gateway probe: ${dc_ws_native:0:500}" + +if echo "$dc_ws_native" | grep -q "^UPGRADE$" \ + && echo "$dc_ws_native" | grep -q "^HELLO$" \ + && echo "$dc_ws_native" | grep -q "^IDENTIFY_SENT_PLACEHOLDER$" \ + && echo "$dc_ws_native" | grep -q "^READY$" \ + && echo "$dc_ws_native" | grep -q "^HEARTBEAT_ACK$"; then + pass "Discord Gateway HELLO, placeholder IDENTIFY, READY, and heartbeat ACK completed" +else + fail "Discord Gateway protocol proof incomplete: ${dc_ws_native:0:400}" +fi + +capture_check=$(check_fake_discord_gateway_capture 2>&1 || true) +if [ "$capture_check" = "OK" ]; then + pass "Fake Discord Gateway saw rewritten host-side token, not the sandbox placeholder" +else + fail "Fake Discord Gateway capture did not prove token rewriting: ${capture_check:0:300}" +fi + +section "Phase 4: Hermetic Discord pairing request" + +gateway_issue_script=$( + cat <<'SCRIPT' + set -a + [ -f /tmp/nemoclaw-proxy-env.sh ] && . /tmp/nemoclaw-proxy-env.sh + set +a + discord_pairing_user="$1" + discord_dm_channel="$2" + : "${OPENCLAW_HOME:?OPENCLAW_HOME missing from runtime shell env}" + : "${OPENCLAW_STATE_DIR:?OPENCLAW_STATE_DIR missing from runtime shell env}" + : "${OPENCLAW_CONFIG_PATH:?OPENCLAW_CONFIG_PATH missing from runtime shell env}" + : "${OPENCLAW_OAUTH_DIR:?OPENCLAW_OAUTH_DIR missing from runtime shell env}" + printf 'GATEWAY_OPENCLAW_ENV uid=%s gid=%s OPENCLAW_STATE_DIR=%s OPENCLAW_OAUTH_DIR=%s\n' "$(id -u)" "$(id -g)" "$OPENCLAW_STATE_DIR" "$OPENCLAW_OAUTH_DIR" + exec env \ + HOME=/sandbox \ + OPENCLAW_HOME="$OPENCLAW_HOME" \ + OPENCLAW_STATE_DIR="$OPENCLAW_STATE_DIR" \ + OPENCLAW_CONFIG_PATH="$OPENCLAW_CONFIG_PATH" \ + OPENCLAW_OAUTH_DIR="$OPENCLAW_OAUTH_DIR" \ + HTTP_PROXY="${HTTP_PROXY:-}" \ + HTTPS_PROXY="${HTTPS_PROXY:-}" \ + http_proxy="${http_proxy:-}" \ + https_proxy="${https_proxy:-}" \ + NO_PROXY="${NO_PROXY:-}" \ + no_proxy="${no_proxy:-}" \ + NODE_OPTIONS="${NODE_OPTIONS:-}" \ + DISCORD_PAIRING_USER="$discord_pairing_user" \ + DISCORD_DM_CHANNEL="$discord_dm_channel" \ + node --input-type=module <<'NODE' +import fs from "node:fs"; +import path from "node:path"; +import { execFileSync } from "node:child_process"; +import { pathToFileURL } from "node:url"; + +function findOpenClawPackageRootFromBinary() { + let binary = ""; + try { + binary = execFileSync("sh", ["-lc", "command -v openclaw"], { encoding: "utf8" }).trim(); + } catch { + return null; + } + if (!binary) return null; + + let current = ""; + try { + current = fs.realpathSync(binary); + } catch { + return null; + } + if (fs.statSync(current).isFile()) current = path.dirname(current); + + for (let depth = 0; depth < 8; depth += 1) { + const manifest = path.join(current, "package.json"); + if (fs.existsSync(manifest)) { + try { + const pkg = JSON.parse(fs.readFileSync(manifest, "utf8")); + if (pkg?.name === "openclaw") return current; + } catch { + // Keep walking toward the filesystem root. + } + } + const parent = path.dirname(current); + if (parent === current) break; + current = parent; + } + return null; +} + +function loadConversationRuntime() { + const candidates = []; + const binaryRoot = findOpenClawPackageRootFromBinary(); + if (binaryRoot) candidates.push(binaryRoot); + try { + const globalRoot = execFileSync("npm", ["root", "-g"], { encoding: "utf8" }).trim(); + if (globalRoot) candidates.push(path.join(globalRoot, "openclaw")); + } catch { + // Keep the explicit global-root fallbacks below. + } + candidates.push( + "/usr/local/lib/node_modules/openclaw", + "/usr/lib/node_modules/openclaw", + ); + const uniqueCandidates = [...new Set(candidates)]; + for (const root of uniqueCandidates) { + const runtime = path.join(root, "dist/plugin-sdk/conversation-runtime.js"); + if (fs.existsSync(runtime)) return import(pathToFileURL(runtime).href); + } + throw new Error(`OpenClaw conversation runtime not found; checked: ${uniqueCandidates.join(", ")}`); +} + +const { + issuePairingChallenge, + upsertChannelPairingRequest, +} = await loadConversationRuntime(); + +const senderId = process.env.DISCORD_PAIRING_USER; +const channelId = process.env.DISCORD_DM_CHANNEL; +let replyText = ""; + +const result = await issuePairingChallenge({ + channel: "discord", + senderId, + senderIdLine: `Discord user id: ${senderId}`, + meta: { + accountId: "default", + channelId, + isDirectMessage: true, + }, + upsertPairingRequest: async ({ id, meta }) => upsertChannelPairingRequest({ + channel: "discord", + id, + accountId: "default", + meta, + }), + sendPairingReply: async (text) => { + replyText = text; + }, +}); + +if (!result.created || !result.code) { + throw new Error(`pairing challenge was not created: ${JSON.stringify(result)}`); +} + +console.log(`DISCORD_PAIRING_E2E_RESULT ${JSON.stringify({ + code: result.code, + senderId, + channelId, + replyText, +})}`); +NODE +SCRIPT +) + +gateway_issue_output=$(sandbox_exec_sh_script "$gateway_issue_script" "$DISCORD_PAIRING_USER" "$DISCORD_DM_CHANNEL" 2>&1) +gateway_issue_status=$? +info "Discord pairing issue output: ${gateway_issue_output:0:700}" +if [ $gateway_issue_status -eq 0 ] && echo "$gateway_issue_output" | grep -q '^DISCORD_PAIRING_E2E_RESULT '; then + pass "OpenClaw runtime created a Discord pending pairing request" +else + fail "OpenClaw runtime did not create a Discord pending pairing request" +fi + +pairing_result_line=$(printf '%s\n' "$gateway_issue_output" | grep '^DISCORD_PAIRING_E2E_RESULT ' | tail -1 || true) +pairing_json="${pairing_result_line#DISCORD_PAIRING_E2E_RESULT }" +pairing_code=$(node -e 'const data = JSON.parse(process.argv[1]); process.stdout.write(data.code || "");' "$pairing_json" 2>/dev/null || true) +if [ -n "$pairing_code" ]; then + pass "Pairing code extracted from fake Discord reply path" +else + fail "Failed to extract Discord pairing code" + pairing_code="__missing_pairing_code__" +fi + +if echo "$pairing_json" | grep -qF "$DISCORD_PAIRING_USER" \ + && echo "$pairing_json" | grep -qF "$pairing_code"; then + pass "Discord pairing reply includes the code and sender identity" +else + fail "Discord pairing reply did not include expected code/user" +fi + +section "Phase 5: Connect-shell approval" + +pending_file_check=$(sandbox_exec "test -f /sandbox/.openclaw/credentials/discord-pairing.json && grep -F '$pairing_code' /sandbox/.openclaw/credentials/discord-pairing.json && grep -F '$DISCORD_PAIRING_USER' /sandbox/.openclaw/credentials/discord-pairing.json") +pending_file_status=$? +if [ $pending_file_status -eq 0 ] \ + && echo "$pending_file_check" | grep -qF "$pairing_code" \ + && echo "$pending_file_check" | grep -qF "$DISCORD_PAIRING_USER"; then + pass "Runtime-created Discord pending request is in the shared OpenClaw state root" +else + fail "Discord pending request missing from /sandbox/.openclaw/credentials/discord-pairing.json" +fi + +pairing_list=$(sandbox_exec 'openclaw pairing list discord --json 2>&1') +pairing_list_status=$? +info "Pairing list after fake Discord event: ${pairing_list:0:500}" +if [ $pairing_list_status -eq 0 ] \ + && echo "$pairing_list" | grep -qF "$pairing_code" \ + && echo "$pairing_list" | grep -qF "$DISCORD_PAIRING_USER"; then + pass "Connect-shell openclaw pairing list sees runtime-created Discord request" +else + fail "Connect-shell openclaw pairing list does not see the Discord request" +fi + +approve_output=$(sandbox_exec "openclaw pairing approve discord '$pairing_code' 2>&1") +approve_status=$? +info "Pairing approve output: ${approve_output:0:500}" +if [ $approve_status -eq 0 ] \ + && echo "$approve_output" | grep -q "Approved" \ + && echo "$approve_output" | grep -qF "$DISCORD_PAIRING_USER"; then + pass "Connect-shell openclaw pairing approve approved the Discord request" +else + fail "Connect-shell openclaw pairing approve failed: ${approve_output:0:500}" +fi + +pairing_list_after=$(sandbox_exec 'openclaw pairing list discord --json 2>&1') +pairing_list_after_status=$? +if [ $pairing_list_after_status -ne 0 ]; then + fail "openclaw pairing list discord failed after approval: ${pairing_list_after:0:300}" +elif echo "$pairing_list_after" | grep -qF "$pairing_code"; then + fail "Approved Discord pairing code is still pending" +else + pass "Approved Discord pairing code was consumed" +fi + +allow_from_check=$(sandbox_exec "test -f /sandbox/.openclaw/credentials/discord-default-allowFrom.json && grep -F '$DISCORD_PAIRING_USER' /sandbox/.openclaw/credentials/discord-default-allowFrom.json") +allow_from_status=$? +if [ $allow_from_status -eq 0 ] \ + && echo "$allow_from_check" | grep -qF "$DISCORD_PAIRING_USER"; then + pass "Discord allowFrom store contains the approved user" +else + fail "Discord allowFrom store missing approved user" +fi + +repeat_approve=$(sandbox_exec "openclaw pairing approve discord '$pairing_code' 2>&1") +repeat_approve_status=$? +if [ $repeat_approve_status -ne 0 ] \ + && echo "$repeat_approve" | grep -q "No pending pairing request found"; then + pass "Second approval fails closed after request consumption" +else + fail "Second approval did not report missing pending request: ${repeat_approve:0:300}" +fi + +section "Phase 6: Cleanup" + +if [[ "${NEMOCLAW_E2E_KEEP_SANDBOX:-}" = "1" ]]; then + skip "Cleanup: NEMOCLAW_E2E_KEEP_SANDBOX=1 - leaving sandbox '$SANDBOX_NAME' for inspection" +else + nemoclaw "$SANDBOX_NAME" destroy --yes 2>/dev/null || true + openshell sandbox delete "$SANDBOX_NAME" 2>/dev/null || true +fi + +if [[ "${NEMOCLAW_E2E_KEEP_SANDBOX:-}" = "1" ]]; then + pass "Cleanup: Sandbox '$SANDBOX_NAME' intentionally kept" +elif openshell sandbox list 2>&1 | grep -q "$SANDBOX_NAME"; then + fail "Cleanup: Sandbox '$SANDBOX_NAME' still present after cleanup" +else + pass "Cleanup: Sandbox '$SANDBOX_NAME' removed" +fi + +echo "" +echo "==========================================" +echo " OpenClaw Discord Pairing E2E Results:" +echo " Passed: $PASS" +echo " Failed: $FAIL" +echo " Skipped: $SKIP" +echo " Total: $TOTAL" +echo "==========================================" + +if [ "$FAIL" -eq 0 ]; then + printf '\n\033[1;32m OpenClaw Discord pairing E2E PASSED.\033[0m\n' + exit 0 +else + printf '\n\033[1;31m %d test(s) FAILED.\033[0m\n' "$FAIL" + exit 1 +fi diff --git a/test/e2e-vpn/test-openclaw-inference-switch.sh b/test/e2e-vpn/test-openclaw-inference-switch.sh new file mode 100755 index 00000000000..1d8d17a1f2a --- /dev/null +++ b/test/e2e-vpn/test-openclaw-inference-switch.sh @@ -0,0 +1,524 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# OpenClaw inference switch E2E. +# +# Installs NemoClaw with the default OpenClaw agent, switches the running +# sandbox with `nemoclaw inference set`, verifies OpenShell and OpenClaw config +# state, then sends live requests through inference.local and OpenClaw. +# +# Prerequisites: +# - Docker running +# - NVIDIA_API_KEY set for hosted inference +# - NEMOCLAW_NON_INTERACTIVE=1 +# - NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 + +# Do not use errexit because this test records pass/fail counts and exits +# explicitly after critical failures or at the final summary. +set -uo pipefail + +PASS=0 +FAIL=0 +SKIP=0 +TOTAL=0 + +pass() { + ((PASS++)) + ((TOTAL++)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} +fail() { + ((FAIL++)) + ((TOTAL++)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} +skip() { + ((SKIP++)) + ((TOTAL++)) + printf '\033[33m SKIP: %s\033[0m\n' "$1" +} +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } + +is_transient_live_http_code() { + case "${1:-}" in + 502 | 503 | 504) return 0 ;; + *) return 1 ;; + esac +} + +http_status_from_response() { + sed -n 's/^__NEMOCLAW_HTTP_STATUS__=//p' <<<"$1" | tail -1 +} + +http_body_from_response() { + sed '/^__NEMOCLAW_HTTP_STATUS__=/d' <<<"$1" +} + +run_with_timeout() { + local seconds="$1" + shift + if command -v timeout >/dev/null 2>&1; then + timeout "$seconds" "$@" + elif command -v gtimeout >/dev/null 2>&1; then + gtimeout "$seconds" "$@" + else + "$@" + fi +} + +parse_chat_content() { + python3 -c " +import json, sys +try: + r = json.load(sys.stdin) + c = r['choices'][0]['message'] + content = c.get('content') or c.get('reasoning_content') or c.get('reasoning') or '' + print(content.strip()) +except Exception as e: + print(f'PARSE_ERROR: {e}', file=sys.stderr) + sys.exit(1) +" +} + +openclaw_gateway_pid() { + # shellcheck disable=SC2016 # awk runs inside the sandbox. + openshell sandbox exec --name "$SANDBOX_NAME" -- sh -lc \ + 'ps -eo pid=,comm=,args= 2>/dev/null | awk '"'"'$2 != "sh" && $2 != "bash" && $2 != "awk" && $0 ~ /openclaw/ && $0 ~ /gateway run/ { print $1; exit }'"'"'' \ + 2>/dev/null || true +} + +get_route_output() { + local output + if output=$(openshell inference get -g nemoclaw 2>&1); then + printf '%s\n' "$output" + return 0 + fi + openshell inference get 2>&1 +} + +strip_ansi() { + python3 -c 'import re, sys; sys.stdout.write(re.sub(r"\x1b\[[0-9;]*m", "", sys.stdin.read()))' +} + +assert_route() { + local output plain_output + if ! output=$(get_route_output); then + fail "OpenShell inference get failed: ${output:0:240}" + return + fi + plain_output=$(printf '%s' "$output" | strip_ansi) + + if grep -Fq "Provider: ${SWITCH_PROVIDER}" <<<"$plain_output" \ + && grep -Fq "Model: ${SWITCH_MODEL}" <<<"$plain_output"; then + pass "OpenShell route points at ${SWITCH_PROVIDER} / ${SWITCH_MODEL}" + else + fail "OpenShell route did not switch to ${SWITCH_PROVIDER} / ${SWITCH_MODEL}: ${plain_output:0:400}" + fi +} + +assert_registry_session() { + local probe + probe=$( + SANDBOX_NAME="$SANDBOX_NAME" EXPECTED_PROVIDER="$SWITCH_PROVIDER" EXPECTED_MODEL="$SWITCH_MODEL" python3 - <<'PY' +import json +import os +from pathlib import Path + +home = Path.home() +name = os.environ["SANDBOX_NAME"] +provider = os.environ["EXPECTED_PROVIDER"] +model = os.environ["EXPECTED_MODEL"] +errors = [] + +registry_path = home / ".nemoclaw" / "sandboxes.json" +try: + registry = json.loads(registry_path.read_text(encoding="utf-8")) + sandbox = (registry.get("sandboxes") or {}).get(name) +except Exception as exc: + sandbox = None + errors.append(f"could not read registry: {exc}") + +if not sandbox: + errors.append(f"sandbox {name} missing from registry") +else: + if sandbox.get("provider") != provider: + errors.append(f"registry provider={sandbox.get('provider')!r}") + if sandbox.get("model") != model: + errors.append(f"registry model={sandbox.get('model')!r}") + +session_path = home / ".nemoclaw" / "onboard-session.json" +try: + session = json.loads(session_path.read_text(encoding="utf-8")) +except Exception as exc: + session = None + errors.append(f"could not read onboard session: {exc}") + +if session is not None: + if not isinstance(session, dict) or not session: + errors.append("onboard session is empty or invalid") + else: + if session.get("sandboxName") != name: + errors.append(f"session sandboxName={session.get('sandboxName')!r}") + if session.get("provider") != provider: + errors.append(f"session provider={session.get('provider')!r}") + if session.get("model") != model: + errors.append(f"session model={session.get('model')!r}") + +if errors: + print("; ".join(errors)) + raise SystemExit(1) +print("OK") +PY + ) || { + fail "Registry/session were not updated for switch: ${probe:0:400}" + return + } + pass "Registry and onboard session record the switched provider/model" +} + +assert_openclaw_config() { + local config probe hash_check + config=$(openshell sandbox exec --name "$SANDBOX_NAME" -- cat /sandbox/.openclaw/openclaw.json 2>&1) || { + fail "Could not read /sandbox/.openclaw/openclaw.json: ${config:0:240}" + return + } + + probe=$(EXPECTED_MODEL="$SWITCH_MODEL" EXPECTED_INFERENCE_API="$SWITCH_INFERENCE_API" python3 -c ' +import json +import os +import sys + +expected = os.environ["EXPECTED_MODEL"] +expected_api = os.environ["EXPECTED_INFERENCE_API"] +doc = json.load(sys.stdin) +errors = [] +primary = (((doc.get("agents") or {}).get("defaults") or {}).get("model") or {}).get("primary") +expected_provider_key = "anthropic" if expected_api == "anthropic-messages" else "inference" +expected_primary = f"{expected_provider_key}/{expected}" +if primary != expected_primary: + errors.append(f"primary={primary!r}") + +provider = (((doc.get("models") or {}).get("providers") or {}).get(expected_provider_key) or {}) +expected_base = "https://inference.local" if expected_api == "anthropic-messages" else "https://inference.local/v1" +if provider.get("baseUrl") != expected_base: + errors.append("baseUrl={!r}".format(provider.get("baseUrl"))) +if provider.get("api") != expected_api: + errors.append("api={!r}".format(provider.get("api"))) +models = provider.get("models") or [] +if not models or models[0].get("id") != expected: + errors.append("model id={!r}".format(models[0].get("id") if models else None)) +if not models or models[0].get("name") != expected_primary: + errors.append("model name={!r}".format(models[0].get("name") if models else None)) + +if errors: + print("; ".join(errors)) + raise SystemExit(1) +print("OK") +' <<<"$config" 2>&1) || { + fail "OpenClaw config was not patched correctly: ${probe:0:400}" + return + } + pass "OpenClaw config uses ${SWITCH_INFERENCE_API} route for ${SWITCH_MODEL}" + + hash_check=$(openshell sandbox exec --name "$SANDBOX_NAME" -- sh -lc \ + 'cd /sandbox/.openclaw && sha256sum -c .config-hash --status && echo OK' 2>&1 || true) + if grep -qx "OK" <<<"$hash_check"; then + pass "OpenClaw config hash matches openclaw.json" + else + fail "OpenClaw config hash check failed: ${hash_check:0:240}" + fi +} + +check_sandbox_inference() { + local payload payload_arg response rc content attempt last_fail http_code body remote transient=0 + payload=$(SWITCH_MODEL="$SWITCH_MODEL" SWITCH_INFERENCE_API="$SWITCH_INFERENCE_API" python3 -c ' +import json +import os +if os.environ["SWITCH_INFERENCE_API"] == "anthropic-messages": + print(json.dumps({ + "model": os.environ["SWITCH_MODEL"], + "messages": [{"role": "user", "content": "Reply with exactly one word: PONG"}], + "max_tokens": 32, + })) +else: + print(json.dumps({ + "model": os.environ["SWITCH_MODEL"], + "messages": [{"role": "user", "content": "Reply with exactly one word: PONG"}], + "max_tokens": 100, + })) +') + payload_arg="$(printf '%q' "$payload")" + if [ "$SWITCH_INFERENCE_API" = "anthropic-messages" ]; then + remote="tmp=\$(mktemp); code=\$(curl -sS -o \"\$tmp\" -w '%{http_code}' --max-time 90 https://inference.local/v1/messages -H 'Content-Type: application/json' -H 'anthropic-version: 2023-06-01' -d $payload_arg); rc=\$?; cat \"\$tmp\"; rm -f \"\$tmp\"; printf '\n__NEMOCLAW_HTTP_STATUS__=%s\n' \"\${code:-000}\"; exit \"\$rc\"" + else + remote="tmp=\$(mktemp); code=\$(curl -sS -o \"\$tmp\" -w '%{http_code}' --max-time 90 https://inference.local/v1/chat/completions -H 'Content-Type: application/json' -d $payload_arg); rc=\$?; cat \"\$tmp\"; rm -f \"\$tmp\"; printf '\n__NEMOCLAW_HTTP_STATUS__=%s\n' \"\${code:-000}\"; exit \"\$rc\"" + fi + last_fail="" + + for attempt in 1 2 3; do + rc=0 + transient=0 + response=$(openshell sandbox exec --name "$SANDBOX_NAME" -- sh -lc "$remote" 2>&1) || rc=$? + http_code=$(http_status_from_response "$response") + [ -n "$http_code" ] || http_code="000" + body=$(http_body_from_response "$response") + + if [ "$rc" -ne 0 ]; then + [ "$rc" -eq 28 ] && transient=1 + last_fail="curl failed with exit ${rc}; HTTP ${http_code}: ${body:0:300}" + elif is_transient_live_http_code "$http_code"; then + transient=1 + last_fail="transient HTTP ${http_code}: ${body:0:300}" + elif [ "$http_code" != "200" ]; then + last_fail="HTTP ${http_code}: ${body:0:300}" + else + if [ "$SWITCH_INFERENCE_API" = "anthropic-messages" ]; then + content=$(printf '%s' "$body" | parse_anthropic_content 2>/dev/null) || content="" + else + content=$(printf '%s' "$body" | parse_chat_content 2>/dev/null) || content="" + fi + if grep -qi "PONG" <<<"$content"; then + pass "Sandbox inference.local returned PONG with ${SWITCH_MODEL}" + return + fi + last_fail="expected PONG, got ${content:0:300}" + fi + + [ "$attempt" -ge 3 ] || { + info "Sandbox inference attempt ${attempt}/3 failed: ${last_fail}" + sleep 5 + } + done + + if [ "$transient" -eq 1 ]; then + skip "Sandbox inference.local transient failure after switch; route/config checks already passed" + else + fail "Sandbox inference.local did not work after switch: ${last_fail}" + fi +} + +check_openclaw_agent_turn() { + local ssh_config session_id raw stderr_file rc reply warnings + ssh_config="$(mktemp)" + if ! openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config" 2>/dev/null; then + rm -f "$ssh_config" + fail "Could not get SSH config for OpenClaw agent turn" + return + fi + + session_id="e2e-inference-switch-openclaw-$(date +%s)-$$" + stderr_file="$(mktemp)" + rc=0 + raw=$(run_with_timeout 120 ssh -F "$ssh_config" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" \ + "openclaw agent --agent main --json --session-id '${session_id}' -m 'Reply with exactly one word: PONG'" \ + 2>"$stderr_file") || rc=$? + warnings="$(cat "$stderr_file" 2>/dev/null || true)" + rm -f "$ssh_config" + rm -f "$stderr_file" + + reply=$(printf '%s' "$raw" | parse_openclaw_agent_text 2>/dev/null) || true + + if [ "$rc" -eq 0 ] && grep -qi "PONG" <<<"$reply"; then + pass "OpenClaw agent answered through the switched inference route" + elif [ "$rc" -eq 124 ]; then + skip "OpenClaw agent turn timed out after switch; route/config checks already passed" + else + fail "OpenClaw agent turn failed after switch (exit ${rc}); reply='${reply:0:200}', raw='${raw:0:200}', stderr='${warnings:0:200}'" + fi +} + +if [ -d /workspace ] && [ -f /workspace/install.sh ]; then + REPO="/workspace" +elif [ -f "$(cd "$(dirname "$0")/../.." && pwd)/install.sh" ]; then + REPO="$(cd "$(dirname "$0")/../.." && pwd)" +else + echo "ERROR: Cannot find repo root." + exit 1 +fi + +E2E_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=test/e2e-vpn/lib/openclaw-json.sh +. "${E2E_DIR}/lib/openclaw-json.sh" +# shellcheck source=test/e2e-vpn/lib/inference-switch-retry.sh +. "${E2E_DIR}/lib/inference-switch-retry.sh" +# shellcheck source=test/e2e-vpn/lib/anthropic-switch-provider.sh +. "${E2E_DIR}/lib/anthropic-switch-provider.sh" +# shellcheck source=test/e2e-vpn/lib/ci-compatible-inference.sh +. "${E2E_DIR}/lib/ci-compatible-inference.sh" +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-openclaw-inference-switch}" +if nemoclaw_e2e_using_compatible_inference; then + SWITCH_PROVIDER="${NEMOCLAW_SWITCH_PROVIDER:-$(nemoclaw_e2e_expected_route_provider)}" + SWITCH_MODEL="${NEMOCLAW_SWITCH_MODEL:-$(nemoclaw_e2e_hosted_inference_model)}" +else + SWITCH_PROVIDER="${NEMOCLAW_SWITCH_PROVIDER:-compatible-endpoint}" + SWITCH_MODEL="${NEMOCLAW_SWITCH_MODEL:-z-ai/glm-5.1}" +fi +SWITCH_INFERENCE_API="${NEMOCLAW_SWITCH_INFERENCE_API:-openai-completions}" +# shellcheck disable=SC2034 # consumed by anthropic-switch-provider.sh helpers +SWITCH_ENDPOINT_URL="${NEMOCLAW_SWITCH_ENDPOINT_URL:-}" +# shellcheck disable=SC2034 # consumed by anthropic-switch-provider.sh helpers +SWITCH_MOCK_ANTHROPIC="${NEMOCLAW_SWITCH_MOCK_ANTHROPIC:-0}" +# shellcheck disable=SC2034 # consumed by anthropic-switch-provider.sh helpers +SWITCH_MOCK_PORT="${NEMOCLAW_SWITCH_MOCK_PORT:-18767}" +INSTALL_LOG="/tmp/nemoclaw-e2e-openclaw-inference-switch-install.log" + +# shellcheck source=test/e2e-vpn/lib/sandbox-teardown.sh +. "${E2E_DIR}/lib/sandbox-teardown.sh" +trap 'stop_mock_anthropic_switch_provider; _nemoclaw_sandbox_teardown' EXIT +# shellcheck source=test/e2e-vpn/lib/install-path-refresh.sh +. "${E2E_DIR}/lib/install-path-refresh.sh" +register_sandbox_for_teardown "$SANDBOX_NAME" +nemoclaw_e2e_configure_compatible_inference || exit 1 + +section "Phase 0: Pre-cleanup" +if command -v nemoclaw >/dev/null 2>&1; then + nemoclaw "$SANDBOX_NAME" destroy --yes 2>/dev/null || true +fi +if command -v openshell >/dev/null 2>&1; then + openshell sandbox delete "$SANDBOX_NAME" 2>/dev/null || true + openshell gateway destroy -g nemoclaw 2>/dev/null || true +fi +pass "Pre-cleanup complete" + +section "Phase 1: Prerequisites" +if docker info >/dev/null 2>&1; then + pass "Docker is running" +else + fail "Docker is not running" + exit 1 +fi + +if ! nemoclaw_e2e_require_hosted_inference_key; then + exit 1 +fi + +if [ "${NEMOCLAW_NON_INTERACTIVE:-}" = "1" ]; then + pass "NEMOCLAW_NON_INTERACTIVE=1" +else + fail "NEMOCLAW_NON_INTERACTIVE=1 is required" + exit 1 +fi + +if [ "${NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE:-}" = "1" ]; then + pass "Third-party software acceptance is set" +else + fail "NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 is required" + exit 1 +fi + +section "Phase 2: Install and onboard OpenClaw" +cd "$REPO" || { + fail "Could not cd to repo root: $REPO" + exit 1 +} + +export NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" +export NEMOCLAW_RECREATE_SANDBOX="${NEMOCLAW_RECREATE_SANDBOX:-1}" + +info "Running install.sh --non-interactive for sandbox ${SANDBOX_NAME}..." +bash install.sh --non-interactive --yes-i-accept-third-party-software >"$INSTALL_LOG" 2>&1 & +install_pid=$! +tail -f "$INSTALL_LOG" --pid=$install_pid 2>/dev/null & +tail_pid=$! +wait "$install_pid" +install_exit=$? +kill "$tail_pid" 2>/dev/null || true +wait "$tail_pid" 2>/dev/null || true + +nemoclaw_refresh_install_env +export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" +# shellcheck source=/dev/null +[ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" +nemoclaw_ensure_local_bin_on_path + +if [ "$install_exit" -eq 0 ]; then + pass "install.sh completed" +else + fail "install.sh failed (exit ${install_exit})" + tail -80 "$INSTALL_LOG" || true + exit 1 +fi + +command -v nemoclaw >/dev/null 2>&1 || { + fail "nemoclaw not found on PATH" + exit 1 +} +command -v openshell >/dev/null 2>&1 || { + fail "openshell not found on PATH" + exit 1 +} +pass "nemoclaw and openshell are on PATH" +ensure_compatible_anthropic_switch_provider || exit 1 + +section "Phase 3: Switch inference" +pid_before="$(openclaw_gateway_pid)" +info "Switching ${SANDBOX_NAME} to ${SWITCH_PROVIDER} / ${SWITCH_MODEL}..." +switch_output=$(run_inference_set_with_retry nemoclaw inference set --provider "$SWITCH_PROVIDER" --model "$SWITCH_MODEL" --sandbox "$SANDBOX_NAME") +switch_rc=$? +if [ "$switch_rc" -eq 0 ]; then + pass "nemoclaw inference set completed" +else + fail "nemoclaw inference set failed (exit ${switch_rc}): ${switch_output:0:500}" + exit 1 +fi + +pid_after="$(openclaw_gateway_pid)" +if [ -n "$pid_before" ] && [ -n "$pid_after" ]; then + if [ "$pid_before" = "$pid_after" ]; then + pass "OpenClaw gateway process stayed running during switch" + else + fail "OpenClaw gateway process changed during switch (${pid_before} -> ${pid_after})" + fi +else + skip "Could not capture OpenClaw gateway PID before and after switch" +fi + +assert_route +assert_openclaw_config +assert_registry_session + +section "Phase 4: Live requests after switch" +check_sandbox_inference +check_openclaw_agent_turn + +section "Phase 5: Cleanup" +if [ "${NEMOCLAW_E2E_KEEP_SANDBOX:-}" != "1" ]; then + nemoclaw "$SANDBOX_NAME" destroy --yes 2>&1 | tail -3 || true + openshell gateway destroy -g nemoclaw 2>/dev/null || true + + registry_file="${HOME}/.nemoclaw/sandboxes.json" + if [ -f "$registry_file" ] && grep -Fq "\"${SANDBOX_NAME}\"" "$registry_file"; then + fail "Sandbox ${SANDBOX_NAME} still in registry after destroy" + else + pass "Sandbox ${SANDBOX_NAME} removed" + fi +else + skip "Sandbox ${SANDBOX_NAME} kept; removal check skipped" +fi + +echo "" +echo "========================================" +echo " OpenClaw inference switch E2E Results:" +echo " Passed: $PASS" +echo " Failed: $FAIL" +echo " Skipped: $SKIP" +echo " Total: $TOTAL" +echo "========================================" + +if [ "$FAIL" -eq 0 ]; then + printf '\n\033[1;32m OpenClaw inference switch E2E PASSED.\033[0m\n' + exit 0 +fi + +printf '\n\033[1;31m %d test(s) failed.\033[0m\n' "$FAIL" +exit 1 diff --git a/test/e2e-vpn/test-openclaw-plugin-runtime-exdev.sh b/test/e2e-vpn/test-openclaw-plugin-runtime-exdev.sh new file mode 100755 index 00000000000..e286ff942a7 --- /dev/null +++ b/test/e2e-vpn/test-openclaw-plugin-runtime-exdev.sh @@ -0,0 +1,209 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Coverage guard for #3513 / #3127 — a fresh sandbox must be able to run the +# first OpenClaw CLI invocation without bundled plugin runtime-deps failing on +# EXDEV cross-device rename. + +set -uo pipefail + +PASS=0 +FAIL=0 +TOTAL=0 + +pass() { + ((PASS++)) + ((TOTAL++)) + echo " OK: $1" +} +fail() { + ((FAIL++)) + ((TOTAL++)) + echo " ERROR: $1" +} +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO="$(cd "${SCRIPT_DIR}/../.." && pwd)" +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-openclaw-plugin-exdev}" +ONBOARD_LOG="${E2E_OPENCLAW_PLUGIN_EXDEV_ONBOARD_LOG:-/tmp/nemoclaw-e2e-openclaw-plugin-exdev-onboard.log}" +AGENT_LOG="${E2E_OPENCLAW_PLUGIN_EXDEV_AGENT_LOG:-/tmp/nemoclaw-e2e-openclaw-plugin-exdev-agent.log}" +DF_LOG="${E2E_OPENCLAW_PLUGIN_EXDEV_DF_LOG:-/tmp/nemoclaw-e2e-openclaw-plugin-exdev-df.log}" +TIMEOUT_CMD="${TIMEOUT_CMD:-timeout}" + +# shellcheck source=test/e2e-vpn/lib/install-path-refresh.sh +. "${SCRIPT_DIR}/lib/install-path-refresh.sh" +# shellcheck source=test/e2e-vpn/lib/sandbox-teardown.sh +. "${SCRIPT_DIR}/lib/sandbox-teardown.sh" +register_sandbox_for_teardown "$SANDBOX_NAME" + +redact_file() { + local file="$1" + [ -f "$file" ] || return 0 + python3 - "$file" <<'PY' +import os, sys +path = sys.argv[1] +secrets = [os.environ.get("NVIDIA_API_KEY", ""), os.environ.get("NEMOCLAW_PROVIDER_KEY", "")] +text = open(path, "r", errors="replace").read() +for secret in filter(None, secrets): + text = text.replace(secret, "") +open(path, "w").write(text) +PY +} + +redact_logs() { + redact_file "$ONBOARD_LOG" + redact_file "$AGENT_LOG" + redact_file "$DF_LOG" +} +trap redact_logs EXIT + +section "Prerequisites" +if docker info >/dev/null 2>&1; then + pass "Docker is running" +else + fail "Docker is not running" + exit 1 +fi + +if [ -n "${NVIDIA_API_KEY:-}" ] && [[ "${NVIDIA_API_KEY}" == nvapi-* ]]; then + pass "NVIDIA_API_KEY is set" +else + fail "NVIDIA_API_KEY is required and must start with nvapi-" + exit 1 +fi + +section "Install NemoClaw from checkout" +if ! command -v nemoclaw >/dev/null 2>&1; then + NEMOCLAW_NON_INTERACTIVE=1 \ + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ + bash "${REPO}/install.sh" --non-interactive --yes-i-accept-third-party-software >"$ONBOARD_LOG" 2>&1 || true + nemoclaw_refresh_install_env +fi + +if command -v nemoclaw >/dev/null 2>&1; then + pass "nemoclaw is available: $(nemoclaw --version 2>/dev/null || echo unknown)" +else + fail "nemoclaw not found after install" + exit 1 +fi + +section "Fresh sandbox onboard" +rm -f "$HOME/.nemoclaw/onboard.lock" 2>/dev/null || true +nemoclaw "$SANDBOX_NAME" destroy --yes >/dev/null 2>&1 || true + +python3 - "${REPO}" <<'PY' +import sys +from pathlib import Path +repo = Path(sys.argv[1]) +policy_paths = [ + repo / "agents/openclaw/policy-permissive.yaml", + repo / "nemoclaw-blueprint/policies/openclaw-sandbox.yaml", + repo / "nemoclaw-blueprint/policies/openclaw-sandbox-permissive.yaml", +] +for path in policy_paths: + text = path.read_text() + needle = " read_write:\n - /tmp\n" + if needle not in text: + raise SystemExit(f"could not find read_write /tmp anchor in {path}") + additions = "" + for entry in ["/dev", "/dev/shm"]: + if f" - {entry}\n" not in text: + additions += f" - {entry}\n" + if additions: + path.write_text(text.replace(needle, needle + additions, 1)) +PY +env \ + NEMOCLAW_PROVIDER_KEY="$NVIDIA_API_KEY" \ + NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" \ + NEMOCLAW_NON_INTERACTIVE=1 \ + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ + NEMOCLAW_POLICY_MODE="skip" \ + NEMOCLAW_PROVIDER="custom" \ + NVIDIA_API_KEY="$NVIDIA_API_KEY" \ + "$TIMEOUT_CMD" 1500 nemoclaw onboard --fresh --non-interactive --yes-i-accept-third-party-software --agent openclaw --from "$REPO/Dockerfile" \ + >"$ONBOARD_LOG" 2>&1 +onboard_rc=$? +redact_logs +if [ "$onboard_rc" -eq 0 ]; then + pass "fresh sandbox onboard completed" +else + fail "fresh sandbox onboard failed (exit ${onboard_rc}); see ${ONBOARD_LOG}" + exit 1 +fi + +section "Filesystem layout evidence" +openshell sandbox exec --name "$SANDBOX_NAME" -- sh -lc 'df -PT / /tmp /dev/shm /sandbox /sandbox/.openclaw/plugin-runtime-deps 2>&1' \ + >"$DF_LOG" 2>&1 || true +redact_logs +info "Filesystem layout captured in ${DF_LOG}" + +section "Bundled plugin runtime-deps cross-device replacement" +agent_rc=0 +# Reproduce the precise #3513 failure mode without depending on OpenClaw's +# broader CLI temp/log initialization: the vulnerable helper copies dependency +# contents into a staging dir adjacent to the source and then renameSyncs that +# staged node_modules dir into the final plugin-runtime-deps target. When source +# is on tmpfs (/dev/shm) and target is under /sandbox, unfixed code throws EXDEV. +remote_script_b64=$( + cat <<'REMOTE' | base64 | tr -d '\n' +set -eu +rm -rf /sandbox/.openclaw/plugin-runtime-deps/exdev-guard 2>/dev/null || true +rm -rf /dev/shm/nemoclaw-exdev-source 2>/dev/null || true +mkdir -p /dev/shm/nemoclaw-exdev-source +printf 'ok\n' >/dev/shm/nemoclaw-exdev-source/package.txt +node --input-type=module - <<'NODE' +import fs from 'node:fs'; +import path from 'node:path'; +function replaceNodeModulesDir(targetDir, sourceDir) { + const parentDir = path.dirname(sourceDir); + fs.mkdirSync(path.dirname(targetDir), { recursive: true }); + const tempDir = fs.mkdtempSync(path.join(parentDir, '.openclaw-runtime-deps-copy-')); + const stagedDir = path.join(tempDir, 'node_modules'); + try { + fs.cpSync(sourceDir, stagedDir, { recursive: true }); + fs.rmSync(targetDir, { recursive: true, force: true }); + fs.renameSync(stagedDir, targetDir); + } finally { + try { fs.rmSync(tempDir, { recursive: true, force: true }); } catch {} + } +} +replaceNodeModulesDir('/sandbox/.openclaw/plugin-runtime-deps/exdev-guard/node_modules', '/dev/shm/nemoclaw-exdev-source'); +console.log('runtime deps replacement completed'); +NODE +REMOTE +) +remote_cmd="printf '%s' '${remote_script_b64}' | base64 -d > /tmp/nemoclaw-exdev-guard.sh && sh /tmp/nemoclaw-exdev-guard.sh" +"$TIMEOUT_CMD" 60 openshell sandbox exec --name "$SANDBOX_NAME" -- sh -lc "$remote_cmd" \ + >"$AGENT_LOG" 2>&1 || agent_rc=$? +redact_logs + +if grep -qiE 'EXDEV: cross-device link not permitted|cross-device link not permitted' "$AGENT_LOG"; then + fail "OpenClaw-style plugin runtime deps replacement hit #3513 EXDEV failure" + info "Runtime-deps log excerpt: $(grep -iE 'EXDEV|cross-device link not permitted' "$AGENT_LOG" | head -5 | tr '\n' ' ')" + exit 1 +fi + +if [ "$agent_rc" -ne 0 ]; then + fail "runtime deps replacement exited ${agent_rc}; see ${AGENT_LOG}" + exit 1 +fi + +if grep -q 'runtime deps replacement completed' "$AGENT_LOG"; then + pass "OpenClaw-style plugin runtime-deps replacement completed across filesystems" +else + fail "runtime deps replacement exited 0 but success marker was missing; see ${AGENT_LOG}" + exit 1 +fi + +section "Summary" +if [ "$FAIL" -eq 0 ]; then + pass "OpenClaw plugin runtime-deps EXDEV guard passed" + exit 0 +fi +exit 1 diff --git a/test/e2e-vpn/test-openclaw-skill-cli-e2e.sh b/test/e2e-vpn/test-openclaw-skill-cli-e2e.sh new file mode 100755 index 00000000000..05d131ce7db --- /dev/null +++ b/test/e2e-vpn/test-openclaw-skill-cli-e2e.sh @@ -0,0 +1,341 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# OpenClaw skills install/list E2E — direct CLI roundtrip inside sandbox. +# +# Asserts that when a user runs `openclaw skills install ` directly +# inside a NemoClaw sandbox, the installed skill is enumerated by +# `openclaw skills list`. The sandbox onboard flow pins OPENCLAW_HOME, +# OPENCLAW_STATE_DIR, and OPENCLAW_WORKSPACE_DIR so install and list resolve +# the same workspace dir. +# +# Unlike test-skill-agent-e2e.sh, this script does NOT exercise the agent — +# it exercises the CLI contract only, so it has no LLM dependency and no +# retry/fuzzy-match logic. +# +# Prerequisites: +# - Docker running +# - NVIDIA_API_KEY set (needed to onboard the sandbox) +# - NEMOCLAW_NON_INTERACTIVE=1, NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 +# +# Environment: +# NEMOCLAW_SANDBOX_NAME — sandbox name (default: e2e-openclaw-skill-cli) +# NEMOCLAW_RECREATE_SANDBOX=1 — recreate if exists +# +# Usage: +# NEMOCLAW_NON_INTERACTIVE=1 NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ +# NVIDIA_API_KEY=... bash test/e2e-vpn/test-openclaw-skill-cli-e2e.sh + +# shellcheck disable=SC2317 +set -uo pipefail + +PASS=0 +FAIL=0 +TOTAL=0 + +pass() { + ((PASS++)) + ((TOTAL++)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} +fail() { + ((FAIL++)) + ((TOTAL++)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } + +# ── Repo root ── +_script_dir="$(cd "$(dirname "$0")" && pwd)" +_candidate="$(cd "${_script_dir}/../.." && pwd)" +if [ -d /workspace ] && [ -f /workspace/package.json ] && [ -d /workspace/test/e2e ]; then + REPO="/workspace" +elif [ -f "${_candidate}/package.json" ] && [ -d "${_candidate}/test/e2e" ]; then + REPO="${_candidate}" +else + echo "ERROR: Cannot find repo root." + exit 1 +fi +unset _script_dir _candidate + +E2E_DIR="$(cd "$(dirname "$0")" && pwd)" +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-openclaw-skill-cli}" +SKILL_ID="openclaw-skill-cli-fixture" +SKILL_DESCRIPTION="E2E fixture proving openclaw skills install + list roundtrip" + +# Source shared teardown helper +# shellcheck source=test/e2e-vpn/lib/sandbox-teardown.sh +. "${E2E_DIR}/lib/sandbox-teardown.sh" +# shellcheck source=test/e2e-vpn/lib/ci-compatible-inference.sh +. "${E2E_DIR}/lib/ci-compatible-inference.sh" +register_sandbox_for_teardown "$SANDBOX_NAME" +nemoclaw_e2e_configure_compatible_inference || exit 1 + +# ══════════════════════════════════════════════════════════════════════ +# Phase 1: Install + Prerequisites +# ══════════════════════════════════════════════════════════════════════ +section "Phase 1: Install + Prerequisites" + +if ! docker info >/dev/null 2>&1; then + fail "Docker is not running" + exit 1 +fi +pass "Docker is running" + +if ! nemoclaw_e2e_require_hosted_inference_key; then + exit 1 +fi + +cd "$REPO" || { + fail "Could not cd to repo root" + exit 1 +} + +export NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" +export NEMOCLAW_RECREATE_SANDBOX="${NEMOCLAW_RECREATE_SANDBOX:-1}" + +info "Installing NemoClaw via install.sh --non-interactive..." +INSTALL_LOG="/tmp/nemoclaw-e2e-openclaw-skill-cli-install.log" +bash install.sh --non-interactive --yes-i-accept-third-party-software >"$INSTALL_LOG" 2>&1 & +install_pid=$! +tail -f "$INSTALL_LOG" --pid=$install_pid 2>/dev/null & +tail_pid=$! +wait "$install_pid" +install_exit=$? +kill "$tail_pid" 2>/dev/null || true +wait "$tail_pid" 2>/dev/null || true + +if [ -f "$HOME/.bashrc" ]; then + # shellcheck source=/dev/null + source "$HOME/.bashrc" 2>/dev/null || true +fi +export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" +# shellcheck source=/dev/null +[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" +[ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]] && export PATH="$HOME/.local/bin:$PATH" + +if [ "$install_exit" -ne 0 ]; then + fail "install.sh failed (exit $install_exit)" + tail -30 "$INSTALL_LOG" + exit 1 +fi +pass "NemoClaw installed" + +command -v openshell >/dev/null 2>&1 || { + fail "openshell not on PATH" + exit 1 +} +pass "openshell on PATH" + +# ══════════════════════════════════════════════════════════════════════ +# Phase 2: Pre-flight — verify the OPENCLAW_* runtime env pins reach +# the sandbox's runtime shell rc. Drift here means the workaround in +# src/lib/onboard.ts never propagates past `nemoclaw-start` and +# `openclaw skills list` will fall back to a hardcoded default workspace. +# ══════════════════════════════════════════════════════════════════════ +section "Phase 2: Pre-flight runtime env propagation check" + +set +e +# Single-quote the inner script so the OPENCLAW_* variables expand inside the +# sandbox shell, not on the host. +# shellcheck disable=SC2016 +env_check_out=$(openshell sandbox exec --name "$SANDBOX_NAME" -- sh -lc 'printf "OPENCLAW_HOME=%s\nOPENCLAW_STATE_DIR=%s\nOPENCLAW_WORKSPACE_DIR=%s\n" "${OPENCLAW_HOME:-}" "${OPENCLAW_STATE_DIR:-}" "${OPENCLAW_WORKSPACE_DIR:-}"' 2>&1) +env_check_rc=$? +set -uo pipefail + +if [ "$env_check_rc" -ne 0 ]; then + fail "Failed to read OPENCLAW_* env vars from sandbox runtime shell (exit ${env_check_rc})" + printf '%s\n' "$env_check_out" + exit 1 +fi + +for required_var in OPENCLAW_HOME OPENCLAW_STATE_DIR OPENCLAW_WORKSPACE_DIR; do + if ! printf '%s\n' "$env_check_out" | grep -Eq "^${required_var}=.+"; then + fail "${required_var} not exported in sandbox runtime shell" + printf '%s\n' "$env_check_out" + exit 1 + fi +done +pass "OPENCLAW_HOME, OPENCLAW_STATE_DIR, and OPENCLAW_WORKSPACE_DIR are exported in sandbox runtime shell" + +# ══════════════════════════════════════════════════════════════════════ +# Phase 3: Write a skill fixture into the sandbox under /tmp and install +# it through the OpenClaw CLI from a non-managed source path. +# ══════════════════════════════════════════════════════════════════════ +section "Phase 3: Install skill via 'openclaw skills install ' inside sandbox" + +remote_skill_dir="/tmp/${SKILL_ID}" +# openshell sandbox exec rejects command arguments that contain newlines or CRs +# ("InvalidArgument: command argument N contains newline or carriage return +# characters"), so the SKILL.md payload is base64-encoded on the host and decoded +# inside the sandbox. The encoder uses base64 -w0 (or tr -d) so the encoded +# payload is itself single-line. +skill_payload=$(printf '%s\n' \ + "---" \ + "name: \"${SKILL_ID}\"" \ + "description: \"${SKILL_DESCRIPTION}\"" \ + "---" \ + "" \ + "# OpenClaw skill CLI roundtrip fixture" \ + "" \ + "Written by test/e2e-vpn/test-openclaw-skill-cli-e2e.sh.") +skill_payload_b64=$(printf '%s' "$skill_payload" | base64 | tr -d '\n') +write_skill_cmd="rm -rf $(printf "%q" "$remote_skill_dir") && mkdir -p $(printf "%q" "$remote_skill_dir") && printf '%s' '${skill_payload_b64}' | base64 -d > $(printf "%q" "${remote_skill_dir}/SKILL.md")" + +set +e +write_out=$(openshell sandbox exec --name "$SANDBOX_NAME" -- sh -lc "$write_skill_cmd" 2>&1) +write_rc=$? +set -uo pipefail +if [ "$write_rc" -ne 0 ]; then + fail "Failed to write skill fixture into sandbox (exit ${write_rc})" + printf '%s\n' "$write_out" + exit 1 +fi +pass "Wrote skill fixture into sandbox at ${remote_skill_dir}" + +set +e +install_out=$(openshell sandbox exec --name "$SANDBOX_NAME" -- sh -lc "openclaw skills install $(printf "%q" "$remote_skill_dir")" 2>&1) +install_rc=$? +set -uo pipefail +if [ "$install_rc" -ne 0 ]; then + fail "openclaw skills install failed (exit ${install_rc})" + printf '%s\n' "$install_out" + exit 1 +fi +pass "openclaw skills install completed (exit 0)" +info "install output:" +printf '%s\n' "$install_out" + +# ══════════════════════════════════════════════════════════════════════ +# Phase 4: Disk verification — install must land under the workspace dir +# the runtime env pin advertises, NOT under the managed dir or a host +# fallback. The reporter's repro on disk was ls /sandbox/.openclaw/workspace/skills/. +# ══════════════════════════════════════════════════════════════════════ +section "Phase 4: Verify install landed under \${OPENCLAW_WORKSPACE_DIR}/skills/" + +expected_disk_path="/sandbox/.openclaw/workspace/skills/${SKILL_ID}/SKILL.md" +set +e +disk_out=$(openshell sandbox exec --name "$SANDBOX_NAME" -- sh -lc "ls -1 \"\${OPENCLAW_WORKSPACE_DIR}/skills/${SKILL_ID}/\" 2>&1 ; test -f \"\${OPENCLAW_WORKSPACE_DIR}/skills/${SKILL_ID}/SKILL.md\" && echo SKILL_MD_PRESENT" 2>&1) +disk_rc=$? +set -uo pipefail +if [ "$disk_rc" -ne 0 ] || ! printf '%s' "$disk_out" | grep -Fq "SKILL_MD_PRESENT"; then + fail "Installed skill not present at \${OPENCLAW_WORKSPACE_DIR}/skills/${SKILL_ID}/SKILL.md (expected ${expected_disk_path})" + printf '%s\n' "$disk_out" + exit 1 +fi +pass "SKILL.md present on disk at \${OPENCLAW_WORKSPACE_DIR}/skills/${SKILL_ID}/" + +# ══════════════════════════════════════════════════════════════════════ +# Phase 5: List skills via 'openclaw skills list --json' and assert the +# installed fixture is enumerated. This is the contract the issue reports as +# broken when the runtime env pin is missing; passing here proves the +# install path and the list path agree on the workspace dir. +# ══════════════════════════════════════════════════════════════════════ +section "Phase 5: Verify 'openclaw skills list' surfaces the installed skill" + +set +e +list_out=$(openshell sandbox exec --name "$SANDBOX_NAME" -- sh -lc 'openclaw skills list --json' 2>&1) +list_rc=$? +set -uo pipefail +if [ "$list_rc" -ne 0 ]; then + fail "openclaw skills list --json failed (exit ${list_rc})" + printf '%s\n' "$list_out" + exit 1 +fi +pass "openclaw skills list --json completed (exit 0)" + +if ! printf '%s' "$list_out" | grep -Fq "\"${SKILL_ID}\""; then + fail "Installed skill '${SKILL_ID}' did not appear in 'openclaw skills list --json' output" + printf '%s\n' "$list_out" | tail -c 8000 + exit 1 +fi +pass "Installed skill '${SKILL_ID}' is enumerated by 'openclaw skills list --json'" + +# Assert the list entry's source labels it as openclaw-workspace (not +# openclaw-managed or openclaw-extra) so we know the skill came from the +# workspace install path and not a fallback location. +if ! printf '%s' "$list_out" | grep -Fq "openclaw-workspace"; then + fail "Expected at least one entry with source 'openclaw-workspace' in 'openclaw skills list --json' output" + printf '%s\n' "$list_out" | tail -c 8000 + exit 1 +fi +pass "list output includes an entry with source 'openclaw-workspace'" + +# ══════════════════════════════════════════════════════════════════════ +# Phase 6: 'openclaw skills info ' must resolve the same skill that +# install wrote and report its on-disk location. This catches drift +# between the install resolver and the per-skill info resolver. +# ══════════════════════════════════════════════════════════════════════ +section "Phase 6: Verify 'openclaw skills info ${SKILL_ID}' resolves the workspace path" + +set +e +info_out=$(openshell sandbox exec --name "$SANDBOX_NAME" -- sh -lc "openclaw skills info $(printf "%q" "$SKILL_ID") --json" 2>&1) +info_rc=$? +set -uo pipefail +if [ "$info_rc" -ne 0 ]; then + fail "openclaw skills info ${SKILL_ID} --json failed (exit ${info_rc})" + printf '%s\n' "$info_out" + exit 1 +fi +pass "openclaw skills info ${SKILL_ID} --json completed (exit 0)" + +if ! printf '%s' "$info_out" | grep -Fq "${SKILL_ID}"; then + fail "'openclaw skills info' output did not include the skill id" + printf '%s\n' "$info_out" | tail -c 8000 + exit 1 +fi +if ! printf '%s' "$info_out" | grep -Fq "/.openclaw/workspace/skills/${SKILL_ID}"; then + fail "'openclaw skills info' did not report the workspace install path" + printf '%s\n' "$info_out" | tail -c 8000 + exit 1 +fi +pass "'openclaw skills info' reports the skill at the workspace install path" + +# ══════════════════════════════════════════════════════════════════════ +# Phase 7: 'openclaw skills check' is the eligibility report users run to +# diagnose missing skills. The installed fixture must appear there too so +# users do not see a partial view of their workspace. +# ══════════════════════════════════════════════════════════════════════ +section "Phase 7: Verify 'openclaw skills check' includes the installed skill" + +set +e +check_out=$(openshell sandbox exec --name "$SANDBOX_NAME" -- sh -lc 'openclaw skills check --json' 2>&1) +check_rc=$? +set -uo pipefail +if [ "$check_rc" -ne 0 ]; then + fail "openclaw skills check --json failed (exit ${check_rc})" + printf '%s\n' "$check_out" + exit 1 +fi +pass "openclaw skills check --json completed (exit 0)" + +if ! printf '%s' "$check_out" | grep -Fq "\"${SKILL_ID}\""; then + fail "Installed skill '${SKILL_ID}' did not appear in 'openclaw skills check --json' output" + printf '%s\n' "$check_out" | tail -c 8000 + exit 1 +fi +pass "Installed skill '${SKILL_ID}' is enumerated by 'openclaw skills check --json'" + +# ══════════════════════════════════════════════════════════════════════ +# Summary +# ══════════════════════════════════════════════════════════════════════ +echo "" +echo "========================================" +echo " OpenClaw skill CLI E2E Results:" +echo " Passed: $PASS" +echo " Failed: $FAIL" +echo " Total: $TOTAL" +echo "========================================" + +if [ "$FAIL" -eq 0 ]; then + printf '\033[1;32m\n OpenClaw skill CLI E2E PASSED.\033[0m\n' + exit 0 +else + printf '\033[1;31m\n %d test(s) failed.\033[0m\n' "$FAIL" + exit 1 +fi diff --git a/test/e2e-vpn/test-openclaw-slack-pairing.sh b/test/e2e-vpn/test-openclaw-slack-pairing.sh new file mode 100755 index 00000000000..db366f8d199 --- /dev/null +++ b/test/e2e-vpn/test-openclaw-slack-pairing.sh @@ -0,0 +1,860 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# OpenClaw Slack pairing E2E (#3730/#3737). +# +# This test keeps Slack hermetic while covering the failure boundary from the +# DGX Spark report: +# 1. Slack-style Socket Mode event reaches sandbox code over native websocket +# policy with xapp placeholder rewriting. +# 2. OpenShell-tracked Slack Socket Mode flow writes a Slack pending request. +# 3. Connect-shell `openclaw pairing approve slack ` finds and approves +# the request created by the runtime flow. +# 4. Approval creates the Slack allowFrom store entry where OpenClaw resolves it. +# +# Environment variables: +# NEMOCLAW_NON_INTERACTIVE=1 - required +# NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 - required +# NVIDIA_API_KEY - required for onboarding +# NEMOCLAW_SANDBOX_NAME - sandbox name (default: e2e-openclaw-slack-pairing) +# SLACK_BOT_TOKEN - defaults to a fake xoxb- token +# SLACK_APP_TOKEN - defaults to a fake xapp- token +# +# Usage: +# NEMOCLAW_NON_INTERACTIVE=1 NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ +# NVIDIA_API_KEY=nvapi-... bash test/e2e-vpn/test-openclaw-slack-pairing.sh + +# shellcheck disable=SC2016 +# SC2016: Single-quoted strings are intentional for commands evaluated inside +# the sandbox rather than on the host. + +set -uo pipefail + +PASS=0 +FAIL=0 +SKIP=0 +TOTAL=0 + +pass() { + ((PASS++)) + ((TOTAL++)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} +fail() { + ((FAIL++)) + ((TOTAL++)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} +skip() { + ((SKIP++)) + ((TOTAL++)) + printf '\033[33m SKIP: %s\033[0m\n' "$1" +} +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } +is_fake_slack_token() { + case "${1:-}" in + xoxb-fake-* | xoxb-test-* | xapp-fake-* | xapp-test-*) return 0 ;; + *) return 1 ;; + esac +} + +run_with_timeout() { + local seconds="$1" + shift + if command -v timeout >/dev/null 2>&1; then + timeout "$seconds" "$@" + elif command -v gtimeout >/dev/null 2>&1; then + gtimeout "$seconds" "$@" + else + "$@" + fi +} + +if [ -d /workspace ] && [ -f /workspace/install.sh ]; then + REPO="/workspace" +elif [ -f "$(cd "$(dirname "$0")/../.." && pwd)/install.sh" ]; then + REPO="$(cd "$(dirname "$0")/../.." && pwd)" +else + echo "ERROR: Cannot find repo root." + exit 1 +fi + +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-openclaw-slack-pairing}" +OPENSHELL_BIN="${NEMOCLAW_OPENSHELL_BIN:-openshell}" +SLACK_TOKEN="${SLACK_BOT_TOKEN:-xoxb-fake-slack-pairing-e2e}" +SLACK_APP="${SLACK_APP_TOKEN:-xapp-fake-slack-pairing-e2e}" +SLACK_PAIRING_USER="${NEMOCLAW_SLACK_PAIRING_USER:-U3730E2E}" + +export NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" +export NEMOCLAW_RECREATE_SANDBOX=1 +export NEMOCLAW_FRESH=1 +export NEMOCLAW_POLICY_TIER="${NEMOCLAW_POLICY_TIER:-open}" +export SLACK_BOT_TOKEN="$SLACK_TOKEN" +export SLACK_APP_TOKEN="$SLACK_APP" +if [ -z "${NEMOCLAW_SKIP_SLACK_AUTH_VALIDATION:-}" ] \ + && { is_fake_slack_token "$SLACK_TOKEN" || is_fake_slack_token "$SLACK_APP"; }; then + export NEMOCLAW_SKIP_SLACK_AUTH_VALIDATION=1 + info "Skipping onboarding Slack auth validation for fake-token E2E" +fi + +openshell() { + if [ "$OPENSHELL_BIN" = "openshell" ]; then + command openshell "$@" + else + "$OPENSHELL_BIN" "$@" + fi +} + +sandbox_exec() { + local cmd="$1" + local ssh_config + ssh_config="$(mktemp)" + openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config" 2>/dev/null + + local result status + result=$(run_with_timeout 60 ssh -F "$ssh_config" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" \ + "$cmd" \ + 2>&1) + status=$? + + rm -f "$ssh_config" + printf '%s\n' "$result" + return "$status" +} + +quote_for_remote_sh() { + local value="${1:-}" + printf "'%s'" "$(printf '%s' "$value" | sed "s/'/'\\\\''/g")" +} + +sandbox_exec_sh_script() { + local script="$1" + shift + local encoded remote_cmd arg + encoded="$(printf '%s' "$script" | base64 | tr -d '\n')" + remote_cmd="tmp=\$(mktemp); trap 'rm -f \"\$tmp\"' EXIT; printf %s $(quote_for_remote_sh "$encoded") | base64 -d > \"\$tmp\"; sh \"\$tmp\"" + for arg in "$@"; do + remote_cmd+=" $(quote_for_remote_sh "$arg")" + done + openshell sandbox exec --name "$SANDBOX_NAME" -- sh -lc "$remote_cmd" +} + +# shellcheck source=test/e2e-vpn/lib/sandbox-teardown.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib/sandbox-teardown.sh" +register_sandbox_for_teardown "$SANDBOX_NAME" + +# shellcheck source=test/e2e-vpn/lib/slack-api-proof.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib/slack-api-proof.sh" + +check_fake_slack_pairing_capture() { + node - "$FAKE_SLACK_API_CAPTURE_FILE" <<'NODE' +const fs = require("fs"); +const file = process.argv[2]; +const rows = fs + .readFileSync(file, "utf8") + .trim() + .split(/\n+/) + .filter(Boolean) + .map((line) => JSON.parse(line)); + +const ws = rows + .filter((row) => row.event === "websocket-message" && row.messageType === "socket_mode_client_hello") + .at(-1); +if (!ws) { + console.log("NO_WEBSOCKET_MESSAGE"); + process.exit(2); +} +if (ws.tokenMatchesExpected !== true) { + console.log("BAD_WEBSOCKET_TOKEN_REWRITE"); + process.exit(3); +} +if (ws.tokenLooksPlaceholder) { + console.log("WEBSOCKET_PLACEHOLDER_LEAK"); + process.exit(4); +} + +const post = rows + .filter((row) => row.event === "request" && row.path === "/api/chat.postMessage") + .at(-1); +if (!post) { + console.log("NO_CHAT_POSTMESSAGE"); + process.exit(5); +} +if (post.authorization !== undefined || post.body !== undefined) { + console.log("RAW_CAPTURE_LEAK"); + process.exit(6); +} +if (post.tokenMatchesExpected !== true || post.bodyMatchesExpected !== true) { + console.log("BAD_CHAT_POSTMESSAGE_TOKEN_REWRITE"); + process.exit(7); +} +if (post.tokenLooksPlaceholder) { + console.log("CHAT_POSTMESSAGE_PLACEHOLDER_LEAK"); + process.exit(8); +} +console.log("OK"); +NODE +} + +section "Phase 0: Prerequisites" + +if [ -z "${NVIDIA_API_KEY:-}" ]; then + fail "NVIDIA_API_KEY not set" + exit 1 +fi +pass "NVIDIA_API_KEY is set" + +if ! docker info >/dev/null 2>&1; then + fail "Docker is not running" + exit 1 +fi +pass "Docker is running" + +info "Sandbox name: $SANDBOX_NAME" +info "Slack bot token: configured (${#SLACK_TOKEN} chars)" +info "Slack app token: configured (${#SLACK_APP} chars)" + +section "Phase 1: Install NemoClaw with Slack enabled" + +cd "$REPO" || exit 1 + +info "Pre-cleanup..." +if command -v nemoclaw >/dev/null 2>&1; then + nemoclaw "$SANDBOX_NAME" destroy --yes 2>/dev/null || true +fi +if openshell --version >/dev/null 2>&1; then + openshell sandbox delete "$SANDBOX_NAME" 2>/dev/null || true + openshell gateway destroy -g nemoclaw 2>/dev/null || true +fi +pass "Pre-cleanup complete" + +# Keep this in sync with the Slack boot-time pre-merge in +# test-messaging-providers.sh. Slack presets normally apply after the sandbox +# first starts; pre-merging avoids a slow first-boot Slack SDK CONNECT failure. +BASE_POLICY="$REPO/nemoclaw-blueprint/policies/openclaw-sandbox.yaml" +SLACK_PRESET="$REPO/nemoclaw-blueprint/policies/presets/slack.yaml" +if [ -f "$BASE_POLICY" ] && [ -f "$SLACK_PRESET" ] && ! grep -q "api.slack.com" "$BASE_POLICY"; then + BASE_POLICY_BAK="$(mktemp)" + cp "$BASE_POLICY" "$BASE_POLICY_BAK" + _previous_exit_trap=$(trap -p EXIT | sed "s/^trap -- '//;s/' EXIT$//") + trap ''"${_previous_exit_trap:+$_previous_exit_trap;}"' cp "$BASE_POLICY_BAK" "$BASE_POLICY" 2>/dev/null || true; rm -f "$BASE_POLICY_BAK"' EXIT + info "Pre-merging Slack network policy into base sandbox policy..." + cat >>"$BASE_POLICY" <<'SLACK_POLICY_EOF' + + # ── Slack — pre-merged for Slack pairing E2E (#3730) ────────── + slack: + name: slack + endpoints: + - host: slack.com + port: 443 + protocol: rest + enforcement: enforce + rules: + - allow: { method: GET, path: "/**" } + - allow: { method: POST, path: "/**" } + - host: api.slack.com + port: 443 + protocol: rest + enforcement: enforce + rules: + - allow: { method: GET, path: "/**" } + - allow: { method: POST, path: "/**" } + - host: hooks.slack.com + port: 443 + protocol: rest + enforcement: enforce + rules: + - allow: { method: GET, path: "/**" } + - allow: { method: POST, path: "/**" } + - host: wss-primary.slack.com + port: 443 + protocol: websocket + enforcement: enforce + rules: + - allow: { method: GET, path: "/**" } + - allow: { method: WEBSOCKET_TEXT, path: "/**" } + - host: wss-backup.slack.com + port: 443 + protocol: websocket + enforcement: enforce + rules: + - allow: { method: GET, path: "/**" } + - allow: { method: WEBSOCKET_TEXT, path: "/**" } + binaries: + - { path: /usr/local/bin/node } + - { path: /usr/bin/node } +SLACK_POLICY_EOF + pass "Slack network policy pre-merged into base policy" +else + if grep -q "api.slack.com" "$BASE_POLICY" 2>/dev/null; then + info "Slack policy already present in base policy — skipping pre-merge" + else + fail "Cannot pre-merge Slack policy: missing base policy or preset file" + exit 1 + fi +fi + +INSTALL_LOG="/tmp/nemoclaw-e2e-openclaw-slack-pairing-install.log" +info "Running install.sh --non-interactive..." +bash install.sh --non-interactive >"$INSTALL_LOG" 2>&1 & +install_pid=$! +tail -f "$INSTALL_LOG" --pid=$install_pid 2>/dev/null & +tail_pid=$! +wait $install_pid +install_exit=$? +kill $tail_pid 2>/dev/null || true +wait $tail_pid 2>/dev/null || true + +if [ -f "$HOME/.bashrc" ]; then + # shellcheck source=/dev/null + source "$HOME/.bashrc" 2>/dev/null || true +fi +export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" +if [ -s "$NVM_DIR/nvm.sh" ]; then + # shellcheck source=/dev/null + . "$NVM_DIR/nvm.sh" +fi +if [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]]; then + export PATH="$HOME/.local/bin:$PATH" +fi + +if [ $install_exit -eq 0 ]; then + pass "Install completed" +else + fail "install.sh failed (exit $install_exit)" + info "Last 40 lines of install log:" + tail -40 "$INSTALL_LOG" 2>/dev/null || true + exit 1 +fi + +sandbox_list=$(openshell sandbox list 2>&1 || true) +if echo "$sandbox_list" | grep -q "$SANDBOX_NAME.*Ready"; then + pass "Sandbox '$SANDBOX_NAME' is Ready" +else + fail "Sandbox '$SANDBOX_NAME' not Ready (list: ${sandbox_list:0:300})" + exit 1 +fi + +if openshell provider get "${SANDBOX_NAME}-slack-bridge" >/dev/null 2>&1 \ + && openshell provider get "${SANDBOX_NAME}-slack-app" >/dev/null 2>&1; then + pass "Slack bot/app providers exist in OpenShell" +else + fail "Slack bot/app providers missing in OpenShell" +fi + +section "Phase 2: Runtime state root contract" + +state_env=$(sandbox_exec 'printf "OPENCLAW_HOME=%s\nOPENCLAW_STATE_DIR=%s\nOPENCLAW_CONFIG_PATH=%s\nOPENCLAW_OAUTH_DIR=%s\n" "$OPENCLAW_HOME" "$OPENCLAW_STATE_DIR" "$OPENCLAW_CONFIG_PATH" "$OPENCLAW_OAUTH_DIR"') +state_env_status=$? +info "OpenClaw env from connect shell: ${state_env//$'\n'/; }" +if [ $state_env_status -eq 0 ] \ + && echo "$state_env" | grep -q '^OPENCLAW_HOME=/sandbox$' \ + && echo "$state_env" | grep -q '^OPENCLAW_STATE_DIR=/sandbox/.openclaw$' \ + && echo "$state_env" | grep -q '^OPENCLAW_CONFIG_PATH=/sandbox/.openclaw/openclaw.json$' \ + && echo "$state_env" | grep -q '^OPENCLAW_OAUTH_DIR=/sandbox/.openclaw/credentials$'; then + pass "Connect-shell OpenClaw env resolves to /sandbox/.openclaw" +else + fail "Connect-shell OpenClaw env does not resolve to the shared state root" +fi + +pairing_list_empty=$(sandbox_exec 'openclaw pairing list slack --json 2>&1') +pairing_list_empty_status=$? +info "Initial pairing list: ${pairing_list_empty:0:300}" +if [ $pairing_list_empty_status -eq 0 ] \ + && echo "$pairing_list_empty" | grep -q '"channel"[[:space:]]*:[[:space:]]*"slack"'; then + pass "openclaw pairing list slack works in connect shell" +else + fail "openclaw pairing list slack failed before request creation: ${pairing_list_empty:0:300}" +fi + +section "Phase 3: Hermetic Slack Socket Mode pairing request" + +if start_fake_slack_api "$SLACK_TOKEN" "$SLACK_APP"; then + pass "Hermetic fake Slack API started on host port ${FAKE_SLACK_API_PORT}" +else + fail "Failed to start hermetic fake Slack API" + exit 1 +fi + +if apply_fake_slack_api_policy "$SANDBOX_NAME" "$FAKE_SLACK_API_PORT" >/tmp/nemoclaw-fake-slack-pairing-rest-policy.log 2>&1; then + pass "Applied REST policy for fake Slack chat.postMessage" +else + fail "Failed to apply fake Slack REST policy: $(tail -20 /tmp/nemoclaw-fake-slack-pairing-rest-policy.log 2>/dev/null | tr '\n' ' ' | cut -c1-300)" +fi + +if apply_fake_slack_socket_mode_policy "$SANDBOX_NAME" "$FAKE_SLACK_API_PORT" >/tmp/nemoclaw-fake-slack-pairing-ws-policy.log 2>&1; then + pass "Applied websocket policy for fake Slack Socket Mode" +else + fail "Failed to apply fake Slack websocket policy: $(tail -20 /tmp/nemoclaw-fake-slack-pairing-ws-policy.log 2>/dev/null | tr '\n' ' ' | cut -c1-300)" +fi + +gateway_issue_script=$( + cat <<'SCRIPT' + set -a + [ -f /tmp/nemoclaw-proxy-env.sh ] && . /tmp/nemoclaw-proxy-env.sh + set +a + fake_slack_api_port="$1" + slack_pairing_user="$2" + fake_slack_api_host="$3" + pairing_e2e_mode="$4" + : "${OPENCLAW_HOME:?OPENCLAW_HOME missing from runtime shell env}" + : "${OPENCLAW_STATE_DIR:?OPENCLAW_STATE_DIR missing from runtime shell env}" + : "${OPENCLAW_CONFIG_PATH:?OPENCLAW_CONFIG_PATH missing from runtime shell env}" + : "${OPENCLAW_OAUTH_DIR:?OPENCLAW_OAUTH_DIR missing from runtime shell env}" + printf 'GATEWAY_OPENCLAW_ENV uid=%s gid=%s OPENCLAW_STATE_DIR=%s OPENCLAW_OAUTH_DIR=%s\n' "$(id -u)" "$(id -g)" "$OPENCLAW_STATE_DIR" "$OPENCLAW_OAUTH_DIR" + exec env \ + HOME=/sandbox \ + OPENCLAW_HOME="$OPENCLAW_HOME" \ + OPENCLAW_STATE_DIR="$OPENCLAW_STATE_DIR" \ + OPENCLAW_CONFIG_PATH="$OPENCLAW_CONFIG_PATH" \ + OPENCLAW_OAUTH_DIR="$OPENCLAW_OAUTH_DIR" \ + HTTP_PROXY="${HTTP_PROXY:-}" \ + HTTPS_PROXY="${HTTPS_PROXY:-}" \ + http_proxy="${http_proxy:-}" \ + https_proxy="${https_proxy:-}" \ + NO_PROXY="${NO_PROXY:-}" \ + no_proxy="${no_proxy:-}" \ + NODE_OPTIONS="${NODE_OPTIONS:-}" \ + FAKE_SLACK_API_HOST="$fake_slack_api_host" \ + FAKE_SLACK_API_PORT="$fake_slack_api_port" \ + SLACK_PAIRING_USER="$slack_pairing_user" \ + PAIRING_E2E_MODE="$pairing_e2e_mode" \ + node --input-type=module <<'NODE' +import crypto from "node:crypto"; +import fs from "node:fs"; +import http from "node:http"; +import net from "node:net"; +import path from "node:path"; +import { execFileSync } from "node:child_process"; +import { pathToFileURL } from "node:url"; + +function findOpenClawPackageRootFromBinary() { + let binary = ""; + try { + binary = execFileSync("sh", ["-lc", "command -v openclaw"], { encoding: "utf8" }).trim(); + } catch { + return null; + } + if (!binary) return null; + + let current = ""; + try { + current = fs.realpathSync(binary); + } catch { + return null; + } + if (fs.statSync(current).isFile()) current = path.dirname(current); + + for (let depth = 0; depth < 8; depth += 1) { + const manifest = path.join(current, "package.json"); + if (fs.existsSync(manifest)) { + try { + const pkg = JSON.parse(fs.readFileSync(manifest, "utf8")); + if (pkg?.name === "openclaw") return current; + } catch { + // Keep walking toward the filesystem root. + } + } + const parent = path.dirname(current); + if (parent === current) break; + current = parent; + } + return null; +} + +function loadConversationRuntime() { + const candidates = []; + const binaryRoot = findOpenClawPackageRootFromBinary(); + if (binaryRoot) candidates.push(binaryRoot); + try { + const globalRoot = execFileSync("npm", ["root", "-g"], { encoding: "utf8" }).trim(); + if (globalRoot) candidates.push(path.join(globalRoot, "openclaw")); + } catch { + // Keep the explicit global-root fallbacks below. + } + candidates.push( + "/usr/local/lib/node_modules/openclaw", + "/usr/lib/node_modules/openclaw", + ); + const uniqueCandidates = [...new Set(candidates)]; + for (const root of uniqueCandidates) { + const runtime = path.join(root, "dist/plugin-sdk/conversation-runtime.js"); + if (fs.existsSync(runtime)) return import(pathToFileURL(runtime).href); + } + throw new Error(`OpenClaw conversation runtime not found; checked: ${uniqueCandidates.join(", ")}`); +} + +function parseProxyTarget() { + const raw = process.env.HTTP_PROXY || process.env.http_proxy || ""; + if (!raw) return null; + try { + const parsed = new URL(raw); + if (parsed.protocol !== "http:") return null; + return { host: parsed.hostname, port: Number(parsed.port || "80") }; + } catch { + return null; + } +} + +function encodeClientText(payload) { + const body = Buffer.from(payload, "utf8"); + const mask = crypto.randomBytes(4); + const masked = Buffer.alloc(body.length); + for (let i = 0; i < body.length; i += 1) masked[i] = body[i] ^ mask[i % 4]; + if (body.length < 126) { + return Buffer.concat([Buffer.from([0x81, 0x80 | body.length]), mask, masked]); + } + const header = Buffer.alloc(4); + header[0] = 0x81; + header[1] = 0x80 | 126; + header.writeUInt16BE(body.length, 2); + return Buffer.concat([header, mask, masked]); +} + +function decodeServerFrame(buffer) { + if (buffer.length < 2) return null; + const opcode = buffer[0] & 0x0f; + let payloadLength = buffer[1] & 0x7f; + let offset = 2; + if (payloadLength === 126) { + if (buffer.length < 4) return null; + payloadLength = buffer.readUInt16BE(2); + offset = 4; + } else if (payloadLength === 127) { + if (buffer.length < 10) return null; + payloadLength = Number(buffer.readBigUInt64BE(2)); + offset = 10; + } + if (buffer.length < offset + payloadLength) return null; + return { + opcode, + payload: buffer.slice(offset, offset + payloadLength), + totalLength: offset + payloadLength, + }; +} + +function receiveSlackSocketEvent() { + const host = process.env.FAKE_SLACK_API_HOST || "host.openshell.internal"; + const port = Number(process.env.FAKE_SLACK_API_PORT); + const proxy = parseProxyTarget(); + + return new Promise((resolve, reject) => { + const socket = proxy + ? net.createConnection({ host: proxy.host, port: proxy.port }) + : net.createConnection({ host, port }); + const timer = setTimeout(() => { + socket.destroy(); + reject(new Error("timed out waiting for fake Slack Socket Mode event")); + }, 30000); + + let handshake = Buffer.alloc(0); + let framed = Buffer.alloc(0); + let upgraded = false; + + socket.on("connect", () => { + const key = crypto.randomBytes(16).toString("base64"); + const requestTarget = proxy + ? `http://${host}:${port}/socket-mode` + : "/socket-mode"; + socket.write([ + `GET ${requestTarget} HTTP/1.1`, + `Host: ${host}:${port}`, + "Upgrade: websocket", + "Connection: Upgrade", + `Sec-WebSocket-Key: ${key}`, + "Sec-WebSocket-Version: 13", + "\r\n", + ].join("\r\n")); + }); + + socket.on("data", (chunk) => { + if (!upgraded) { + handshake = Buffer.concat([handshake, chunk]); + const end = handshake.indexOf("\r\n\r\n"); + if (end === -1) return; + const statusLine = handshake.slice(0, end).toString("latin1").split("\r\n")[0] || ""; + if (!statusLine.includes("101")) { + clearTimeout(timer); + socket.destroy(); + reject(new Error(`fake Slack websocket upgrade failed: ${statusLine}`)); + return; + } + upgraded = true; + framed = Buffer.concat([framed, handshake.slice(end + 4)]); + socket.write(encodeClientText(JSON.stringify({ + type: "socket_mode_client_hello", + token: "xapp-OPENSHELL-RESOLVE-ENV-SLACK_APP_TOKEN", + }))); + } else { + framed = Buffer.concat([framed, chunk]); + } + + while (framed.length > 0) { + const frame = decodeServerFrame(framed); + if (!frame) break; + framed = framed.slice(frame.totalLength); + if (frame.opcode !== 1) continue; + const envelope = JSON.parse(frame.payload.toString("utf8")); + socket.write(encodeClientText(JSON.stringify({ envelope_id: envelope.envelope_id }))); + clearTimeout(timer); + socket.end(); + socket.destroy(); + resolve(envelope); + return; + } + }); + + socket.on("error", (error) => { + clearTimeout(timer); + reject(error); + }); + }); +} + +function postPairingReply(text, channel) { + const host = process.env.FAKE_SLACK_API_HOST || "host.openshell.internal"; + const port = Number(process.env.FAKE_SLACK_API_PORT); + const token = "xoxb-OPENSHELL-RESOLVE-ENV-SLACK_BOT_TOKEN"; + const data = new URLSearchParams({ token, channel, text }).toString(); + + return new Promise((resolve, reject) => { + const req = http.request({ + hostname: host, + port, + path: "/api/chat.postMessage", + method: "POST", + headers: { + Authorization: `Bearer ${token}`, + "Content-Type": "application/x-www-form-urlencoded", + "Content-Length": Buffer.byteLength(data), + }, + timeout: 30000, + }, (res) => { + let body = ""; + res.on("data", (chunk) => { + body += chunk; + }); + res.on("end", () => { + if (res.statusCode !== 200) { + reject(new Error(`chat.postMessage failed: ${res.statusCode} ${body.slice(0, 200)}`)); + return; + } + resolve(body); + }); + }); + req.on("error", reject); + req.on("timeout", () => { + req.destroy(new Error("chat.postMessage timed out")); + }); + req.write(data); + req.end(); + }); +} + +const { + issuePairingChallenge, + upsertChannelPairingRequest, +} = await loadConversationRuntime(); + +const mode = process.env.PAIRING_E2E_MODE || "full"; +const directGateway = mode === "direct-gateway"; +const socketProbeOnly = mode === "socket-probe"; +const envelope = directGateway + ? { + payload: { + team_id: "T3730E2E", + event: { + type: "message", + channel: "D3730E2E", + user: process.env.SLACK_PAIRING_USER, + }, + }, + } + : await receiveSlackSocketEvent(); +const event = envelope?.payload?.event; +if (!event || event.type !== "message" || !event.user || !event.channel) { + throw new Error(`unexpected fake Slack envelope: ${JSON.stringify(envelope).slice(0, 400)}`); +} +if (event.user !== process.env.SLACK_PAIRING_USER) { + throw new Error(`unexpected fake Slack user: ${event.user}`); +} + +if (socketProbeOnly) { + await postPairingReply("Slack pairing E2E websocket probe", event.channel); + console.log(`SLACK_SOCKET_PROBE_RESULT ${JSON.stringify({ + senderId: event.user, + channelId: event.channel, + })}`); + process.exit(0); +} + +let replyText = ""; +const result = await issuePairingChallenge({ + channel: "slack", + senderId: event.user, + senderIdLine: `Slack user ID: ${event.user}`, + meta: { + accountId: "default", + channelId: event.channel, + teamId: envelope.payload?.team_id || "", + }, + upsertPairingRequest: async ({ id, meta }) => upsertChannelPairingRequest({ + channel: "slack", + id, + accountId: "default", + meta, + }), + sendPairingReply: async (text) => { + if (directGateway) { + replyText = text; + } else { + await postPairingReply(text, event.channel); + } + }, +}); + +if (!result.created || !result.code) { + throw new Error(`pairing challenge was not created: ${JSON.stringify(result)}`); +} + +console.log(`PAIRING_E2E_RESULT ${JSON.stringify({ + code: result.code, + senderId: event.user, + channelId: event.channel, + replyText, +})}`); +NODE +SCRIPT +) +# Drive the hermetic Slack flow through OpenShell's tracked sandbox execution +# path so the request lands in the same state root that the approval CLI reads. +# The gateway-user env inheritance is covered by nemoclaw-start regression tests. +gateway_issue_output=$(sandbox_exec_sh_script "$gateway_issue_script" "$FAKE_SLACK_API_PORT" "$SLACK_PAIRING_USER" "$FAKE_SLACK_API_HOST" full 2>&1) +gateway_issue_status=$? +info "Slack pairing issue output: ${gateway_issue_output:0:600}" +if [ $gateway_issue_status -eq 0 ] && echo "$gateway_issue_output" | grep -q '^PAIRING_E2E_RESULT '; then + pass "OpenShell-tracked Slack Socket Mode handler created a pairing request" +else + fail "OpenShell-tracked Slack Socket Mode pairing request creation failed" +fi + +pairing_result_line=$(printf '%s\n' "$gateway_issue_output" | grep '^PAIRING_E2E_RESULT ' | tail -1 || true) +pairing_json="${pairing_result_line#PAIRING_E2E_RESULT }" +pairing_code=$(node -e 'const data = JSON.parse(process.argv[1]); process.stdout.write(data.code || "");' "$pairing_json" 2>/dev/null || true) +if [ -n "$pairing_code" ]; then + pass "Pairing code extracted from fake Slack reply path" +else + fail "Failed to extract pairing code" + pairing_code="__missing_pairing_code__" +fi + +capture_check=$(check_fake_slack_pairing_capture 2>&1 || true) +if [ "$capture_check" = "OK" ]; then + pass "Fake Slack saw rewritten xapp websocket frame and xoxb chat.postMessage" +else + fail "Fake Slack capture did not prove Slack token rewriting: ${capture_check:0:300}" +fi + +section "Phase 4: Connect-shell approval" + +pending_file_check=$(sandbox_exec "test -f /sandbox/.openclaw/credentials/slack-pairing.json && grep -F '$pairing_code' /sandbox/.openclaw/credentials/slack-pairing.json && grep -F '$SLACK_PAIRING_USER' /sandbox/.openclaw/credentials/slack-pairing.json") +pending_file_status=$? +if [ $pending_file_status -eq 0 ] \ + && echo "$pending_file_check" | grep -qF "$pairing_code" \ + && echo "$pending_file_check" | grep -qF "$SLACK_PAIRING_USER"; then + pass "Runtime-created Slack pending request is in the shared OpenClaw state root" +else + fail "Slack pending request missing from /sandbox/.openclaw/credentials/slack-pairing.json" +fi + +pairing_list=$(sandbox_exec 'openclaw pairing list slack --json 2>&1') +pairing_list_status=$? +info "Pairing list after fake Slack event: ${pairing_list:0:500}" +if [ $pairing_list_status -eq 0 ] \ + && echo "$pairing_list" | grep -qF "$pairing_code" \ + && echo "$pairing_list" | grep -qF "$SLACK_PAIRING_USER"; then + pass "Connect-shell openclaw pairing list sees runtime-created Slack request" +else + fail "Connect-shell openclaw pairing list does not see the Slack request" +fi + +approve_output=$(sandbox_exec "openclaw pairing approve slack '$pairing_code' 2>&1") +approve_status=$? +info "Pairing approve output: ${approve_output:0:500}" +if [ $approve_status -eq 0 ] \ + && echo "$approve_output" | grep -q "Approved" \ + && echo "$approve_output" | grep -qF "$SLACK_PAIRING_USER"; then + pass "Connect-shell openclaw pairing approve approved the Slack request" +else + fail "Connect-shell openclaw pairing approve failed: ${approve_output:0:500}" +fi + +pairing_list_after=$(sandbox_exec 'openclaw pairing list slack --json 2>&1') +pairing_list_after_status=$? +if [ $pairing_list_after_status -ne 0 ]; then + fail "openclaw pairing list slack failed after approval: ${pairing_list_after:0:300}" +elif echo "$pairing_list_after" | grep -qF "$pairing_code"; then + fail "Approved Slack pairing code is still pending" +else + pass "Approved Slack pairing code was consumed" +fi + +allow_from_check=$(sandbox_exec "test -f /sandbox/.openclaw/credentials/slack-default-allowFrom.json && grep -F '$SLACK_PAIRING_USER' /sandbox/.openclaw/credentials/slack-default-allowFrom.json") +allow_from_status=$? +if [ $allow_from_status -eq 0 ] \ + && echo "$allow_from_check" | grep -qF "$SLACK_PAIRING_USER"; then + pass "Slack allowFrom store contains the approved user" +else + fail "Slack allowFrom store missing approved user" +fi + +repeat_approve=$(sandbox_exec "openclaw pairing approve slack '$pairing_code' 2>&1") +if echo "$repeat_approve" | grep -q "No pending pairing request found"; then + pass "Second approval fails closed after request consumption" +else + fail "Second approval did not report missing pending request: ${repeat_approve:0:300}" +fi + +section "Phase 5: Cleanup" + +if [[ "${NEMOCLAW_E2E_KEEP_SANDBOX:-}" = "1" ]]; then + skip "Cleanup: NEMOCLAW_E2E_KEEP_SANDBOX=1 — leaving sandbox '$SANDBOX_NAME' for inspection" +else + nemoclaw "$SANDBOX_NAME" destroy --yes 2>/dev/null || true + openshell sandbox delete "$SANDBOX_NAME" 2>/dev/null || true +fi + +if [[ "${NEMOCLAW_E2E_KEEP_SANDBOX:-}" = "1" ]]; then + pass "Cleanup: Sandbox '$SANDBOX_NAME' intentionally kept" +elif openshell sandbox list 2>&1 | grep -q "$SANDBOX_NAME"; then + fail "Cleanup: Sandbox '$SANDBOX_NAME' still present after cleanup" +else + pass "Cleanup: Sandbox '$SANDBOX_NAME' removed" +fi + +echo "" +echo "========================================" +echo " OpenClaw Slack Pairing E2E Results:" +echo " Passed: $PASS" +echo " Failed: $FAIL" +echo " Skipped: $SKIP" +echo " Total: $TOTAL" +echo "========================================" + +if [ "$FAIL" -eq 0 ]; then + printf '\n\033[1;32m OpenClaw Slack pairing E2E PASSED.\033[0m\n' + exit 0 +else + printf '\n\033[1;31m %d test(s) FAILED.\033[0m\n' "$FAIL" + exit 1 +fi diff --git a/test/e2e-vpn/test-openclaw-tui-chat-correlation.sh b/test/e2e-vpn/test-openclaw-tui-chat-correlation.sh new file mode 100755 index 00000000000..8e7281bee05 --- /dev/null +++ b/test/e2e-vpn/test-openclaw-tui-chat-correlation.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Validation-only E2E for release-blocker close calls: +# #2603 - previous TUI/chat message disappears after reconnect/scroll +# #3145 - rapid sequential TUI messages duplicate or arrive out of order +# +# The Vitest live harness drives OpenClaw's gateway websocket directly against a +# real sandbox. This wrapper creates a fresh cloud-backed OpenClaw sandbox first +# so CI evidence is not dependent on a developer machine's stale sandbox state. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO="$(cd "${SCRIPT_DIR}/../.." && pwd)" + +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-openclaw-tui-correlation}" +INSTALL_LOG="${E2E_OPENCLAW_TUI_CORRELATION_INSTALL_LOG:-/tmp/nemoclaw-e2e-openclaw-tui-correlation-install.log}" + +cleanup() { + if [ "${NEMOCLAW_E2E_SKIP_CLEANUP:-0}" = "1" ]; then + return + fi + SANDBOX_NAME="$SANDBOX_NAME" bash "${SCRIPT_DIR}/e2e-cloud-experimental/cleanup.sh" --verify >/dev/null 2>&1 || true +} +trap cleanup EXIT + +export NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" +export E2E_CLOUD_ONBOARD_INSTALL_LOG="$INSTALL_LOG" +export NEMOCLAW_E2E_KEEP_SANDBOX=1 +export NEMOCLAW_NON_INTERACTIVE="${NEMOCLAW_NON_INTERACTIVE:-1}" +export NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE="${NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE:-1}" +export NEMOCLAW_RECREATE_SANDBOX="${NEMOCLAW_RECREATE_SANDBOX:-1}" + +bash "${SCRIPT_DIR}/test-cloud-onboard-e2e.sh" + +# Pick up PATH changes from the public installer in this shell. +# shellcheck source=test/e2e-vpn/lib/install-path-refresh.sh +. "${SCRIPT_DIR}/lib/install-path-refresh.sh" +nemoclaw_refresh_install_env +nemoclaw_ensure_local_bin_on_path +export PATH="/usr/local/bin:${HOME}/.local/bin:${PATH}" + +openclaw_version="$( + openshell sandbox exec --name "$SANDBOX_NAME" -- openclaw --version 2>&1 || true +)" +echo "Sandbox OpenClaw version: ${openclaw_version}" +if ! grep -Fq "2026.5.27" <<<"$openclaw_version"; then + echo "Expected fresh sandbox to run OpenClaw 2026.5.27" >&2 + exit 1 +fi + +cd "$REPO" + +if [ ! -x ./node_modules/.bin/vitest ]; then + echo "Restoring repository dev dependencies for the live Vitest harness" + npm ci --include=dev +fi + +NEMOCLAW_ISSUE_2603_LIVE=1 \ + NEMOCLAW_ISSUE_2603_SANDBOX="$SANDBOX_NAME" \ + ./node_modules/.bin/vitest run test/openclaw-tui-chat-correlation.test.ts --reporter=verbose diff --git a/test/e2e-vpn/test-openshell-gateway-upgrade.sh b/test/e2e-vpn/test-openshell-gateway-upgrade.sh new file mode 100755 index 00000000000..2d27f9e0f28 --- /dev/null +++ b/test/e2e-vpn/test-openshell-gateway-upgrade.sh @@ -0,0 +1,718 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Regression coverage for PR #3001 upgrade installs: +# 1. If a user already has a working claw on the previous OpenShell release, +# the current install/onboard path must back up the old claw before replacing +# the incompatible OpenShell gateway, recreate it under the current gateway, +# restore durable agent state, and leave the same agent type running. +# 2. If a macOS arm64 user already has the current OpenShell CLI but not the +# standalone openshell-gateway binary, the installer must fetch the Darwin +# gateway asset instead of accepting the incomplete CLI-only install. + +set -euo pipefail + +LOG_FILE="/tmp/nemoclaw-e2e-openshell-gateway-upgrade.log" +INSTALL_LOG="/tmp/nemoclaw-e2e-openshell-gateway-install.log" +OLD_INSTALL_LOG="/tmp/nemoclaw-e2e-openshell-gateway-old-install.log" +CURRENT_INSTALL_LOG="/tmp/nemoclaw-e2e-openshell-gateway-current-install.log" +START_LOG="/tmp/nemoclaw-e2e-openshell-gateway-start.log" +GATEWAY_LOG="/tmp/nemoclaw-e2e-openshell-gateway-process.log" +MOCK_LOG="/tmp/nemoclaw-e2e-openshell-gateway-compatible-mock.log" +OLD_DOCKER_WRAPPER_DIR="" +OLD_DOCKER_WRAPPER_LOG="/tmp/nemoclaw-e2e-openshell-gateway-old-docker.log" +exec > >(tee "$LOG_FILE") 2>&1 + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +pass() { echo -e "${GREEN}[PASS]${NC} $1"; } +info() { echo -e "${YELLOW}[INFO]${NC} $1"; } +diag() { echo -e "${YELLOW}[DIAG]${NC} $1"; } +fail() { + echo -e "${RED}[FAIL]${NC} $1" >&2 + diag "openshell status: $(openshell status 2>&1 || true)" + diag "gateway info: $(openshell gateway info -g nemoclaw 2>&1 || true)" + diag "pid file: $(cat "$PID_FILE" 2>/dev/null || echo missing)" + if command -v openshell >/dev/null 2>&1 && [ -n "${SURVIVOR_SANDBOX:-}" ]; then + diag "survivor agent state: $(survivor_agent_probe 2>&1 || true)" + diag "survivor agent log tail:" + openshell sandbox exec --name "$SURVIVOR_SANDBOX" -- \ + sh -lc 'tail -40 /tmp/nemoclaw-e2e-agent.log 2>/dev/null || true' 2>/dev/null || true + fi + diag "gateway log tail:" + tail -100 "$GATEWAY_LOG" 2>/dev/null || true + exit 1 +} + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +# shellcheck source=test/e2e-vpn/lib/openai-compatible-api-proof.sh +source "${SCRIPT_DIR}/lib/openai-compatible-api-proof.sh" +STATE_DIR="${NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR:-$HOME/.local/state/nemoclaw/openshell-docker-gateway}" +PID_FILE="${STATE_DIR}/openshell-gateway.pid" +OLD_NEMOCLAW_REF="${NEMOCLAW_OLD_NEMOCLAW_REF:-v0.0.36}" +OLD_OPENSHELL_VERSION="${NEMOCLAW_OLD_OPENSHELL_VERSION:-0.0.36}" +OLD_SANDBOX_BASE_IMAGE_REF="${NEMOCLAW_OLD_SANDBOX_BASE_IMAGE_REF:-ghcr.io/nvidia/nemoclaw/sandbox-base@sha256:104151ffadc2ff0b6c815e3c95c2783ced61aee0d0f83fc327cc02be9b7e14e6}" +OLD_OPENCLAW_VERSION="${NEMOCLAW_OLD_OPENCLAW_VERSION:-2026.4.24}" +CURRENT_OPENSHELL_VERSION="${NEMOCLAW_CURRENT_OPENSHELL_VERSION:-0.0.44}" +SURVIVOR_SANDBOX="${NEMOCLAW_GATEWAY_UPGRADE_SURVIVOR_NAME:-e2e-gateway-upgrade-survivor}" +SURVIVOR_MARKER="gateway-upgrade-survivor-$(date +%s)" +SURVIVOR_MARKER_PATH="/sandbox/.openclaw/workspace/nemoclaw-gateway-upgrade-marker" +REGISTRY_FILE="$HOME/.nemoclaw/sandboxes.json" +FAKE_BASE_URL="" +SURVIVOR_AGENT_PID="" + +load_shell_path() { + if [ -f "$HOME/.bashrc" ]; then + # shellcheck source=/dev/null + source "$HOME/.bashrc" 2>/dev/null || true + fi + export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" + if [ -s "$NVM_DIR/nvm.sh" ]; then + # shellcheck source=/dev/null + . "$NVM_DIR/nvm.sh" + fi + if [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]]; then + export PATH="$HOME/.local/bin:$PATH" + fi +} + +survivor_agent_probe() { + local probe + # shellcheck disable=SC2016 + probe='pid="$(cat /tmp/nemoclaw-e2e-agent.pid 2>/dev/null || true)"; [ -n "$pid" ] || exit 1; kill -0 "$pid" 2>/dev/null || exit 1; counter="$(sed -n "s/^[^ ]* \([0-9][0-9]*\).*/\1/p" /tmp/nemoclaw-e2e-agent.heartbeat 2>/dev/null | head -1)"; cmdline="$(tr "\000" " " <"/proc/${pid}/cmdline" 2>/dev/null || true)"; case "$cmdline" in *nemoclaw-e2e-agent*) ;; *) exit 1 ;; esac; printf "%s %s %s\n" "$pid" "${counter:-0}" "$cmdline"' + openshell sandbox exec --name "$SURVIVOR_SANDBOX" -- sh -lc "$probe" +} + +wait_for_survivor_agent_ready() { + for _i in $(seq 1 60); do + if survivor_agent_probe >/dev/null 2>&1; then + return 0 + fi + sleep 1 + done + return 1 +} + +survivor_agent_pid() { + survivor_agent_probe | awk '{print $1}' +} + +survivor_agent_counter() { + survivor_agent_probe | awk '{print $2}' +} + +cleanup_pid() { + local pid="$1" + [ -n "$pid" ] || return 0 + if kill -0 "$pid" 2>/dev/null; then + kill "$pid" 2>/dev/null || true + sleep 1 + kill -9 "$pid" 2>/dev/null || true + fi +} + +create_old_docker_wrapper() { + OLD_DOCKER_WRAPPER_DIR="$(mktemp -d)" + rm -f "$OLD_DOCKER_WRAPPER_LOG" + cat >"${OLD_DOCKER_WRAPPER_DIR}/docker" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +real_docker="${NEMOCLAW_REAL_DOCKER:-/usr/bin/docker}" +base_ref="${NEMOCLAW_OLD_SANDBOX_BASE_IMAGE_REF:?}" +old_openclaw="${NEMOCLAW_OLD_OPENCLAW_VERSION:?}" +log_file="${NEMOCLAW_OLD_DOCKER_WRAPPER_LOG:-/tmp/nemoclaw-e2e-openshell-gateway-old-docker.log}" +base_tag="ghcr.io/nvidia/nemoclaw/sandbox-base:latest" +if [ "${1:-}" = "pull" ]; then + for arg in "$@"; do + if [ "$arg" = "$base_tag" ]; then + printf 'rewrite pull %s -> %s\n' "$base_tag" "$base_ref" >>"$log_file" + "$real_docker" pull "$base_ref" + "$real_docker" tag "$base_ref" "$base_tag" + exit 0 + fi + done +fi +if [ "${1:-}" != "build" ]; then + exec "$real_docker" "$@" +fi + +args=() +rewrote_openclaw=0 +rewrote_base=0 +while [ "$#" -gt 0 ]; do + case "$1" in + --build-arg) + if [ "$#" -ge 2 ] && [ "${2#BASE_IMAGE=}" != "$2" ]; then + rewrote_base=1 + fi + if [ "$#" -ge 2 ] && [ "${2#OPENCLAW_VERSION=}" != "$2" ]; then + args+=("--build-arg" "OPENCLAW_VERSION=${old_openclaw}") + rewrote_openclaw=1 + printf 'rewrite build-arg %s -> OPENCLAW_VERSION=%s\n' "$2" "$old_openclaw" >>"$log_file" + shift 2 + continue + fi + if [ "$#" -ge 2 ] && [ "${2#BASE_IMAGE=}" != "$2" ]; then + args+=("--build-arg" "BASE_IMAGE=${base_ref}") + rewrote_base=1 + printf 'rewrite build-arg %s -> BASE_IMAGE=%s\n' "$2" "$base_ref" >>"$log_file" + shift 2 + continue + fi + ;; + --build-arg=OPENCLAW_VERSION=*) + args+=("--build-arg=OPENCLAW_VERSION=${old_openclaw}") + rewrote_openclaw=1 + printf 'rewrite build-arg %s -> OPENCLAW_VERSION=%s\n' "$1" "$old_openclaw" >>"$log_file" + shift + continue + ;; + --build-arg=BASE_IMAGE=*) + args+=("--build-arg=BASE_IMAGE=${base_ref}") + rewrote_base=1 + printf 'rewrite build-arg %s -> BASE_IMAGE=%s\n' "$1" "$base_ref" >>"$log_file" + shift + continue + ;; + --build-arg=BASE_IMAGE=*) + rewrote_base=1 + ;; + esac + args+=("$1") + shift +done +if [ "$rewrote_openclaw" = "0" ]; then + args+=("--build-arg" "OPENCLAW_VERSION=${old_openclaw}") + printf 'add build-arg OPENCLAW_VERSION=%s\n' "$old_openclaw" >>"$log_file" +fi +if [ "$rewrote_base" = "0" ]; then + args+=("--build-arg" "BASE_IMAGE=${base_ref}") + printf 'add build-arg BASE_IMAGE=%s\n' "$base_ref" >>"$log_file" +fi +exec "$real_docker" "${args[@]}" +EOF + chmod 755 "${OLD_DOCKER_WRAPPER_DIR}/docker" +} + +patch_old_installer_fixture() { + local installer="$1" + python3 - "$installer" <<'PY' +from pathlib import Path +import sys + +path = Path(sys.argv[1]) +text = path.read_text(encoding="utf-8") +needle = ' legacy_script="${source_root}/install.sh"\n' +insertion = r""" if [[ -n "${NEMOCLAW_OLD_OPENCLAW_VERSION:-}" && -f "$payload_script" ]]; then + python3 - "$payload_script" <<'NEMOCLAW_OLD_PAYLOAD_PIN_PY' +from pathlib import Path +import sys + +path = Path(sys.argv[1]) +text = path.read_text(encoding="utf-8") +needle = ' spin "Cloning ${_CLI_DISPLAY} source" clone_nemoclaw_ref "$release_ref" "$nemoclaw_src"\n' +hook = r''' if [[ -n "${NEMOCLAW_OLD_OPENCLAW_VERSION:-}" ]]; then + python3 - "$nemoclaw_src/Dockerfile" "$NEMOCLAW_OLD_OPENCLAW_VERSION" <<'NEMOCLAW_OLD_DOCKERFILE_PIN_PY' +from pathlib import Path +import sys + +path = Path(sys.argv[1]) +version = sys.argv[2] +text = path.read_text(encoding="utf-8") +marker = "RUN set -eu; \\\n MIN_VER=$(grep -m 1 'min_openclaw_version'" +injection = ( + "# E2E old-upgrade fixture: force the historical OpenClaw before the old Dockerfile's version gate.\n" + "RUN rm -rf /usr/local/lib/node_modules/openclaw /usr/local/bin/openclaw \\\n" + f" && npm install -g --no-audit --no-fund --no-progress \"openclaw@{version}\" \\\n" + " && openclaw --version\n\n" +) +if injection not in text: + if marker not in text: + raise SystemExit(f"{path}: old OpenClaw version gate not found") + text = text.replace(marker, injection + marker, 1) + path.write_text(text, encoding="utf-8") +print(f"INFO: Forced OpenClaw {version} in old upgrade fixture Dockerfile", flush=True) +NEMOCLAW_OLD_DOCKERFILE_PIN_PY + fi +''' +if hook not in text: + if needle not in text: + raise SystemExit(f"{path}: old source clone hook not found") + text = text.replace(needle, needle + hook, 1) + path.write_text(text, encoding="utf-8") +NEMOCLAW_OLD_PAYLOAD_PIN_PY + fi +""" +if insertion not in text: + if needle not in text: + raise SystemExit(f"{path}: old bootstrap payload hook not found") + text = text.replace(needle, needle + insertion, 1) + path.write_text(text, encoding="utf-8") +PY +} + +cleanup() { + set +e + stop_fake_openai_compatible_api + if command -v openshell >/dev/null 2>&1; then + openshell sandbox delete "$SURVIVOR_SANDBOX" >/dev/null 2>&1 || true + openshell gateway remove nemoclaw >/dev/null 2>&1 || true + fi + rm -f "$PID_FILE" + if [ -n "$OLD_DOCKER_WRAPPER_DIR" ]; then + rm -rf "$OLD_DOCKER_WRAPPER_DIR" + fi +} +trap cleanup EXIT + +exercise_macos_gateway_installer_regression() { + local tmp fake_bin curl_log install_out install_err + tmp="$(mktemp -d)" + fake_bin="$tmp/bin" + curl_log="$tmp/curl.log" + install_out="$tmp/install.out" + install_err="$tmp/install.err" + mkdir -p "$fake_bin" + + cat >"$fake_bin/uname" <<'EOF' +#!/usr/bin/env bash +if [ "${1:-}" = "-m" ]; then + printf 'arm64\n' +else + printf 'Darwin\n' +fi +EOF + + cat >"$fake_bin/openshell" <<'EOF' +#!/usr/bin/env bash +# request-body-credential-rewrite +# websocket-credential-rewrite +if [ "${1:-}" = "--version" ]; then + printf 'openshell 0.0.44\n' + exit 0 +fi +exit 99 +# request-body-credential-rewrite websocket-credential-rewrite +EOF + + cat >"$fake_bin/gh" <<'EOF' +#!/usr/bin/env bash +exit 1 +EOF + + cat >"$fake_bin/curl" <<'EOF' +#!/usr/bin/env bash +out="" +prev="" +for arg in "$@"; do + if [ "$prev" = "-o" ]; then + out="$arg" + break + fi + prev="$arg" +done +printf '%s\n' "$*" >>"$NEMOCLAW_FAKE_CURL_LOG" +if [ -n "$out" ]; then + printf 'fake payload\n' >"$out" +fi +exit 0 +EOF + + chmod +x "$fake_bin"/* + + if PATH="$fake_bin:/usr/bin:/bin" \ + NEMOCLAW_OPENSHELL_CHANNEL=stable \ + NEMOCLAW_FAKE_CURL_LOG="$curl_log" \ + bash scripts/install-openshell.sh >"$install_out" 2>"$install_err"; then + rm -rf "$tmp" + fail "macOS incomplete OpenShell install unexpectedly succeeded with fake payloads" + fi + + if ! grep -q "missing Docker-driver binaries" "$install_out"; then + diag "installer stdout:" + cat "$install_out" + diag "installer stderr:" + cat "$install_err" + rm -rf "$tmp" + fail "macOS installer did not detect missing openshell-gateway" + fi + + if ! grep -q "openshell-gateway-aarch64-apple-darwin.tar.gz" "$curl_log"; then + diag "curl log:" + cat "$curl_log" 2>/dev/null || true + rm -rf "$tmp" + fail "macOS installer did not request the Darwin openshell-gateway asset" + fi + if grep -q "openshell-driver-vm-aarch64-apple-darwin.tar.gz" "$curl_log"; then + diag "curl log:" + cat "$curl_log" 2>/dev/null || true + rm -rf "$tmp" + fail "macOS installer still requested the Darwin openshell-driver-vm asset" + fi + + rm -rf "$tmp" + pass "macOS OpenShell ${CURRENT_OPENSHELL_VERSION} incomplete install fetches Darwin gateway asset" +} + +exercise_macos_vm_driver_entitlement_not_required() { + local tmp fake_bin state_file sign_log install_out install_err + tmp="$(mktemp -d)" + fake_bin="$tmp/bin" + state_file="$tmp/codesign-state" + sign_log="$tmp/codesign.log" + install_out="$tmp/install.out" + install_err="$tmp/install.err" + mkdir -p "$fake_bin" + + cat >"$fake_bin/uname" <<'EOF' +#!/usr/bin/env bash +if [ "${1:-}" = "-m" ]; then + printf 'arm64\n' +else + printf 'Darwin\n' +fi +EOF + + cat >"$fake_bin/openshell" <<'EOF' +#!/usr/bin/env bash +# request-body-credential-rewrite +# websocket-credential-rewrite +if [ "${1:-}" = "--version" ]; then + printf 'openshell 0.0.44\n' + exit 0 +fi +exit 99 +# request-body-credential-rewrite websocket-credential-rewrite +EOF + + cat >"$fake_bin/openshell-gateway" <<'EOF' +#!/usr/bin/env bash +exit 0 +EOF + + cat >"$fake_bin/openshell-driver-vm" <<'EOF' +#!/usr/bin/env bash +exit 0 +EOF + + cat >"$fake_bin/codesign" <<'EOF' +#!/usr/bin/env bash +if [ "${1:-}" = "-d" ]; then + if [ -f "$NEMOCLAW_FAKE_CODESIGN_STATE" ]; then + printf '%s\n' 'com.apple.security.hypervisor' + fi + exit 0 +fi +printf '%s\n' "$*" >>"$NEMOCLAW_FAKE_CODESIGN_LOG" +: >"$NEMOCLAW_FAKE_CODESIGN_STATE" +exit 0 +EOF + + chmod +x "$fake_bin"/* + + if ! PATH="$fake_bin:/usr/bin:/bin" \ + NEMOCLAW_OPENSHELL_CHANNEL=stable \ + NEMOCLAW_FAKE_CODESIGN_LOG="$sign_log" \ + NEMOCLAW_FAKE_CODESIGN_STATE="$state_file" \ + bash scripts/install-openshell.sh >"$install_out" 2>"$install_err"; then + diag "installer stdout:" + cat "$install_out" 2>/dev/null || true + diag "installer stderr:" + cat "$install_err" 2>/dev/null || true + rm -rf "$tmp" + fail "macOS installer still required openshell-driver-vm Hypervisor entitlement" + fi + + if [ -s "$sign_log" ] && grep -q -- "--force --sign - --entitlements" "$sign_log"; then + diag "codesign log:" + cat "$sign_log" 2>/dev/null || true + rm -rf "$tmp" + fail "macOS installer still codesigned openshell-driver-vm" + fi + + if grep -q "Installing OpenShell from release" "$install_out"; then + diag "installer stdout:" + cat "$install_out" 2>/dev/null || true + rm -rf "$tmp" + fail "macOS installer reinstalled instead of repairing an otherwise complete OpenShell install" + fi + + rm -rf "$tmp" + pass "macOS OpenShell ${CURRENT_OPENSHELL_VERSION} installer does not require VM driver Hypervisor entitlement" +} + +exercise_macos_docker_rootfs_permission_regression() { + grep -q "ARG NEMOCLAW_DARWIN_VM_COMPAT=0" Dockerfile \ + || fail "Dockerfile is missing the macOS VM rootfs compatibility ARG" + grep -Fq "ARG NEMOCLAW_DARWIN_VM_COMPAT=\${sanitizeDockerArg(darwinVmCompat ? \"1\" : \"0\")}" src/lib/onboard/dockerfile-patch.ts \ + || fail "Dockerfile patch helper does not patch the macOS VM rootfs compatibility ARG" + grep -Fq "const darwinVmCompat = false;" src/lib/onboard/sandbox-dockerfile-patch-flow.ts \ + || fail "onboard does not keep macOS Docker sandbox builds out of the VM rootfs compatibility path" + grep -q "chmod -R a+rwX /sandbox/.openclaw" Dockerfile \ + || fail "Dockerfile does not relax OpenClaw state permissions for macOS VM rootfs remapping" + grep -q "ARG NEMOCLAW_DARWIN_VM_COMPAT=0" agents/hermes/Dockerfile \ + || fail "Hermes Dockerfile is missing the macOS VM rootfs compatibility ARG" + grep -q "chmod -R a+rwX /sandbox/.hermes" agents/hermes/Dockerfile \ + || fail "Hermes Dockerfile does not relax Hermes state permissions for macOS VM rootfs remapping" + grep -q "chmod a+rw /sandbox/.bashrc /sandbox/.profile" agents/hermes/Dockerfile \ + || fail "Hermes Dockerfile does not relax trusted rc files for macOS VM ownership repair" + pass "macOS Docker sandbox builds keep VM rootfs compatibility disabled" +} + +wait_for_survivor_ready() { + for _i in $(seq 1 60); do + if openshell sandbox list 2>/dev/null | grep -q "${SURVIVOR_SANDBOX}.*Ready"; then + return 0 + fi + sleep 2 + done + return 1 +} + +start_compatible_endpoint_mock() { + export FAKE_OPENAI_HOST="127.0.0.1" + export FAKE_OPENAI_PORT="0" + export FAKE_OPENAI_LOG="$MOCK_LOG" + if start_fake_openai_compatible_api; then + FAKE_BASE_URL="$FAKE_OPENAI_BASE_URL" + pass "Compatible endpoint mock is listening at ${FAKE_BASE_URL}" + return 0 + fi + fail "compatible endpoint mock did not start" +} + +run_installer_payload() { + local label="$1" ref="$2" installer="$3" log_file="$4" + info "Running ${label} NemoClaw installer from ${ref}" + rm -f "$log_file" + local docker_path_env=() + if [ -n "$OLD_DOCKER_WRAPPER_DIR" ] && [[ "$label" == old\ * ]]; then + docker_path_env=( + PATH="${OLD_DOCKER_WRAPPER_DIR}:$PATH" + NEMOCLAW_REAL_DOCKER="$(command -v docker)" + NEMOCLAW_OLD_SANDBOX_BASE_IMAGE_REF="$OLD_SANDBOX_BASE_IMAGE_REF" + NEMOCLAW_OLD_OPENCLAW_VERSION="$OLD_OPENCLAW_VERSION" + NEMOCLAW_OLD_DOCKER_WRAPPER_LOG="$OLD_DOCKER_WRAPPER_LOG" + ) + fi + + env \ + "${docker_path_env[@]}" \ + COMPATIBLE_API_KEY=dummy \ + NEMOCLAW_NON_INTERACTIVE=1 \ + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ + NEMOCLAW_ACCEPT_EXPERIMENTAL_OPENSHELL_UPGRADE=1 \ + NEMOCLAW_BOOTSTRAP_PAYLOAD=1 \ + NEMOCLAW_INSTALL_REF="$ref" \ + NEMOCLAW_INSTALL_TAG="$ref" \ + NEMOCLAW_PROVIDER=custom \ + NEMOCLAW_ENDPOINT_URL="$FAKE_BASE_URL" \ + NEMOCLAW_MODEL=test-model \ + NEMOCLAW_SANDBOX_NAME="$SURVIVOR_SANDBOX" \ + NEMOCLAW_POLICY_MODE=skip \ + NEMOCLAW_DASHBOARD_PORT= \ + CHAT_UI_URL= \ + bash "$installer" --non-interactive --yes-i-accept-third-party-software \ + >"$log_file" 2>&1 || { + diag "${label} installer log tail:" + tail -120 "$log_file" 2>/dev/null || true + if [ -f "$OLD_DOCKER_WRAPPER_LOG" ]; then + diag "old installer docker wrapper activity:" + cat "$OLD_DOCKER_WRAPPER_LOG" || true + fi + fail "${label} NemoClaw installer failed" + } + load_shell_path +} + +download_old_curl_installer() { + local target="$1" + curl -fsSL "https://raw.githubusercontent.com/NVIDIA/NemoClaw/${OLD_NEMOCLAW_REF}/install.sh" \ + -o "$target" + chmod 755 "$target" +} + +install_old_nemoclaw_and_claw() { + local installer + installer="$(mktemp)" + create_old_docker_wrapper + info "Pinning old ${OLD_NEMOCLAW_REF} OpenClaw base build to ${OLD_OPENCLAW_VERSION}" + download_old_curl_installer "$installer" + patch_old_installer_fixture "$installer" + run_installer_payload "old ${OLD_NEMOCLAW_REF}" "$OLD_NEMOCLAW_REF" "$installer" "$OLD_INSTALL_LOG" + if [ -f "$OLD_DOCKER_WRAPPER_LOG" ]; then + diag "old installer docker wrapper activity:" + cat "$OLD_DOCKER_WRAPPER_LOG" || true + fi + local wrong_old_openclaw + wrong_old_openclaw="$( + grep -Eo "OpenClaw [0-9]{4}\\.[0-9]+\\.[0-9]+ is current \\(>= ${OLD_OPENCLAW_VERSION}\\)" "$OLD_INSTALL_LOG" 2>/dev/null \ + | awk '{print $2}' \ + | grep -v "^${OLD_OPENCLAW_VERSION}$" \ + | head -n 1 || true + )" + if [ -n "$wrong_old_openclaw" ]; then + fail "old ${OLD_NEMOCLAW_REF} fixture used OpenClaw ${wrong_old_openclaw} instead of pinned ${OLD_OPENCLAW_VERSION}" + fi + if ! grep -q "OpenClaw ${OLD_OPENCLAW_VERSION}\\|openclaw@${OLD_OPENCLAW_VERSION}" "$OLD_INSTALL_LOG" 2>/dev/null; then + fail "old ${OLD_NEMOCLAW_REF} fixture did not show pinned OpenClaw ${OLD_OPENCLAW_VERSION}" + fi + rm -f "$installer" + + if ! openshell --version 2>&1 | grep -q "$OLD_OPENSHELL_VERSION"; then + fail "old NemoClaw install did not leave OpenShell ${OLD_OPENSHELL_VERSION}: $(openshell --version 2>&1 || true)" + fi + pass "Old NemoClaw install selected $(openshell --version)" + + if [ -d "$HOME/.nemoclaw/source/.git" ]; then + local old_head expected_head + old_head="$(git -C "$HOME/.nemoclaw/source" rev-parse HEAD 2>/dev/null || true)" + expected_head="$(git ls-remote https://github.com/NVIDIA/NemoClaw.git "refs/tags/${OLD_NEMOCLAW_REF}" | awk '{print $1}')" + if [ -z "$old_head" ] || [ "$old_head" != "$expected_head" ]; then + fail "old installer source is ${old_head:-unknown}, expected ${expected_head:-$OLD_NEMOCLAW_REF}" + fi + pass "Old NemoClaw source is ${OLD_NEMOCLAW_REF} (${old_head:0:12})" + fi + + wait_for_survivor_ready || fail "survivor sandbox did not become Ready before gateway upgrade" + if nemoclaw list 2>&1 | grep -Fq "$SURVIVOR_SANDBOX"; then + pass "Old NemoClaw install registered survivor claw ${SURVIVOR_SANDBOX}" + else + fail "old NemoClaw install did not register survivor claw ${SURVIVOR_SANDBOX}" + fi +} + +start_survivor_agent_in_existing_claw() { + info "Starting survivor agent inside old NemoClaw claw" + openshell sandbox exec --name "$SURVIVOR_SANDBOX" -- \ + sh -lc "mkdir -p /sandbox/.openclaw/workspace && printf '%s\n' '$SURVIVOR_MARKER' >'$SURVIVOR_MARKER_PATH'" \ + || fail "failed to write survivor marker before gateway upgrade" + + local agent_payload remote_setup + agent_payload="$( + cat <<'AGENT' | base64 | tr -d '\n' +#!/bin/sh +set -eu +pid_file="/tmp/nemoclaw-e2e-agent.pid" +heartbeat_file="/tmp/nemoclaw-e2e-agent.heartbeat" +events_file="/tmp/nemoclaw-e2e-agent.events" +printf '%s\n' "$$" >"$pid_file" +printf 'started %s\n' "$$" >>"$events_file" +counter=0 +trap 'printf "stopped %s\n" "$$" >>"$events_file"; exit 0' TERM INT +while true; do + counter=$((counter + 1)) + printf '%s %s %s\n' "$$" "$counter" "$(date +%s)" >"$heartbeat_file" + sleep 1 +done +AGENT + )" + remote_setup="printf '%s' '$agent_payload' | base64 -d >/tmp/nemoclaw-e2e-agent; chmod 755 /tmp/nemoclaw-e2e-agent; rm -f /tmp/nemoclaw-e2e-agent.pid /tmp/nemoclaw-e2e-agent.heartbeat /tmp/nemoclaw-e2e-agent.events /tmp/nemoclaw-e2e-agent.log; nohup /tmp/nemoclaw-e2e-agent >/tmp/nemoclaw-e2e-agent.log 2>&1 &" + + openshell sandbox exec --name "$SURVIVOR_SANDBOX" -- sh -lc "$remote_setup" \ + || fail "failed to start survivor agent before gateway upgrade" + wait_for_survivor_agent_ready || fail "survivor agent did not become healthy before gateway upgrade" + SURVIVOR_AGENT_PID="$(survivor_agent_pid)" + [ -n "$SURVIVOR_AGENT_PID" ] || fail "survivor agent pid was empty before gateway upgrade" + + pass "Old NemoClaw claw has live agent activity (pid ${SURVIVOR_AGENT_PID}) before gateway upgrade" +} + +install_current_nemoclaw_upgrade() { + local current_ref + current_ref="${NEMOCLAW_CURRENT_NEMOCLAW_REF:-$(git rev-parse HEAD 2>/dev/null || printf '%s' "${GITHUB_SHA:-}")}" + [ -n "$current_ref" ] || fail "could not determine current NemoClaw ref" + run_installer_payload "current ${current_ref:0:12}" "$current_ref" "${REPO_ROOT}/scripts/install.sh" "$CURRENT_INSTALL_LOG" + grep -Fq "Accepted experimental OpenShell gateway upgrade" "$CURRENT_INSTALL_LOG" \ + || fail "current installer did not exercise the experimental OpenShell gateway upgrade acceptance path" + + if ! openshell --version 2>&1 | grep -q "$CURRENT_OPENSHELL_VERSION"; then + fail "current NemoClaw install did not upgrade OpenShell to ${CURRENT_OPENSHELL_VERSION}: $(openshell --version 2>&1 || true)" + fi + pass "Current NemoClaw install selected $(openshell --version)" + + local status_output + status_output="$(openshell status 2>&1 || true)" + if ! grep -q "Version:.*${CURRENT_OPENSHELL_VERSION}" <<<"$status_output"; then + diag "openshell status after current install:" + printf '%s\n' "$status_output" + fail "gateway server did not report OpenShell ${CURRENT_OPENSHELL_VERSION} after upgrade" + fi + pass "Gateway server reports OpenShell ${CURRENT_OPENSHELL_VERSION} after upgrade" + + if grep -Fq "Pre-upgrade backup: 1 backed up, 0 failed, 0 skipped" "$CURRENT_INSTALL_LOG"; then + pass "Current installer backed up the old running claw before replacing OpenShell" + else + diag "current installer backup lines:" + grep -n "Pre-upgrade backup\\|Backing up\\|Skipping '${SURVIVOR_SANDBOX}'" "$CURRENT_INSTALL_LOG" || true + fail "current installer did not back up the old running claw before replacing OpenShell" + fi +} + +assert_survivor_sandbox_after_upgrade() { + local agent_check marker + info "Verifying survivor sandbox after OpenShell gateway upgrade" + wait_for_survivor_ready || fail "survivor sandbox is not Ready after gateway upgrade" + + marker="$( + openshell sandbox exec --name "$SURVIVOR_SANDBOX" -- \ + cat "$SURVIVOR_MARKER_PATH" 2>/dev/null || true + )" + [ "$marker" = "$SURVIVOR_MARKER" ] \ + || fail "survivor marker changed after gateway upgrade: got '${marker}'" + pass "Durable OpenClaw workspace state was restored after gateway upgrade" + + agent_check="$( + openshell sandbox exec --name "$SURVIVOR_SANDBOX" -- \ + sh -lc 'command -v openclaw >/dev/null && test -s /sandbox/.openclaw/openclaw.json && openclaw --version 2>/dev/null' \ + || true + )" + [ -n "$agent_check" ] || fail "OpenClaw agent is not installed/configured after gateway upgrade" + pass "OpenClaw agent is installed and configured after gateway upgrade" + + if [ -f "$REGISTRY_FILE" ] && grep -Fq "\"${SURVIVOR_SANDBOX}\"" "$REGISTRY_FILE"; then + pass "NemoClaw registry retained survivor sandbox after gateway upgrade" + else + fail "NemoClaw registry lost survivor sandbox after gateway upgrade" + fi + + local list_output + if list_output="$(nemoclaw list 2>&1)" && grep -Fq "$SURVIVOR_SANDBOX" <<<"$list_output"; then + pass "nemoclaw list still shows survivor sandbox after gateway upgrade" + else + fail "nemoclaw list does not show survivor sandbox after gateway upgrade: ${list_output:0:200}" + fi + + pass "Survivor claw state remained reachable after OpenShell gateway upgrade" +} + +cd "$REPO_ROOT" +load_shell_path + +if [ "$(uname -s)" != "Linux" ]; then + exercise_macos_gateway_installer_regression + exercise_macos_vm_driver_entitlement_not_required + exercise_macos_docker_rootfs_permission_regression + pass "Skipping live Docker-driver gateway restart regression on non-Linux host" + exit 0 +fi + +info "Preparing real old-install upgrade scenario" +rm -f "$INSTALL_LOG" "$OLD_INSTALL_LOG" "$CURRENT_INSTALL_LOG" "$START_LOG" "$GATEWAY_LOG" +start_compatible_endpoint_mock +install_old_nemoclaw_and_claw +start_survivor_agent_in_existing_claw + +info "Running current NemoClaw installer/onboard against old working claw" +install_current_nemoclaw_upgrade +assert_survivor_sandbox_after_upgrade +pass "Current NemoClaw installer upgraded old ${OLD_NEMOCLAW_REF} claw, restored state, and kept OpenClaw running on OpenShell ${CURRENT_OPENSHELL_VERSION}" + +exercise_macos_gateway_installer_regression +exercise_macos_vm_driver_entitlement_not_required +exercise_macos_docker_rootfs_permission_regression diff --git a/test/e2e-vpn/test-openshell-version-pin.sh b/test/e2e-vpn/test-openshell-version-pin.sh new file mode 100755 index 00000000000..dd4132ab4e2 --- /dev/null +++ b/test/e2e-vpn/test-openshell-version-pin.sh @@ -0,0 +1,288 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Coverage guard for #3474 — a host with an already-installed OpenShell newer +# than NemoClaw's max supported version must not get stuck in an uninstall / +# reinstall loop. The installer should replace the too-new OpenShell with the +# pinned compatible version instead of failing before the reinstall path. +# +# Expected result on unfixed main: FAIL. scripts/install-openshell.sh sees the +# fake installed `openshell 0.0.45`, compares it to MAX_VERSION=0.0.44, and +# exits with "above the maximum" before downloading the pinned 0.0.44 release. +# +# Expected result after the fix: PASS. The script warns about the too-new +# installed OpenShell, downloads v0.0.44, replaces openshell plus helper +# binaries, and exits successfully. + +set -euo pipefail + +LOG_FILE="/tmp/nemoclaw-e2e-openshell-version-pin.log" +INSTALL_LOG="/tmp/nemoclaw-e2e-openshell-version-pin-install.log" +DOWNLOAD_LOG="/tmp/nemoclaw-e2e-openshell-version-pin-downloads.log" +FAKE_BIN="/tmp/nemoclaw-e2e-openshell-version-pin-bin" + +exec > >(tee "$LOG_FILE") 2>&1 + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +pass() { echo -e "${GREEN}[PASS]${NC} $1"; } +info() { echo -e "${YELLOW}[INFO]${NC} $1"; } +diag() { echo -e "${YELLOW}[DIAG]${NC} $1"; } +fail() { + echo -e "${RED}[FAIL]${NC} $1" >&2 + diag "install log tail:" + tail -120 "$INSTALL_LOG" 2>/dev/null || true + diag "download log:" + cat "$DOWNLOAD_LOG" 2>/dev/null || true + exit 1 +} + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" + +cleanup() { + rm -rf "$FAKE_BIN" +} +trap cleanup EXIT + +write_executable() { + local target="$1" + cat >"$target" + chmod 755 "$target" +} + +mkdir -p "$FAKE_BIN" +: >"$DOWNLOAD_LOG" + +# Force Linux/x86_64 asset selection so this guard is stable on any host that +# dispatches the regression workflow. +write_executable "$FAKE_BIN/uname" <<'SH' +#!/usr/bin/env bash +if [ "${1:-}" = "-m" ]; then + echo "x86_64" +else + echo "Linux" +fi +SH + +# Existing sticky OpenShell: newer than NemoClaw's MAX_VERSION. This is the +# Margaret/Aaron failure mode we want the eventual fix to repair by reinstalling +# the pinned compatible release. +write_executable "$FAKE_BIN/openshell" <<'SH' +#!/usr/bin/env bash +if [ "${1:-}" = "--version" ]; then echo "openshell 0.0.45"; exit 0; fi +# request-body-credential-rewrite websocket-credential-rewrite +exit 0 +SH + +# Helper binaries exist so the only reason to reinstall is the too-new version, +# not missing Docker-driver helpers. +write_executable "$FAKE_BIN/openshell-gateway" <<'SH' +#!/usr/bin/env bash +exit 0 +SH +write_executable "$FAKE_BIN/openshell-sandbox" <<'SH' +#!/usr/bin/env bash +exit 0 +SH + +write_executable "$FAKE_BIN/gh" <<'SH' +#!/usr/bin/env bash +set -euo pipefail +write_asset() { + local asset_name="$1" + local asset_path="$2" + printf 'fake OpenShell release asset: %s\n' "$asset_name" >"$asset_path" +} +sha256_digest() { + if [ -x /usr/bin/sha256sum ]; then + /usr/bin/sha256sum "$1" | awk '{print $1}' + elif [ -x /bin/sha256sum ]; then + /bin/sha256sum "$1" | awk '{print $1}' + elif [ -x /usr/bin/shasum ]; then + /usr/bin/shasum -a 256 "$1" | awk '{print $1}' + else + exit 3 + fi +} +write_checksum() { + local checksum_file="$1" + local asset_name="$2" + local asset_path="$3" + [ -f "$asset_path" ] || write_asset "$asset_name" "$asset_path" + printf '%s %s\n' "$(sha256_digest "$asset_path")" "$asset_name" >"$checksum_file" +} +if [ "${1:-}" = "release" ] && [ "${2:-}" = "download" ]; then + tag="${3:-}" + pattern="" + dir="" + while [ "$#" -gt 0 ]; do + case "$1" in + --pattern) shift; pattern="${1:-}" ;; + --dir) shift; dir="${1:-}" ;; + esac + shift || true + done + [ -n "$tag" ] && [ -n "$pattern" ] && [ -n "$dir" ] || exit 2 + printf 'gh download %s %s\n' "$tag" "$pattern" >> "${DOWNLOAD_LOG:?}" + mkdir -p "$dir" + case "$pattern" in + openshell-checksums-sha256.txt) + asset_name="openshell-x86_64-unknown-linux-musl.tar.gz" + write_checksum "$dir/$pattern" "$asset_name" "$dir/$asset_name" + ;; + openshell-gateway-checksums-sha256.txt) + asset_name="openshell-gateway-x86_64-unknown-linux-gnu.tar.gz" + write_checksum "$dir/$pattern" "$asset_name" "$dir/$asset_name" + ;; + openshell-sandbox-checksums-sha256.txt) + asset_name="openshell-sandbox-x86_64-unknown-linux-gnu.tar.gz" + write_checksum "$dir/$pattern" "$asset_name" "$dir/$asset_name" + ;; + *) + write_asset "$pattern" "$dir/$pattern" + ;; + esac + exit 0 +fi +exit 1 +SH + +write_executable "$FAKE_BIN/curl" <<'SH' +#!/usr/bin/env bash +set -euo pipefail +write_asset() { + local asset_name="$1" + local asset_path="$2" + printf 'fake OpenShell release asset: %s\n' "$asset_name" >"$asset_path" +} +sha256_digest() { + if [ -x /usr/bin/sha256sum ]; then + /usr/bin/sha256sum "$1" | awk '{print $1}' + elif [ -x /bin/sha256sum ]; then + /bin/sha256sum "$1" | awk '{print $1}' + elif [ -x /usr/bin/shasum ]; then + /usr/bin/shasum -a 256 "$1" | awk '{print $1}' + else + exit 3 + fi +} +write_checksum() { + local checksum_file="$1" + local asset_name="$2" + local asset_path="$3" + [ -f "$asset_path" ] || write_asset "$asset_name" "$asset_path" + printf '%s %s\n' "$(sha256_digest "$asset_path")" "$asset_name" >"$checksum_file" +} +printf 'curl %s\n' "$*" >> "${DOWNLOAD_LOG:?}" +out="" +while [ "$#" -gt 0 ]; do + if [ "$1" = "-o" ]; then + shift + out="${1:-}" + fi + shift || true +done +[ -n "$out" ] || exit 0 +case "$(basename "$out")" in + openshell-checksums-sha256.txt) + asset_name="openshell-x86_64-unknown-linux-musl.tar.gz" + write_checksum "$out" "$asset_name" "$(dirname "$out")/$asset_name" + ;; + openshell-gateway-checksums-sha256.txt) + asset_name="openshell-gateway-x86_64-unknown-linux-gnu.tar.gz" + write_checksum "$out" "$asset_name" "$(dirname "$out")/$asset_name" + ;; + openshell-sandbox-checksums-sha256.txt) + asset_name="openshell-sandbox-x86_64-unknown-linux-gnu.tar.gz" + write_checksum "$out" "$asset_name" "$(dirname "$out")/$asset_name" + ;; + *) + write_asset "$(basename "$out")" "$out" + ;; +esac +SH + +write_executable "$FAKE_BIN/shasum" <<'SH' +#!/usr/bin/env bash +cat >/dev/null +echo "checksum OK" +exit 0 +SH + +# The installer extracts three archives. Create the binary each archive would +# have produced. The replacement openshell reports 0.0.44 and contains the +# feature strings checked by install-openshell.sh. +write_executable "$FAKE_BIN/tar" <<'SH' +#!/usr/bin/env bash +set -euo pipefail +outdir="" +prev="" +for arg in "$@"; do + if [ "$prev" = "-C" ]; then + outdir="$arg" + break + fi + prev="$arg" +done +[ -n "$outdir" ] || exit 1 +case "$*" in + *openshell-gateway*) name="openshell-gateway" ;; + *openshell-sandbox*) name="openshell-sandbox" ;; + *) name="openshell" ;; +esac +cat > "$outdir/$name" <<'EOS' +#!/usr/bin/env bash +if [ "${1:-}" = "--version" ]; then echo "openshell 0.0.44"; exit 0; fi +# request-body-credential-rewrite websocket-credential-rewrite +exit 0 +EOS +chmod 755 "$outdir/$name" +SH + +# Keep the feature-probe hermetic. It only needs to see the marker comments in +# the fake installed binary. +write_executable "$FAKE_BIN/strings" <<'SH' +#!/usr/bin/env bash +cat "$@" 2>/dev/null || true +SH + +cd "$REPO_ROOT" +info "Running install-openshell.sh with sticky openshell 0.0.45 and max 0.0.44" +set +e +env \ + PATH="$FAKE_BIN:/usr/bin:/bin" \ + HOME="${HOME}" \ + DOWNLOAD_LOG="$DOWNLOAD_LOG" \ + bash scripts/install-openshell.sh >"$INSTALL_LOG" 2>&1 +install_rc=$? +set -e + +if [ "$install_rc" -ne 0 ]; then + if grep -q "openshell 0.0.45 is above the maximum (0.0.44)" "$INSTALL_LOG"; then + fail "Installer hard-failed on sticky OpenShell 0.0.45 instead of reinstalling pinned 0.0.44 (#3474)" + fi + fail "install-openshell.sh failed before proving sticky-version recovery (exit ${install_rc})" +fi +pass "install-openshell.sh completed" + +if ! grep -q "v0.0.44" "$DOWNLOAD_LOG"; then + fail "Expected installer to download pinned OpenShell v0.0.44" +fi +pass "Installer downloaded pinned OpenShell v0.0.44" + +if grep -q "v0.0.45" "$DOWNLOAD_LOG"; then + fail "Installer downloaded OpenShell v0.0.45 despite NemoClaw max 0.0.44" +fi +pass "Installer did not download too-new OpenShell v0.0.45" + +if ! "$FAKE_BIN/openshell" --version 2>&1 | grep -q "0.0.44"; then + fail "openshell binary was not replaced with pinned 0.0.44" +fi +pass "Sticky openshell 0.0.45 was replaced with pinned 0.0.44" + +info "OpenShell sticky-version pin guard complete" diff --git a/test/e2e-vpn/test-overlayfs-autofix.sh b/test/e2e-vpn/test-overlayfs-autofix.sh new file mode 100755 index 00000000000..af407a63582 --- /dev/null +++ b/test/e2e-vpn/test-overlayfs-autofix.sh @@ -0,0 +1,549 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# E2E: Docker 26+ overlayfs nested-mount auto-fix (NemoClaw#2481) +# +# Validates that NemoClaw transparently builds a fuse-overlayfs cluster +# image and routes around the kernel-level nested-overlay limitation when +# the host runs Docker 26+ with the containerd image store enabled. Also +# validates the negative path: with NEMOCLAW_DISABLE_OVERLAY_FIX=1 the +# original failure mode reproduces, proving the auto-fix is the +# load-bearing piece (not coincidence). +# +# This test is **TEMPORARY**. It exists to guard the workaround in +# src/lib/cluster-image-patch.ts while OpenShell roadmap #873 lands a +# non-k3s sandbox driver. Remove this script, the +# overlayfs-autofix-e2e workflow job, and the matching notify-on-failure +# needs entry in the same PR that deletes src/lib/cluster-image-patch.ts. +# +# Test phases: +# 1. Prerequisites — Docker running, NVIDIA_API_KEY, sudo, etc. +# 2. Setup — flip /etc/docker/daemon.json to enable containerd-snapshotter, +# restart Docker, verify the conflict config is active. Auto-skip on +# runners whose Docker does not support the feature flag. +# 3. Pre-cleanup — destroy any leftover sandbox/gateway/patched image. +# 4. Positive — install + onboard, expect the auto-fix to trigger and +# the gateway to come up on the patched image. +# 5. Idempotency — call ensurePatchedClusterImage directly via Node and +# verify the local Docker cache hit returns the same tag without +# re-invoking docker pull/build. We deliberately do NOT re-run +# install.sh here because the OpenClaw sandbox-image build step is +# independently flaky on GitHub Actions runner kernels (nested +# overlayfs limitations) and would make this phase a coin toss. +# 6. Negative — onboard with NEMOCLAW_DISABLE_OVERLAY_FIX=1, expect +# install.sh to fail within a bounded timeout. Three-way result: +# - nested-overlay signature in cluster or install log → PASS +# (canonical k3s string, "CreateDiff: Canceled", or +# "failed to mount overlay") +# - signature absent AND `timeout` fired (exit 124) → SKIP +# (this runner instance did not reproduce the bug) +# - signature absent AND a different non-zero exit → FAIL +# (likely an unrelated flake) +# 7. Final teardown — revert daemon.json, restart Docker, destroy sandbox. +# +# Prerequisites: +# - Docker installed (any version that supports `features.containerd-snapshotter`, +# i.e. Docker 23+; the test skips cleanly on older versions) +# - Passwordless sudo (for editing /etc/docker/daemon.json + restarting Docker) +# - NVIDIA_API_KEY set (real key; required by install.sh) +# +# Environment variables: +# NEMOCLAW_NON_INTERACTIVE=1 — required +# NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 — required +# NVIDIA_API_KEY — required +# NEMOCLAW_SANDBOX_NAME — sandbox name (default: e2e-overlayfs) +# NEMOCLAW_E2E_TIMEOUT_SECONDS — overall timeout (default: 1500) +# NEMOCLAW_OVERLAYFS_E2E_NEGATIVE_TIMEOUT — negative-phase k3s wait (default: 300) +# +# Usage: +# NEMOCLAW_NON_INTERACTIVE=1 \ +# NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ +# NVIDIA_API_KEY=... \ +# bash test/e2e-vpn/test-overlayfs-autofix.sh + +# ShellCheck cannot see EXIT trap invocations of cleanup helpers in this E2E script. +# shellcheck disable=SC2317 +set -uo pipefail + +export NEMOCLAW_E2E_DEFAULT_TIMEOUT=1500 +SCRIPT_DIR_TIMEOUT="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +# shellcheck source=test/e2e-vpn/e2e-timeout.sh +source "${SCRIPT_DIR_TIMEOUT}/e2e-timeout.sh" +# shellcheck source=test/e2e-vpn/lib/ci-compatible-inference.sh +. "${SCRIPT_DIR_TIMEOUT}/lib/ci-compatible-inference.sh" + +PASS=0 +FAIL=0 +SKIP=0 +TOTAL=0 + +pass() { + ((PASS++)) + ((TOTAL++)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} +fail() { + ((FAIL++)) + ((TOTAL++)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} +skip() { + ((SKIP++)) + ((TOTAL++)) + printf '\033[33m SKIP: %s\033[0m\n' "$1" +} +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } +print_summary() { + echo "" + printf '\033[1;33m=== Test summary ===\033[0m\n' + echo " PASS: $PASS" + echo " FAIL: $FAIL" + echo " SKIP: $SKIP" + echo " TOTAL: $TOTAL" + echo "" +} + +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-overlayfs}" +NEGATIVE_TIMEOUT="${NEMOCLAW_OVERLAYFS_E2E_NEGATIVE_TIMEOUT:-300}" +GATEWAY_CONTAINER="openshell-cluster-nemoclaw" +DAEMON_JSON="/etc/docker/daemon.json" + +# Use a private temp directory for daemon-state files. The previous +# fixed-name paths under /tmp were predictable enough that a pre-created +# symlink at /tmp/nemoclaw-e2e-daemon.json.bak could redirect the +# subsequent `sudo cp` into an attacker-chosen path on a shared runner. +# `mktemp -d` returns a per-run directory with mode 0700, so neither the +# backup nor the absent-marker path is guessable. +STATE_DIR="$(mktemp -d "${TMPDIR:-/tmp}/nemoclaw-overlayfs-e2e.XXXXXX")" +DAEMON_JSON_BACKUP="${STATE_DIR}/daemon.json.bak" +DAEMON_JSON_ABSENT_MARKER="${STATE_DIR}/daemon.json.absent" +INSTALL_LOG="${NEMOCLAW_E2E_INSTALL_LOG:-/tmp/nemoclaw-e2e-install.log}" +ONBOARD_LOG_POSITIVE="/tmp/nemoclaw-e2e-onboard-positive.log" +ONBOARD_LOG_NEGATIVE="/tmp/nemoclaw-e2e-onboard-negative.log" + +# shellcheck source=test/e2e-vpn/lib/sandbox-teardown.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib/sandbox-teardown.sh" +register_sandbox_for_teardown "$SANDBOX_NAME" +nemoclaw_e2e_configure_compatible_inference || exit 1 + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" + +if [ "$(uname -s)" = "Linux" ] && grep -q 'platform === "linux"' "$REPO_ROOT/src/lib/onboard/docker-driver-platform.ts"; then + section "Applicability" + skip "OpenShell Docker-driver onboarding is active on Linux; k3s overlayfs auto-fix is not in the runtime path" + print_summary + exit 0 +fi + +# ── Daemon revert ─────────────────────────────────────────────────── +# Always restore the original daemon.json on exit so we don't leave the +# runner in a degraded state if the test crashes mid-flight. +# shellcheck disable=SC2329 # invoked via the EXIT trap below +revert_daemon_config() { + if [ -f "$DAEMON_JSON_ABSENT_MARKER" ]; then + # No original file existed; remove whatever we wrote so the daemon + # falls back to defaults on restart. + info "Removing test-generated $DAEMON_JSON (no original to restore)..." + sudo rm -f "$DAEMON_JSON" 2>/dev/null || true + sudo systemctl restart docker 2>/dev/null || true + elif [ -f "$DAEMON_JSON_BACKUP" ]; then + info "Reverting Docker daemon configuration..." + sudo cp "$DAEMON_JSON_BACKUP" "$DAEMON_JSON" 2>/dev/null || true + sudo systemctl restart docker 2>/dev/null || true + fi + # Always wipe the private state dir on exit. mktemp -d created it 0700, + # so this is per-run cleanup without affecting other concurrent tests. + rm -rf "$STATE_DIR" 2>/dev/null || true +} +trap revert_daemon_config EXIT + +# ══════════════════════════════════════════════════════════════════ +# Phase 0: Prerequisites +# ══════════════════════════════════════════════════════════════════ +section "Phase 0: Prerequisites" + +if docker info >/dev/null 2>&1; then + pass "Docker is running" +else + fail "Docker is not running — cannot continue" + exit 1 +fi + +if ! nemoclaw_e2e_require_hosted_inference_key; then + exit 1 +fi + +if [ "${NEMOCLAW_NON_INTERACTIVE:-}" != "1" ]; then + fail "NEMOCLAW_NON_INTERACTIVE=1 is required" + exit 1 +fi + +if [ "${NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE:-}" != "1" ]; then + fail "NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 is required" + exit 1 +fi + +if sudo -n true 2>/dev/null; then + pass "Passwordless sudo available" +else + fail "Passwordless sudo required to edit $DAEMON_JSON" + exit 1 +fi + +if [ ! -f "$REPO_ROOT/install.sh" ]; then + fail "Cannot find install.sh at $REPO_ROOT/install.sh" + exit 1 +fi +pass "Repo root found: $REPO_ROOT" + +DOCKER_VERSION=$(docker info --format '{{.ServerVersion}}' 2>/dev/null || echo "unknown") +DOCKER_MAJOR=$(echo "$DOCKER_VERSION" | cut -d. -f1) +info "Docker server version: $DOCKER_VERSION" +if [ "${DOCKER_MAJOR:-0}" -lt 23 ] 2>/dev/null; then + skip "Docker $DOCKER_VERSION predates the containerd-snapshotter feature flag — nothing to validate" + echo "" + printf '\033[1;33m=== Test summary ===\033[0m\n' + echo " PASS: $PASS" + echo " FAIL: $FAIL" + echo " SKIP: $SKIP" + exit 0 +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 1: Force the bug-triggering Docker configuration +# ══════════════════════════════════════════════════════════════════ +section "Phase 1: Enable containerd image store on the host" + +# Back up whatever's there (or note its absence) so the EXIT trap can restore it. +# Both paths live inside the per-run STATE_DIR (mode 0700, mktemp-allocated), +# so neither is guessable for symlink redirects. +if [ -f "$DAEMON_JSON" ]; then + sudo cp "$DAEMON_JSON" "$DAEMON_JSON_BACKUP" + info "Backed up existing $DAEMON_JSON to $DAEMON_JSON_BACKUP" +else + # Marker file (separate from the backup path) tells revert there was no + # original to restore — never write a non-JSON sentinel into the backup + # itself, since that would corrupt $DAEMON_JSON on revert. + : >"${DAEMON_JSON_ABSENT_MARKER}.tmp" + mv "${DAEMON_JSON_ABSENT_MARKER}.tmp" "$DAEMON_JSON_ABSENT_MARKER" + info "No existing $DAEMON_JSON; flagged for removal on revert" +fi + +# Write a minimal daemon.json that enables the containerd-snapshotter feature. +# We deliberately do NOT merge with any user keys — the GitHub runner only +# owns this daemon for the duration of the job. +sudo tee "$DAEMON_JSON" >/dev/null <<'EOF' +{ + "features": { "containerd-snapshotter": true } +} +EOF +info "Wrote new $DAEMON_JSON enabling containerd-snapshotter" + +if ! sudo systemctl restart docker; then + fail "Failed to restart Docker after daemon.json change" + exit 1 +fi + +# Give Docker a moment to settle. +for _ in 1 2 3 4 5 6 7 8 9 10; do + if docker info >/dev/null 2>&1; then break; fi + sleep 2 +done + +if ! docker info >/dev/null 2>&1; then + fail "Docker did not come back up after restart" + exit 1 +fi + +DOCKER_INFO_JSON=$(docker info --format '{{json .}}' 2>/dev/null || echo "{}") + +if echo "$DOCKER_INFO_JSON" | grep -q '"Driver":"overlayfs"'; then + pass "Docker storage Driver is now overlayfs" +else + driver=$(echo "$DOCKER_INFO_JSON" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("Driver","?"))' 2>/dev/null || echo "?") + skip "Docker reports Driver=$driver — runner did not switch to overlayfs (containerd-snapshotter may be disabled in this image)" + echo "" + printf '\033[1;33m=== Test summary ===\033[0m\n' + echo " PASS: $PASS" + echo " FAIL: $FAIL" + echo " SKIP: $SKIP" + exit 0 +fi + +if echo "$DOCKER_INFO_JSON" | grep -q 'io.containerd.snapshotter.v1'; then + pass "DriverStatus reports io.containerd.snapshotter.v1 (the bug-triggering config)" +else + skip "Docker overlayfs is active but DriverStatus does not advertise the v1 snapshotter — host may not exhibit the nested-overlay break" + echo "" + printf '\033[1;33m=== Test summary ===\033[0m\n' + echo " PASS: $PASS" + echo " FAIL: $FAIL" + echo " SKIP: $SKIP" + exit 0 +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 2: Pre-cleanup +# ══════════════════════════════════════════════════════════════════ +section "Phase 2: Pre-cleanup" + +if command -v nemoclaw >/dev/null 2>&1; then + nemoclaw "$SANDBOX_NAME" destroy --yes 2>/dev/null || true +fi +if command -v openshell >/dev/null 2>&1; then + openshell sandbox delete "$SANDBOX_NAME" 2>/dev/null || true + openshell gateway destroy -g nemoclaw 2>/dev/null || true +fi +docker rm -f "$GATEWAY_CONTAINER" 2>/dev/null || true +# Drop any patched cluster images from previous runs so we measure first-build behavior. +patched_images=$(docker image ls --format '{{.Repository}}:{{.Tag}}' | grep -E '^nemoclaw-cluster:' || true) +if [ -n "$patched_images" ]; then + echo "$patched_images" | xargs -r docker rmi -f >/dev/null 2>&1 || true +fi +rm -f "$HOME/.nemoclaw/onboard.lock" 2>/dev/null || true +pass "Pre-cleanup complete" + +# ══════════════════════════════════════════════════════════════════ +# Phase 3: Positive — install + onboard with auto-fix on +# ══════════════════════════════════════════════════════════════════ +section "Phase 3: Install + onboard (auto-fix on)" + +cd "$REPO_ROOT" || { + fail "Could not cd to repo root: $REPO_ROOT" + exit 1 +} + +# Hermetic env: explicitly unset the auto-fix override knobs so a caller +# that already exports NEMOCLAW_DISABLE_OVERLAY_FIX=1 or +# NEMOCLAW_OVERLAY_SNAPSHOTTER=native can't silently change the path the +# positive phase is asserting on (lines 325-345 below). +env -u NEMOCLAW_DISABLE_OVERLAY_FIX -u NEMOCLAW_OVERLAY_SNAPSHOTTER \ + NEMOCLAW_NON_INTERACTIVE=1 \ + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ + NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" \ + NEMOCLAW_RECREATE_SANDBOX=1 \ + bash install.sh --non-interactive >"$INSTALL_LOG" 2>&1 & +install_pid=$! +tail -f "$INSTALL_LOG" --pid=$install_pid 2>/dev/null & +tail_pid=$! +wait $install_pid +install_exit=$? +kill $tail_pid 2>/dev/null || true +wait $tail_pid 2>/dev/null || true + +# Source nvm/PATH so a fresh installer becomes visible to subsequent commands. +if [ -f "$HOME/.bashrc" ]; then + # shellcheck source=/dev/null + source "$HOME/.bashrc" 2>/dev/null || true +fi +export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" +if [ -s "$NVM_DIR/nvm.sh" ]; then + # shellcheck source=/dev/null + . "$NVM_DIR/nvm.sh" +fi +if [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]]; then + export PATH="$HOME/.local/bin:$PATH" +fi + +if [ $install_exit -eq 0 ]; then + pass "install.sh + onboard completed (exit 0)" +else + fail "install.sh + onboard failed (exit $install_exit)" + exit 1 +fi + +# Capture the install log into a phase-specific file so later phases can +# overwrite it without losing the positive-phase signal. +cp "$INSTALL_LOG" "$ONBOARD_LOG_POSITIVE" 2>/dev/null || true + +# ── Auto-fix signals ───────────────────────────────────────────── +if grep -q "Detected Docker 26+ containerd-snapshotter overlayfs" "$ONBOARD_LOG_POSITIVE"; then + pass "Onboard log contains the auto-fix detection message" +else + fail "Onboard log missing 'Detected Docker 26+ containerd-snapshotter overlayfs'" +fi + +patched_tag=$(docker image ls --format '{{.Repository}}:{{.Tag}}' | grep -E '^nemoclaw-cluster:.*-fuse-overlayfs-[0-9a-f]{8}$' | head -1) +if [ -n "$patched_tag" ]; then + pass "Patched cluster image present: $patched_tag" +else + fail "No nemoclaw-cluster:*-fuse-overlayfs-* image found after onboard" +fi + +# Only assert image-equality + log-cleanliness when we actually found a +# patched tag. Without this guard, an empty `gateway_image` could equal an +# empty `patched_tag` and silently PASS, and the log-grep would scan the +# wrong (empty / non-existent) container. +if [ -n "$patched_tag" ]; then + gateway_image=$(docker inspect --format '{{.Config.Image}}' "$GATEWAY_CONTAINER" 2>/dev/null || echo "") + if [ "$gateway_image" = "$patched_tag" ]; then + pass "Gateway container is running the patched image" + else + fail "Gateway image '$gateway_image' does not match patched tag '$patched_tag'" + fi +fi + +# Cluster log must NOT carry the original error string. +if docker logs "$GATEWAY_CONTAINER" 2>&1 | grep -q "overlayfs.*snapshotter cannot be enabled"; then + fail "Cluster log still contains the nested-overlay error after auto-fix" +else + pass "Cluster log clean of the nested-overlay error" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 4: Idempotency — ensurePatchedClusterImage no-ops when cached +# ══════════════════════════════════════════════════════════════════ +# We deliberately do NOT re-run install.sh here. install.sh would +# rebuild the OpenClaw sandbox image from scratch, and that build +# step is independently flaky on GitHub-Actions runner kernels (see +# the negative phase below for the same failure mode). The behavior +# we actually want to validate is narrower: when the cluster image is +# already in the local Docker cache, calling ensurePatchedClusterImage +# again must return the same tag without invoking docker build. That's +# a property of the patch module, not of install.sh, and it's most +# precisely tested by calling the module directly. +section "Phase 4: Idempotency check" + +if [ -z "$patched_tag" ]; then + skip "Idempotency check skipped (no patched image from phase 3)" +else + before_created=$(docker inspect --format '{{.Created}}' "$patched_tag" 2>/dev/null || echo "") + + # Derive the upstream image from patched_tag (format: + # `nemoclaw-cluster:--`). + openshell_version=$(printf '%s\n' "$patched_tag" | sed -E 's|^nemoclaw-cluster:([^-]+)-.*|\1|') + upstream_image="ghcr.io/nvidia/openshell/cluster:${openshell_version}" + + # Invoke ensurePatchedClusterImage a second time. With the patched + # image already in the local cache, it must return the same tag and + # invoke neither docker pull nor docker build. + cd "$REPO_ROOT" || exit 1 + second_tag=$(node -e ' + const m = require("./dist/lib/cluster-image-patch"); + const tag = m.ensurePatchedClusterImage({ + upstreamImage: process.argv[1], + logger: () => {}, + }); + console.log(tag); + ' "$upstream_image" 2>&1 | tail -1) + + after_created=$(docker inspect --format '{{.Created}}' "$patched_tag" 2>/dev/null || echo "") + + if [ "$second_tag" = "$patched_tag" ]; then + pass "ensurePatchedClusterImage returned the same tag on second invocation: $second_tag" + else + fail "ensurePatchedClusterImage tag mismatch (first=$patched_tag second=$second_tag)" + fi + + if [ -n "$before_created" ] && [ "$before_created" = "$after_created" ]; then + pass "Patched image was reused (Created timestamp unchanged: $before_created)" + else + fail "Patched image was rebuilt unexpectedly (before=$before_created after=$after_created)" + fi +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 5: Negative — opt out of the auto-fix, expect the original failure +# ══════════════════════════════════════════════════════════════════ +section "Phase 5: Negative path (NEMOCLAW_DISABLE_OVERLAY_FIX=1)" + +if command -v nemoclaw >/dev/null 2>&1; then + nemoclaw "$SANDBOX_NAME" destroy --yes 2>/dev/null || true +fi +if command -v openshell >/dev/null 2>&1; then + openshell sandbox delete "$SANDBOX_NAME" 2>/dev/null || true + openshell gateway destroy -g nemoclaw 2>/dev/null || true +fi +docker rm -f "$GATEWAY_CONTAINER" 2>/dev/null || true + +# The script header sets `set -uo pipefail` only — errexit is NOT enabled, +# so a non-zero exit from `timeout` won't terminate us. The previous +# `set +e` / `set -e` toggle was both unnecessary and unsafe: forcing +# `set -e` after the timeout would have made later `((PASS++))` calls fatal +# whenever the counter starts at zero (post-increment returns 0, which bash +# interprets as exit 1 under errexit). Just don't touch errexit here. +env \ + NEMOCLAW_NON_INTERACTIVE=1 \ + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ + NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" \ + NEMOCLAW_RECREATE_SANDBOX=1 \ + NEMOCLAW_DISABLE_OVERLAY_FIX=1 \ + timeout "$NEGATIVE_TIMEOUT" bash install.sh --non-interactive >"$ONBOARD_LOG_NEGATIVE" 2>&1 +negative_exit=$? + +if [ $negative_exit -ne 0 ]; then + pass "Onboard with auto-fix disabled exited non-zero (exit $negative_exit) within $NEGATIVE_TIMEOUT s" +else + fail "Onboard unexpectedly succeeded with NEMOCLAW_DISABLE_OVERLAY_FIX=1" +fi + +# Negative-phase characterization. Three-way result, distinguished by +# whether a known nested-overlay failure signature shows up AND by the +# `timeout` exit code (124 = our wrapper fired, anything else = install.sh +# exited under its own steam): +# +# - signature present → PASS (confirmed reproduction) +# - signature absent + exit == 124 → SKIP (this runner instance did +# not reproduce the bug; +# we hit our 300s timeout +# while install.sh was +# making progress past the +# gateway and sandbox build) +# - signature absent + exit != 124 → FAIL (install.sh exited for an +# unrelated reason — likely +# an unrelated flake) +# +# GitHub-Actions ubuntu-latest runners vary kernel and Docker patchlevels +# enough that some runs just don't reproduce the bug at all; the SKIP +# path keeps the gate honest without papering over real failures. The +# unit + idempotency phases still validate the auto-fix on every run. +# +# Recognized signatures, in either the cluster container log or the +# install.sh log: +# - "overlayfs snapshotter cannot be enabled" (k3s init — user's report) +# - "CreateDiff: Canceled" (sandbox image build — alt manifestation) +# - "failed to mount overlay" (catch-all) +overlay_signatures='overlayfs.*snapshotter cannot be enabled|CreateDiff: Canceled|failed to mount overlay' +overlay_evidence="" + +if docker ps -a --format '{{.Names}}' | grep -q "^${GATEWAY_CONTAINER}$"; then + if + cluster_match=$(docker logs "$GATEWAY_CONTAINER" 2>&1 | grep -m1 -E "$overlay_signatures" || true) + [ -n "$cluster_match" ] + then + overlay_evidence="cluster log: $cluster_match" + fi +fi + +if [ -z "$overlay_evidence" ] && [ -f "$ONBOARD_LOG_NEGATIVE" ]; then + if + install_match=$(grep -m1 -E "$overlay_signatures" "$ONBOARD_LOG_NEGATIVE" || true) + [ -n "$install_match" ] + then + overlay_evidence="install log: $install_match" + fi +fi + +if [ -n "$overlay_evidence" ]; then + pass "Cluster/install logs surface a nested-overlay failure signature ($overlay_evidence)" +elif [ "$negative_exit" -eq 124 ]; then + skip "This runner did not reproduce the nested-overlay bug under the upstream image (no signature; install.sh hit our $NEGATIVE_TIMEOUT s timeout). Auto-fix correctness is still validated by phases 3 and 4." +else + fail "Negative phase exited $negative_exit (not our timeout, no overlay signature) — likely unrelated flake" +fi + +# ══════════════════════════════════════════════════════════════════ +# Test summary +# ══════════════════════════════════════════════════════════════════ +print_summary + +if [ $FAIL -gt 0 ]; then + exit 1 +fi +exit 0 diff --git a/test/e2e-vpn/test-rebuild-hermes.sh b/test/e2e-vpn/test-rebuild-hermes.sh new file mode 100755 index 00000000000..238139d55d5 --- /dev/null +++ b/test/e2e-vpn/test-rebuild-hermes.sh @@ -0,0 +1,450 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Hermes rebuild upgrade E2E — same upgrade scenario as OpenClaw but for Hermes: +# +# 1. Install NemoClaw (install.sh) +# 2. Build a Hermes base image with an OLDER version (v2026.4.13) +# 3. Build a minimal Hermes sandbox image (no current-Dockerfile patches) +# 4. Create sandbox via openshell directly +# 5. Write marker files into Hermes state dirs +# 6. Restore the current Hermes base image +# 7. Run `nemoclaw rebuild --yes` +# 8. Verify marker files survived + version upgraded +# +# Set NEMOCLAW_HERMES_STALE_BASE_REBUILD_E2E=1 to leave the cached +# ghcr.io/nvidia/nemoclaw/hermes-sandbox-base:latest tag on the older Hermes +# base before rebuild. That mode is the regression coverage for issue #3025. +# +# Prerequisites: +# - Docker running +# - NVIDIA_API_KEY set (real key, starts with nvapi-) +# +# Environment variables: +# NEMOCLAW_NON_INTERACTIVE=1 — required +# NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 — required +# NVIDIA_API_KEY — required + +set -euo pipefail + +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-rebuild-hm}" + +# shellcheck source=test/e2e-vpn/lib/sandbox-teardown.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib/sandbox-teardown.sh" +register_sandbox_for_teardown "$SANDBOX_NAME" + +OLD_HERMES_VERSION="v2026.4.13" +OLD_HERMES_REGISTRY_VERSION="${OLD_HERMES_VERSION#v}" +OLD_HERMES_TARBALL_SHA256="5e4529b8cb6e4821eb916b81517e48125109b1764d6d1e68a204a9f0ddf2d98c" +STALE_BASE_REBUILD="${NEMOCLAW_HERMES_STALE_BASE_REBUILD_E2E:-0}" +MARKER_FILE="/sandbox/.hermes/memories/rebuild-marker.txt" +MARKER_CONTENT="REBUILD_HM_E2E_$(date +%s)" +DISCORD_PLACEHOLDER="openshell:resolve:env:DISCORD_BOT_TOKEN" +DISCORD_FAKE_TOKEN="test-fake-discord-token-rebuild-e2e" +REGISTRY_FILE="$HOME/.nemoclaw/sandboxes.json" +SESSION_FILE="$HOME/.nemoclaw/onboard-session.json" + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +pass() { echo -e "${GREEN}[PASS]${NC} $1"; } +fail() { + echo -e "${RED}[FAIL]${NC} $1" >&2 + echo -e "${YELLOW}[DIAG]${NC} --- Failure diagnostics ---" >&2 + echo -e "${YELLOW}[DIAG]${NC} Registry: $(cat "${REGISTRY_FILE}" 2>/dev/null || echo 'not found')" >&2 + echo -e "${YELLOW}[DIAG]${NC} Session: $(cat "${SESSION_FILE}" 2>/dev/null || echo 'not found')" >&2 + echo -e "${YELLOW}[DIAG]${NC} Sandboxes: $(openshell sandbox list 2>&1 || echo 'openshell unavailable')" >&2 + echo -e "${YELLOW}[DIAG]${NC} Docker: $(docker ps --format '{{.Names}} {{.Image}} {{.Status}}' 2>&1 | head -5)" >&2 + dump_hermes_sandbox_logs >&2 || true + echo -e "${YELLOW}[DIAG]${NC} --- End diagnostics ---" >&2 + exit 1 +} +info() { echo -e "${YELLOW}[INFO]${NC} $1"; } +diag() { echo -e "${YELLOW}[DIAG]${NC} $1"; } + +dump_hermes_sandbox_logs() { + command -v openshell >/dev/null 2>&1 || { + diag "openshell is not available for sandbox log diagnostics" + return + } + openshell sandbox list 2>&1 | grep -Fq -- "$SANDBOX_NAME" || { + diag "sandbox '${SANDBOX_NAME}' is not visible to openshell" + return + } + + local diag_script + diag_script='set +e' + diag_script+='; echo "== identity =="; id 2>&1 || true' + diag_script+='; echo "== listening sockets =="; ss -tlnp 2>&1 || ss -tln 2>&1 || true' + diag_script+='; echo "== log and state paths =="; ls -ld /tmp /sandbox/.hermes /sandbox/.hermes/logs 2>&1 || true; ls -l /tmp/nemoclaw-start.log /tmp/gateway.log 2>&1 || true' + diag_script+='; echo "== hermes-related processes =="' + # shellcheck disable=SC2016 # script is intentionally evaluated inside the sandbox + diag_script+='; for p in /proc/[0-9]*; do cmd=$(tr "\000" " " < "$p/cmdline" 2>/dev/null || true); case "$cmd" in *hermes*|*socat*) echo "$(basename "$p") $cmd" ;; esac; done' + diag_script+='; echo "== /tmp/nemoclaw-start.log tail =="; tail -n 80 /tmp/nemoclaw-start.log 2>&1 || true' + diag_script+='; echo "== /tmp/gateway.log tail =="; tail -n 120 /tmp/gateway.log 2>&1 || true' + + diag "Hermes sandbox runtime logs:" + openshell sandbox exec -n "$SANDBOX_NAME" -- sh -lc "$diag_script" 2>&1 | sed 's/^/[DIAG] /' +} + +export NEMOCLAW_REBUILD_VERBOSE=1 + +# ── Preflight ─────────────────────────────────────────────────────── +[ -n "${NVIDIA_API_KEY:-}" ] || fail "NVIDIA_API_KEY is required" +[ "${NEMOCLAW_NON_INTERACTIVE:-}" = "1" ] || fail "NEMOCLAW_NON_INTERACTIVE=1 is required" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +EXPECTED_HERMES_VERSION="$(grep -E '^expected_version:' "${REPO_ROOT}/agents/hermes/manifest.yaml" | sed -E 's/.*"([^"]+)".*/\1/')" +[ -n "${EXPECTED_HERMES_VERSION}" ] || fail "Could not parse expected Hermes version from manifest" + +if [ "${STALE_BASE_REBUILD}" = "1" ]; then + info "Hermes stale-base rebuild E2E (old: ${OLD_HERMES_VERSION}, expected: ${EXPECTED_HERMES_VERSION}, sandbox: ${SANDBOX_NAME})" +else + info "Hermes rebuild upgrade E2E (old: ${OLD_HERMES_VERSION}, expected: ${EXPECTED_HERMES_VERSION}, sandbox: ${SANDBOX_NAME})" +fi + +# ── Phase 1: Install NemoClaw ─────────────────────────────────────── +info "Phase 1: Installing NemoClaw via install.sh..." + +export NEMOCLAW_NON_INTERACTIVE=1 +export NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 +export NEMOCLAW_SANDBOX_NAME="${SANDBOX_NAME}" +export NEMOCLAW_RECREATE_SANDBOX=1 +export NEMOCLAW_AGENT=hermes + +INSTALL_LOG="/tmp/nemoclaw-e2e-install.log" +if ! bash "${REPO_ROOT}/install.sh" --non-interactive >"$INSTALL_LOG" 2>&1; then + info "install.sh exited non-zero (may be expected on re-install). Checking for nemoclaw..." +fi + +# Source shell profile to pick up nvm/PATH changes +if [ -f "$HOME/.bashrc" ]; then + # shellcheck source=/dev/null + source "$HOME/.bashrc" 2>/dev/null || true +fi +export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" +if [ -s "$NVM_DIR/nvm.sh" ]; then + # shellcheck source=/dev/null + . "$NVM_DIR/nvm.sh" +fi +if [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]]; then + export PATH="$HOME/.local/bin:$PATH" +fi + +command -v nemoclaw >/dev/null 2>&1 || fail "nemoclaw not found on PATH after install" +command -v openshell >/dev/null 2>&1 || fail "openshell not found on PATH after install" +pass "NemoClaw installed" + +# Delete the sandbox that install.sh created — we'll make our own old one. +# Use openshell directly to preserve the 'nemoclaw' gateway for the rebuild. +openshell sandbox delete "${SANDBOX_NAME}" 2>/dev/null || true +# Raw OpenShell deletion can leave the prior Hermes API/dashboard forward +# bound for a short window. The rebuild create path intentionally rolls back if +# the baked dashboard port is host-bound after image build, so make this phase +# cleanup synchronous before creating the old fixture sandbox. +openshell forward stop 8642 >/dev/null 2>&1 || true +diag "Deleted Phase 1 sandbox, gateway preserved: $(docker ps --filter name=openshell --format '{{.Names}} {{.Status}}' 2>/dev/null)" + +# ── Phase 2: Build old Hermes base image ─────────────────────────── +info "Phase 2: Building Hermes base image with ${OLD_HERMES_VERSION}..." + +OLD_BASE_TAG="nemoclaw-hermes-old-base:e2e-rebuild" + +docker build \ + --build-arg "HERMES_VERSION=${OLD_HERMES_VERSION}" \ + --build-arg "HERMES_TARBALL_SHA256=${OLD_HERMES_TARBALL_SHA256}" \ + --build-arg "HERMES_UV_EXTRAS=messaging" \ + -f "${REPO_ROOT}/agents/hermes/Dockerfile.base" \ + -t "${OLD_BASE_TAG}" \ + "${REPO_ROOT}" \ + || fail "Failed to build old Hermes base image" + +pass "Old Hermes base image built (${OLD_HERMES_VERSION})" + +if [ "${STALE_BASE_REBUILD}" = "1" ]; then + docker tag "${OLD_BASE_TAG}" "ghcr.io/nvidia/nemoclaw/hermes-sandbox-base:latest" + pass "Cached Hermes base tag now points at old version" +fi + +# ── Phase 3: Create old sandbox via openshell ─────────────────────── +info "Phase 3: Creating sandbox with old Hermes via openshell..." + +# Build a minimal Dockerfile — NOT the full agents/hermes/Dockerfile which +# patches files that may not exist in the old Hermes version. +TESTDIR=$(mktemp -d) +cat >"${TESTDIR}/Dockerfile" < /sandbox/.hermes/config.yaml \ + && printf '%s\n' \ + 'API_SERVER_PORT=18642' \ + 'API_SERVER_HOST=127.0.0.1' \ + 'DISCORD_BOT_TOKEN=${DISCORD_PLACEHOLDER}' \ + > /sandbox/.hermes/.env +CMD ["/bin/bash"] +DOCKERFILE + +DISCORD_BOT_TOKEN="${DISCORD_FAKE_TOKEN}" \ + openshell provider create --name "${SANDBOX_NAME}-discord-bridge" --type generic --credential DISCORD_BOT_TOKEN \ + >/dev/null 2>&1 || DISCORD_BOT_TOKEN="${DISCORD_FAKE_TOKEN}" \ + openshell provider update "${SANDBOX_NAME}-discord-bridge" --credential DISCORD_BOT_TOKEN \ + >/dev/null 2>&1 +openshell sandbox create \ + --name "${SANDBOX_NAME}" \ + --from "${TESTDIR}/Dockerfile" \ + --gateway nemoclaw \ + --provider "${SANDBOX_NAME}-discord-bridge" \ + --no-tty \ + -- true +rm -rf "${TESTDIR}" + +# Wait for Ready +for _i in $(seq 1 30); do + if openshell sandbox list 2>/dev/null | grep -q "${SANDBOX_NAME}.*Ready"; then + break + fi + sleep 5 +done +openshell sandbox list 2>/dev/null | grep -q "${SANDBOX_NAME}.*Ready" || fail "Sandbox did not become Ready" + +pass "Old Hermes sandbox created" + +# ── Phase 4: Write markers + register ─────────────────────────────── +info "Phase 4: Writing markers and registering sandbox..." + +openshell sandbox exec --name "${SANDBOX_NAME}" -- \ + sh -c "mkdir -p /sandbox/.hermes/memories && echo '${MARKER_CONTENT}' > ${MARKER_FILE}" \ + || fail "Failed to write marker file" + +VERIFY=$(openshell sandbox exec --name "${SANDBOX_NAME}" -- cat "${MARKER_FILE}" 2>/dev/null || true) +[ "$VERIFY" = "${MARKER_CONTENT}" ] || fail "Marker verification failed" +PRE_REBUILD_ENV=$(openshell sandbox exec --name "${SANDBOX_NAME}" -- cat /sandbox/.hermes/.env 2>/dev/null || true) +echo "$PRE_REBUILD_ENV" | grep -Fq "DISCORD_BOT_TOKEN=${DISCORD_PLACEHOLDER}" \ + || fail "Pre-rebuild Hermes .env missing Discord placeholder" +PRE_REBUILD_CONFIG=$(openshell sandbox exec --name "${SANDBOX_NAME}" -- cat /sandbox/.hermes/config.yaml 2>/dev/null || true) +echo "$PRE_REBUILD_CONFIG" | grep -Fq "discord:" \ + || fail "Pre-rebuild Hermes config.yaml missing platforms.discord" + +# Register in NemoClaw registry +python3 -c " +import hashlib, json, os +sess_path = '${SESSION_FILE}' +try: + with open(sess_path) as f: + sess = json.load(f) +except Exception: + sess = {} +env_provider = (os.environ.get('NEMOCLAW_PROVIDER') or '').strip() +if env_provider == 'custom': + env_provider = 'compatible-endpoint' +provider = sess.get('provider') or env_provider or 'compatible-endpoint' +model = ( + sess.get('model') + or os.environ.get('NEMOCLAW_MODEL') + or os.environ.get('NEMOCLAW_COMPAT_MODEL') + or 'nvidia/nvidia/nemotron-3-super-v3' +) +credential_hash = hashlib.sha256('${DISCORD_FAKE_TOKEN}'.encode()).hexdigest() +plan = { + 'schemaVersion': 1, + 'sandboxName': '${SANDBOX_NAME}', + 'agent': 'hermes', + 'workflow': 'onboard', + 'channels': [{ + 'channelId': 'discord', + 'displayName': 'discord', + 'authMode': 'token-paste', + 'active': True, + 'selected': True, + 'configured': True, + 'disabled': False, + 'inputs': [], + 'hooks': [], + }], + 'disabledChannels': [], + 'credentialBindings': [{ + 'channelId': 'discord', + 'credentialId': 'discordBotToken', + 'sourceInput': 'botToken', + 'providerName': '${SANDBOX_NAME}-discord-bridge', + 'providerEnvKey': 'DISCORD_BOT_TOKEN', + 'placeholder': '${DISCORD_PLACEHOLDER}', + 'credentialAvailable': True, + 'credentialHash': credential_hash, + }], + 'networkPolicy': {'presets': ['discord'], 'entries': []}, + 'agentRender': [], + 'buildSteps': [], + 'stateUpdates': [], + 'healthChecks': [], +} +reg = {'sandboxes': {'${SANDBOX_NAME}': { + 'name': '${SANDBOX_NAME}', + 'createdAt': '$(date -u +%Y-%m-%dT%H:%M:%SZ)', + 'model': model, + 'provider': provider, + 'gpuEnabled': False, + 'policies': [], + 'policyTier': None, + 'agent': 'hermes', + 'agentVersion': '${OLD_HERMES_REGISTRY_VERSION}', + 'messaging': {'schemaVersion': 1, 'plan': plan} +}}, 'defaultSandbox': '${SANDBOX_NAME}'} +with open('${REGISTRY_FILE}', 'w') as f: + json.dump(reg, f, indent=2) + +sess['sandboxName'] = '${SANDBOX_NAME}' +sess['agent'] = 'hermes' +sess['status'] = 'complete' +for key in ('messagingChannels', 'messagingChannelConfig', 'disabledChannels'): + sess.pop(key, None) +sess['messagingPlan'] = plan +with open(sess_path, 'w') as f: + json.dump(sess, f, indent=2) +print('Registry and session updated') +" + +pass "Markers written, sandbox registered" + +# ── Phase 5: Prepare current base-image cache state ───────────────── +if [ "${STALE_BASE_REBUILD}" = "1" ]; then + info "Phase 5: Leaving cached Hermes base image stale..." + diag "Cached ghcr.io/nvidia/nemoclaw/hermes-sandbox-base:latest intentionally points at ${OLD_HERMES_VERSION}; rebuild must refresh it from agents/hermes/Dockerfile.base." +else + info "Phase 5: Building current Hermes base image..." + + docker build \ + -f "${REPO_ROOT}/agents/hermes/Dockerfile.base" \ + -t "ghcr.io/nvidia/nemoclaw/hermes-sandbox-base:latest" \ + "${REPO_ROOT}" \ + || fail "Failed to build current Hermes base image" + + pass "Current Hermes base image built" +fi + +# ── Phase 6: Rebuild ──────────────────────────────────────────────── +info "Phase 6: Running nemoclaw rebuild..." +unset DISCORD_BOT_TOKEN + +diag "Pre-rebuild state:" +diag " Registry: $(python3 -c "import json; d=json.load(open('${REGISTRY_FILE}')); print(json.dumps({k: {'agent': v.get('agent'), 'agentVersion': v.get('agentVersion')} for k,v in d.get('sandboxes',{}).items()}))" 2>/dev/null)" +diag " Session: $(python3 -c "import json; s=json.load(open('${SESSION_FILE}')); print(f'name={s.get(\"sandboxName\")} status={s.get(\"status\")} resumable={s.get(\"resumable\")} agent={s.get(\"agent\")} provider={s.get(\"provider\")}')" 2>/dev/null)" +diag " Live sandboxes: $(openshell sandbox list 2>&1 | grep -v NAME || echo none)" +diag " Gateway: $(docker ps --filter name=openshell --format '{{.Names}} {{.Status}}' 2>/dev/null || echo 'not running')" + +diag "Calling: nemoclaw ${SANDBOX_NAME} rebuild --yes --verbose" +nemoclaw "${SANDBOX_NAME}" rebuild --yes --verbose || fail "Rebuild failed" + +pass "Rebuild completed" + +# ── Phase 7: Verify ───────────────────────────────────────────────── +info "Phase 7: Verifying results..." + +# Marker file survived +RESTORED=$(openshell sandbox exec --name "${SANDBOX_NAME}" -- cat "${MARKER_FILE}" 2>/dev/null || true) +if [ "$RESTORED" = "${MARKER_CONTENT}" ]; then + pass "Marker file survived rebuild" +else + fail "Marker file lost: got '${RESTORED}', expected '${MARKER_CONTENT}'" +fi + +# Actual Hermes binary version updated +HERMES_VERSION_OUTPUT=$(openshell sandbox exec --name "${SANDBOX_NAME}" -- hermes --version 2>&1 || true) +diag "Hermes version after rebuild: ${HERMES_VERSION_OUTPUT//$'\n'/ | }" +if echo "${HERMES_VERSION_OUTPUT}" | grep -Fq "${OLD_HERMES_REGISTRY_VERSION}"; then + fail "Hermes binary still reports old version ${OLD_HERMES_REGISTRY_VERSION}" +fi +if echo "${HERMES_VERSION_OUTPUT}" | grep -Fq "${EXPECTED_HERMES_VERSION}"; then + pass "Hermes binary reports expected version ${EXPECTED_HERMES_VERSION}" +else + fail "Hermes binary version mismatch: expected output to contain '${EXPECTED_HERMES_VERSION}'" +fi + +# Hermes messaging config survived through non-interactive rebuild without +# requiring the Discord token to be re-exported on the host. +RESTORED_ENV=$(openshell sandbox exec --name "${SANDBOX_NAME}" -- cat /sandbox/.hermes/.env 2>/dev/null || true) +if echo "$RESTORED_ENV" | grep -Fq "DISCORD_BOT_TOKEN=${DISCORD_PLACEHOLDER}"; then + pass "Hermes .env preserved Discord token placeholder" +else + fail "Hermes .env lost Discord placeholder after rebuild: ${RESTORED_ENV}" +fi + +RESTORED_CONFIG=$(openshell sandbox exec --name "${SANDBOX_NAME}" -- cat /sandbox/.hermes/config.yaml 2>/dev/null || true) +if echo "$RESTORED_CONFIG" | grep -Fq "discord:"; then + pass "Hermes config.yaml preserved platforms.discord" +else + fail "Hermes config.yaml lost platforms.discord after rebuild: ${RESTORED_CONFIG}" +fi + +# Inference works after rebuild (proves credential chain is intact) +info "Verifying inference after rebuild..." +POST_REBUILD_INFERENCE_MODEL="${NEMOCLAW_MODEL:-${NEMOCLAW_COMPAT_MODEL:-nvidia/nvidia/nemotron-3-super-v3}}" +INFERENCE_RESPONSE=$(openshell sandbox exec --name "${SANDBOX_NAME}" -- \ + curl -s --max-time 60 https://inference.local/v1/chat/completions \ + -H 'Content-Type: application/json' \ + -d "{\"model\":\"${POST_REBUILD_INFERENCE_MODEL}\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with exactly one word: PONG\"}],\"max_tokens\":100}" \ + 2>&1 || true) +if echo "${INFERENCE_RESPONSE}" | python3 -c "import json,sys; r=json.load(sys.stdin); c=r['choices'][0]['message']; print(c.get('content',''))" 2>/dev/null | grep -qi "PONG"; then + pass "Inference works after rebuild (NVIDIA API key + provider chain intact)" +else + # Non-fatal — inference depends on external API availability and Hermes gateway being up + info "Inference check inconclusive (may be API timeout or gateway not started): ${INFERENCE_RESPONSE:0:200}" +fi + +# Registry updated +REGISTRY_VERSION=$(python3 -c " +import json +with open('${REGISTRY_FILE}') as f: + data = json.load(f) +sb = data.get('sandboxes', {}).get('${SANDBOX_NAME}', {}) +print(sb.get('agentVersion', 'null')) +" 2>/dev/null || echo "error") +if [ "$REGISTRY_VERSION" != "null" ] && [ "$REGISTRY_VERSION" != "error" ] && [ "$REGISTRY_VERSION" != "$OLD_HERMES_REGISTRY_VERSION" ]; then + pass "Registry agentVersion updated to ${REGISTRY_VERSION}" +else + fail "Registry agentVersion not updated: got '${REGISTRY_VERSION}', expected != '${OLD_HERMES_REGISTRY_VERSION}'" +fi + +# No credentials in backup +BACKUP_DIR="$HOME/.nemoclaw/rebuild-backups/${SANDBOX_NAME}" +if [ -d "$BACKUP_DIR" ]; then + CRED_LEAKS=$(find "$BACKUP_DIR" \( -name "*.json" -o -name "*.yaml" -o -name "*.env" -o -name ".env" \) -exec grep -l "nvapi-\|sk-\|Bearer " {} \; 2>/dev/null || true) + if [ -z "$CRED_LEAKS" ]; then + pass "No credentials in backup" + else + fail "Credentials found: $CRED_LEAKS" + fi +else + fail "Backup directory missing: $BACKUP_DIR" +fi + +# ── Cleanup ───────────────────────────────────────────────────────── +info "Cleaning up..." +[[ "${NEMOCLAW_E2E_KEEP_SANDBOX:-}" = "1" ]] || nemoclaw "${SANDBOX_NAME}" destroy --yes 2>/dev/null || true +docker rmi "${OLD_BASE_TAG}" 2>/dev/null || true + +echo "" +if [ "${STALE_BASE_REBUILD}" = "1" ]; then + echo -e "${GREEN}Hermes stale-base rebuild E2E passed.${NC}" +else + echo -e "${GREEN}Hermes rebuild upgrade E2E passed.${NC}" +fi diff --git a/test/e2e-vpn/test-rebuild-openclaw.sh b/test/e2e-vpn/test-rebuild-openclaw.sh new file mode 100755 index 00000000000..2b2f926de93 --- /dev/null +++ b/test/e2e-vpn/test-rebuild-openclaw.sh @@ -0,0 +1,552 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# OpenClaw rebuild upgrade E2E — reproduces the exact NVBug 6076156 scenario: +# +# 1. Install NemoClaw (install.sh) +# 2. Build a base image with an OLDER OpenClaw version (2026.3.11) +# 3. Create a sandbox from that old image via openshell directly +# 4. Write marker files into workspace state dirs +# 4.5 Apply policy presets (npm, pypi) and verify they are active (#1952) +# 5. Restore the current base image +# 6. Run `nemoclaw rebuild --yes` +# 7. Verify marker files survived the rebuild +# 8. Verify the sandbox now reports the CURRENT version +# 9. Verify the OpenClaw gateway auth token rotated (#4517) +# 10. Verify no credentials leaked into the local backup +# 11. Verify policy presets survived the rebuild (#1952) +# +# Prerequisites: +# - Docker running +# - NVIDIA_API_KEY set (real key, starts with nvapi-) +# +# Environment variables: +# NEMOCLAW_NON_INTERACTIVE=1 — required +# NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 — required +# NVIDIA_API_KEY — required + +set -euo pipefail + +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-rebuild-oc}" + +# shellcheck source=test/e2e-vpn/lib/sandbox-teardown.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib/sandbox-teardown.sh" +register_sandbox_for_teardown "$SANDBOX_NAME" + +OLD_OPENCLAW_VERSION="2026.3.11" +MARKER_FILE="/sandbox/.openclaw/workspace/rebuild-marker.txt" +MARKER_CONTENT="REBUILD_OC_E2E_$(date +%s)" +PRE_REBUILD_GATEWAY_TOKEN="nemoclaw-e2e-old-gateway-token-${MARKER_CONTENT}" +REGISTRY_FILE="$HOME/.nemoclaw/sandboxes.json" +SESSION_FILE="$HOME/.nemoclaw/onboard-session.json" + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +pass() { echo -e "${GREEN}[PASS]${NC} $1"; } +fail() { + echo -e "${RED}[FAIL]${NC} $1" >&2 + # Dump diagnostic state on failure + echo -e "${YELLOW}[DIAG]${NC} --- Failure diagnostics ---" >&2 + echo -e "${YELLOW}[DIAG]${NC} Registry: $(cat "${REGISTRY_FILE}" 2>/dev/null || echo 'not found')" >&2 + echo -e "${YELLOW}[DIAG]${NC} Session: $(cat "${SESSION_FILE}" 2>/dev/null || echo 'not found')" >&2 + echo -e "${YELLOW}[DIAG]${NC} Sandboxes: $(openshell sandbox list 2>&1 || echo 'openshell unavailable')" >&2 + echo -e "${YELLOW}[DIAG]${NC} Docker: $(docker ps --format '{{.Names}} {{.Image}} {{.Status}}' 2>&1 | head -5)" >&2 + echo -e "${YELLOW}[DIAG]${NC} --- End diagnostics ---" >&2 + exit 1 +} +info() { echo -e "${YELLOW}[INFO]${NC} $1"; } +diag() { echo -e "${YELLOW}[DIAG]${NC} $1"; } + +read_sandbox_gateway_token() { + openshell sandbox exec --name "${SANDBOX_NAME}" -- \ + python3 -c 'import json; cfg = json.load(open("/sandbox/.openclaw/openclaw.json")); print(cfg.get("gateway", {}).get("auth", {}).get("token", ""), end="")' +} + +read_sandbox_runtime_gateway_token() { + # shellcheck disable=SC2016 # OPENCLAW_GATEWAY_TOKEN must expand inside the sandbox. + openshell sandbox exec --name "${SANDBOX_NAME}" -- \ + bash -lc '. /tmp/nemoclaw-proxy-env.sh >/dev/null 2>&1 || exit 1; printf "%s" "${OPENCLAW_GATEWAY_TOKEN:-}"' +} + +read_sandbox_config_hash() { + openshell sandbox exec --name "${SANDBOX_NAME}" -- cat /sandbox/.openclaw/.config-hash +} + +# Enable verbose logging in rebuild command +export NEMOCLAW_REBUILD_VERBOSE=1 + +# ── Preflight ─────────────────────────────────────────────────────── +[ -n "${NVIDIA_API_KEY:-}" ] || fail "NVIDIA_API_KEY is required" +[ "${NEMOCLAW_NON_INTERACTIVE:-}" = "1" ] || fail "NEMOCLAW_NON_INTERACTIVE=1 is required" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" + +info "OpenClaw rebuild upgrade E2E (old: ${OLD_OPENCLAW_VERSION}, sandbox: ${SANDBOX_NAME})" + +# ── Phase 1: Install NemoClaw ─────────────────────────────────────── +info "Phase 1: Installing NemoClaw via install.sh..." + +export NEMOCLAW_NON_INTERACTIVE=1 +export NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 +export NEMOCLAW_SANDBOX_NAME="${SANDBOX_NAME}" +export NEMOCLAW_RECREATE_SANDBOX=1 + +INSTALL_LOG="/tmp/nemoclaw-e2e-install.log" +if ! bash "${REPO_ROOT}/install.sh" --non-interactive >"$INSTALL_LOG" 2>&1; then + info "install.sh exited non-zero (may be expected on re-install). Checking for nemoclaw..." +fi + +# Source shell profile to pick up nvm/PATH changes +if [ -f "$HOME/.bashrc" ]; then + # shellcheck source=/dev/null + source "$HOME/.bashrc" 2>/dev/null || true +fi +export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" +if [ -s "$NVM_DIR/nvm.sh" ]; then + # shellcheck source=/dev/null + . "$NVM_DIR/nvm.sh" +fi +if [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]]; then + export PATH="$HOME/.local/bin:$PATH" +fi + +command -v nemoclaw >/dev/null 2>&1 || fail "nemoclaw not found on PATH after install" +command -v openshell >/dev/null 2>&1 || fail "openshell not found on PATH after install" +pass "NemoClaw installed" + +# Delete the sandbox that install.sh created — we'll make our own old one. +# Use openshell directly to preserve the 'nemoclaw' gateway for the rebuild. +openshell sandbox delete "${SANDBOX_NAME}" 2>/dev/null || true +diag "Deleted Phase 1 sandbox, gateway preserved: $(docker ps --filter name=openshell --format '{{.Names}} {{.Status}}' 2>/dev/null)" + +# ── Phase 2: Build old base image ────────────────────────────────── +info "Phase 2: Building base image with OpenClaw ${OLD_OPENCLAW_VERSION}..." + +OLD_BASE_TAG="nemoclaw-old-base:e2e-rebuild" +BLUEPRINT="${REPO_ROOT}/nemoclaw-blueprint/blueprint.yaml" +BLUEPRINT_BAK="${BLUEPRINT}.bak" + +# Dockerfile.base validates OPENCLAW_VERSION >= min_openclaw_version. +# Temporarily lower the minimum so the old version builds. +cp "${BLUEPRINT}" "${BLUEPRINT_BAK}" +# sed -i behaves differently on macOS vs Linux; use a temp file for portability +sed "s/min_openclaw_version:.*/min_openclaw_version: \"${OLD_OPENCLAW_VERSION}\"/" "${BLUEPRINT}" >"${BLUEPRINT}.tmp" +mv "${BLUEPRINT}.tmp" "${BLUEPRINT}" + +docker build \ + --build-arg "OPENCLAW_VERSION=${OLD_OPENCLAW_VERSION}" \ + -f "${REPO_ROOT}/Dockerfile.base" \ + -t "${OLD_BASE_TAG}" \ + "${REPO_ROOT}" +BUILD_RC=$? + +mv "${BLUEPRINT_BAK}" "${BLUEPRINT}" +[ "$BUILD_RC" -eq 0 ] || fail "Failed to build old base image" + +pass "Old base image built (OpenClaw ${OLD_OPENCLAW_VERSION})" + +# ── Phase 3: Create old sandbox via openshell ─────────────────────── +info "Phase 3: Creating sandbox with old OpenClaw via openshell..." + +# Build a minimal Dockerfile that uses the old base +TESTDIR=$(mktemp -d) +cat >"${TESTDIR}/Dockerfile" < /sandbox/.openclaw/openclaw.json +CMD ["/bin/bash"] +DOCKERFILE + +openshell sandbox create --name "${SANDBOX_NAME}" --from "${TESTDIR}/Dockerfile" --gateway nemoclaw --no-tty -- true +rm -rf "${TESTDIR}" + +# Wait for Ready +for _i in $(seq 1 30); do + if openshell sandbox list 2>/dev/null | grep -q "${SANDBOX_NAME}.*Ready"; then + break + fi + sleep 5 +done +openshell sandbox list 2>/dev/null | grep -q "${SANDBOX_NAME}.*Ready" || fail "Sandbox did not become Ready" + +# Verify old version +SANDBOX_VERSION=$(openshell sandbox exec --name "${SANDBOX_NAME}" -- openclaw --version 2>&1 || true) +echo "${SANDBOX_VERSION}" | grep -q "${OLD_OPENCLAW_VERSION}" || info "Version: ${SANDBOX_VERSION}" + +pass "Old sandbox created (OpenClaw ${OLD_OPENCLAW_VERSION})" + +# ── Phase 4: Write marker files + register ────────────────────────── +info "Phase 4: Writing markers and registering sandbox..." + +openshell sandbox exec --name "${SANDBOX_NAME}" -- \ + sh -c "mkdir -p /sandbox/.openclaw/workspace && echo '${MARKER_CONTENT}' > ${MARKER_FILE}" \ + || fail "Failed to write marker file" + +# Seed an existing gateway token so the rebuild path must rotate it. +openshell sandbox exec --name "${SANDBOX_NAME}" -- \ + env "PRE_REBUILD_GATEWAY_TOKEN=${PRE_REBUILD_GATEWAY_TOKEN}" \ + python3 -c 'import json, os; path = "/sandbox/.openclaw/openclaw.json"; cfg = json.load(open(path)); cfg.setdefault("gateway", {}).setdefault("auth", {})["token"] = os.environ["PRE_REBUILD_GATEWAY_TOKEN"]; f = open(path, "w"); json.dump(cfg, f, indent=2); f.write("\n"); f.close()' \ + || fail "Failed to seed old gateway token" +openshell sandbox exec --name "${SANDBOX_NAME}" -- \ + sh -c "cd /sandbox/.openclaw && sha256sum openclaw.json > .config-hash" \ + || fail "Failed to write pre-rebuild config hash" +PRE_REBUILD_CONFIG_HASH="$(read_sandbox_config_hash 2>/dev/null || true)" +SEEDED_GATEWAY_TOKEN="$(read_sandbox_gateway_token 2>/dev/null || true)" +if [ "${SEEDED_GATEWAY_TOKEN}" = "${PRE_REBUILD_GATEWAY_TOKEN}" ]; then + pass "Old gateway token seeded before rebuild" +else + fail "Failed to verify seeded gateway token before rebuild" +fi +if [ -n "${PRE_REBUILD_CONFIG_HASH}" ] && echo "${PRE_REBUILD_CONFIG_HASH}" | grep -q "openclaw.json"; then + pass "Pre-rebuild config hash recorded" +else + fail "Pre-rebuild config hash missing openclaw.json" +fi + +# Verify +VERIFY=$(openshell sandbox exec --name "${SANDBOX_NAME}" -- cat "${MARKER_FILE}" 2>/dev/null || true) +[ "$VERIFY" = "${MARKER_CONTENT}" ] || fail "Marker verification failed: got '${VERIFY}'" + +# Register in NemoClaw registry with old version +python3 -c " +import json, os +sess_path = '${SESSION_FILE}' +try: + with open(sess_path) as f: + sess = json.load(f) +except Exception: + sess = {} +env_provider = (os.environ.get('NEMOCLAW_PROVIDER') or '').strip() +if env_provider == 'custom': + env_provider = 'compatible-endpoint' +provider = sess.get('provider') or env_provider or 'compatible-endpoint' +model = ( + sess.get('model') + or os.environ.get('NEMOCLAW_MODEL') + or os.environ.get('NEMOCLAW_COMPAT_MODEL') + or 'nvidia/nvidia/nemotron-3-super-v3' +) +reg = {'sandboxes': {'${SANDBOX_NAME}': { + 'name': '${SANDBOX_NAME}', + 'createdAt': '$(date -u +%Y-%m-%dT%H:%M:%SZ)', + 'model': model, + 'provider': provider, + 'gpuEnabled': False, + 'policies': ['npm', 'pypi'], + 'policyTier': None, + 'agent': None, + 'agentVersion': '${OLD_OPENCLAW_VERSION}' +}}, 'defaultSandbox': '${SANDBOX_NAME}'} +with open('${REGISTRY_FILE}', 'w') as f: + json.dump(reg, f, indent=2) + +# Update session to point at this sandbox. +# Mark preflight and gateway steps as complete so that rebuild's +# onboard --resume skips them (the gateway is already running and +# port 8080 is legitimately in use). +sess['sandboxName'] = '${SANDBOX_NAME}' +sess['status'] = 'complete' +sess['resumable'] = True +sess['lastCompletedStep'] = 'gateway' +sess['failure'] = None +now = __import__('datetime').datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S.000Z') +complete = {'status': 'complete', 'startedAt': now, 'completedAt': now, 'error': None} +pending = {'status': 'pending', 'startedAt': None, 'completedAt': None, 'error': None} +sess['steps'] = { + 'preflight': complete, + 'gateway': complete, + 'sandbox': pending, + 'provider_selection': pending, + 'inference': pending, + 'openclaw': pending, + 'agent_setup': pending, + 'policies': pending, +} +with open(sess_path, 'w') as f: + json.dump(sess, f, indent=2) +print('Registry and session updated') +" + +pass "Markers written, sandbox registered" + +# ── Phase 4.5: Apply policy presets (#1952) ───────────────────────── +info "Phase 4.5: Applying policy presets (npm, pypi) to sandbox..." + +# Apply each preset to the live gateway policy engine. Resolve the NemoClaw +# module directory from the `nemoclaw` binary on PATH (portable across +# install methods: npm link, npm -g, source checkout). +NEMOCLAW_BIN="$(command -v nemoclaw)" +# nemoclaw is a shell wrapper; extract the real node binary path from it +# to find the node_modules root. +NEMOCLAW_MODULE_DIR="$(node -e " + try { console.log(require.resolve('nemoclaw/package.json').replace('/package.json','')); } + catch(e) { + // Fallback: walk up from the nemoclaw bin wrapper + const fs = require('fs'), path = require('path'); + const wrapper = fs.readFileSync('${NEMOCLAW_BIN}', 'utf-8'); + const m = wrapper.match(/exec\\s+\"?([^\"\\s]+node)\"?/); + if (m) { + const nodeDir = path.dirname(path.dirname(m[1])); + const candidate = path.join(nodeDir, 'lib/node_modules/nemoclaw'); + if (fs.existsSync(path.join(candidate, 'dist/lib/policy/index.js'))) { + console.log(candidate); + process.exit(0); + } + } + // Last resort: relative to the repo root + const repoCandidate = '${REPO_ROOT}'; + if (fs.existsSync(path.join(repoCandidate, 'dist/lib/policy/index.js'))) { + console.log(repoCandidate); + process.exit(0); + } + console.error('Cannot locate nemoclaw module directory'); + process.exit(1); + } +" 2>/dev/null)" || fail "Cannot locate nemoclaw module directory" +diag "NemoClaw module dir: ${NEMOCLAW_MODULE_DIR}" + +for preset in npm pypi; do + info " Applying preset: ${preset}" + node -e " + const policies = require('${NEMOCLAW_MODULE_DIR}/dist/lib/policy/index.js'); + const ok = policies.applyPreset('${SANDBOX_NAME}', '${preset}'); + if (!ok) { console.error('applyPreset returned false for ${preset}'); process.exit(1); } + " || fail "Failed to apply preset: ${preset}" +done + +# Verify presets are in the live gateway policy +PRE_REBUILD_POLICY=$(openshell policy get --full "${SANDBOX_NAME}" 2>&1 || true) +if echo "${PRE_REBUILD_POLICY}" | grep -qi "npm\|registry.npmjs.org"; then + pass "npm preset active in gateway policy" +else + fail "npm preset not found in live gateway policy before rebuild" +fi +if echo "${PRE_REBUILD_POLICY}" | grep -qi "pypi\|pypi.org"; then + pass "pypi preset active in gateway policy" +else + fail "pypi preset not found in live gateway policy before rebuild" +fi + +# Verify presets in registry +PRE_REBUILD_PRESETS=$(python3 -c " +import json +with open('${REGISTRY_FILE}') as f: + data = json.load(f) +sb = data.get('sandboxes', {}).get('${SANDBOX_NAME}', {}) +print(','.join(sb.get('policies', []))) +" 2>/dev/null || echo "error") +diag "Pre-rebuild registry policies: ${PRE_REBUILD_PRESETS}" + +pass "Policy presets applied and verified" + +# Diagnostic dump before rebuild +diag "Pre-rebuild state:" +diag " Registry: $(python3 -c "import json; d=json.load(open('${REGISTRY_FILE}')); print(json.dumps({k: {'agent': v.get('agent'), 'agentVersion': v.get('agentVersion')} for k,v in d.get('sandboxes',{}).items()}))" 2>/dev/null)" +diag " Session: $(python3 -c "import json; s=json.load(open('${SESSION_FILE}')); print(f'name={s.get(\"sandboxName\")} status={s.get(\"status\")} resumable={s.get(\"resumable\")} provider={s.get(\"provider\")} model={s.get(\"model\")}')" 2>/dev/null)" +diag " Live sandboxes: $(openshell sandbox list 2>&1 | grep -v NAME || echo none)" +diag " Gateway: $(docker ps --filter name=openshell --format '{{.Names}} {{.Status}}' 2>/dev/null || echo 'not running')" + +# ── Phase 5: Restore current base image ───────────────────────────── +info "Phase 5: Restoring current base image..." + +docker build \ + -f "${REPO_ROOT}/Dockerfile.base" \ + -t "ghcr.io/nvidia/nemoclaw/sandbox-base:latest" \ + "${REPO_ROOT}" \ + || fail "Failed to build current base image" + +pass "Current base image restored" + +# ── Phase 6: Rebuild ──────────────────────────────────────────────── +info "Phase 6: Running nemoclaw rebuild..." + +diag "Calling: nemoclaw ${SANDBOX_NAME} rebuild --yes --verbose" +nemoclaw "${SANDBOX_NAME}" rebuild --yes --verbose || fail "Rebuild failed" + +pass "Rebuild completed" + +# ── Phase 7: Verify ───────────────────────────────────────────────── +info "Phase 7: Verifying results..." + +# Marker file survived +RESTORED=$(openshell sandbox exec --name "${SANDBOX_NAME}" -- cat "${MARKER_FILE}" 2>/dev/null || true) +if [ "$RESTORED" = "${MARKER_CONTENT}" ]; then + pass "Marker file survived rebuild" +else + fail "Marker file lost: got '${RESTORED}', expected '${MARKER_CONTENT}'" +fi + +# Version upgraded +NEW_VERSION=$(openshell sandbox exec --name "${SANDBOX_NAME}" -- openclaw --version 2>&1 || true) +if [ -z "${NEW_VERSION}" ]; then + fail "Could not get OpenClaw version from sandbox (empty output)" +elif echo "${NEW_VERSION}" | grep -q "${OLD_OPENCLAW_VERSION}"; then + fail "Version still old after rebuild: ${NEW_VERSION}" +else + pass "OpenClaw version upgraded: ${NEW_VERSION}" +fi + +# Registry updated +REGISTRY_VERSION=$(python3 -c " +import json +with open('${REGISTRY_FILE}') as f: + data = json.load(f) +sb = data.get('sandboxes', {}).get('${SANDBOX_NAME}', {}) +print(sb.get('agentVersion', 'null')) +" 2>/dev/null || echo "error") +if [ "$REGISTRY_VERSION" != "null" ] && [ "$REGISTRY_VERSION" != "error" ] && [ "$REGISTRY_VERSION" != "${OLD_OPENCLAW_VERSION}" ]; then + pass "Registry agentVersion updated to ${REGISTRY_VERSION}" +else + fail "Registry agentVersion not updated: got '${REGISTRY_VERSION}', expected != '${OLD_OPENCLAW_VERSION}'" +fi + +# Gateway token rotated and runtime env matches (#4517) +POST_REBUILD_GATEWAY_TOKEN="$(read_sandbox_gateway_token 2>/dev/null || true)" +if [ -n "${POST_REBUILD_GATEWAY_TOKEN}" ]; then + pass "Gateway auth token present after rebuild" +else + fail "Gateway auth token missing after rebuild — issue #4517" +fi +if [ "${POST_REBUILD_GATEWAY_TOKEN}" != "${PRE_REBUILD_GATEWAY_TOKEN}" ]; then + pass "Gateway auth token rotated after rebuild" +else + fail "Gateway auth token did not rotate after rebuild — issue #4517" +fi +RUNTIME_GATEWAY_TOKEN="$(read_sandbox_runtime_gateway_token 2>/dev/null || true)" +if [ "${RUNTIME_GATEWAY_TOKEN}" = "${POST_REBUILD_GATEWAY_TOKEN}" ]; then + pass "Runtime proxy env exports the rotated gateway token" +elif [ "${RUNTIME_GATEWAY_TOKEN}" = "${PRE_REBUILD_GATEWAY_TOKEN}" ]; then + fail "Runtime proxy env still exports the old gateway token — issue #4517" +else + fail "Runtime proxy env gateway token does not match openclaw.json — issue #4517" +fi +POST_REBUILD_CONFIG_HASH="$(read_sandbox_config_hash 2>/dev/null || true)" +if [ -n "${POST_REBUILD_CONFIG_HASH}" ] && echo "${POST_REBUILD_CONFIG_HASH}" | grep -q "openclaw.json"; then + pass "Post-rebuild config hash references openclaw.json" +else + fail "Post-rebuild config hash missing openclaw.json — issue #4517" +fi +if [ "${POST_REBUILD_CONFIG_HASH}" != "${PRE_REBUILD_CONFIG_HASH}" ]; then + pass "Config hash changed after gateway token rotation" +else + fail "Config hash did not change after gateway token rotation — issue #4517" +fi +if openshell sandbox exec --name "${SANDBOX_NAME}" -- \ + sh -c "cd /sandbox/.openclaw && sha256sum -c .config-hash --status" 2>/dev/null; then + pass "Config hash validates after gateway token rotation" +else + fail "Config hash does not validate after gateway token rotation — issue #4517" +fi + +# Inference works after rebuild (proves credential chain is intact) +info "Verifying inference after rebuild..." +POST_REBUILD_INFERENCE_MODEL="${NEMOCLAW_MODEL:-${NEMOCLAW_COMPAT_MODEL:-nvidia/nvidia/nemotron-3-super-v3}}" +INFERENCE_RESPONSE=$(openshell sandbox exec --name "${SANDBOX_NAME}" -- \ + curl -s --max-time 60 https://inference.local/v1/chat/completions \ + -H 'Content-Type: application/json' \ + -d "{\"model\":\"${POST_REBUILD_INFERENCE_MODEL}\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with exactly one word: PONG\"}],\"max_tokens\":100}" \ + 2>&1 || true) +if echo "${INFERENCE_RESPONSE}" | python3 -c "import json,sys; r=json.load(sys.stdin); c=r['choices'][0]['message']; print(c.get('content',''))" 2>/dev/null | grep -qi "PONG"; then + pass "Inference works after rebuild (NVIDIA API key + provider chain intact)" +else + # Non-fatal — inference depends on external API availability + info "Inference check inconclusive (may be API timeout): ${INFERENCE_RESPONSE:0:200}" +fi + +# No credentials in backup +BACKUP_DIR="$HOME/.nemoclaw/rebuild-backups/${SANDBOX_NAME}" +if [ -d "$BACKUP_DIR" ]; then + # Dependency lockfiles can contain public package metadata matching coarse + # token patterns; the product snapshot filter excludes them too. + CRED_LEAKS=$(find "$BACKUP_DIR" \ + \( -name "package-lock.json" -o -name "npm-shrinkwrap.json" -o -name "yarn.lock" -o -name "pnpm-lock.yaml" -o -name "pnpm-lock.yml" \) -prune -o \ + \( -name "*.json" -o -name "*.env" -o -name ".env" \) -type f \ + -exec grep -l "nvapi-\|sk-\|Bearer " {} \; 2>/dev/null || true) + if [ -z "$CRED_LEAKS" ]; then + pass "No credentials in backup" + else + fail "Credentials found: $CRED_LEAKS" + fi + GATEWAY_TOKEN_LEAKS=$(find "$BACKUP_DIR" -type f \ + -exec grep -F -l "${PRE_REBUILD_GATEWAY_TOKEN}" {} \; 2>/dev/null || true) + if [ -z "${GATEWAY_TOKEN_LEAKS}" ]; then + pass "Old gateway token absent from backup" + else + fail "Old gateway token found in backup: ${GATEWAY_TOKEN_LEAKS}" + fi +else + fail "Backup directory missing: $BACKUP_DIR" +fi + +# ── Phase 7b: Verify policy presets survived rebuild (#1952) ──────── +info "Verifying policy presets survived rebuild..." + +# Check registry still has the presets +POST_REBUILD_PRESETS=$(python3 -c " +import json +with open('${REGISTRY_FILE}') as f: + data = json.load(f) +sb = data.get('sandboxes', {}).get('${SANDBOX_NAME}', {}) +print(','.join(sb.get('policies', []))) +" 2>/dev/null || echo "error") +diag "Post-rebuild registry policies: ${POST_REBUILD_PRESETS}" + +if echo "${POST_REBUILD_PRESETS}" | grep -q "npm"; then + pass "npm preset survived rebuild (in registry)" +else + fail "npm preset LOST after rebuild — issue #1952" +fi +if echo "${POST_REBUILD_PRESETS}" | grep -q "pypi"; then + pass "pypi preset survived rebuild (in registry)" +else + fail "pypi preset LOST after rebuild — issue #1952" +fi + +# Check the live gateway policy still has the preset endpoints +POST_REBUILD_POLICY=$(openshell policy get --full "${SANDBOX_NAME}" 2>&1 || true) +if echo "${POST_REBUILD_POLICY}" | grep -qi "npm\|registry.npmjs.org"; then + pass "npm preset active in gateway policy after rebuild" +else + fail "npm preset not in live gateway policy after rebuild — issue #1952" +fi +if echo "${POST_REBUILD_POLICY}" | grep -qi "pypi\|pypi.org"; then + pass "pypi preset active in gateway policy after rebuild" +else + fail "pypi preset not in live gateway policy after rebuild — issue #1952" +fi + +# Check backup manifest recorded the presets +if [ -d "$BACKUP_DIR" ]; then + MANIFEST_PRESETS=$(find "$BACKUP_DIR" -mindepth 1 -maxdepth 1 -type d 2>/dev/null \ + | sort -r | head -1 \ + | xargs -I{} python3 -c " +import json, sys +try: + with open('{}/rebuild-manifest.json') as f: + m = json.load(f) + presets = m.get('policyPresets', []) + print(','.join(presets) if presets else 'NONE') +except Exception as e: + print('ERROR: ' + str(e)) +" 2>/dev/null || echo "error") + if echo "${MANIFEST_PRESETS}" | grep -q "npm" \ + && echo "${MANIFEST_PRESETS}" | grep -q "pypi"; then + pass "Backup manifest contains policyPresets: ${MANIFEST_PRESETS}" + else + fail "Backup manifest missing expected policyPresets (npm,pypi): got '${MANIFEST_PRESETS}' — issue #1952" + fi +fi + +# ── Cleanup ───────────────────────────────────────────────────────── +info "Cleaning up..." +[[ "${NEMOCLAW_E2E_KEEP_SANDBOX:-}" = "1" ]] || nemoclaw "${SANDBOX_NAME}" destroy --yes 2>/dev/null || true +docker rmi "${OLD_BASE_TAG}" 2>/dev/null || true + +echo "" +echo -e "${GREEN}OpenClaw rebuild upgrade E2E passed.${NC}" diff --git a/test/e2e-vpn/test-runtime-overrides.sh b/test/e2e-vpn/test-runtime-overrides.sh new file mode 100755 index 00000000000..b89b437ed77 --- /dev/null +++ b/test/e2e-vpn/test-runtime-overrides.sh @@ -0,0 +1,337 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# E2E test for runtime config overrides (NEMOCLAW_MODEL_OVERRIDE, CORS, etc.). +# Builds the sandbox image once, then runs each override scenario as a short-lived +# container. Each test starts the entrypoint, reads the patched openclaw.json, +# and verifies the expected field changed while other fields are untouched. +# +# Designed for parallel CI execution — no shared state between tests. +# +# Requires: docker, jq +# Usage: bash test/e2e-vpn/test-runtime-overrides.sh + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" +IMAGE="${NEMOCLAW_TEST_IMAGE:-nemoclaw-override-test}" + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +pass() { + echo -e "${GREEN}PASS${NC}: $1" + PASSED=$((PASSED + 1)) +} +fail() { + echo -e "${RED}FAIL${NC}: $1" + FAILED=$((FAILED + 1)) +} +info() { echo -e "${YELLOW}TEST${NC}: $1"; } + +PASSED=0 +FAILED=0 + +# ── Log file for CI artifact collection ────────────────────────── +# Create a timestamped log file whose name matches the CI artifact glob +# test-runtime-overrides-*.log so Docker stderr is captured automatically. +LOG_DIR="${REPO_DIR}" +LOG_FILE="${LOG_DIR}/test-runtime-overrides-$(date +%Y%m%dT%H%M%S).log" +: >"$LOG_FILE" +info "Logging Docker stderr to: $LOG_FILE" + +# Helper: run entrypoint with env vars, then read a config field via jq. +# The entrypoint patches config and starts the gateway — we only need the +# config patch, so we override CMD to just cat the config and exit. +# Docker stderr is captured to the log file for CI artifact visibility. +# +# The entrypoint redirects stdout through a tee process substitution. For +# short-lived one-shot commands, container teardown can race tee's flush to +# Docker stdout; write captured payloads to fd 3 to bypass that pipe (#4924). +run_override() { + local env_args=("$@") + docker run --rm "${env_args[@]}" "$IMAGE" \ + bash -c 'cat /sandbox/.openclaw/openclaw.json >&3; printf "\n" >&3' 2>>"$LOG_FILE" +} + +run_config_hash_check() { + local env_args=("$@") + docker run --rm "${env_args[@]}" "$IMAGE" \ + bash -c 'cd /sandbox/.openclaw && if sha256sum -c .config-hash --status; then printf "OK\n" >&3; else printf "FAIL\n" >&3; fi' 2>>"$LOG_FILE" +} + +valid_config() { + jq -e ' + type == "object" + and (.agents.defaults.model.primary | type == "string" and length > 0) + and (.models.providers | type == "object" and length > 0) + and ((.models.providers | to_entries[0].value.models[0].contextWindow) | type == "number") + and ((.models.providers | to_entries[0].value.models[0].maxTokens) | type == "number") + and ((.models.providers | to_entries[0].value.models[0].reasoning) | type == "boolean") + and (.gateway.controlUi.allowedOrigins | type == "array") + ' >/dev/null || return 1 +} + +capture_config() { + local label="$1" + shift + local cfg="" + local attempt=1 + local run_rc=0 + local valid_rc=0 + + while [ "$attempt" -le 3 ]; do + set +e + cfg=$(run_override "$@") + run_rc=$? + if [ "$run_rc" -eq 0 ]; then + printf '%s' "$cfg" | valid_config 2>>"$LOG_FILE" + valid_rc=$? + else + valid_rc=1 + fi + set -e + + if [ "$run_rc" -eq 0 ] && [ "$valid_rc" -eq 0 ]; then + printf '%s\n' "$cfg" + return 0 + fi + + if [ "$attempt" -lt 3 ]; then + printf '%b\n' "${YELLOW}TEST${NC}: ${label} config capture attempt ${attempt} returned invalid output; retrying" >&2 + sleep "$attempt" + fi + attempt=$((attempt + 1)) + done + + printf '%b\n' "${RED}FAIL${NC}: ${label} config capture failed after 3 attempts; Docker stderr tail follows" >&2 + tail -80 "$LOG_FILE" >&2 || true + return 1 +} + +jq_config() { + local cfg="$1" + local filter="$2" + printf '%s' "$cfg" | jq -r \ + "[$filter] | if length == 0 or .[0] == null then error(\"missing required config field\") else .[0] end" +} + +# Helper: run entrypoint with env vars and capture stderr for validation messages. +run_override_stderr() { + local env_args=("$@") + local tmpfile + tmpfile="$(mktemp)" + docker run --rm "${env_args[@]}" "$IMAGE" \ + bash -c 'true' >/dev/null 2>"$tmpfile" || true + cat "$tmpfile" + # Also append to the main log file for CI artifact capture + cat "$tmpfile" >>"$LOG_FILE" + rm -f "$tmpfile" +} + +# ── Build the image ────────────────────────────────────────────── + +if docker image inspect "$IMAGE" >/dev/null 2>&1; then + info "Using pre-built image: $IMAGE" +else + info "Building test image: $IMAGE" + docker build -t "$IMAGE" -f "$REPO_DIR/Dockerfile" "$REPO_DIR" \ + --build-arg NEMOCLAW_DISABLE_DEVICE_AUTH=1 \ + --build-arg "NEMOCLAW_BUILD_ID=$(date +%s)" \ + --quiet +fi + +# ── Capture baseline config ────────────────────────────────────── + +info "Capturing baseline config (no overrides)" +if ! BASELINE=$(capture_config "baseline"); then + fail "baseline container failed before config capture" + info "Docker stderr tail:" + tail -80 "$LOG_FILE" || true + exit 1 +fi +BASELINE_MODEL=$(jq_config "$BASELINE" '.agents.defaults.model.primary') +BASELINE_CTX=$(jq_config "$BASELINE" '.models.providers | to_entries[0].value.models[0].contextWindow') +BASELINE_MAX=$(jq_config "$BASELINE" '.models.providers | to_entries[0].value.models[0].maxTokens') +BASELINE_REASONING=$(jq_config "$BASELINE" '.models.providers | to_entries[0].value.models[0].reasoning') +BASELINE_ORIGINS=$(jq_config "$BASELINE" '.gateway.controlUi.allowedOrigins | length') + +info "Baseline: model=$BASELINE_MODEL ctx=$BASELINE_CTX max=$BASELINE_MAX reasoning=$BASELINE_REASONING origins=$BASELINE_ORIGINS" + +# ── Test 1: No-op baseline ─────────────────────────────────────── + +info "1. No overrides — config matches build-time defaults" +HASH_CHECK=$(run_config_hash_check) +if [ "$HASH_CHECK" = "OK" ]; then + pass "baseline config hash valid" +else + fail "baseline config hash invalid" +fi + +# ── Test 2: Model override ─────────────────────────────────────── + +info "2. NEMOCLAW_MODEL_OVERRIDE patches model" +OVERRIDE_MODEL="anthropic/claude-sonnet-4-6" +CFG=$(capture_config "model override" -e "NEMOCLAW_MODEL_OVERRIDE=$OVERRIDE_MODEL") +ACTUAL=$(jq_config "$CFG" '.agents.defaults.model.primary') +if [ "$ACTUAL" = "$OVERRIDE_MODEL" ]; then + pass "model overridden to $OVERRIDE_MODEL" +else + fail "expected model=$OVERRIDE_MODEL, got $ACTUAL" +fi + +# Verify hash was recomputed +HASH_CHECK=$(run_config_hash_check -e "NEMOCLAW_MODEL_OVERRIDE=$OVERRIDE_MODEL") +if [ "$HASH_CHECK" = "OK" ]; then + pass "config hash valid after model override" +else + fail "config hash invalid after model override" +fi + +# ── Test 3: Context window override ────────────────────────────── +# NEMOCLAW_CONTEXT_WINDOW only takes effect alongside a model override +# (standalone values are baked at build time). Ref: #2653 Phase 2. + +info "3. NEMOCLAW_CONTEXT_WINDOW patches contextWindow (with model override)" +CFG=$(capture_config "context window override" -e "NEMOCLAW_MODEL_OVERRIDE=$OVERRIDE_MODEL" -e "NEMOCLAW_CONTEXT_WINDOW=32768") +ACTUAL=$(jq_config "$CFG" '.models.providers | to_entries[0].value.models[0].contextWindow') +if [ "$ACTUAL" = "32768" ]; then + pass "contextWindow overridden to 32768" +else + fail "expected contextWindow=32768, got $ACTUAL" +fi + +# ── Test 4: Max tokens override ────────────────────────────────── + +info "4. NEMOCLAW_MAX_TOKENS patches maxTokens (with model override)" +CFG=$(capture_config "max tokens override" -e "NEMOCLAW_MODEL_OVERRIDE=$OVERRIDE_MODEL" -e "NEMOCLAW_MAX_TOKENS=16384") +ACTUAL=$(jq_config "$CFG" '.models.providers | to_entries[0].value.models[0].maxTokens') +if [ "$ACTUAL" = "16384" ]; then + pass "maxTokens overridden to 16384" +else + fail "expected maxTokens=16384, got $ACTUAL" +fi + +# ── Test 5: Reasoning override ─────────────────────────────────── + +info "5. NEMOCLAW_REASONING=true patches reasoning (with model override)" +CFG=$(capture_config "reasoning override" -e "NEMOCLAW_MODEL_OVERRIDE=$OVERRIDE_MODEL" -e "NEMOCLAW_REASONING=true") +ACTUAL=$(jq_config "$CFG" '.models.providers | to_entries[0].value.models[0].reasoning') +if [ "$ACTUAL" = "true" ]; then + pass "reasoning overridden to true" +else + fail "expected reasoning=true, got $ACTUAL" +fi + +# ── Test 6: CORS origin override ───────────────────────────────── + +info "6. NEMOCLAW_CORS_ORIGIN adds to allowedOrigins" +CORS="https://custom.example.com:9999" +CFG=$(capture_config "CORS origin override" -e "NEMOCLAW_CORS_ORIGIN=$CORS") +HAS_ORIGIN=$(echo "$CFG" | jq --arg o "$CORS" '.gateway.controlUi.allowedOrigins | index($o) != null') +NEW_LEN=$(jq_config "$CFG" '.gateway.controlUi.allowedOrigins | length') +if [ "$HAS_ORIGIN" = "true" ] && [ "$NEW_LEN" -gt "$BASELINE_ORIGINS" ]; then + pass "CORS origin added: $CORS" +else + ORIGINS=$(echo "$CFG" | jq -c '.gateway.controlUi.allowedOrigins // []' 2>/dev/null || printf '%s' "$CFG") + fail "CORS origin not found in allowedOrigins: ${ORIGINS}" +fi + +# ── Test 7: Combined overrides ─────────────────────────────────── + +info "7. Multiple overrides applied together" +CFG=$(capture_config "combined overrides" \ + -e "NEMOCLAW_MODEL_OVERRIDE=nvidia/llama-3.3-nemotron-super-49b-v1.5" \ + -e "NEMOCLAW_CONTEXT_WINDOW=65536" \ + -e "NEMOCLAW_MAX_TOKENS=8192" \ + -e "NEMOCLAW_REASONING=true" \ + -e "NEMOCLAW_CORS_ORIGIN=https://multi.example.com") +M=$(jq_config "$CFG" '.agents.defaults.model.primary') +C=$(jq_config "$CFG" '.models.providers | to_entries[0].value.models[0].contextWindow') +T=$(jq_config "$CFG" '.models.providers | to_entries[0].value.models[0].maxTokens') +R=$(jq_config "$CFG" '.models.providers | to_entries[0].value.models[0].reasoning') +O=$(echo "$CFG" | jq --arg o "https://multi.example.com" '.gateway.controlUi.allowedOrigins | index($o) != null') +if [ "$M" = "nvidia/llama-3.3-nemotron-super-49b-v1.5" ] \ + && [ "$C" = "65536" ] && [ "$T" = "8192" ] \ + && [ "$R" = "true" ] && [ "$O" = "true" ]; then + pass "all 5 overrides applied correctly" +else + fail "combined override mismatch: model=$M ctx=$C max=$T reasoning=$R cors=$O" +fi + +# ── Test 8-12: Validation rejections ───────────────────────────── + +info "8. NEMOCLAW_MODEL_OVERRIDE with control chars is rejected" +STDERR=$(run_override_stderr -e $'NEMOCLAW_MODEL_OVERRIDE=bad\x01model') +if echo "$STDERR" | grep -q "control characters"; then + pass "model override with control chars rejected" +else + fail "model override with control chars was not rejected" +fi + +info "9. NEMOCLAW_CONTEXT_WINDOW with non-integer is rejected" +STDERR=$(run_override_stderr -e "NEMOCLAW_MODEL_OVERRIDE=test" -e "NEMOCLAW_CONTEXT_WINDOW=notanumber") +if echo "$STDERR" | grep -q "must be a positive integer"; then + pass "non-integer context window rejected" +else + fail "non-integer context window was not rejected" +fi + +info "10. NEMOCLAW_MAX_TOKENS with non-integer is rejected" +STDERR=$(run_override_stderr -e "NEMOCLAW_MODEL_OVERRIDE=test" -e "NEMOCLAW_MAX_TOKENS=abc") +if echo "$STDERR" | grep -q "must be a positive integer"; then + pass "non-integer max tokens rejected" +else + fail "non-integer max tokens was not rejected" +fi + +info "11. NEMOCLAW_REASONING with invalid value is rejected" +STDERR=$(run_override_stderr -e "NEMOCLAW_MODEL_OVERRIDE=test" -e "NEMOCLAW_REASONING=maybe") +if echo "$STDERR" | grep -q 'must be "true" or "false"'; then + pass "invalid reasoning value rejected" +else + fail "invalid reasoning value was not rejected" +fi + +info "12. NEMOCLAW_CORS_ORIGIN without http/https is rejected" +STDERR=$(run_override_stderr -e "NEMOCLAW_CORS_ORIGIN=ftp://evil.com") +if echo "$STDERR" | grep -q "must start with http"; then + pass "non-http CORS origin rejected" +else + fail "non-http CORS origin was not rejected" +fi + +info "13. NEMOCLAW_INFERENCE_API_OVERRIDE with invalid type is rejected" +STDERR=$(run_override_stderr -e "NEMOCLAW_MODEL_OVERRIDE=test" -e "NEMOCLAW_INFERENCE_API_OVERRIDE=graphql") +if echo "$STDERR" | grep -q "openai-completions"; then + pass "invalid inference API type rejected" +else + fail "invalid inference API type was not rejected" +fi + +# ── Test 14: Original config unchanged after rejected override ─── + +info "14. Config unchanged after rejected override" +CFG=$(capture_config "rejected override" -e "NEMOCLAW_MODEL_OVERRIDE=test" -e "NEMOCLAW_CONTEXT_WINDOW=notanumber") +ACTUAL_CTX=$(jq_config "$CFG" '.models.providers | to_entries[0].value.models[0].contextWindow') +ACTUAL_MODEL=$(jq_config "$CFG" '.agents.defaults.model.primary') +if [ "$ACTUAL_CTX" = "$BASELINE_CTX" ] && [ "$ACTUAL_MODEL" = "$BASELINE_MODEL" ]; then + pass "config unchanged after rejected override" +else + fail "config was modified despite rejected override: model=$ACTUAL_MODEL ctx=$ACTUAL_CTX (expected model=$BASELINE_MODEL ctx=$BASELINE_CTX)" +fi + +# ── Summary ────────────────────────────────────────────────────── + +echo "" +echo "────────────────────────────────────────────────" +echo -e "Results: ${GREEN}${PASSED} passed${NC}, ${RED}${FAILED} failed${NC}" +echo "────────────────────────────────────────────────" + +if [ "$FAILED" -gt 0 ]; then + exit 1 +fi diff --git a/test/e2e-vpn/test-sandbox-operations.sh b/test/e2e-vpn/test-sandbox-operations.sh new file mode 100755 index 00000000000..c7d03fbb4e4 --- /dev/null +++ b/test/e2e-vpn/test-sandbox-operations.sh @@ -0,0 +1,884 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# ============================================================================= +# test-sandbox-operations.sh +# NemoClaw Sandbox Operations E2E Test Suite +# +# Covers: TC-SBX-01 through TC-SBX-11 +# Assumes: NemoClaw is installed, no sandbox is currently onboarded +# +# Test ordering: +# Phase 1 — Basic operations (sandbox A alive) +# Phase 2 — Non-destructive recovery (sandbox A alive) +# Phase 3 — Multi-sandbox (onboards sandbox B alongside A) +# Phase 4 — Cleanup verification (destroys sandbox B) +# Phase 5 — Gateway kill recovery (destructive — runs last) +# ============================================================================= + +set -euo pipefail + +# ── Overall timeout (prevents hung CI jobs) ────────────────────────────────── +export NEMOCLAW_E2E_DEFAULT_TIMEOUT=1800 +SCRIPT_DIR_TIMEOUT="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +# shellcheck source=test/e2e-vpn/e2e-timeout.sh +source "${SCRIPT_DIR_TIMEOUT}/e2e-timeout.sh" +# shellcheck source=test/e2e-vpn/lib/openclaw-json.sh +source "${SCRIPT_DIR_TIMEOUT}/lib/openclaw-json.sh" + +# ── Config ─────────────────────────────────────────────────────────────────── +SANDBOX_A="test-sbx-a" +SANDBOX_B="test-sbx-b" +LOG_FILE="test-sandbox-operations-$(date +%Y%m%d-%H%M%S).log" + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +CYAN='\033[0;36m' +NC='\033[0m' + +# ── Counters ───────────────────────────────────────────────────────────────── +PASS=0 +FAIL=0 +SKIP=0 +TOTAL=0 + +# ── Helpers ────────────────────────────────────────────────────────────────── +log() { echo -e "${CYAN}[$(date +%H:%M:%S)]${NC} $*" | tee -a "$LOG_FILE"; } +pass() { + ((PASS += 1)) + ((TOTAL += 1)) + echo -e "${GREEN} PASS${NC} $1" | tee -a "$LOG_FILE" +} +fail() { + ((FAIL += 1)) + ((TOTAL += 1)) + echo -e "${RED} FAIL${NC} $1 — $2" | tee -a "$LOG_FILE" +} +skip() { + ((SKIP += 1)) + ((TOTAL += 1)) + echo -e "${YELLOW} SKIP${NC} $1 — $2" | tee -a "$LOG_FILE" +} + +# Check that a sandbox is registered; skip the named test case if not. +# Usage: require_sandbox "$SANDBOX_A" "TC-SBX-02" || return +require_sandbox() { + if ! nemoclaw list 2>/dev/null | grep -q "$1"; then + skip "$2" "sandbox '$1' not available" + return 1 + fi + return 0 +} + +# Run a command inside a named sandbox via SSH. Returns the command output. +# Logs warnings on SSH config failure, empty config, timeout, or non-zero exit. +sandbox_exec_for() { + local name="$1" cmd="$2" + local ssh_cfg + ssh_cfg="$(mktemp)" + if ! openshell sandbox ssh-config "$name" >"$ssh_cfg" 2>/dev/null; then + log " [sandbox_exec] Failed to get SSH config for '$name'" + rm -f "$ssh_cfg" + echo "" + return 1 + fi + if [[ ! -s "$ssh_cfg" ]]; then + log " [sandbox_exec] SSH config for '$name' is empty" + rm -f "$ssh_cfg" + echo "" + return 1 + fi + local result exit_code=0 + result=$(run_with_timeout 60 ssh -F "$ssh_cfg" \ + -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 -o LogLevel=ERROR \ + "openshell-${name}" "$cmd" 2>&1) || exit_code=$? + rm -f "$ssh_cfg" + if [[ $exit_code -eq 124 ]]; then + log " [sandbox_exec] SSH command timed out after 60s for '$name'" + elif [[ $exit_code -ne 0 && -z "$result" ]]; then + log " [sandbox_exec] SSH command failed (exit $exit_code) for '$name'" + fi + echo "$result" +} + +# Shorthand: run a command inside sandbox A. +sandbox_exec() { + sandbox_exec_for "$SANDBOX_A" "$1" +} + +is_onboard_import_stream_reset() { + local output_file="$1" + [[ -f "$output_file" ]] || return 1 + + grep -q "Connection reset by peer (os error 104)" "$output_file" \ + && grep -Eq "The image appears to have reached the gateway before the stream failed|Recovery: nemoclaw onboard --resume" "$output_file" +} + +is_transient_onboard_resume_error() { + local output_file="$1" + [[ -f "$output_file" ]] || return 1 + + grep -Eq "Connection reset by peer \(os error 104\)|transport error|gateway unavailable|No active gateway|No gateway metadata found" "$output_file" +} + +resume_onboard_after_import_stream_reset() { + local name="$1" output_file="$2" + if ! is_onboard_import_stream_reset "$output_file"; then + return 1 + fi + + log " [onboard] Image reached gateway but import stream reset; retrying with nemoclaw onboard --resume..." + + local attempt delay resume_exit resume_output + for attempt in 1 2 3; do + rm -f "$HOME/.nemoclaw/onboard.lock" 2>/dev/null || true + resume_exit=0 + resume_output="$(mktemp)" + log " [onboard] Resume attempt ${attempt}/3..." + NEMOCLAW_SANDBOX_NAME="$name" \ + NEMOCLAW_NON_INTERACTIVE=1 \ + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ + nemoclaw onboard --resume --non-interactive --yes-i-accept-third-party-software \ + 2>&1 | tee -a "$LOG_FILE" "$resume_output" || resume_exit=$? + + if [[ $resume_exit -eq 0 ]]; then + rm -f "$resume_output" + return 0 + fi + + log " [onboard] nemoclaw onboard --resume attempt ${attempt}/3 exited with code $resume_exit" + if ((attempt < 3)) && is_transient_onboard_resume_error "$resume_output"; then + delay=$((attempt * 15)) + log " [onboard] Gateway transport still settling; retrying resume in ${delay}s..." + rm -f "$resume_output" + sleep "$delay" + continue + fi + rm -f "$resume_output" + return 1 + done + return 1 +} + +# Onboard a sandbox by name. Removes stale locks, runs nemoclaw onboard in +# non-interactive mode, and returns 0 if the sandbox appears in nemoclaw list. +onboard_sandbox() { + local name="$1" + log " Onboarding sandbox '$name'..." + + # Remove stale lock from previous crashed runs + rm -f "$HOME/.nemoclaw/onboard.lock" 2>/dev/null || true + + local onboard_exit=0 onboard_output + onboard_output="$(mktemp)" + NEMOCLAW_SANDBOX_NAME="$name" \ + NEMOCLAW_NON_INTERACTIVE=1 \ + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ + NEMOCLAW_RECREATE_SANDBOX=1 \ + nemoclaw onboard --non-interactive --yes-i-accept-third-party-software \ + 2>&1 | tee -a "$LOG_FILE" "$onboard_output" || onboard_exit=$? + + if [[ $onboard_exit -ne 0 ]]; then + log " [onboard_sandbox] nemoclaw onboard exited with code $onboard_exit" + if resume_onboard_after_import_stream_reset "$name" "$onboard_output"; then + onboard_exit=0 + else + rm -f "$onboard_output" + return 1 + fi + fi + rm -f "$onboard_output" + + if ! nemoclaw list 2>/dev/null | grep -q "$name"; then + log " [onboard_sandbox] Sandbox '$name' not found in nemoclaw list after onboard" + return 1 + fi + return 0 +} + +# ── Resolve repo root ──────────────────────────────────────────────────────── +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +if [ -f "$SCRIPT_DIR/../../install.sh" ]; then + REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +elif [ -f "./install.sh" ]; then + REPO_ROOT="$(pwd)" +else + echo "ERROR: Cannot find install.sh — run from the repo root or test/e2e-vpn/" + exit 1 +fi + +# ── Install NemoClaw if not present ────────────────────────────────────────── +# Matches the pattern from test-sandbox-survival.sh and test-full-e2e.sh: +# each E2E test installs NemoClaw from source so it runs on a fresh CI runner. +install_nemoclaw() { + if command -v nemoclaw &>/dev/null; then + log "nemoclaw already installed: $(nemoclaw --version 2>/dev/null || echo 'unknown')" + return 0 + fi + + log "=== Installing NemoClaw via install.sh ===" + + local install_exit=0 install_output + install_output="$(mktemp)" + bash "$REPO_ROOT/install.sh" --non-interactive --yes-i-accept-third-party-software \ + 2>&1 | tee -a "$LOG_FILE" "$install_output" || install_exit=$? + + # Source shell profile to pick up PATH changes from install.sh + if [ -f "$HOME/.bashrc" ]; then + # shellcheck source=/dev/null + source "$HOME/.bashrc" 2>/dev/null || true + fi + export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" + if [ -s "$NVM_DIR/nvm.sh" ]; then + # shellcheck source=/dev/null + . "$NVM_DIR/nvm.sh" + fi + if [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]]; then + export PATH="$HOME/.local/bin:$PATH" + fi + + if [[ $install_exit -ne 0 ]]; then + local install_sandbox + install_sandbox="${NEMOCLAW_SANDBOX_NAME:-my-assistant}" + if resume_onboard_after_import_stream_reset "$install_sandbox" "$install_output"; then + install_exit=0 + fi + fi + rm -f "$install_output" + + if [[ $install_exit -ne 0 ]]; then + echo -e "${RED}FATAL: install.sh failed (exit $install_exit)${NC}" + exit 1 + fi + + if ! command -v nemoclaw &>/dev/null; then + echo -e "${RED}FATAL: nemoclaw not found on PATH after install${NC}" + exit 1 + fi + + log "nemoclaw installed: $(nemoclaw --version 2>/dev/null || echo 'unknown')" + + # Destroy the sandbox that install.sh created (we create our own) + local install_sandbox + install_sandbox="${NEMOCLAW_SANDBOX_NAME:-my-assistant}" + if nemoclaw list 2>/dev/null | grep -q "$install_sandbox"; then + log "Destroying install sandbox '$install_sandbox'..." + nemoclaw "$install_sandbox" destroy --yes 2>/dev/null || true + fi +} + +# ── Pre-flight ─────────────────────────────────────────────────────────────── +# Verify prerequisites (Docker, API key), install NemoClaw if needed, and +# clean up leftover sandboxes and stale locks from previous crashed runs. +preflight() { + log "=== Pre-flight checks ===" + + if ! docker info &>/dev/null; then + echo -e "${RED}ERROR: Docker is not running.${NC}" + exit 1 + fi + log "Docker is running" + + if [[ -z "${NVIDIA_API_KEY:-}" && -z "${OPENAI_API_KEY:-}" && -z "${ANTHROPIC_API_KEY:-}" ]]; then + echo -e "${YELLOW}WARNING: No API key detected.${NC}" + fi + + install_nemoclaw + + log "nemoclaw: $(nemoclaw --version 2>/dev/null || echo 'unknown')" + log "openshell: $(openshell --version 2>&1 | head -1 || echo 'unknown')" + log "timeout: $TIMEOUT_CMD" + + # Remove stale onboard lock from previous crashed runs + if [[ -f "$HOME/.nemoclaw/onboard.lock" ]]; then + log "Removing stale onboard lock" + rm -f "$HOME/.nemoclaw/onboard.lock" + fi + + for sb in "$SANDBOX_A" "$SANDBOX_B"; do + if nemoclaw list 2>/dev/null | grep -q "$sb"; then + log "Cleaning up leftover sandbox: $sb" + nemoclaw "$sb" destroy --yes 2>/dev/null || true + fi + done + + log "Pre-flight complete" + echo "" +} + +# ── Setup: Onboard sandbox A ──────────────────────────────────────────────── +# Create the primary test sandbox. Exits the script on failure since all +# subsequent test cases depend on sandbox A being available. +setup_sandbox_a() { + log "=== Setup: Onboarding sandbox '$SANDBOX_A' ===" + log "This may take a few minutes..." + + if ! onboard_sandbox "$SANDBOX_A"; then + echo -e "${RED}FATAL: Onboard failed — sandbox '$SANDBOX_A' not found.${NC}" + exit 1 + fi + + log "Sandbox '$SANDBOX_A' onboarded successfully" + echo "" +} + +# ============================================================================= +# Phase 1: Basic operations (sandbox A alive) +# ============================================================================= + +# ── TC-SBX-01: List Sandboxes ─────────────────────────────────────────────── +test_sbx_01_list_sandboxes() { + log "=== TC-SBX-01: List Sandboxes ===" + + local output + output=$(nemoclaw list 2>&1) + + if echo "$output" | grep -q "$SANDBOX_A"; then + pass "TC-SBX-01: nemoclaw list shows '$SANDBOX_A'" + else + fail "TC-SBX-01: List Sandboxes" "'$SANDBOX_A' not found in nemoclaw list output" + fi +} + +# ── TC-SBX-02: Connect & Chat ─────────────────────────────────────────────── +# Drives one openclaw-mediated turn through the sandbox and asserts the +# model produced a real answer. Three properties keep this honest: +# +# 1. Uses `openclaw agent --json`, which calls routeLogsToStderr() in +# openclaw/src/commands/agent-via-gateway.ts:57 so stdout is a clean +# JSON envelope. Merged stdout/stderr is preserved for failure +# diagnostics, but assertions only read JSON payload text. +# 2. The expected token (the integer 42) is not a literal substring of +# the prompt, so an error path that quoted the prompt back cannot +# false-positive the grep — which is what masked the openclaw 4.9 +# SSRF regression from the prior `Say exactly: HELLO_E2E` assertion. +# 3. Asserts on parsed model reply text from the JSON envelope, not on +# merged stdout/stderr or a single brittle envelope shape. +# 4. Relies on generated `thinkingDefault: off` config so the first-turn +# smoke contract is not delayed by model-catalog inferred reasoning +# defaults without depending on transient CLI flags. +test_sbx_02_connect_chat() { + log "=== TC-SBX-02: Connect & Chat ===" + require_sandbox "$SANDBOX_A" "TC-SBX-02" || return + + log " Sending one-shot message to agent via SSH (openclaw agent --json)..." + local session_id raw ssh_cfg rc + session_id="e2e-sbx-02-$(date +%s)-$$" + # Use a direct ssh invocation rather than sandbox_exec() so the JSON envelope + # is easy to parse while still preserving stderr in failure output. + ssh_cfg="$(mktemp)" + if ! openshell sandbox ssh-config "$SANDBOX_A" >"$ssh_cfg" 2>/dev/null; then + rm -f "$ssh_cfg" + fail "TC-SBX-02: Connect & Chat" "Failed to fetch SSH config for '$SANDBOX_A'" + return + fi + rc=0 + raw=$(run_with_timeout 90 ssh -F "$ssh_cfg" \ + -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 -o LogLevel=ERROR \ + "openshell-${SANDBOX_A}" \ + "openclaw agent --agent main --json --session-id '${session_id}' -m 'What is 6 multiplied by 7? Reply with only the integer, no extra words.'" \ + 2>&1) || rc=$? + rm -f "$ssh_cfg" + + local reply + reply=$(printf '%s' "$raw" | parse_openclaw_agent_text 2>/dev/null) || true + + if [[ $rc -eq 0 && -n "$reply" ]] && echo "$reply" | grep -qE "(^|[^0-9])42([^0-9]|$)"; then + pass "TC-SBX-02: Agent computed 6×7=42 through openclaw → inference.local" + else + fail "TC-SBX-02: Connect & Chat" "Expected '42' in agent reply (rc=$rc); reply='${reply:0:200}'; raw output='${raw:0:200}'" + fi +} + +# ── TC-SBX-03: Status Fields ──────────────────────────────────────────────── +test_sbx_03_status_fields() { + log "=== TC-SBX-03: Status Fields ===" + require_sandbox "$SANDBOX_A" "TC-SBX-03" || return + + local output + output=$(nemoclaw "$SANDBOX_A" status 2>&1) + + local all_good=true + for field in "Sandbox" "Model" "Provider" "GPU"; do + if echo "$output" | grep -qi "$field"; then + log " Found field: $field" + else + log " MISSING field: $field" + all_good=false + fi + done + + if $all_good; then + pass "TC-SBX-03: Status output contains all expected fields" + else + fail "TC-SBX-03: Status Fields" "Missing expected fields. Output: $(echo "$output" | head -10)" + fi +} + +# ── TC-SBX-04: Log Streaming ──────────────────────────────────────────────── +test_sbx_04_log_streaming() { + log "=== TC-SBX-04: Log Streaming ===" + require_sandbox "$SANDBOX_A" "TC-SBX-04" || return + + local output logs_exit=0 + output=$(run_with_timeout 10 nemoclaw "$SANDBOX_A" logs 2>&1) || logs_exit=$? + + if [[ $logs_exit -ne 0 ]]; then + fail "TC-SBX-04: Log Streaming" "nemoclaw logs exited with code $logs_exit" + elif [[ -n "$output" ]]; then + pass "TC-SBX-04: Log streaming produced output ($(echo "$output" | wc -l | tr -d ' ') lines)" + else + fail "TC-SBX-04: Log Streaming" "nemoclaw logs succeeded but produced no output" + fi + + run_with_timeout 5 nemoclaw "$SANDBOX_A" logs --follow &>/dev/null & + local pid=$! + sleep 3 + + if ! ps -p "$pid" &>/dev/null; then + fail "TC-SBX-04: Log --follow" "Process exited before kill (was not streaming)" + else + kill "$pid" 2>/dev/null || true + wait "$pid" 2>/dev/null || true + if ps -p "$pid" &>/dev/null; then + fail "TC-SBX-04: Log --follow cleanup" "Orphaned log process still running" + else + pass "TC-SBX-04: Log --follow exited cleanly after kill" + fi + fi +} + +# ── TC-SBX-09: Tmux Session Flow ──────────────────────────────────────────── +# OpenClaw's bundled tmux-session flow shells out to `tmux` inside the sandbox. +# The sandbox image must ship tmux (issue #4513) AND the sandbox landlock policy +# must grant the devpts PTY devices so tmux can actually allocate a window. +# +# History: #4606 installed tmux but the lifecycle drive then failed with +# `create window failed: fork failed: Permission denied`; #4640 degraded that +# branch to a soft skip. The real cause was landlock denying /dev/ptmx + +# /dev/pts (EACCES on forkpty()), not a fork/seccomp/nproc limit. The base +# policy now grants /dev/pts, so this drive MUST pass — a `fork failed` here is +# a hard regression of #4513, never a skip. +test_sbx_09_tmux_session_flow() { + log "=== TC-SBX-09: Tmux Session Flow ===" + require_sandbox "$SANDBOX_A" "TC-SBX-09" || return + + local which_out + which_out=$(sandbox_exec "command -v tmux || echo TMUX_MISSING" 2>&1) || true + if echo "$which_out" | grep -q "TMUX_MISSING"; then + fail "TC-SBX-09: Tmux Session Flow" "tmux not found inside sandbox (issue #4513)" + return + fi + pass "TC-SBX-09: tmux is installed in the sandbox ($(echo "$which_out" | head -1))" + + # Pin the #4513 root cause directly: PTY allocation must succeed. This opens + # /dev/ptmx and a /dev/pts/ slave the same way tmux's forkpty() does, and + # fails fast with the underlying EACCES if the devpts grant ever regresses. + # Guarded on python3 (a diagnostic) so a missing interpreter cannot be + # mistaken for a devpts regression — the tmux lifecycle below is the gate. + # The PTY_OK sentinel is emitted by a shell `&& echo` only after openpty() + # exits 0 — it is deliberately NOT a literal inside the python source, because + # a Python traceback echoes the failing source line and would otherwise make + # `grep PTY_OK` match on the very EACCES regression this probe guards against. + # PY3_MISSING is gated solely on `command -v`, so an openpty() failure with + # python3 present falls through to `fail`, never to the soft skip. + local pty_out + pty_out=$(sandbox_exec "if command -v python3 >/dev/null 2>&1; then \ + python3 -c 'import os; _,s=os.openpty(); print(os.ttyname(s))' && echo PTY_OK; \ + else echo PY3_MISSING; fi" 2>&1) || true + if echo "$pty_out" | grep -q "PTY_OK"; then + pass "TC-SBX-09: PTY allocation works (slave $(echo "$pty_out" | grep -E '^/dev/pts/' | head -1))" + elif echo "$pty_out" | grep -q "PY3_MISSING"; then + log "TC-SBX-09: python3 unavailable; skipping direct openpty() probe (tmux lifecycle still gates devpts)" + else + fail "TC-SBX-09: PTY allocation" "openpty() failed — devpts not granted by sandbox policy (#4513): $(echo "$pty_out" | head -3)" + fi + + # Drive a detached session lifecycle the way the bundled flow does. tmux needs + # a writable socket dir; /tmp is on the sandbox write set. + local sess="nemoclaw-e2e-tmux-$$" + local flow_out + flow_out=$(sandbox_exec "TMUX_TMPDIR=/tmp tmux new-session -d -s '${sess}' 'sleep 30' \ + && TMUX_TMPDIR=/tmp tmux list-sessions \ + && TMUX_TMPDIR=/tmp tmux kill-session -t '${sess}' \ + && echo TMUX_FLOW_OK" 2>&1) || true + + if echo "$flow_out" | grep -q "TMUX_FLOW_OK" && echo "$flow_out" | grep -q "${sess}"; then + pass "TC-SBX-09: tmux new/list/kill session lifecycle works" + else + # Best-effort cleanup in case kill-session never ran. A `fork failed` + # message here means the devpts grant regressed — fail loudly (#4513), + # do not skip. + sandbox_exec "TMUX_TMPDIR=/tmp tmux kill-session -t '${sess}' 2>/dev/null || true" >/dev/null 2>&1 || true + fail "TC-SBX-09: Tmux Session Flow" "Session lifecycle failed: $(echo "$flow_out" | head -5)" + fi +} + +# ============================================================================= +# Phase 2: Non-destructive recovery (sandbox A stays alive) +# ============================================================================= + +# ── TC-SBX-07: Registry Rebuild ───────────────────────────────────────────── +test_sbx_07_registry_rebuild() { + log "=== TC-SBX-07: Registry Rebuild ===" + require_sandbox "$SANDBOX_A" "TC-SBX-07" || return + + local registry="$HOME/.nemoclaw/sandboxes.json" + if [[ ! -f "$registry" ]]; then + skip "TC-SBX-07" "sandboxes.json not found" + return + fi + + cp "$registry" "${registry}.bak" + log " Backed up and deleted sandboxes.json" + rm -f "$registry" + + local output + output=$(run_with_timeout 60 nemoclaw list 2>&1) || true + + if echo "$output" | grep -q "$SANDBOX_A"; then + pass "TC-SBX-07: Registry rebuilt — '$SANDBOX_A' found after deletion" + rm -f "${registry}.bak" + else + fail "TC-SBX-07: Registry Rebuild" "Not found after rebuild. Restoring backup." + mv "${registry}.bak" "$registry" + fi +} + +# ── TC-SBX-08: Process Recovery ───────────────────────────────────────────── +test_sbx_08_process_recovery() { + log "=== TC-SBX-08: Process Recovery ===" + require_sandbox "$SANDBOX_A" "TC-SBX-08" || return + + log " Killing OpenClaw gateway process inside sandbox..." + local kill_output + kill_output=$(sandbox_exec "pkill -9 -f 'openclaw gateway' 2>/dev/null || kill -9 \$(pgrep -f 'openclaw gateway') 2>/dev/null || kill -9 \$(ps aux | grep 'openclaw.*gateway' | grep -v grep | awk '{print \$2}') 2>/dev/null; echo EXIT_\$?" 2>&1) || true + + if echo "$kill_output" | grep -q "EXIT_0"; then + log " Process kill confirmed" + else + log " WARNING: Could not confirm process was killed (output: $kill_output)" + fi + sleep 5 + + log " Running nemoclaw status (expect process recovery)..." + local status_output status_exit=0 + status_output=$(run_with_timeout 120 nemoclaw "$SANDBOX_A" status 2>&1) || status_exit=$? + + if [[ $status_exit -ne 0 ]]; then + fail "TC-SBX-08: Process Recovery (status)" "nemoclaw status exited with code $status_exit" + elif echo "$status_output" | grep -qiE "recover|running|healthy|OpenClaw"; then + pass "TC-SBX-08: Status detected and recovered dead OpenClaw process" + else + fail "TC-SBX-08: Process Recovery (status)" "Output: $(echo "$status_output" | head -5)" + fi + + log " Verifying SSH still works..." + local check + check=$(sandbox_exec "echo process-recovery-ok" 2>&1) || true + if echo "$check" | grep -q "process-recovery-ok"; then + pass "TC-SBX-08: SSH works after process recovery" + else + fail "TC-SBX-08: Process Recovery (SSH)" "Cannot SSH after recovery" + fi +} + +# ── TC-SBX-05: Destroy Cleanup ────────────────────────────────────────────── +test_sbx_05_destroy_cleanup() { + log "=== TC-SBX-05: Destroy Cleanup ===" + local target="$1" + + if ! nemoclaw list 2>/dev/null | grep -q "$target"; then + skip "TC-SBX-05" "Sandbox '$target' not present" + return + fi + + log " Destroying sandbox '$target'..." + local destroy_exit=0 + nemoclaw "$target" destroy --yes 2>&1 | tee -a "$LOG_FILE" || destroy_exit=$? + + if [[ $destroy_exit -ne 0 ]]; then + fail "TC-SBX-05: Destroy ($target)" "nemoclaw destroy exited with code $destroy_exit" + fi + + if nemoclaw list 2>/dev/null | grep -q "$target"; then + fail "TC-SBX-05: Destroy ($target)" "Still in nemoclaw list after destroy (exit $destroy_exit)" + else + pass "TC-SBX-05: '$target' removed from nemoclaw list" + fi + + if openshell sandbox list 2>/dev/null | grep -q "$target"; then + fail "TC-SBX-05: Destroy ($target)" "Still in openshell sandbox list after destroy" + else + pass "TC-SBX-05: '$target' removed from openshell sandbox list" + fi +} + +# ============================================================================= +# Phase 5: Gateway kill recovery (destructive — runs last) +# ============================================================================= + +test_sbx_06_gateway_recovery() { + log "=== TC-SBX-06: Gateway Auto-Recovery ===" + require_sandbox "$SANDBOX_A" "TC-SBX-06" || return + + local container="openshell-cluster-nemoclaw" + if ! docker ps -q --filter "name=$container" | grep -q .; then + skip "TC-SBX-06" "Gateway container '$container' not running" + return + fi + + log " Killing gateway container (simulates Docker crash)..." + docker kill "$container" 2>/dev/null || true + sleep 5 + + local container_state + container_state=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "removed") + log " Container state after kill: $container_state" + if [[ "$container_state" == "true" ]]; then + skip "TC-SBX-06" "Container still running after docker kill" + return + fi + + local status_output + status_output=$(mktemp /tmp/sbx06-status-output.XXXXXX) + + log " Running nemoclaw status in background..." + nemoclaw "$SANDBOX_A" status >"$status_output" 2>&1 & + local status_pid=$! + + local recovered=false + local docker_restarted=false + for i in $(seq 1 40); do + sleep 15 + local cstate + cstate=$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || echo "removed") + [[ "$cstate" == "true" ]] && docker_restarted=true + + if ! kill -0 "$status_pid" 2>/dev/null; then + local exit_code=0 + wait "$status_pid" 2>/dev/null || exit_code=$? + log " nemoclaw status exited with code $exit_code after $((i * 15))s" + if [[ $exit_code -eq 0 ]]; then + recovered=true + fi + break + fi + log " [${i}] +$((i * 15))s | container: $cstate" + done + + if kill -0 "$status_pid" 2>/dev/null; then + log " nemoclaw status still running after 10 min — killing" + kill "$status_pid" 2>/dev/null || true + wait "$status_pid" 2>/dev/null || true + fi + + log " Output:" + head -20 "$status_output" 2>/dev/null | while IFS= read -r line; do log " $line"; done + rm -f "$status_output" + + if $recovered; then + pass "TC-SBX-06: Gateway recovered after docker kill" + elif ! $docker_restarted; then + skip "TC-SBX-06" "Docker did not restart gateway container on this runner" + else + fail "TC-SBX-06: Gateway Recovery" "nemoclaw status did not recover the gateway" + fi +} + +# ============================================================================= +# Phase 3: Multi-sandbox (onboards sandbox B alongside A) +# ============================================================================= + +test_sbx_10_multi_sandbox_metadata() { + log "=== TC-SBX-10: Multi-Sandbox Metadata ===" + require_sandbox "$SANDBOX_A" "TC-SBX-10" || return + + log " Onboarding second sandbox '$SANDBOX_B'..." + if ! CHAT_UI_URL="http://127.0.0.1:18790" onboard_sandbox "$SANDBOX_B"; then + fail "TC-SBX-10: Multi-Sandbox" "Sandbox '$SANDBOX_B' failed to onboard" + return + fi + + local output + output=$(nemoclaw list 2>&1) + + local found_a=false found_b=false + echo "$output" | grep -q "$SANDBOX_A" && found_a=true + echo "$output" | grep -q "$SANDBOX_B" && found_b=true + + if $found_a && $found_b; then + pass "TC-SBX-10: Both sandboxes visible in nemoclaw list" + else + fail "TC-SBX-10: Multi-Sandbox" "Missing sandbox (A=$found_a, B=$found_b)" + return + fi + + local meta_ok=true + for sb in "$SANDBOX_A" "$SANDBOX_B"; do + local sb_meta + sb_meta=$(echo "$output" | grep -A1 "$sb" | tail -1) + if [[ -z "$sb_meta" ]] || ! echo "$sb_meta" | grep -q "model:"; then + log " $sb: metadata line missing or no model field" + meta_ok=false + elif echo "$sb_meta" | grep -q "model: unknown"; then + log " $sb: model is unknown" + meta_ok=false + fi + if [[ -z "$sb_meta" ]] || ! echo "$sb_meta" | grep -q "provider:"; then + log " $sb: metadata line missing or no provider field" + meta_ok=false + elif echo "$sb_meta" | grep -q "provider: unknown"; then + log " $sb: provider is unknown" + meta_ok=false + fi + done + + if $meta_ok; then + pass "TC-SBX-10: Both sandboxes have non-empty metadata" + else + fail "TC-SBX-10: Multi-Sandbox Metadata" "One or more sandboxes have unknown model/provider" + fi +} + +test_sbx_11_network_isolation() { + log "=== TC-SBX-11: Sandbox Network Isolation ===" + require_sandbox "$SANDBOX_A" "TC-SBX-11" || return + require_sandbox "$SANDBOX_B" "TC-SBX-11" || return + + # Use node (always available) instead of curl (removed by hardening). + # Isolation is enforced by the OpenShell proxy — blocked requests return + # HTTP 403. Connection errors (ENOTFOUND, ECONNREFUSED, TIMEOUT) also + # count as isolation. Only HTTP 200 would indicate a breach. + log " Testing: sandbox A cannot reach sandbox B by hostname..." + local probe_a + probe_a=$(sandbox_exec_for "$SANDBOX_A" "node -e \" +const http = require('http'); +const req = http.get('http://${SANDBOX_B}:18789/', (res) => { + console.log('STATUS_' + res.statusCode); + res.resume(); +}); +req.on('error', (e) => console.log('ERROR: ' + e.message)); +req.setTimeout(5000, () => { req.destroy(); console.log('TIMEOUT'); }); +\"" 2>&1) || true + + if [[ -z "$probe_a" ]]; then + fail "TC-SBX-11: Isolation (A→B)" "Empty response — SSH or infrastructure failure" + elif echo "$probe_a" | grep -qiE "STATUS_403|ERROR|TIMEOUT"; then + pass "TC-SBX-11: Sandbox A cannot reach sandbox B ($(echo "$probe_a" | grep -oE 'STATUS_[0-9]+|ERROR|TIMEOUT' | head -1))" + elif echo "$probe_a" | grep -qE "STATUS_[0-9]+"; then + fail "TC-SBX-11: Isolation (A→B)" "Sandbox A reached sandbox B ($(echo "$probe_a" | grep -oE 'STATUS_[0-9]+' | head -1))" + else + fail "TC-SBX-11: Isolation (A→B)" "Unexpected probe output: $(echo "$probe_a" | head -3)" + fi + + log " Testing reverse: sandbox B cannot reach sandbox A..." + local probe_b + probe_b=$(sandbox_exec_for "$SANDBOX_B" "node -e \" +const http = require('http'); +const req = http.get('http://${SANDBOX_A}:18789/', (res) => { + console.log('STATUS_' + res.statusCode); + res.resume(); +}); +req.on('error', (e) => console.log('ERROR: ' + e.message)); +req.setTimeout(5000, () => { req.destroy(); console.log('TIMEOUT'); }); +\"" 2>&1) || true + + if [[ -z "$probe_b" ]]; then + fail "TC-SBX-11: Isolation (B→A)" "Empty response — SSH or infrastructure failure" + elif echo "$probe_b" | grep -qiE "STATUS_403|ERROR|TIMEOUT"; then + pass "TC-SBX-11: Sandbox B cannot reach sandbox A ($(echo "$probe_b" | grep -oE 'STATUS_[0-9]+|ERROR|TIMEOUT' | head -1))" + elif echo "$probe_b" | grep -qE "STATUS_[0-9]+"; then + fail "TC-SBX-11: Isolation (B→A)" "Sandbox B reached sandbox A ($(echo "$probe_b" | grep -oE 'STATUS_[0-9]+' | head -1))" + else + fail "TC-SBX-11: Isolation (B→A)" "Unexpected probe output: $(echo "$probe_b" | head -3)" + fi +} + +# ── Teardown ───────────────────────────────────────────────────────────────── +teardown() { + # Disable errexit during teardown — cleanup must be best-effort + set +e + log "" + log "=== Teardown ===" + for sb in "$SANDBOX_B" "$SANDBOX_A"; do + if nemoclaw list 2>/dev/null | grep -q "$sb"; then + log "Destroying sandbox '$sb'..." + nemoclaw "$sb" destroy --yes 2>/dev/null || true + fi + done + # Clean up gateway if no sandboxes remain + openshell gateway destroy -g nemoclaw 2>/dev/null || true + # Do not unlink ~/.nemoclaw/onboard.lock: see rationale in + # test/e2e-vpn/lib/sandbox-teardown.sh — the lock is PID-ownership-aware + # and onboard cleans up stale locks itself. + log "Teardown complete" + set -e +} + +# ── Summary ────────────────────────────────────────────────────────────────── +summary() { + echo "" + echo "============================================================" + echo " TEST SUMMARY" + echo "============================================================" + echo -e " ${GREEN}PASS: $PASS${NC}" + echo -e " ${RED}FAIL: $FAIL${NC}" + echo -e " ${YELLOW}SKIP: $SKIP${NC}" + echo " TOTAL: $TOTAL" + echo "============================================================" + echo " Log: $LOG_FILE" + echo "============================================================" + echo "" + + if [[ $FAIL -gt 0 ]]; then + exit 1 + fi + exit 0 +} + +# ── Main ───────────────────────────────────────────────────────────────────── +main() { + echo "" + echo "============================================================" + echo " NemoClaw Sandbox Operations E2E Test Suite" + echo " $(date)" + echo "============================================================" + echo "" + + preflight + setup_sandbox_a + + # Phase 1: Basic operations (sandbox A alive) + test_sbx_01_list_sandboxes + test_sbx_02_connect_chat + test_sbx_03_status_fields + test_sbx_04_log_streaming + test_sbx_09_tmux_session_flow + + # Phase 2: Non-destructive recovery (sandbox A stays alive) + test_sbx_07_registry_rebuild + test_sbx_08_process_recovery + + # Phase 3: Multi-sandbox (onboards sandbox B alongside A) + test_sbx_10_multi_sandbox_metadata + test_sbx_11_network_isolation + + # Phase 4: Cleanup verification (destroys sandbox B) + test_sbx_05_destroy_cleanup "$SANDBOX_B" + + # Phase 5: Gateway kill recovery (destructive — runs last) + test_sbx_06_gateway_recovery + + # Report — teardown runs via EXIT trap, no need to call explicitly + trap - EXIT + teardown + summary +} + +trap teardown EXIT +main "$@" diff --git a/test/e2e-vpn/test-sandbox-rebuild.sh b/test/e2e-vpn/test-sandbox-rebuild.sh new file mode 100755 index 00000000000..62e8f9750bc --- /dev/null +++ b/test/e2e-vpn/test-sandbox-rebuild.sh @@ -0,0 +1,197 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Sandbox rebuild — end-to-end proof. +# +# Validates the rebuild lifecycle from NVBug 6076156: +# 1. Version detection: nemoclaw status shows agent version +# 2. Staleness warning: connect warns when sandbox version < expected +# 3. Rebuild preserves state: marker files survive backup→destroy→create→restore +# 4. Rebuild aborts safely when backup fails (sandbox not running) +# 5. Credential stripping: API keys are removed from local backups +# 6. Registry updated: agentVersion reflects new version after rebuild +# +# Prerequisites: +# - Docker running +# - NVIDIA_API_KEY set (real key, starts with nvapi-) +# - Network access to inference.nvidia.com +# +# Environment variables: +# NEMOCLAW_NON_INTERACTIVE=1 — required +# NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 — required +# NVIDIA_API_KEY — required +# NEMOCLAW_SANDBOX_NAME — sandbox name (default: e2e-rebuild) +# NEMOCLAW_E2E_TIMEOUT_SECONDS — overall timeout (default: 1200) +# +# Usage: +# NEMOCLAW_NON_INTERACTIVE=1 \ +# NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ +# NVIDIA_API_KEY=nvapi-... \ +# bash test/e2e-vpn/test-sandbox-rebuild.sh + +set -euo pipefail + +# ── Config ────────────────────────────────────────────────────────── +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-rebuild}" + +# shellcheck source=test/e2e-vpn/lib/sandbox-teardown.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib/sandbox-teardown.sh" +register_sandbox_for_teardown "$SANDBOX_NAME" + +TIMEOUT="${NEMOCLAW_E2E_TIMEOUT_SECONDS:-1200}" +MARKER_FILE="/sandbox/.openclaw/workspace/rebuild-marker.txt" +MARKER_CONTENT="REBUILD_E2E_$(date +%s)" +REGISTRY_FILE="$HOME/.nemoclaw/sandboxes.json" + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +pass() { echo -e "${GREEN}[PASS]${NC} $1"; } +fail() { + echo -e "${RED}[FAIL]${NC} $1" >&2 + exit 1 +} +info() { echo -e "${YELLOW}[INFO]${NC} $1"; } + +# ── Preflight ─────────────────────────────────────────────────────── +[ -n "${NVIDIA_API_KEY:-}" ] || fail "NVIDIA_API_KEY is required" +[ "${NEMOCLAW_NON_INTERACTIVE:-}" = "1" ] || fail "NEMOCLAW_NON_INTERACTIVE=1 is required" + +info "Starting rebuild E2E test (sandbox: ${SANDBOX_NAME}, timeout: ${TIMEOUT}s)" + +# ── Step 1: Create sandbox via onboard ────────────────────────────── +info "Step 1: Creating sandbox via onboard..." + +export NEMOCLAW_NON_INTERACTIVE=1 +export NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 +export NEMOCLAW_RECREATE_SANDBOX=1 + +# Use a timeout wrapper for the full test +timeout_cmd() { + if command -v timeout >/dev/null 2>&1; then + timeout "$TIMEOUT" "$@" + else + "$@" + fi +} + +nemoclaw onboard \ + --sandbox-name "$SANDBOX_NAME" \ + --non-interactive \ + --accept-third-party-software \ + --recreate-sandbox \ + || fail "Onboard failed" + +pass "Sandbox created" + +# ── Step 2: Verify version shows in status ────────────────────────── +info "Step 2: Checking version detection in status..." + +STATUS_OUTPUT=$(nemoclaw "$SANDBOX_NAME" status 2>&1 || true) +if echo "$STATUS_OUTPUT" | grep -qiE "Agent:.*v[0-9]+\.[0-9]+"; then + pass "Version detection: agent version visible in status" +else + info "Status output: $STATUS_OUTPUT" + info "Version may not be cached yet (first run) — acceptable" +fi + +# ── Step 3: Write marker files into sandbox ───────────────────────── +info "Step 3: Writing marker files into sandbox workspace..." + +openshell sandbox exec --name "$SANDBOX_NAME" -- \ + sh -c "mkdir -p /sandbox/.openclaw/workspace && echo '${MARKER_CONTENT}' > ${MARKER_FILE}" \ + || fail "Failed to write marker file" + +# Verify the marker file was written +VERIFY=$(openshell sandbox exec --name "$SANDBOX_NAME" -- cat "$MARKER_FILE" 2>/dev/null || true) +[ "$VERIFY" = "$MARKER_CONTENT" ] || fail "Marker file verification failed: got '$VERIFY'" + +pass "Marker file written and verified" + +# ── Step 4: Simulate staleness and check warning ──────────────────── +info "Step 4: Simulating stale version in registry..." + +# Patch the registry to set an old agentVersion +python3 -c " +import json, sys +with open('$REGISTRY_FILE') as f: + data = json.load(f) +if '$SANDBOX_NAME' in data.get('sandboxes', {}): + data['sandboxes']['$SANDBOX_NAME']['agentVersion'] = '0.0.1' + with open('$REGISTRY_FILE', 'w') as f: + json.dump(data, f, indent=2) + print('Patched agentVersion to 0.0.1') +else: + print('Sandbox not found in registry', file=sys.stderr) + sys.exit(1) +" + +# Check that connect warns about staleness (use timeout to avoid blocking on shell) +CONNECT_OUTPUT=$(timeout 10 nemoclaw "$SANDBOX_NAME" connect <<<"exit" 2>&1 || true) +if echo "$CONNECT_OUTPUT" | grep -qi "rebuild"; then + pass "Staleness warning appears on connect" +else + info "Connect output: $CONNECT_OUTPUT" + info "Warning may not appear if sandbox is not live — acceptable for CI" +fi + +# ── Step 5: Run rebuild ───────────────────────────────────────────── +info "Step 5: Running rebuild..." + +nemoclaw "$SANDBOX_NAME" rebuild --yes \ + || fail "Rebuild failed" + +pass "Rebuild completed" + +# ── Step 6: Verify marker files survived ──────────────────────────── +info "Step 6: Verifying marker files survived rebuild..." + +RESTORED=$(openshell sandbox exec --name "$SANDBOX_NAME" -- cat "$MARKER_FILE" 2>/dev/null || true) +if [ "$RESTORED" = "$MARKER_CONTENT" ]; then + pass "Marker file survived rebuild" +else + fail "Marker file missing or changed after rebuild: got '$RESTORED', expected '$MARKER_CONTENT'" +fi + +# ── Step 7: Verify registry updated ──────────────────────────────── +info "Step 7: Checking registry has updated agentVersion..." + +REGISTRY_VERSION=$(python3 -c " +import json +with open('$REGISTRY_FILE') as f: + data = json.load(f) +sb = data.get('sandboxes', {}).get('$SANDBOX_NAME', {}) +print(sb.get('agentVersion', 'null')) +" 2>/dev/null || echo "error") + +if [ "$REGISTRY_VERSION" != "null" ] && [ "$REGISTRY_VERSION" != "0.0.1" ] && [ "$REGISTRY_VERSION" != "error" ]; then + pass "Registry agentVersion updated to $REGISTRY_VERSION" +else + fail "Registry agentVersion not updated: got '$REGISTRY_VERSION'" +fi + +# ── Step 8: Verify no credentials in backup ───────────────────────── +info "Step 8: Checking backup directory for leaked credentials..." + +BACKUP_DIR="$HOME/.nemoclaw/rebuild-backups/$SANDBOX_NAME" +if [ -d "$BACKUP_DIR" ]; then + # Search for common credential patterns in JSON files + CRED_LEAKS=$(find "$BACKUP_DIR" -name "*.json" -exec grep -l "nvapi-\|sk-\|Bearer " {} \; 2>/dev/null || true) + if [ -z "$CRED_LEAKS" ]; then + pass "No credentials found in backup directory" + else + fail "Credentials found in backup files: $CRED_LEAKS" + fi +else + info "No backup directory found (may have been cleaned up) — skipping" +fi + +# ── Cleanup ───────────────────────────────────────────────────────── +info "Cleaning up..." +[[ "${NEMOCLAW_E2E_KEEP_SANDBOX:-}" = "1" ]] || nemoclaw "$SANDBOX_NAME" destroy --yes 2>/dev/null || true + +echo "" +echo -e "${GREEN}All rebuild E2E tests passed.${NC}" diff --git a/test/e2e-vpn/test-sandbox-survival.sh b/test/e2e-vpn/test-sandbox-survival.sh new file mode 100755 index 00000000000..e326c2859fd --- /dev/null +++ b/test/e2e-vpn/test-sandbox-survival.sh @@ -0,0 +1,796 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Sandbox survival across gateway restart — end-to-end proof. +# +# Validates EVERY complaint from NVIDIA/NemoClaw#486, #888, #859, #1086: +# 1. Sandbox is discoverable after restart (not "No sandboxes registered") +# 2. SSH connectivity resumes (no handshake verification failure) +# 3. Workspace files in /sandbox/ persist +# 4. OpenClaw agent data persists (/sandbox/.openclaw/) +# 5. No re-onboard required (nemoclaw status/connect work) +# 6. Live inference works end-to-end after restart +# 7. NemoClaw registry retains sandbox entry +# 8. Gateway stop/start is non-destructive +# +# This test uses NemoClaw's own install.sh to set up everything including +# OpenShell — we are the installer, we test the installer. +# +# Requires OpenShell >= 0.0.24 (gateway resume + SSH secret persistence + +# sandbox state persistence: NVIDIA/OpenShell#488, #739). +# +# Prerequisites: +# - Docker running +# - NVIDIA_API_KEY set for hosted inference +# - Network access to inference.nvidia.com +# +# Environment variables: +# NEMOCLAW_NON_INTERACTIVE=1 — required +# NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 — required +# NVIDIA_API_KEY — required for hosted inference +# NEMOCLAW_SANDBOX_NAME — sandbox name (default: e2e-survival) +# NEMOCLAW_E2E_TIMEOUT_SECONDS — overall timeout (default: 900) +# +# Usage: +# NEMOCLAW_NON_INTERACTIVE=1 \ +# NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ +# NVIDIA_API_KEY=... \ +# bash test/e2e-vpn/test-sandbox-survival.sh + +set -uo pipefail + +export NEMOCLAW_E2E_DEFAULT_TIMEOUT=900 +SCRIPT_DIR_TIMEOUT="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +# shellcheck source=test/e2e-vpn/e2e-timeout.sh +source "${SCRIPT_DIR_TIMEOUT}/e2e-timeout.sh" +# shellcheck source=test/e2e-vpn/lib/ci-compatible-inference.sh +. "${SCRIPT_DIR_TIMEOUT}/lib/ci-compatible-inference.sh" + +PASS=0 +FAIL=0 +SKIP=0 +TOTAL=0 + +pass() { + ((PASS++)) + ((TOTAL++)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} +fail() { + ((FAIL++)) + ((TOTAL++)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} +skip() { + ((SKIP++)) + ((TOTAL++)) + printf '\033[33m SKIP: %s\033[0m\n' "$1" +} +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } + +# Parse chat completion response — handles both content and reasoning_content +# (nemotron-3-super is a reasoning model that may put output in reasoning_content) +parse_chat_content() { + python3 -c " +import json, sys +try: + r = json.load(sys.stdin) + c = r['choices'][0]['message'] + content = c.get('content') or c.get('reasoning_content') or '' + print(content.strip()) +except Exception as e: + print(f'PARSE_ERROR: {e}', file=sys.stderr) + sys.exit(1) +" +} + +# Compare semver: returns 0 if $1 >= $2 +version_gte() { + [ "$(printf '%s\n%s\n' "$2" "$1" | sort -V | head -1)" = "$2" ] +} + +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-survival}" +nemoclaw_e2e_configure_compatible_inference || exit 1 +HOSTED_INFERENCE_BASE_URL="$(nemoclaw_e2e_hosted_inference_base_url)" +MODEL="$(nemoclaw_e2e_hosted_inference_model)" + +# shellcheck source=test/e2e-vpn/lib/sandbox-teardown.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib/sandbox-teardown.sh" +register_sandbox_for_teardown "$SANDBOX_NAME" + +REGISTRY="$HOME/.nemoclaw/sandboxes.json" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +MIN_OPENSHELL="0.0.24" + +# SSH helper — sets up SSH config and common options for sandbox access +# Sets: ssh_config, SSH_OPTS, SSH_TARGET +setup_ssh() { + ssh_config="$(mktemp)" + if ! openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config" 2>/dev/null; then + rm -f "$ssh_config" + ssh_config="" + return 1 + fi + SSH_OPTS=(-F "$ssh_config" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=10 -o LogLevel=ERROR) + SSH_TARGET="openshell-${SANDBOX_NAME}" + return 0 +} + +cleanup_ssh() { + [ -n "${ssh_config:-}" ] && rm -f "$ssh_config" + ssh_config="" +} + +docker_driver_gateway_pid_file() { + printf '%s/.local/state/nemoclaw/openshell-docker-gateway/openshell-gateway.pid\n' "$HOME" +} + +gateway_runtime_id() { + local pid_file pid cid + pid_file="$(docker_driver_gateway_pid_file)" + if [ -f "$pid_file" ]; then + pid="$(tr -d '[:space:]' <"$pid_file" 2>/dev/null || true)" + if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then + printf 'pid:%s\n' "$pid" + return 0 + fi + fi + + cid="$(docker ps -qf "name=openshell-cluster-nemoclaw" 2>/dev/null | head -1)" + if [ -n "$cid" ]; then + printf 'container:%s\n' "$cid" + return 0 + fi + + return 1 +} + +stop_gateway_runtime() { + local pid_file pid cid + openshell forward stop 18789 2>/dev/null || true + openshell gateway stop -g nemoclaw 2>/dev/null || true + + pid_file="$(docker_driver_gateway_pid_file)" + if [ -f "$pid_file" ]; then + pid="$(tr -d '[:space:]' <"$pid_file" 2>/dev/null || true)" + if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then + kill "$pid" 2>/dev/null || true + for _ in $(seq 1 10); do + kill -0 "$pid" 2>/dev/null || break + sleep 1 + done + if kill -0 "$pid" 2>/dev/null; then + kill -9 "$pid" 2>/dev/null || true + fi + fi + fi + + cid="$(docker ps -qf "name=openshell-cluster-nemoclaw" 2>/dev/null | head -1)" + if [ -n "$cid" ]; then + docker stop "$cid" >/dev/null 2>&1 || true + fi +} + +start_gateway_runtime() { + local previous_runtime="$1" + if [[ "$previous_runtime" == pid:* ]]; then + local recovery_log + recovery_log="$(mktemp)" + if nemoclaw "$SANDBOX_NAME" status >"$recovery_log" 2>&1; then + pass "Gateway recovered through NemoClaw status" + else + info "NemoClaw status recovery returned non-zero; polling gateway health" + sed 's/^/ /' "$recovery_log" | tail -40 || true + fi + rm -f "$recovery_log" + return 0 + fi + + if openshell gateway start --name nemoclaw 2>&1; then + pass "Gateway start command succeeded" + else + info "Gateway start returned non-zero — checking health..." + fi +} + +# ══════════════════════════════════════════════════════════════════ +# Phase 0: Prerequisites +# ══════════════════════════════════════════════════════════════════ +section "Phase 0: Prerequisites" + +if docker info >/dev/null 2>&1; then + pass "Docker is running" +else + fail "Docker is not running — cannot continue" + exit 1 +fi + +if ! nemoclaw_e2e_require_hosted_inference_key; then + exit 1 +fi + +if nemoclaw_e2e_probe_hosted_inference; then + pass "Network access to ${HOSTED_INFERENCE_BASE_URL}" +else + fail "Cannot reach ${HOSTED_INFERENCE_BASE_URL}" + exit 1 +fi + +if [ "${NEMOCLAW_NON_INTERACTIVE:-}" != "1" ]; then + fail "NEMOCLAW_NON_INTERACTIVE=1 is required" + exit 1 +fi + +if [ "${NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE:-}" != "1" ]; then + fail "NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 is required" + exit 1 +fi + +if [ ! -f "$REPO_ROOT/install.sh" ]; then + fail "Cannot find install.sh at $REPO_ROOT/install.sh" + exit 1 +fi +pass "Repo root found: $REPO_ROOT" + +# ══════════════════════════════════════════════════════════════════ +# Phase 1: Pre-cleanup +# ══════════════════════════════════════════════════════════════════ +section "Phase 1: Pre-cleanup" + +info "Destroying any leftover sandbox/gateway from previous runs..." +if command -v nemoclaw >/dev/null 2>&1; then + nemoclaw "$SANDBOX_NAME" destroy --yes 2>/dev/null || true +fi +if command -v openshell >/dev/null 2>&1; then + openshell sandbox delete "$SANDBOX_NAME" 2>/dev/null || true + stop_gateway_runtime + openshell gateway destroy -g nemoclaw 2>/dev/null || true +fi +rm -f "$HOME/.nemoclaw/onboard.lock" 2>/dev/null || true +pass "Pre-cleanup complete" + +# ══════════════════════════════════════════════════════════════════ +# Phase 2: Install NemoClaw (which installs OpenShell) +# ══════════════════════════════════════════════════════════════════ +section "Phase 2: Install NemoClaw via install.sh" + +info "Running install.sh --non-interactive (installs Node.js, OpenShell, NemoClaw, runs onboard)..." + +cd "$REPO_ROOT" || { + fail "Could not cd to repo root: $REPO_ROOT" + exit 1 +} + +INSTALL_LOG="$(mktemp)" +env \ + NEMOCLAW_NON_INTERACTIVE=1 \ + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ + NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" \ + NEMOCLAW_RECREATE_SANDBOX=1 \ + bash install.sh --non-interactive >"$INSTALL_LOG" 2>&1 & +install_pid=$! +tail -f "$INSTALL_LOG" --pid=$install_pid 2>/dev/null & +tail_pid=$! +wait $install_pid +install_exit=$? +kill $tail_pid 2>/dev/null || true +wait $tail_pid 2>/dev/null || true +rm -f "$INSTALL_LOG" + +# Source shell profile to pick up nvm/PATH changes from install.sh +if [ -f "$HOME/.bashrc" ]; then + # shellcheck source=/dev/null + source "$HOME/.bashrc" 2>/dev/null || true +fi +export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" +if [ -s "$NVM_DIR/nvm.sh" ]; then + # shellcheck source=/dev/null + . "$NVM_DIR/nvm.sh" +fi +if [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]]; then + export PATH="$HOME/.local/bin:$PATH" +fi + +if [ $install_exit -eq 0 ]; then + pass "install.sh completed (exit 0)" +else + fail "install.sh failed (exit $install_exit)" + exit 1 +fi + +# Verify nemoclaw is on PATH +if command -v nemoclaw >/dev/null 2>&1; then + pass "nemoclaw on PATH: $(command -v nemoclaw)" +else + fail "nemoclaw not found on PATH after install" + exit 1 +fi + +# Verify openshell was installed and meets minimum version +if ! command -v openshell >/dev/null 2>&1; then + fail "openshell not found on PATH after install" + exit 1 +fi + +OPENSHELL_VERSION=$(openshell --version 2>&1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1) +if version_gte "$OPENSHELL_VERSION" "$MIN_OPENSHELL"; then + pass "openshell $OPENSHELL_VERSION >= $MIN_OPENSHELL (gateway resume + SSH secret + state persistence)" +else + fail "openshell $OPENSHELL_VERSION < $MIN_OPENSHELL — sandbox survival requires $MIN_OPENSHELL+" + exit 1 +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 3: Verify sandbox is live after install +# ══════════════════════════════════════════════════════════════════ +section "Phase 3: Post-install verification" + +# 3a: NemoClaw registry has it +if [ -f "$REGISTRY" ] && grep -Fq "\"${SANDBOX_NAME}\"" "$REGISTRY"; then + pass "NemoClaw registry contains '$SANDBOX_NAME'" +else + fail "NemoClaw registry missing '$SANDBOX_NAME' — onboard may have failed" + exit 1 +fi + +# 3b: nemoclaw list shows it +if list_output=$(nemoclaw list 2>&1) && grep -Fq "$SANDBOX_NAME" <<<"$list_output"; then + pass "nemoclaw list shows '$SANDBOX_NAME'" +else + fail "nemoclaw list doesn't show '$SANDBOX_NAME': ${list_output:0:200}" + exit 1 +fi + +# 3c: openshell sandbox list shows it +if os_list=$(openshell sandbox list 2>&1) && grep -q "$SANDBOX_NAME" <<<"$os_list"; then + pass "openshell sandbox list shows '$SANDBOX_NAME'" +else + fail "openshell sandbox list doesn't show '$SANDBOX_NAME': ${os_list:0:200}" + exit 1 +fi + +# 3d: nemoclaw status works +if status_output=$(nemoclaw "$SANDBOX_NAME" status 2>&1); then + pass "nemoclaw $SANDBOX_NAME status exits 0" +else + fail "nemoclaw $SANDBOX_NAME status failed: ${status_output:0:200}" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 4: Baseline — prove live inference BEFORE restart +# ══════════════════════════════════════════════════════════════════ +section "Phase 4: Baseline — live inference before restart" + +if ! setup_ssh; then + fail "Could not get SSH config for sandbox" + exit 1 +fi +pass "SSH config obtained" + +# 4a: SSH connectivity +if ssh "${SSH_OPTS[@]}" "$SSH_TARGET" "echo alive" >/dev/null 2>&1; then + pass "SSH into sandbox works (baseline)" +else + fail "SSH into sandbox failed (baseline) — cannot continue" + cleanup_ssh + exit 1 +fi + +# 4b: Live inference through sandbox +info "[LIVE] Baseline inference: user → sandbox → gateway → hosted inference endpoint..." +# shellcheck disable=SC2029 # client-side expansion is intentional +baseline_response=$(run_with_timeout 90 ssh "${SSH_OPTS[@]}" "$SSH_TARGET" \ + "curl -s --max-time 60 https://inference.local/v1/chat/completions \ + -H 'Content-Type: application/json' \ + -d '{\"model\":\"$MODEL\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with exactly one word: PONG\"}],\"max_tokens\":100}'" \ + 2>&1) || true + +# Retry baseline inference up to 3 times — live models are not deterministic +# and the gateway proxy can return unexpected responses on first attempt. (#1969) +baseline_content="" +pong_ok=false +for pong_attempt in 1 2 3; do + baseline_content="" + if [ -n "$baseline_response" ]; then + baseline_content=$(echo "$baseline_response" | parse_chat_content 2>/dev/null) || true + fi + if grep -qi "PONG" <<<"$baseline_content"; then + pong_ok=true + break + fi + info "Baseline attempt ${pong_attempt}/3: got '${baseline_content:0:80}', retrying in 5s..." + [ "$pong_attempt" -lt 3 ] || break + sleep 5 + # shellcheck disable=SC2029 + baseline_response=$(run_with_timeout 90 ssh "${SSH_OPTS[@]}" "$SSH_TARGET" \ + "curl -s --max-time 60 https://inference.local/v1/chat/completions \ + -H 'Content-Type: application/json' \ + -d '{\"model\":\"$MODEL\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with exactly one word: PONG\"}],\"max_tokens\":100}'" \ + 2>&1) || true +done +if $pong_ok; then + pass "[LIVE] Baseline: model responded with PONG through sandbox" +else + fail "[LIVE] Baseline: expected PONG after 3 attempts, got: ${baseline_content:0:200}" + info "Raw response: ${baseline_response:0:300}" + info "Cannot establish baseline — aborting (survival test meaningless without it)" + cleanup_ssh + exit 1 +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 5: Plant state markers inside sandbox +# ══════════════════════════════════════════════════════════════════ +section "Phase 5: Plant state markers in sandbox" + +MARKER_VALUE="nemoclaw-survival-$(date +%s)" + +# 5a: Workspace file in writable agent state directory. +# /sandbox is writable in the mutable-default policy. Use .openclaw for durable +# agent state markers so survival checks validate the configured state path. +# shellcheck disable=SC2029 +if ssh "${SSH_OPTS[@]}" "$SSH_TARGET" "echo ${MARKER_VALUE} > /sandbox/.openclaw/.survival-marker-workspace" 2>/dev/null; then + pass "Planted workspace marker: /sandbox/.openclaw/.survival-marker-workspace" +else + fail "Could not plant workspace marker" +fi + +# Verify read-back before restart +readback=$(ssh "${SSH_OPTS[@]}" "$SSH_TARGET" "cat /sandbox/.openclaw/.survival-marker-workspace" 2>/dev/null) +if [ "$readback" = "$MARKER_VALUE" ]; then + pass "Workspace marker verified before restart" +else + fail "Workspace marker read-back mismatch: expected '$MARKER_VALUE', got '$readback'" +fi + +# 5b: Agent data directory — plant marker in .openclaw if it exists +# This tests the complaint from #1086 and @Koneisto: agent state loss +# shellcheck disable=SC2029 +agent_data_exists=$(ssh "${SSH_OPTS[@]}" "$SSH_TARGET" \ + "[ -d /sandbox/.openclaw ] && echo yes || echo no" 2>/dev/null) +if [ "$agent_data_exists" = "yes" ]; then + # shellcheck disable=SC2029 + if ssh "${SSH_OPTS[@]}" "$SSH_TARGET" \ + "echo ${MARKER_VALUE} > /sandbox/.openclaw/.survival-marker" 2>/dev/null; then + pass "Planted agent data marker: /sandbox/.openclaw/.survival-marker" + else + fail "Could not plant agent data marker" + fi +else + info "No .openclaw directory yet — will check if sandbox itself survives" +fi + +# 5c: Snapshot which agent identity files exist (to verify they survive) +agent_files_before=$(ssh "${SSH_OPTS[@]}" "$SSH_TARGET" \ + "ls -la /sandbox/.openclaw/ 2>/dev/null | head -20" 2>/dev/null) || true +if [ -n "$agent_files_before" ]; then + info "Agent data directory contents before restart:" + echo "$agent_files_before" | while IFS= read -r line; do + info " $line" + done +fi + +# 5d: Record a deeper workspace file to test nested persistence +# Uses the writable .openclaw path for durable agent state. +# shellcheck disable=SC2029 +if ssh "${SSH_OPTS[@]}" "$SSH_TARGET" \ + "mkdir -p /sandbox/.openclaw/test-data && echo ${MARKER_VALUE} > /sandbox/.openclaw/test-data/nested-marker.txt" \ + 2>/dev/null; then + pass "Planted nested marker: /sandbox/.openclaw/test-data/nested-marker.txt" +else + fail "Could not plant nested workspace marker" +fi + +cleanup_ssh + +# ══════════════════════════════════════════════════════════════════ +# Phase 6: Gateway stop/start cycle (simulates reboot) +# ══════════════════════════════════════════════════════════════════ +section "Phase 6: Gateway stop/start cycle (simulates host reboot)" + +# Stop any port forwards first +GATEWAY_RUNTIME_BEFORE="$(gateway_runtime_id || true)" +openshell forward stop 18789 2>/dev/null || true + +info "Stopping gateway (simulates laptop close / VM shutdown)..." +stop_gateway_runtime +if [ -z "$(gateway_runtime_id || true)" ]; then + pass "Gateway runtime stopped" +else + fail "Gateway runtime still appears to be running after stop" + # Non-fatal — continue to see what happens +fi + +# Verify the legacy Docker container is stopped when this run uses the +# legacy k3s gateway; Docker-driver runs use a host openshell-gateway PID. +if [[ "$GATEWAY_RUNTIME_BEFORE" == container:* ]]; then + CONTAINER_NAME="openshell-cluster-nemoclaw" + container_state=$(docker inspect -f '{{.State.Running}}' "$CONTAINER_NAME" 2>/dev/null || echo "missing") + if [ "$container_state" = "false" ]; then + pass "Docker container confirmed stopped" + elif [ "$container_state" = "missing" ]; then + info "Container not found (may have been removed) — resume should handle this" + pass "Docker container not running" + else + fail "Docker container still running: state=$container_state" + fi +else + pass "Docker-driver gateway process is not running" +fi + +info "Waiting 5 seconds to simulate delay (laptop lid close / VM hibernate)..." +sleep 5 + +info "Starting gateway (simulates laptop open / VM boot)..." +start_gateway_runtime "$GATEWAY_RUNTIME_BEFORE" + +# Wait for gateway to become healthy +info "Waiting for gateway to become healthy..." +HEALTHY=0 +for attempt in $(seq 1 60); do + gw_status=$(openshell status 2>&1) + if echo "$gw_status" | grep -qi "Connected" && echo "$gw_status" | grep -qi "nemoclaw"; then + HEALTHY=1 + break + fi + sleep 5 +done + +if [ "$HEALTHY" -eq 1 ]; then + pass "Gateway healthy after restart (attempt $attempt)" +else + fail "Gateway did not become healthy within 300 seconds" + openshell status 2>&1 || true + exit 1 +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 7: Verify sandbox survived — every complaint from #486/#888/#859/#1086 +# ══════════════════════════════════════════════════════════════════ +section "Phase 7: Verify sandbox survived restart" + +# 7a: openshell sandbox list — #486 "No sandboxes found" +if openshell sandbox list 2>&1 | grep -q "$SANDBOX_NAME"; then + pass "openshell sandbox list shows '$SANDBOX_NAME' after restart" +else + fail "openshell sandbox list: '$SANDBOX_NAME' NOT FOUND after restart (#486)" + openshell sandbox list 2>&1 || true +fi + +# 7b: Sandbox pod is running, not just listed +sandbox_phase="" +for attempt in $(seq 1 30); do + sandbox_phase=$(openshell sandbox list 2>&1 | grep "$SANDBOX_NAME" | grep -oiE 'running|ready' | head -1) + if [ -n "$sandbox_phase" ]; then + break + fi + sleep 5 +done + +if [ -n "$sandbox_phase" ]; then + pass "Sandbox pod is '$sandbox_phase' after restart" +else + fail "Sandbox pod did not reach Running/Ready after restart" + openshell sandbox list 2>&1 || true +fi + +# 7c: NemoClaw registry still has it — #486 "No sandboxes registered" +if [ -f "$REGISTRY" ] && grep -Fq "\"${SANDBOX_NAME}\"" "$REGISTRY"; then + pass "NemoClaw registry still contains '$SANDBOX_NAME' after restart" +else + fail "NemoClaw registry lost '$SANDBOX_NAME' after restart (#486)" +fi + +# 7d: nemoclaw list shows it — the actual user-facing command +if list_output=$(nemoclaw list 2>&1) && grep -Fq "$SANDBOX_NAME" <<<"$list_output"; then + pass "nemoclaw list shows '$SANDBOX_NAME' after restart" +else + fail "nemoclaw list doesn't show '$SANDBOX_NAME' after restart: ${list_output:0:200}" +fi + +# 7e: nemoclaw status works — #859 "unclear CLI behavior" +# No special intervention should be required after gateway restart. +# If nemoclaw status hangs, that IS the bug — use timeout to detect it. +# Write to a temp file instead of $() to avoid pipe FD inheritance: +# nemoclaw's SSH recovery can spawn background processes that hold the +# pipe open, preventing $() from returning even after timeout kills nemoclaw. +STATUS_TMP="$(mktemp)" +TIMEOUT_STATUS="" +command -v timeout >/dev/null 2>&1 && TIMEOUT_STATUS="timeout 120" +command -v gtimeout >/dev/null 2>&1 && TIMEOUT_STATUS="gtimeout 120" +$TIMEOUT_STATUS nemoclaw "$SANDBOX_NAME" status >"$STATUS_TMP" 2>&1 +status_exit=$? +status_output=$(cat "$STATUS_TMP") +rm -f "$STATUS_TMP" +if [ "$status_exit" -eq 0 ]; then + pass "nemoclaw $SANDBOX_NAME status exits 0 after restart (no re-onboard needed)" +elif [ "$status_exit" -eq 124 ]; then + fail "nemoclaw $SANDBOX_NAME status TIMED OUT after restart (port forward or SSH recovery hung)" +else + fail "nemoclaw $SANDBOX_NAME status failed after restart (exit $status_exit): ${status_output:0:200}" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 8: Verify SSH connectivity — #888/#1086 handshake failure +# ══════════════════════════════════════════════════════════════════ +section "Phase 8: Verify SSH connectivity after restart" + +if ! setup_ssh; then + fail "Could not get SSH config after restart (#888 handshake failure?)" + skip "Workspace marker check (SSH unavailable)" + skip "Agent data marker check (SSH unavailable)" + skip "Nested marker check (SSH unavailable)" + skip "Post-restart inference (SSH unavailable)" + + # Jump to cleanup + section "Phase 11: Cleanup" + [[ "${NEMOCLAW_E2E_KEEP_SANDBOX:-}" = "1" ]] || nemoclaw "$SANDBOX_NAME" destroy --yes 2>&1 | tail -3 || true + openshell gateway destroy -g nemoclaw 2>/dev/null || true + echo "" + echo "========================================" + echo " Sandbox Survival E2E Results:" + echo " Passed: $PASS" + echo " Failed: $FAIL" + echo " Skipped: $SKIP" + echo " Total: $TOTAL" + echo "========================================" + printf '\n\033[1;31m %d test(s) failed.\033[0m\n' "$FAIL" + exit 1 +fi +pass "SSH config available after restart" + +# 8a: Raw SSH connectivity — the #888/#1086 handshake test +# The sandbox SSH agent may take a few seconds to become reachable after +# the gateway reports healthy (especially with newer OpenClaw versions that +# do more startup work). Retry up to 30 seconds before declaring failure. +SSH_OK=0 +for ssh_attempt in $(seq 1 6); do + if ssh "${SSH_OPTS[@]}" "$SSH_TARGET" "echo alive" >/dev/null 2>&1; then + SSH_OK=1 + break + fi + [ "$ssh_attempt" -lt 6 ] && sleep 5 +done + +if [ "$SSH_OK" -eq 1 ]; then + pass "SSH into sandbox works after restart (attempt $ssh_attempt, no handshake failure — #888/#1086)" +else + fail "SSH into sandbox FAILED after restart — handshake verification likely failed (#888/#1086)" + info "This is the core bug: gateway regenerated secrets, sandbox has stale ones" + # Do NOT call cleanup_ssh here — subsequent phases need the config file + # to attempt marker reads and produce meaningful diagnostics. + nemoclaw "$SANDBOX_NAME" logs 2>&1 | grep -i "handshake" | head -5 || true +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 9: Verify workspace and agent state persisted — #1086/@Koneisto +# ══════════════════════════════════════════════════════════════════ +section "Phase 9: Verify state persisted across restart" + +# 9a: Workspace marker +post_restart_marker=$(ssh "${SSH_OPTS[@]}" "$SSH_TARGET" "cat /sandbox/.openclaw/.survival-marker-workspace" 2>/dev/null) +if [ "$post_restart_marker" = "$MARKER_VALUE" ]; then + pass "Workspace marker survived restart: $MARKER_VALUE" +else + fail "Workspace marker LOST: expected '$MARKER_VALUE', got '${post_restart_marker:-}' (#1086 state loss)" +fi + +# 9b: Agent data marker +if [ "$agent_data_exists" = "yes" ]; then + agent_marker=$(ssh "${SSH_OPTS[@]}" "$SSH_TARGET" "cat /sandbox/.openclaw/.survival-marker" 2>/dev/null) + if [ "$agent_marker" = "$MARKER_VALUE" ]; then + pass "Agent data marker survived restart" + else + fail "Agent data marker LOST: expected '$MARKER_VALUE', got '${agent_marker:-}' (agent state destroyed)" + fi +fi + +# 9c: Nested workspace file +nested_marker=$(ssh "${SSH_OPTS[@]}" "$SSH_TARGET" "cat /sandbox/.openclaw/test-data/nested-marker.txt" 2>/dev/null) +if [ "$nested_marker" = "$MARKER_VALUE" ]; then + pass "Nested workspace marker survived restart" +else + fail "Nested workspace marker LOST: expected '$MARKER_VALUE', got '${nested_marker:-}'" +fi + +# 9d: Agent data directory still populated (not wiped to image defaults) +if [ "$agent_data_exists" = "yes" ]; then + agent_files_after=$(ssh "${SSH_OPTS[@]}" "$SSH_TARGET" \ + "ls -la /sandbox/.openclaw/ 2>/dev/null | head -20" 2>/dev/null) || true + if [ -n "$agent_files_after" ]; then + info "Agent data directory contents after restart:" + echo "$agent_files_after" | while IFS= read -r line; do + info " $line" + done + pass "Agent data directory still populated after restart" + else + fail "Agent data directory is empty after restart (@Koneisto overlay wipe)" + fi +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 10: Prove live inference works AFTER restart (the definitive proof) +# ══════════════════════════════════════════════════════════════════ +section "Phase 10: Live inference after restart (THE definitive test)" + +info "[LIVE] Post-restart inference: user → sandbox → gateway → hosted inference endpoint..." +# shellcheck disable=SC2029 +post_response=$(run_with_timeout 90 ssh "${SSH_OPTS[@]}" "$SSH_TARGET" \ + "curl -s --max-time 60 https://inference.local/v1/chat/completions \ + -H 'Content-Type: application/json' \ + -d '{\"model\":\"$MODEL\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with exactly one word: PONG\"}],\"max_tokens\":100}'" \ + 2>&1) || true + +# Retry post-restart inference up to 3 times. (#1969) +post_content="" +pong_ok=false +for pong_attempt in 1 2 3; do + post_content="" + if [ -n "$post_response" ]; then + post_content=$(echo "$post_response" | parse_chat_content 2>/dev/null) || true + fi + if grep -qi "PONG" <<<"$post_content"; then + pong_ok=true + break + fi + info "Post-restart attempt ${pong_attempt}/3: got '${post_content:0:80}', retrying in 5s..." + [ "$pong_attempt" -lt 3 ] || break + sleep 5 + # shellcheck disable=SC2029 + post_response=$(run_with_timeout 90 ssh "${SSH_OPTS[@]}" "$SSH_TARGET" \ + "curl -s --max-time 60 https://inference.local/v1/chat/completions \ + -H 'Content-Type: application/json' \ + -d '{\"model\":\"$MODEL\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with exactly one word: PONG\"}],\"max_tokens\":100}'" \ + 2>&1) || true +done +if $pong_ok; then + pass "[LIVE] Post-restart: model responded with PONG through sandbox" + info "Full path proven: user → sandbox → openshell gateway (resumed) → hosted inference endpoint → response" + info "This proves #859's ask: reliable non-destructive gateway lifecycle with working inference" +else + fail "[LIVE] Post-restart: expected PONG after 3 attempts, got: ${post_content:0:200}" + info "Raw response: ${post_response:0:300}" +fi + +cleanup_ssh + +# ══════════════════════════════════════════════════════════════════ +# Phase 11: Cleanup +# ══════════════════════════════════════════════════════════════════ +section "Phase 11: Cleanup" + +[[ "${NEMOCLAW_E2E_KEEP_SANDBOX:-}" = "1" ]] || nemoclaw "$SANDBOX_NAME" destroy --yes 2>&1 | tail -3 || true +openshell gateway destroy -g nemoclaw 2>/dev/null || true + +if [ -f "$REGISTRY" ] && grep -Fq "\"${SANDBOX_NAME}\"" "$REGISTRY"; then + fail "Sandbox '$SANDBOX_NAME' still in registry after destroy" +else + pass "Sandbox '$SANDBOX_NAME' cleaned up" +fi + +# ══════════════════════════════════════════════════════════════════ +# Summary +# ══════════════════════════════════════════════════════════════════ +echo "" +echo "========================================" +echo " Sandbox Survival E2E Results:" +echo " Passed: $PASS" +echo " Failed: $FAIL" +echo " Skipped: $SKIP" +echo " Total: $TOTAL" +echo "========================================" + +if [ "$FAIL" -eq 0 ]; then + printf '\n\033[1;32m Sandbox survival PASSED — all state persisted, live inference verified before AND after gateway restart.\033[0m\n' + printf '\033[1;32m Issues validated: #486, #888, #859, #1086\033[0m\n' + exit 0 +else + printf '\n\033[1;31m %d test(s) failed.\033[0m\n' "$FAIL" + exit 1 +fi diff --git a/test/e2e-vpn/test-sessions-agents-cli.sh b/test/e2e-vpn/test-sessions-agents-cli.sh new file mode 100755 index 00000000000..ada108cf833 --- /dev/null +++ b/test/e2e-vpn/test-sessions-agents-cli.sh @@ -0,0 +1,501 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# ============================================================================= +# test-sessions-agents-cli.sh +# NemoClaw `sessions` and `agents` subcommand E2E tests +# +# Covers the host-side CLI surface for the sandbox `sessions` and `agents` +# subcommand groups. The end-to-end recovery semantics for the in-sandbox +# session store (stale `.jsonl.lock`, corrupt `sessions.json`, the clean +# follow-up message after a reset) live with the OpenClaw gateway upstream +# — see the scope-boundary note in `src/lib/actions/sandbox/sessions/reset.ts`. +# This script exercises only what NemoClaw owns: argv translation, gateway +# dispatch, and JSON envelope handling. +# TC-SESS-01: `nemoclaw sessions --json` +# (parent default = `openclaw sessions` list) +# TC-SESS-02: `nemoclaw sessions list --json` +# TC-SESS-03: `nemoclaw sessions reset ` via gateway RPC +# TC-SESS-04: `nemoclaw sessions list --json` after reset +# TC-AGENT-01: `nemoclaw agents add work --model gpt-4o` +# (passthrough wizard; --non-interactive bypass) +# TC-AGENT-03: `nemoclaw agents list --json` +# (passthrough lister; OpenClaw owns gateway dispatch) +# TC-AGENT-02: `nemoclaw agents delete work --force --json` +# (passthrough delete; OpenClaw owns workspace removal) +# TC-SESS-05: `nemoclaw sessions delete ` on a non-main session +# +# Prerequisites: +# - Docker running +# - NVIDIA_API_KEY set (real key or fake OpenAI endpoint) +# - NEMOCLAW_NON_INTERACTIVE=1, NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 +# +# Usage: +# NEMOCLAW_NON_INTERACTIVE=1 NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ +# NVIDIA_API_KEY=nvapi-... bash test/e2e-vpn/test-sessions-agents-cli.sh +# ============================================================================= + +set -uo pipefail + +# Silence Node.js experimental-feature warnings (e.g. UNDICI-EHPA "EnvHttpProxyAgent +# is experimental") from every `nemoclaw` invocation. These warnings go to stderr +# but every JSON-capture below uses `2>&1` for diagnostics-on-failure, so without +# this they would prefix the JSON payload and break `python -c json.loads`. +export NODE_NO_WARNINGS=1 + +export NEMOCLAW_E2E_DEFAULT_TIMEOUT="${NEMOCLAW_E2E_DEFAULT_TIMEOUT:-2400}" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +# shellcheck source=test/e2e-vpn/e2e-timeout.sh +. "${SCRIPT_DIR}/e2e-timeout.sh" + +PASS=0 +FAIL=0 +SKIP=0 +TOTAL=0 +pass() { + PASS=$((PASS + 1)) + TOTAL=$((TOTAL + 1)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} +fail() { + FAIL=$((FAIL + 1)) + TOTAL=$((TOTAL + 1)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} +skip() { + SKIP=$((SKIP + 1)) + TOTAL=$((TOTAL + 1)) + printf '\033[33m SKIP: %s\033[0m\n' "$1" +} +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } +print_summary() { + section "Summary" + echo " Total: $TOTAL Pass: $PASS Fail: $FAIL Skip: $SKIP" + if [ "$FAIL" -gt 0 ]; then + echo "" + echo "FAILED" + exit 1 + fi + echo "" + if [ "$SKIP" -gt 0 ]; then + echo "PASSED (with $SKIP skipped)" + else + echo "ALL PASSED" + fi +} + +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-sessions-agents-cli}" +TEST_AGENT_ID="${NEMOCLAW_E2E_AGENT_ID:-work}" + +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +INSTALL_LOG="${E2E_SESSIONS_AGENTS_INSTALL_LOG:-/tmp/nemoclaw-e2e-install.log}" + +# shellcheck source=test/e2e-vpn/lib/sandbox-teardown.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib/sandbox-teardown.sh" +register_sandbox_for_teardown "$SANDBOX_NAME" + +# shellcheck source=test/e2e-vpn/lib/install-path-refresh.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib/install-path-refresh.sh" + +install_nemoclaw_from_source() { + section "Install NemoClaw from source (install.sh --non-interactive)" + if command -v nemoclaw >/dev/null 2>&1; then + info "nemoclaw already on PATH at $(command -v nemoclaw); skipping install" + pass "install: nemoclaw already available" + return 0 + fi + if [ ! -x "${REPO_ROOT}/install.sh" ]; then + fail "install: ${REPO_ROOT}/install.sh missing or not executable" + print_summary + exit 1 + fi + if ! bash "${REPO_ROOT}/install.sh" --non-interactive >"$INSTALL_LOG" 2>&1; then + info "install.sh exited non-zero (may be benign on re-install); verifying PATH" + fi + nemoclaw_refresh_install_env + if ! command -v nemoclaw >/dev/null 2>&1; then + fail "install: nemoclaw not found on PATH after install.sh (see ${INSTALL_LOG})" + print_summary + exit 1 + fi + pass "install: nemoclaw installed at $(command -v nemoclaw)" +} + +is_valid_json() { + # Tolerate leading non-JSON lines on stdout (Node deprecation warnings, oclif + # banner) by scanning for the first line that begins with `{` or `[` and + # parsing from there. Anchoring at line-start avoids false positives like the + # `[UNDICI-EHPA]` token inside Node warning text. `NODE_NO_WARNINGS=1` already + # suppresses the known UNDICI-EHPA warning, but this stays defensive against + # any future stderr leakage when a JSON-capture uses `2>&1`. + printf '%s' "$1" | python3 -c " +import json, sys +raw = sys.stdin.read() +offset = -1 +cursor = 0 +for line in raw.splitlines(keepends=True): + if line.startswith('{') or line.startswith('['): + offset = cursor + break + cursor += len(line) +if offset < 0: + sys.exit(1) +json.loads(raw[offset:]) +" 2>/dev/null +} + +preflight() { + section "Preflight" + if ! docker info >/dev/null 2>&1; then + fail "preflight: Docker not running" + print_summary + exit 1 + fi + if [ -z "${NVIDIA_API_KEY:-}" ]; then + skip "preflight: NVIDIA_API_KEY not set; sessions/agents E2E requires a working onboard credential" + print_summary + exit 0 + fi + pass "preflight: docker + NVIDIA_API_KEY available" +} + +onboard_sandbox() { + section "Onboard sandbox '${SANDBOX_NAME}'" + rm -f "$HOME/.nemoclaw/onboard.lock" 2>/dev/null || true + NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" \ + NEMOCLAW_NON_INTERACTIVE=1 \ + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ + NEMOCLAW_POLICY_TIER="open" \ + nemoclaw onboard --non-interactive --yes-i-accept-third-party-software 2>&1 || { + fail "onboard: onboard command failed for '${SANDBOX_NAME}'" + print_summary + exit 1 + } + pass "onboard: sandbox '${SANDBOX_NAME}' is up" +} + +# Approve any pending OpenClaw CLI device-pairing / scope-upgrade requests so +# downstream `openclaw gateway call ...` invocations (used by `sessions reset` +# and `sessions delete`) do not fall back to the embedded agent. The +# `nemoclaw-start.sh` auto-pair watcher allowlists `cli` / `openclaw-control-ui` +# clients, but a fresh CI sandbox can race the watcher's slow-mode polling on +# late scope upgrades; this loop is defensive and idempotent. +approve_pending_pairing_requests() { + section "Approve any pending OpenClaw pairing / scope-upgrade requests" + local max_iters=10 + local interval=3 + local i state ids + for ((i = 0; i < max_iters; i++)); do + state="$(nemoclaw "$SANDBOX_NAME" exec -- openclaw devices list --json 2>/dev/null || true)" + if [ -z "$state" ]; then + sleep "$interval" + continue + fi + ids="$(printf '%s' "$state" | python3 -c " +import json, sys +try: + data = json.loads(sys.stdin.read()) +except Exception: + sys.exit(0) +pending = data.get('pending') or [] +for d in pending: + if not isinstance(d, dict): + continue + rid = d.get('requestId') or d.get('id') + if rid: + print(rid) +" 2>/dev/null || true)" + if [ -z "$ids" ]; then + pass "gateway scope: no pending pairing/scope-upgrade requests" + return 0 + fi + while IFS= read -r rid; do + [ -z "$rid" ] && continue + info "approving pending request '$rid'" + nemoclaw "$SANDBOX_NAME" exec -- openclaw devices approve "$rid" --json >/dev/null 2>&1 || true + done <<<"$ids" + sleep "$interval" + done + info "gateway scope: gave up after $((max_iters * interval))s; gateway-RPC tests may still hit pending-scope failures" + return 1 +} + +seed_main_session() { + section "Seed main session by sending one prompt" + if ! nemoclaw "$SANDBOX_NAME" exec -- openclaw agent --agent main -m "ping" 2>&1; then + fail "seed: agent invocation failed; sessions store may not be populated" + return 1 + fi + pass "seed: sent one prompt to agent 'main'" +} + +test_sessions_default_json() { + section "TC-SESS-01: sessions --json (parent default = list)" + local out + out="$(nemoclaw "$SANDBOX_NAME" sessions --json 2>&1)" || { + fail "TC-SESS-01: sessions --json exited non-zero" + info "$out" + return 1 + } + if ! is_valid_json "$out"; then + fail "TC-SESS-01: sessions --json did not return parseable JSON" + info "$out" + return 1 + fi + pass "TC-SESS-01: sessions --json returned valid JSON" +} + +test_sessions_list_json() { + section "TC-SESS-02: sessions list --json" + local out + out="$(nemoclaw "$SANDBOX_NAME" sessions list --json 2>&1)" || { + fail "TC-SESS-02: sessions list --json exited non-zero" + info "$out" + return 1 + } + if ! is_valid_json "$out"; then + fail "TC-SESS-02: sessions list --json did not return parseable JSON" + info "$out" + return 1 + fi + pass "TC-SESS-02: sessions list --json returned valid JSON" +} + +test_sessions_reset_main() { + section "TC-SESS-03: sessions reset agent:main:main --json" + local out exit_code attempt=1 max_attempts=5 backoff=4 + # The first invocation of any new gateway-RPC method (here `sessions.reset`) + # can itself trigger a fresh CLI scope-upgrade request that the auto-pair + # watcher only approves asynchronously. Retry while approving any pending + # requests between attempts, until the gateway accepts the scope or we run + # out of attempts. + while [ "$attempt" -le "$max_attempts" ]; do + if out="$(nemoclaw "$SANDBOX_NAME" sessions reset agent:main:main --json 2>&1)"; then + exit_code=0 + break + else + exit_code=$? + fi + if ! grep -qE "scope upgrade pending|Failed to reach the OpenClaw gateway|pairing required" <<<"$out"; then + break + fi + info "TC-SESS-03: gateway scope still pending (attempt ${attempt}/${max_attempts}); approving and retrying" + approve_pending_pairing_requests >/dev/null 2>&1 || true + sleep "$backoff" + attempt=$((attempt + 1)) + done + if [ "$exit_code" -ne 0 ]; then + fail "TC-SESS-03: sessions reset exited non-zero" + info "$out" + return 1 + fi + if ! is_valid_json "$out"; then + fail "TC-SESS-03: sessions reset --json did not return parseable JSON" + info "$out" + return 1 + fi + pass "TC-SESS-03: sessions reset succeeded and returned JSON" +} + +test_sessions_list_after_reset() { + section "TC-SESS-04: sessions list --json after reset" + local out + out="$(nemoclaw "$SANDBOX_NAME" sessions list --json 2>&1)" || { + fail "TC-SESS-04: sessions list --json exited non-zero after reset" + info "$out" + return 1 + } + if ! is_valid_json "$out"; then + fail "TC-SESS-04: sessions list --json after reset did not return parseable JSON" + info "$out" + return 1 + fi + pass "TC-SESS-04: sessions list --json after reset returned valid JSON" +} + +test_agents_add_passthrough() { + section "TC-AGENT-01: agents add ${TEST_AGENT_ID} (passthrough wizard)" + local add_out + # OpenClaw's `agents add --non-interactive` mandates --workspace; the wizard + # only fills it interactively. Pass the canonical secondary-agent workspace + # path reserved by the in-sandbox layout. + if ! add_out="$(nemoclaw "$SANDBOX_NAME" agents add "$TEST_AGENT_ID" \ + --workspace "/sandbox/.openclaw/workspace-${TEST_AGENT_ID}" \ + --non-interactive 2>&1)"; then + fail "TC-AGENT-01: agents add ${TEST_AGENT_ID} exited non-zero" + info "$add_out" + return 1 + fi + # Assert the agent landed in the in-sandbox OpenClaw agents store. We use + # the host-side `sessions list --agent ` gateway call because it is the + # exact path real users hit and it fails if OpenClaw does not know the + # agent. A passthrough exit status alone is not sufficient evidence that + # `agents add` actually created the agent. + local list_out + if ! list_out="$(nemoclaw "$SANDBOX_NAME" sessions list --agent "$TEST_AGENT_ID" --json 2>&1)"; then + fail "TC-AGENT-01: agent '${TEST_AGENT_ID}' not visible via sessions list after add" + info "$list_out" + return 1 + fi + if ! is_valid_json "$list_out"; then + fail "TC-AGENT-01: sessions list --agent '${TEST_AGENT_ID}' did not return parseable JSON after add" + info "$list_out" + return 1 + fi + pass "TC-AGENT-01: agents add ${TEST_AGENT_ID} passthrough created the agent" +} + +seed_agent_session() { + section "Seed session for agent '${TEST_AGENT_ID}'" + if ! nemoclaw "$SANDBOX_NAME" exec -- openclaw agent --agent "$TEST_AGENT_ID" -m "ping" 2>&1; then + fail "seed: agent '${TEST_AGENT_ID}' invocation failed after agents add succeeded" + return 1 + fi + pass "seed: sent one prompt to agent '${TEST_AGENT_ID}'" +} + +test_sessions_delete_non_main() { + section "TC-SESS-05: sessions delete on a non-main session" + local key + # A session under `--agent ` is by definition non-main: the + # original main session for the primary `main` agent never appears in this + # filtered list. Take the first key for the work agent; an earlier filter + # that excluded keys ending with `:main` mishandled the canonical case where + # the secondary agent's default slot is also `main` (key `agent:work:main`). + key="$(nemoclaw "$SANDBOX_NAME" sessions list --agent "$TEST_AGENT_ID" --json 2>/dev/null \ + | python3 -c "import json,sys; sessions=json.loads(sys.stdin.read()); print(next((s['key'] for s in (sessions if isinstance(sessions, list) else sessions.get('sessions', [])) if s.get('key')), ''))" \ + 2>/dev/null || true)" + if [ -z "$key" ]; then + fail "TC-SESS-05: no session key found for agent '${TEST_AGENT_ID}'; expected the seeded prompt to create one" + return 1 + fi + local del_out del_code attempt=1 max_attempts=5 backoff=4 + while [ "$attempt" -le "$max_attempts" ]; do + if del_out="$(nemoclaw "$SANDBOX_NAME" sessions delete "$key" --json 2>&1)"; then + del_code=0 + break + else + del_code=$? + fi + if ! grep -qE "scope upgrade pending|Failed to reach the OpenClaw gateway|pairing required" <<<"$del_out"; then + break + fi + info "TC-SESS-05: gateway scope still pending (attempt ${attempt}/${max_attempts}); approving and retrying" + approve_pending_pairing_requests >/dev/null 2>&1 || true + sleep "$backoff" + attempt=$((attempt + 1)) + done + if [ "$del_code" -ne 0 ]; then + fail "TC-SESS-05: sessions delete ${key} exited non-zero" + info "$del_out" + return 1 + fi + # Assert the deleted key really is gone, not just that delete returned 0. + local after_keys + after_keys="$(nemoclaw "$SANDBOX_NAME" sessions list --agent "$TEST_AGENT_ID" --json 2>/dev/null \ + | python3 -c "import json,sys; sessions=json.loads(sys.stdin.read()); print('\n'.join([s.get('key', '') for s in (sessions if isinstance(sessions, list) else sessions.get('sessions', []))]))" \ + 2>/dev/null || true)" + if printf '%s\n' "$after_keys" | grep -Fxq "$key"; then + fail "TC-SESS-05: session key '${key}' still present after delete" + return 1 + fi + pass "TC-SESS-05: sessions delete ${key} succeeded and the key is gone" +} + +test_agents_list_passthrough() { + section "TC-AGENT-03: agents list --json (passthrough)" + local out + if ! out="$(nemoclaw "$SANDBOX_NAME" agents list --json 2>&1)"; then + fail "TC-AGENT-03: agents list --json exited non-zero" + info "$out" + return 1 + fi + if ! is_valid_json "$out"; then + fail "TC-AGENT-03: agents list --json did not return parseable JSON" + info "$out" + return 1 + fi + # The previously added secondary agent must appear in the listing. A pure + # exit-status check does not prove the passthrough reached the gateway. + # Mirror `is_valid_json`'s prefix-strip so Node warning lines that escape + # `NODE_NO_WARNINGS=1` (via openshell sub-invocations) don't break the parse. + if ! printf '%s' "$out" | TARGET="$TEST_AGENT_ID" python3 -c " +import json, os, sys +raw = sys.stdin.read() +offset = -1 +cursor = 0 +for line in raw.splitlines(keepends=True): + if line.startswith('{') or line.startswith('['): + offset = cursor + break + cursor += len(line) +if offset < 0: + sys.exit(1) +data = json.loads(raw[offset:]) +entries = data if isinstance(data, list) else data.get('agents', []) +target = os.environ['TARGET'] +sys.exit(0 if any(entry.get('id') == target for entry in entries) else 1) +" 2>/dev/null; then + fail "TC-AGENT-03: agent '${TEST_AGENT_ID}' not present in agents list --json output" + info "$out" + return 1 + fi + pass "TC-AGENT-03: agents list --json surfaced agent '${TEST_AGENT_ID}'" +} + +test_agents_delete_passthrough() { + section "TC-AGENT-02: agents delete ${TEST_AGENT_ID} --force --json" + local out + if ! out="$(nemoclaw "$SANDBOX_NAME" agents delete "$TEST_AGENT_ID" --force --json 2>&1)"; then + fail "TC-AGENT-02: agents delete ${TEST_AGENT_ID} exited non-zero" + info "$out" + return 1 + fi + # Assert the agent is actually gone: a follow-up `sessions list --agent ` + # should no longer return a valid JSON listing for it. A successful exit on + # the delete call alone does not prove the agent was removed. + local follow_out + if follow_out="$(nemoclaw "$SANDBOX_NAME" sessions list --agent "$TEST_AGENT_ID" --json 2>&1)" \ + && is_valid_json "$follow_out"; then + fail "TC-AGENT-02: agent '${TEST_AGENT_ID}' still visible via sessions list after delete" + info "$follow_out" + return 1 + fi + pass "TC-AGENT-02: agents delete ${TEST_AGENT_ID} passthrough removed the agent" +} + +preflight +install_nemoclaw_from_source +onboard_sandbox +approve_pending_pairing_requests +if seed_main_session; then + # The seed prompt itself may have triggered a scope-upgrade request — drain + # again before the first gateway-RPC test (TC-SESS-03) so the call does not + # land on a still-pending scope. + approve_pending_pairing_requests + test_sessions_default_json + test_sessions_list_json + test_sessions_reset_main + test_sessions_list_after_reset +else + skip "TC-SESS-01: skipped (seed_main_session failed)" + skip "TC-SESS-02: skipped (seed_main_session failed)" + skip "TC-SESS-03: skipped (seed_main_session failed)" + skip "TC-SESS-04: skipped (seed_main_session failed)" +fi + +# Agents add/delete and non-main session delete are feature-critical for this +# CLI surface — any failure must fail the whole job rather than be skipped. +test_agents_add_passthrough +test_agents_list_passthrough +seed_agent_session +approve_pending_pairing_requests +test_sessions_delete_non_main +test_agents_delete_passthrough + +print_summary diff --git a/test/e2e-vpn/test-shields-config.sh b/test/e2e-vpn/test-shields-config.sh new file mode 100755 index 00000000000..28de6029f2c --- /dev/null +++ b/test/e2e-vpn/test-shields-config.sh @@ -0,0 +1,671 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Shields & Config E2E — validates the full shields up/down lifecycle and +# config get against a live sandbox: +# +# Phase 1: Install NemoClaw +# Phase 2: Verify config is writable (mutable default) +# Phase 3: shields up — verify config becomes immutable +# Phase 4: config get — read-only inspection +# Phase 5: shields status — shows UP +# Phase 5b: Content-seal drift detection (chmod-write-chmod tamper) +# Phase 6: shields down — verify config returns to writable +# Phase 7: shields status — shows DOWN +# Phase 8: Audit trail completeness +# Phase 9: Auto-restore timer (shields up with short timeout) +# Phase 10: Double shields-up rejected +# +# Prerequisites: +# - Docker running +# - NVIDIA_API_KEY set for hosted inference +# +# Environment variables: +# NEMOCLAW_NON_INTERACTIVE=1 — required +# NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 — required +# NVIDIA_API_KEY — required +# NEMOCLAW_SANDBOX_NAME — sandbox name (default: e2e-shields) +# NEMOCLAW_E2E_TIMEOUT_SECONDS — overall timeout (default: 900) + +set -uo pipefail + +export NEMOCLAW_E2E_DEFAULT_TIMEOUT=900 +SCRIPT_DIR_TIMEOUT="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +# shellcheck source=test/e2e-vpn/e2e-timeout.sh +source "${SCRIPT_DIR_TIMEOUT}/e2e-timeout.sh" +# shellcheck source=test/e2e-vpn/lib/ci-compatible-inference.sh +. "${SCRIPT_DIR_TIMEOUT}/lib/ci-compatible-inference.sh" + +PASS=0 +FAIL=0 +TOTAL=0 + +pass() { + ((PASS++)) + ((TOTAL++)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} +fail() { + ((FAIL++)) + ((TOTAL++)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } + +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-shields}" + +# shellcheck source=test/e2e-vpn/lib/sandbox-teardown.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib/sandbox-teardown.sh" +register_sandbox_for_teardown "$SANDBOX_NAME" +nemoclaw_e2e_configure_compatible_inference || exit 1 + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" + +CONFIG_PATH="/sandbox/.openclaw/openclaw.json" +AUDIT_FILE="$HOME/.nemoclaw/state/shields-audit.jsonl" + +# ══════════════════════════════════════════════════════════════════ +# Phase 0: Prerequisites +# ══════════════════════════════════════════════════════════════════ +section "Phase 0: Prerequisites" + +if docker info >/dev/null 2>&1; then + pass "Docker is running" +else + fail "Docker is not running — cannot continue" + exit 1 +fi + +if ! nemoclaw_e2e_require_hosted_inference_key; then + exit 1 +fi + +if [ "${NEMOCLAW_NON_INTERACTIVE:-}" != "1" ]; then + fail "NEMOCLAW_NON_INTERACTIVE=1 is required" + exit 1 +fi + +if [ "${NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE:-}" != "1" ]; then + fail "NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 is required" + exit 1 +fi + +pass "Prerequisites OK" + +# ══════════════════════════════════════════════════════════════════ +# Phase 1: Install NemoClaw +# ══════════════════════════════════════════════════════════════════ +section "Phase 1: Install NemoClaw" + +info "Pre-cleanup..." +if command -v nemoclaw >/dev/null 2>&1; then + nemoclaw "$SANDBOX_NAME" destroy --yes 2>/dev/null || true +fi +if command -v openshell >/dev/null 2>&1; then + openshell sandbox delete "$SANDBOX_NAME" 2>/dev/null || true + openshell gateway destroy -g nemoclaw 2>/dev/null || true +fi +rm -f "$HOME/.nemoclaw/onboard.lock" 2>/dev/null || true +rm -f "$AUDIT_FILE" 2>/dev/null || true + +info "Running install.sh..." +cd "$REPO_ROOT" || exit 1 + +export NEMOCLAW_NON_INTERACTIVE=1 +export NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 +export NEMOCLAW_SANDBOX_NAME="${SANDBOX_NAME}" +export NEMOCLAW_RECREATE_SANDBOX=1 + +INSTALL_LOG="/tmp/nemoclaw-e2e-shields-install.log" +if ! bash install.sh --non-interactive >"$INSTALL_LOG" 2>&1; then + fail "install.sh failed (see $INSTALL_LOG)" + exit 1 +fi + +# Source shell profile for nvm/PATH +if [ -f "$HOME/.bashrc" ]; then + # shellcheck source=/dev/null + source "$HOME/.bashrc" 2>/dev/null || true +fi +export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" +if [ -s "$NVM_DIR/nvm.sh" ]; then + # shellcheck source=/dev/null + . "$NVM_DIR/nvm.sh" +fi +if [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]]; then + export PATH="$HOME/.local/bin:$PATH" +fi + +command -v nemoclaw >/dev/null 2>&1 || { + fail "nemoclaw not on PATH" + exit 1 +} +command -v openshell >/dev/null 2>&1 || { + fail "openshell not on PATH" + exit 1 +} +pass "NemoClaw installed (sandbox: $SANDBOX_NAME)" + +# ══════════════════════════════════════════════════════════════════ +# Phase 2: Config is writable (mutable default) +# ══════════════════════════════════════════════════════════════════ +section "Phase 2: Config is writable (mutable default)" + +# Verify file permissions — OpenClaw mutable default is group-writable so the +# gateway UID can write through the shared sandbox group. +PERMS=$(openshell sandbox exec --name "${SANDBOX_NAME}" -- \ + stat -c '%a %U:%G' "${CONFIG_PATH}" 2>/dev/null || true) +info "Config perms (default): ${PERMS}" + +if [ "$(echo "$PERMS" | awk '{print $1}')" = "660" ]; then + pass "Config file mode is 660 (mutable default)" +else + fail "Config file should start as mode 660: ${PERMS}" +fi + +if [ "$(echo "$PERMS" | awk '{print $2}')" = "sandbox:sandbox" ]; then + pass "Config file owned by sandbox:sandbox (mutable default)" +else + fail "Config file should be owned by sandbox:sandbox: ${PERMS}" +fi + +DIR_PERMS=$(openshell sandbox exec --name "${SANDBOX_NAME}" -- \ + stat -c '%a %U:%G' "$(dirname "${CONFIG_PATH}")" 2>/dev/null || true) +info "Config dir perms (default): ${DIR_PERMS}" + +if [ "$(echo "$DIR_PERMS" | awk '{print $1}')" = "2770" ]; then + pass "Config directory mode is 2770 (mutable default)" +else + fail "Config directory should be mode 2770: ${DIR_PERMS}" +fi + +if [ "$(echo "$DIR_PERMS" | awk '{print $2}')" = "sandbox:sandbox" ]; then + pass "Config directory owned by sandbox:sandbox (mutable default)" +else + fail "Config directory should be owned by sandbox:sandbox: ${DIR_PERMS}" +fi + +STATUS_DEFAULT=$(nemoclaw "${SANDBOX_NAME}" shields status 2>&1) +echo "$STATUS_DEFAULT" +if echo "$STATUS_DEFAULT" | grep -q "Shields: NOT CONFIGURED"; then + pass "Fresh sandbox status reports default mutable state" +else + fail "Fresh sandbox status should report NOT CONFIGURED mutable default: ${STATUS_DEFAULT}" +fi + +# OpenShell rejects command arguments containing newlines, so keep the probe +# as a single shell argument. +# shellcheck disable=SC2016 # expanded inside the sandbox by sh -c +LAYOUT_PROBE='bad=0; if [ -e /sandbox/.openclaw-data ] || [ -L /sandbox/.openclaw-data ]; then echo "legacy data dir exists: /sandbox/.openclaw-data"; bad=1; fi; for entry in /sandbox/.openclaw/*; do [ -L "$entry" ] || continue; target="$(readlink -f "$entry" 2>/dev/null || readlink "$entry" 2>/dev/null || true)"; case "$target" in /sandbox/.openclaw-data/*) echo "legacy symlink remains: $entry -> $target"; bad=1 ;; esac; done; exit "$bad"' +LAYOUT_CHECK=$(openshell sandbox exec --name "${SANDBOX_NAME}" -- sh -c "$LAYOUT_PROBE" 2>&1) +if [ -z "$LAYOUT_CHECK" ]; then + pass "Unified .openclaw layout has no .openclaw-data mirror or symlink bridge" +else + fail "Legacy .openclaw-data layout should not exist: ${LAYOUT_CHECK}" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 3: shields up — config becomes immutable +# ══════════════════════════════════════════════════════════════════ +section "Phase 3: shields up" + +SHIELDS_UP_OUTPUT=$(nemoclaw "${SANDBOX_NAME}" shields up 2>&1) +echo "$SHIELDS_UP_OUTPUT" + +if echo "$SHIELDS_UP_OUTPUT" | grep -q "Lockdown active"; then + pass "shields up succeeded" +else + fail "shields up did not report success: ${SHIELDS_UP_OUTPUT}" +fi + +# Verify config is now immutable +PERMS_UP=$(openshell sandbox exec --name "${SANDBOX_NAME}" -- \ + stat -c '%a %U:%G' "${CONFIG_PATH}" 2>/dev/null || true) +info "Config perms (shields UP): ${PERMS_UP}" + +if echo "$PERMS_UP" | grep -qE "^4[0-4][0-4]"; then + pass "Config file has restrictive permissions after shields up (${PERMS_UP})" +else + fail "Config file should be locked after shields up: ${PERMS_UP}" +fi + +OWNER_UP=$(echo "$PERMS_UP" | awk '{print $2}') +if echo "$OWNER_UP" | grep -q "root:root"; then + pass "Config file ownership changed to root:root" +else + fail "Config file ownership not changed to root:root: ${OWNER_UP}" +fi + +# Verify the sandbox user cannot write to the config file +WRITE_RESULT=$(openshell sandbox exec --name "${SANDBOX_NAME}" -- \ + sh -c "echo 'TAMPERED' >> ${CONFIG_PATH} 2>&1 && echo WRITABLE || echo BLOCKED" 2>&1) + +if echo "$WRITE_RESULT" | grep -q "BLOCKED"; then + pass "Config file is read-only for sandbox user (shields UP)" +elif echo "$WRITE_RESULT" | grep -q "Permission denied\|Read-only\|Operation not permitted"; then + pass "Config file write rejected by OS (shields UP)" +else + fail "Config file should be immutable but sandbox could write: ${WRITE_RESULT}" +fi + +WORKSPACE_WRITE_RESULT=$(openshell sandbox exec --name "${SANDBOX_NAME}" -- \ + sh -c "touch /sandbox/.openclaw/workspace/.shields-up-probe 2>&1 && echo WRITABLE || echo BLOCKED" 2>&1) + +if echo "$WORKSPACE_WRITE_RESULT" | grep -q "BLOCKED"; then + pass "Workspace state is read-only for sandbox user (shields UP)" +elif echo "$WORKSPACE_WRITE_RESULT" | grep -q "Permission denied\|Read-only\|Operation not permitted"; then + pass "Workspace write rejected by OS (shields UP)" +else + fail "Workspace should be locked after shields up: ${WORKSPACE_WRITE_RESULT}" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 4: config get — read-only inspection +# ══════════════════════════════════════════════════════════════════ +section "Phase 4: config get" + +CONFIG_GET_OUTPUT=$(nemoclaw "${SANDBOX_NAME}" config get 2>&1) + +if echo "$CONFIG_GET_OUTPUT" | grep -q "{"; then + pass "config get returns JSON" +else + fail "config get did not return JSON: ${CONFIG_GET_OUTPUT}" +fi + +# Verify credentials are redacted +if echo "$CONFIG_GET_OUTPUT" | grep -qE "nvapi-|sk-|Bearer "; then + fail "config get leaks credentials" +else + pass "config get output has no credential leaks" +fi + +# Verify gateway section is stripped +if echo "$CONFIG_GET_OUTPUT" | grep -q '"gateway"'; then + fail "config get should strip gateway section" +else + pass "config get strips gateway section" +fi + +# Test dotpath extraction +DOTPATH_OUTPUT=$(nemoclaw "${SANDBOX_NAME}" config get --key inference 2>&1 || true) +if [ -n "$DOTPATH_OUTPUT" ] && [ "$DOTPATH_OUTPUT" != "null" ]; then + pass "config get --key dotpath works" +else + info "dotpath extraction returned empty (inference key may not exist) — non-fatal" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 5: shields status — shows UP +# ══════════════════════════════════════════════════════════════════ +section "Phase 5: shields status" + +STATUS_OUTPUT=$(nemoclaw "${SANDBOX_NAME}" shields status 2>&1) +echo "$STATUS_OUTPUT" + +if echo "$STATUS_OUTPUT" | grep -q "Shields: UP"; then + pass "shields status reports UP" +else + fail "shields status should show UP: ${STATUS_OUTPUT}" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 5b: content-seal drift detection — host-root chmod-write-chmod +# ══════════════════════════════════════════════════════════════════ +# Verifies the SHA-256 content seal: a host-root tamper that rewrites a +# locked file and restores 444 root:root afterwards leaves mode/owner +# clean but produces a new content hash. `shields status` must flag this +# as drift, and `shields up` must refuse to launder the tampered +# baseline into a fresh seal. +section "Phase 5b: content-seal drift detection" + +CTR=$(docker ps --filter "name=openshell-${SANDBOX_NAME}" -q | head -n1) +if [ -z "$CTR" ]; then + fail "Could not find sandbox container for ${SANDBOX_NAME}" +else + # Use a byte-preserving temp file for backup/restore. Bash command + # substitution `$(...)` strips trailing newlines, which would change + # the file's SHA-256 between backup and restore and create false + # drift after the post-restore status check. The temp file is cleaned + # up at the end of the phase — do not install an EXIT trap here + # because `sandbox-teardown.sh` already owns the EXIT trap and a bare + # `trap '...' EXIT` would clobber the sandbox cleanup. + ORIG_CONTENT_FILE=$(mktemp -t nemoclaw-shields-orig.XXXXXX) + if ! docker exec -u 0 "$CTR" cat "$CONFIG_PATH" >"$ORIG_CONTENT_FILE" 2>/dev/null; then + fail "Could not read original ${CONFIG_PATH} content as host root" + elif [ ! -s "$ORIG_CONTENT_FILE" ]; then + fail "Original ${CONFIG_PATH} read returned an empty file" + else + # When shields-up applied `chattr +i`, `chmod 644` alone would EPERM + # and the tamper would no-op — masking the seal check. Drop the + # immutable bit best-effort before the tamper, then restore it after + # so the post-tamper file is indistinguishable from the locked + # baseline by `stat`/`lsattr` alone. Track whether `+i` was applied + # via `lsattr -d` so we only re-apply when it was set before. + LSATTR_BEFORE=$(docker exec -u 0 "$CTR" lsattr -d "$CONFIG_PATH" 2>/dev/null | awk '{print $1}' || true) + HAD_IMMUTABLE_BIT=false + if echo "$LSATTR_BEFORE" | grep -q "i"; then + HAD_IMMUTABLE_BIT=true + fi + docker exec -u 0 "$CTR" sh -c \ + "chattr -i ${CONFIG_PATH} 2>/dev/null || true; \ + chmod 644 ${CONFIG_PATH} && printf ' ' >> ${CONFIG_PATH} && chmod 444 ${CONFIG_PATH}" \ + >/dev/null 2>&1 + TAMPER_EXIT=$? + if [ "$HAD_IMMUTABLE_BIT" = "true" ]; then + docker exec -u 0 "$CTR" chattr +i "$CONFIG_PATH" >/dev/null 2>&1 || true + fi + if [ "$TAMPER_EXIT" = "0" ]; then + pass "Tamper command executed (chmod-write-chmod) without error" + else + fail "Tamper command failed (exit ${TAMPER_EXIT}); cannot validate drift detection" + fi + PERMS_AFTER_TAMPER=$(docker exec "$CTR" stat -c '%a %U:%G' "$CONFIG_PATH" 2>/dev/null || true) + info "Config perms after chmod-write-chmod tamper: ${PERMS_AFTER_TAMPER}" + if [ "$PERMS_AFTER_TAMPER" = "444 root:root" ]; then + pass "Tamper restored 444 root:root (mode/owner alone cannot detect drift)" + else + fail "Expected tamper to leave 444 root:root, got: ${PERMS_AFTER_TAMPER}" + fi + + # The script runs with `set -uo pipefail` (no -e), so `$?` after a + # command substitution gives that command's exit code without + # aborting the script. Toggling `set -e` here would interact badly + # with the `fail()` helper, whose `((FAIL++))` returns a non-zero + # exit when FAIL is 0 and would abort under -e. + STATUS_TAMPER_OUTPUT=$(nemoclaw "${SANDBOX_NAME}" shields status 2>&1) + STATUS_TAMPER_EXIT=$? + echo "$STATUS_TAMPER_OUTPUT" + if [ "$STATUS_TAMPER_EXIT" = "2" ]; then + pass "shields status exits 2 on content drift" + else + fail "shields status should exit 2 on content drift, got ${STATUS_TAMPER_EXIT}" + fi + if echo "$STATUS_TAMPER_OUTPUT" | grep -q "UP (DRIFTED"; then + pass "shields status surfaces DRIFTED on content drift" + else + fail "shields status should surface DRIFTED line on content drift" + fi + if echo "$STATUS_TAMPER_OUTPUT" | grep -q "content drifted"; then + pass "shields status names the drifted file" + else + fail "shields status should name the drifted file" + fi + + REUP_OUTPUT=$(nemoclaw "${SANDBOX_NAME}" shields up 2>&1) + REUP_EXIT=$? + echo "$REUP_OUTPUT" + if [ "$REUP_EXIT" != "0" ]; then + pass "shields up refuses to re-seal a tampered baseline (exit ${REUP_EXIT})" + else + fail "shields up should refuse to re-seal a tampered baseline" + fi + if echo "$REUP_OUTPUT" | grep -q "Refusing to re-seal"; then + pass "shields up surfaces the refuse-to-re-seal message" + else + fail "shields up should surface the refuse-to-re-seal message" + fi + + # Restore the original content as host root so the rest of the suite + # can continue against a clean lock. Drop the immutable bit (if any) + # before the write and re-apply it after so the file ends in the + # same chattr posture it started in. `docker exec -i` keeps stdin + # open and we stream the backup file straight in — no command + # substitution that would strip trailing newlines. + docker exec -i -u 0 "$CTR" sh -c \ + "chattr -i ${CONFIG_PATH} 2>/dev/null || true; \ + chmod 644 ${CONFIG_PATH} && cat > ${CONFIG_PATH} && chmod 444 ${CONFIG_PATH}" \ + <"$ORIG_CONTENT_FILE" >/dev/null 2>&1 + if [ "$HAD_IMMUTABLE_BIT" = "true" ]; then + docker exec -u 0 "$CTR" chattr +i "$CONFIG_PATH" >/dev/null 2>&1 || true + fi + POST_RESTORE_OUTPUT=$(nemoclaw "${SANDBOX_NAME}" shields status 2>&1 || true) + if echo "$POST_RESTORE_OUTPUT" | grep -q "Shields: UP (lockdown active)"; then + pass "shields status clean after content restore" + else + fail "shields status should report clean UP after content restore: ${POST_RESTORE_OUTPUT}" + fi + fi + rm -f "$ORIG_CONTENT_FILE" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 6: shields down — config returns to writable +# ══════════════════════════════════════════════════════════════════ +section "Phase 6: shields down" + +SHIELDS_DOWN_OUTPUT=$(nemoclaw "${SANDBOX_NAME}" shields down \ + --timeout 5m --reason "E2E shields lifecycle test" 2>&1) +echo "$SHIELDS_DOWN_OUTPUT" + +if echo "$SHIELDS_DOWN_OUTPUT" | grep -q "Config unlocked"; then + pass "shields down succeeded" +else + fail "shields down did not report success: ${SHIELDS_DOWN_OUTPUT}" +fi + +# Check permissions changed — OpenClaw shields-down uses sandbox:sandbox +# 660/2770 so the gateway UID can write the mutable config tree. +PERMS_DOWN=$(openshell sandbox exec --name "${SANDBOX_NAME}" -- \ + stat -c '%a %U:%G' "${CONFIG_PATH}" 2>/dev/null || true) +info "Config perms (shields DOWN): ${PERMS_DOWN}" + +if [ "$(echo "$PERMS_DOWN" | awk '{print $1}')" = "660" ]; then + pass "Config file mode is 660 (restored to mutable default)" +else + fail "Config file should be mode 660 after shields down: ${PERMS_DOWN}" +fi + +if [ "$(echo "$PERMS_DOWN" | awk '{print $2}')" = "sandbox:sandbox" ]; then + pass "Config file owned by sandbox:sandbox after shields down" +else + fail "Config file should be owned by sandbox:sandbox: ${PERMS_DOWN}" +fi + +DIR_PERMS_DOWN=$(openshell sandbox exec --name "${SANDBOX_NAME}" -- \ + stat -c '%a %U:%G' "$(dirname "${CONFIG_PATH}")" 2>/dev/null || true) +info "Config dir perms (shields DOWN): ${DIR_PERMS_DOWN}" + +if [ "$(echo "$DIR_PERMS_DOWN" | awk '{print $1}')" = "2770" ]; then + pass "Config directory mode is 2770 (restored to mutable default)" +else + fail "Config directory should be mode 2770 after shields down: ${DIR_PERMS_DOWN}" +fi + +if [ "$(echo "$DIR_PERMS_DOWN" | awk '{print $2}')" = "sandbox:sandbox" ]; then + pass "Config directory owned by sandbox:sandbox after shields down" +else + fail "Config directory should be owned by sandbox:sandbox: ${DIR_PERMS_DOWN}" +fi + +WORKSPACE_DOWN_RESULT=$(openshell sandbox exec --name "${SANDBOX_NAME}" -- \ + sh -c "touch /sandbox/.openclaw/workspace/.shields-down-probe 2>&1 && rm -f /sandbox/.openclaw/workspace/.shields-down-probe && echo WRITABLE || echo BLOCKED" 2>&1) +if echo "$WORKSPACE_DOWN_RESULT" | grep -q "WRITABLE"; then + pass "Workspace state is writable again after shields down" +else + fail "Workspace should be writable after shields down: ${WORKSPACE_DOWN_RESULT}" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 7: shields status — shows DOWN +# ══════════════════════════════════════════════════════════════════ +section "Phase 7: shields status" + +STATUS_DOWN=$(nemoclaw "${SANDBOX_NAME}" shields status 2>&1) +echo "$STATUS_DOWN" + +if echo "$STATUS_DOWN" | grep -q "Shields: DOWN"; then + pass "shields status reports DOWN" +else + fail "shields status should show DOWN: ${STATUS_DOWN}" +fi + +if echo "$STATUS_DOWN" | grep -q "E2E shields lifecycle test"; then + pass "shields status shows reason" +else + fail "shields status should show reason: ${STATUS_DOWN}" +fi + +if echo "$STATUS_DOWN" | grep -q "remaining"; then + pass "shields status shows timeout remaining" +else + info "shields status timeout display not found — non-fatal" +fi + +# Restore shields for the next phase +if RESTORE_UP_OUTPUT=$(nemoclaw "${SANDBOX_NAME}" shields up 2>&1); then + echo "$RESTORE_UP_OUTPUT" + pass "shields up restored for audit trail test" +else + echo "$RESTORE_UP_OUTPUT" + fail "Failed to restore shields up before audit phase: ${RESTORE_UP_OUTPUT}" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 8: Audit trail +# ══════════════════════════════════════════════════════════════════ +section "Phase 8: Audit trail" + +if [ -f "$AUDIT_FILE" ]; then + AUDIT_LINES=$(wc -l <"$AUDIT_FILE") + info "Audit entries: ${AUDIT_LINES}" + + # Should have at least: shields_up, shields_down, shields_up + DOWN_COUNT=$(grep -c '"shields_down"' "$AUDIT_FILE" || true) + UP_COUNT=$(grep -c '"shields_up"' "$AUDIT_FILE" || true) + + if [ "$UP_COUNT" -ge 2 ]; then + pass "Audit has ≥2 shields_up entries (got ${UP_COUNT})" + else + fail "Expected ≥2 shields_up audit entries, got ${UP_COUNT}" + fi + + if [ "$DOWN_COUNT" -ge 1 ]; then + pass "Audit has ≥1 shields_down entries (got ${DOWN_COUNT})" + else + fail "Expected ≥1 shields_down audit entries, got ${DOWN_COUNT}" + fi + + # Verify no credentials in audit + if grep -qE "nvapi-|sk-|Bearer " "$AUDIT_FILE"; then + fail "Audit trail contains credentials" + else + pass "Audit trail is credential-free" + fi + + # Verify each entry is valid JSON + INVALID_JSON=0 + while IFS= read -r line; do + if ! echo "$line" | python3 -c "import sys,json; json.load(sys.stdin)" 2>/dev/null; then + ((INVALID_JSON++)) + fi + done <"$AUDIT_FILE" + + if [ "$INVALID_JSON" -eq 0 ]; then + pass "All audit entries are valid JSON" + else + fail "${INVALID_JSON} audit entries are invalid JSON" + fi +else + fail "Audit file not found: $AUDIT_FILE" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 9: Auto-restore timer +# ══════════════════════════════════════════════════════════════════ +section "Phase 9: Auto-restore timer" + +# shields down with a 10s timeout starts an auto-restore timer that +# re-locks config (shields up) after the timeout expires. +nemoclaw "${SANDBOX_NAME}" shields down --timeout 10s --reason "Auto-restore timer E2E" 2>&1 + +# Verify shields are down +STATUS_TIMER=$(nemoclaw "${SANDBOX_NAME}" shields status 2>&1) +if echo "$STATUS_TIMER" | grep -q "Shields: DOWN"; then + pass "shields down with 10s timeout" +else + fail "shields should be DOWN: ${STATUS_TIMER}" +fi + +info "Polling for auto-restore to shields UP (up to 60s)..." +TIMER_RESTORED=false +for _poll in $(seq 1 12); do + sleep 5 + STATUS_AFTER_TIMER=$(nemoclaw "${SANDBOX_NAME}" shields status 2>&1) + if echo "$STATUS_AFTER_TIMER" | grep -q "Shields: UP"; then + TIMER_RESTORED=true + break + fi +done + +if [ "$TIMER_RESTORED" = "true" ]; then + pass "Auto-restore timer re-locked config after timeout" +else + info "Auto-restore may not have fired (timer runs as detached process)" + info "Status: ${STATUS_AFTER_TIMER}" + fail "Auto-restore timer did not re-lock within 60s" +fi + +# Verify config is locked after auto-restore +PERMS_TIMER=$(openshell sandbox exec --name "${SANDBOX_NAME}" -- \ + stat -c '%a' "${CONFIG_PATH}" 2>/dev/null || true) +if echo "$PERMS_TIMER" | grep -qE "^4[0-4][0-4]"; then + pass "Config locked after auto-restore (${PERMS_TIMER})" +else + fail "Config should be locked after auto-restore, got: ${PERMS_TIMER}" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 10: Double shields-up rejected +# ══════════════════════════════════════════════════════════════════ +section "Phase 10: Double shields-up rejected" + +nemoclaw "${SANDBOX_NAME}" shields up 2>&1 +DOUBLE_UP=$(nemoclaw "${SANDBOX_NAME}" shields up 2>&1 || true) + +if echo "$DOUBLE_UP" | grep -q "already active"; then + pass "Double shields-up rejected" +else + fail "Double shields-up should be rejected: ${DOUBLE_UP}" +fi + +nemoclaw "${SANDBOX_NAME}" shields down --timeout 5m --reason "Cleanup" 2>&1 +pass "Cleanup: shields down" + +# ══════════════════════════════════════════════════════════════════ +# Phase 11: Double shields-down rejected +# ══════════════════════════════════════════════════════════════════ +section "Phase 11: Double shields-down rejected" + +DOUBLE_DOWN=$(nemoclaw "${SANDBOX_NAME}" shields down --timeout 5m --reason "Should fail" 2>&1 || true) + +if echo "$DOUBLE_DOWN" | grep -q "already unlocked"; then + pass "Double shields-down rejected" +else + fail "Double shields-down should be rejected: ${DOUBLE_DOWN}" +fi + +# ══════════════════════════════════════════════════════════════════ +# Cleanup +# ══════════════════════════════════════════════════════════════════ +section "Cleanup" + +[[ "${NEMOCLAW_E2E_KEEP_SANDBOX:-}" = "1" ]] || nemoclaw "${SANDBOX_NAME}" destroy --yes 2>/dev/null || true +pass "Sandbox destroyed" + +# ══════════════════════════════════════════════════════════════════ +# Summary +# ══════════════════════════════════════════════════════════════════ +echo "" +echo "════════════════════════════════════════════" +printf " Total: %d | \033[32mPassed: %d\033[0m | \033[31mFailed: %d\033[0m\n" "$TOTAL" "$PASS" "$FAIL" +echo "════════════════════════════════════════════" + +if [ "$FAIL" -gt 0 ]; then + exit 1 +fi diff --git a/test/e2e-vpn/test-skill-agent-e2e.sh b/test/e2e-vpn/test-skill-agent-e2e.sh new file mode 100755 index 00000000000..2e984fe63cc --- /dev/null +++ b/test/e2e-vpn/test-skill-agent-e2e.sh @@ -0,0 +1,269 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Skill Agent E2E — Skill injection + agent verification +# +# Injects a skill fixture into the sandbox and verifies the agent reads +# the skill's SKILL.md and returns the verification token. Includes retry +# logic and fuzzy matching to handle LLM non-determinism. +# +# Split from the cloud-experimental-e2e monolith (see #2644). +# Former phase: 5d (skill agent verification). +# +# Prerequisites: +# - Docker running +# - NVIDIA_API_KEY set for hosted inference +# - NEMOCLAW_NON_INTERACTIVE=1, NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 +# +# Environment: +# NEMOCLAW_SANDBOX_NAME — sandbox name (default: e2e-skill-agent) +# NEMOCLAW_RECREATE_SANDBOX=1 — recreate if exists +# E2E_SKILL_AGENT_MAX_ATTEMPTS — agent turn retries (default: 3) +# E2E_SKILL_AGENT_RETRY_SLEEP_SEC — seconds between retries (default: 15) +# +# Usage: +# NEMOCLAW_NON_INTERACTIVE=1 NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ +# NVIDIA_API_KEY=... bash test/e2e-vpn/test-skill-agent-e2e.sh + +# ShellCheck cannot see EXIT trap invocations of cleanup helpers in this E2E script. +# shellcheck disable=SC2317 +set -uo pipefail + +PASS=0 +FAIL=0 +SKIP=0 +TOTAL=0 + +pass() { + ((PASS++)) + ((TOTAL++)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} +fail() { + ((FAIL++)) + ((TOTAL++)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} +# shellcheck disable=SC2329 +skip() { + ((SKIP++)) + ((TOTAL++)) + printf '\033[33m SKIP: %s\033[0m\n' "$1" +} +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } + +quote_for_remote_sh() { + local value="${1:-}" + printf "'%s'" "$(printf '%s' "$value" | sed "s/'/'\\\\''/g")" +} + +is_external_agent_verification_flake() { + grep -qiE 'LLM idle timeout|request timed out|fetch timeout|model did not produce a response|tool_search_code failed|describe id must be a string|openclaw\.tools\.[A-Za-z0-9_]+ is not a function|call id must be a string|ReferenceError: require is not defined|ssh/agent exit 124|exit 124' <<<"$1" +} + +verify_skill_fixture_present() { + local token skill remote_cmd + token="$(quote_for_remote_sh "$VERIFY_PHRASE")" + skill="$(quote_for_remote_sh "$SKILL_ID")" + remote_cmd="token=${token}; skill=${skill}; found=0; for path in \"/sandbox/.openclaw/skills/\${skill}/SKILL.md\" \"\${HOME:-/home/sandbox}/.openclaw/skills/\${skill}/SKILL.md\" \"/home/sandbox/.openclaw/skills/\${skill}/SKILL.md\" \"/home/openclaw/.openclaw/skills/\${skill}/SKILL.md\"; do if [ -f \"\$path\" ] && grep -Fq \"\$token\" \"\$path\"; then echo \"SKILL_TOKEN_PATH=\$path\"; found=1; fi; done; test \"\$found\" = 1" + openshell sandbox exec --name "$SANDBOX_NAME" -- sh -lc "$remote_cmd" +} + +# ── Repo root ── +_script_dir="$(cd "$(dirname "$0")" && pwd)" +_candidate="$(cd "${_script_dir}/../.." && pwd)" +if [ -d /workspace ] && [ -f /workspace/package.json ] && [ -d /workspace/test/e2e ]; then + REPO="/workspace" +elif [ -f "${_candidate}/package.json" ] && [ -d "${_candidate}/test/e2e" ]; then + REPO="${_candidate}" +else + echo "ERROR: Cannot find repo root." + exit 1 +fi +unset _script_dir _candidate + +E2E_DIR="$(cd "$(dirname "$0")" && pwd)" +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-skill-agent}" +SKILL_ID="skill-smoke-fixture" +VERIFY_PHRASE="SKILL_SMOKE_VERIFY_K9X2" +MAX_ATTEMPTS="${E2E_SKILL_AGENT_MAX_ATTEMPTS:-3}" +RETRY_SLEEP="${E2E_SKILL_AGENT_RETRY_SLEEP_SEC:-15}" +[[ "$MAX_ATTEMPTS" =~ ^[1-9][0-9]*$ ]] || MAX_ATTEMPTS=3 + +# Source shared teardown helper +# shellcheck source=test/e2e-vpn/lib/sandbox-teardown.sh +. "${E2E_DIR}/lib/sandbox-teardown.sh" +# shellcheck source=test/e2e-vpn/lib/ci-compatible-inference.sh +. "${E2E_DIR}/lib/ci-compatible-inference.sh" +register_sandbox_for_teardown "$SANDBOX_NAME" +nemoclaw_e2e_configure_compatible_inference || exit 1 + +# ══════════════════════════════════════════════════════════════════════ +# Phase 1: Install + Prerequisites +# ══════════════════════════════════════════════════════════════════════ +section "Phase 1: Install + Prerequisites" + +if ! docker info >/dev/null 2>&1; then + fail "Docker is not running" + exit 1 +fi +pass "Docker is running" + +if ! nemoclaw_e2e_require_hosted_inference_key; then + exit 1 +fi + +cd "$REPO" || { + fail "Could not cd to repo root" + exit 1 +} + +export NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" +export NEMOCLAW_RECREATE_SANDBOX="${NEMOCLAW_RECREATE_SANDBOX:-1}" + +info "Installing NemoClaw via install.sh --non-interactive..." +INSTALL_LOG="/tmp/nemoclaw-e2e-skill-agent-install.log" +bash install.sh --non-interactive --yes-i-accept-third-party-software >"$INSTALL_LOG" 2>&1 & +install_pid=$! +tail -f "$INSTALL_LOG" --pid=$install_pid 2>/dev/null & +tail_pid=$! +wait "$install_pid" +install_exit=$? +kill "$tail_pid" 2>/dev/null || true +wait "$tail_pid" 2>/dev/null || true + +# Source shell profile +if [ -f "$HOME/.bashrc" ]; then + # shellcheck source=/dev/null + source "$HOME/.bashrc" 2>/dev/null || true +fi +export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" +# shellcheck source=/dev/null +[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" +[ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]] && export PATH="$HOME/.local/bin:$PATH" + +if [ "$install_exit" -ne 0 ]; then + fail "install.sh failed (exit $install_exit)" + tail -30 "$INSTALL_LOG" + exit 1 +fi +pass "NemoClaw installed" + +command -v nemoclaw >/dev/null 2>&1 || { + fail "nemoclaw not on PATH" + exit 1 +} +command -v openshell >/dev/null 2>&1 || { + fail "openshell not on PATH" + exit 1 +} +pass "CLIs on PATH" + +# ══════════════════════════════════════════════════════════════════════ +# Phase 2: Inject skill fixture +# ══════════════════════════════════════════════════════════════════════ +section "Phase 2: Inject skill fixture" + +info "Injecting ${SKILL_ID} into sandbox '${SANDBOX_NAME}'..." +if ! SANDBOX_NAME="$SANDBOX_NAME" \ + SKILL_ID="$SKILL_ID" \ + SKILL_DESCRIPTION="E2E smoke skill injected for agent verification" \ + bash "$E2E_DIR/e2e-cloud-experimental/features/skill/add-sandbox-skill.sh"; then + fail "Failed to inject ${SKILL_ID}" + exit 1 +fi +pass "${SKILL_ID} injected and queryable" + +# ══════════════════════════════════════════════════════════════════════ +# Phase 3: Agent verification with retry + fuzzy matching +# ══════════════════════════════════════════════════════════════════════ +section "Phase 3: Agent verification (${MAX_ATTEMPTS} attempts, ${RETRY_SLEEP}s between)" + +attempt=1 +agent_ok=0 +last_fail="" +last_agent_out="" + +while [ "$attempt" -le "$MAX_ATTEMPTS" ]; do + info "Attempt ${attempt}/${MAX_ATTEMPTS}: running openclaw agent turn..." + + set +e + agent_out=$( + NVIDIA_API_KEY="$NVIDIA_API_KEY" \ + SANDBOX_NAME="$SANDBOX_NAME" \ + SKILL_ID="$SKILL_ID" \ + VERIFY_TOKEN="$VERIFY_PHRASE" \ + bash "$E2E_DIR/e2e-cloud-experimental/features/skill/verify-sandbox-skill-via-agent.sh" 2>&1 + ) + agent_rc=$? + set -uo pipefail + last_agent_out="$agent_out" + + if [ "$agent_rc" -eq 0 ]; then + pass "Agent returned ${VERIFY_PHRASE} (attempt ${attempt}/${MAX_ATTEMPTS})" + agent_ok=1 + break + fi + + # Fuzzy fallback: check if the token appears in the *agent output section only*, + # not in helper diagnostic/error lines. The helper delimits agent output with + # "--- agent stdout/stderr" / "--- end ---" markers. We extract only that + # section to avoid false positives from error messages that echo the token + # (see Brandon's review on #2647). + agent_section=$(printf '%s' "$agent_out" | sed -n '/--- agent stdout\/stderr/,/--- end ---/p') + if [ -n "$agent_section" ]; then + collapsed=$(printf '%s' "$agent_section" | tr -d '\n\r' | tr -d '`"'\''' | tr '[:upper:]' '[:lower:]') + token_lower=$(printf '%s' "$VERIFY_PHRASE" | tr '[:upper:]' '[:lower:]') + if printf '%s' "$collapsed" | grep -Fq "$token_lower"; then + info "Token found in agent output section (fuzzy match — script exited ${agent_rc} but token present in delimited output)" + pass "Agent returned ${VERIFY_PHRASE} via fuzzy match (attempt ${attempt}/${MAX_ATTEMPTS})" + agent_ok=1 + break + fi + fi + + last_fail="Agent verification failed (exit ${agent_rc})" + + if [ "$attempt" -ge "$MAX_ATTEMPTS" ]; then break; fi + info "Attempt ${attempt}/${MAX_ATTEMPTS} failed — sleeping ${RETRY_SLEEP}s before retry..." + sleep "$RETRY_SLEEP" + attempt=$((attempt + 1)) +done + +if [ "$agent_ok" -ne 1 ]; then + info "Last agent verification output (tail):" + printf '%s\n' "$last_agent_out" | tail -c 12000 + printf '\n' + + if is_external_agent_verification_flake "$last_agent_out" && verify_skill_fixture_present; then + skip "Agent verification inconclusive due to model/tool-call behavior; skill fixture is present and queryable" + else + fail "$last_fail" + exit 1 + fi +fi + +# ══════════════════════════════════════════════════════════════════════ +# Summary +# ══════════════════════════════════════════════════════════════════════ +echo "" +echo "========================================" +echo " Skill Agent E2E Results:" +echo " Passed: $PASS" +echo " Failed: $FAIL" +echo " Skipped: $SKIP" +echo " Total: $TOTAL" +echo "========================================" + +if [ "$FAIL" -eq 0 ]; then + printf '\033[1;32m\n Skill Agent E2E PASSED.\033[0m\n' + exit 0 +else + printf '\033[1;31m\n %d test(s) failed.\033[0m\n' "$FAIL" + exit 1 +fi diff --git a/test/e2e-vpn/test-snapshot-commands.sh b/test/e2e-vpn/test-snapshot-commands.sh new file mode 100755 index 00000000000..d187cfccf76 --- /dev/null +++ b/test/e2e-vpn/test-snapshot-commands.sh @@ -0,0 +1,288 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Snapshot commands E2E — validates the full snapshot create/list/restore lifecycle: +# +# 1. Install NemoClaw (install.sh) +# 2. Write marker files into sandbox workspace +# 3. nemoclaw snapshot create — verify snapshot created +# 4. nemoclaw snapshot list — verify snapshot appears in list +# 5. Delete marker files from sandbox (simulate data loss) +# 6. nemoclaw snapshot restore — verify markers restored +# 7. nemoclaw snapshot restore — verify targeted restore +# 8. No credentials in snapshot directory +# +# Prerequisites: +# - Docker running +# - NVIDIA_API_KEY set (real key, starts with nvapi-) +# +# Environment variables: +# NEMOCLAW_NON_INTERACTIVE=1 — required +# NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 — required +# NVIDIA_API_KEY — required + +set -euo pipefail + +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-snapshot}" + +# shellcheck source=test/e2e-vpn/lib/sandbox-teardown.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib/sandbox-teardown.sh" +register_sandbox_for_teardown "$SANDBOX_NAME" + +MARKER_FILE="/sandbox/.openclaw/workspace/snapshot-marker.txt" +MARKER_CONTENT="SNAPSHOT_E2E_$(date +%s)" +SECOND_MARKER="/sandbox/.openclaw/workspace/snapshot-marker-2.txt" +SECOND_CONTENT="SNAPSHOT_E2E_SECOND_$(date +%s)" + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +pass() { echo -e "${GREEN}[PASS]${NC} $1"; } + +# Shared diagnostics — called by fail() and Phase 2b. +# Intentionally non-reentrant (single-threaded bash). +dump_diagnostics() { + local _fd="${1:-2}" # default to stderr + echo -e "${YELLOW}[DIAG]${NC} --- Diagnostics ---" >&"$_fd" + echo -e "${YELLOW}[DIAG]${NC} nemoclaw path: $(command -v nemoclaw 2>&1 || echo 'not found')" >&"$_fd" + echo -e "${YELLOW}[DIAG]${NC} nemoclaw version: $(nemoclaw --version 2>&1 || echo 'failed')" >&"$_fd" + echo -e "${YELLOW}[DIAG]${NC} node version: $(node --version 2>&1 || echo 'not found')" >&"$_fd" + echo -e "${YELLOW}[DIAG]${NC} Sandboxes: $(openshell sandbox list 2>&1 || echo 'unavailable')" >&"$_fd" + echo -e "${YELLOW}[DIAG]${NC} Backup dir: $(ls -la "$HOME/.nemoclaw/rebuild-backups/${SANDBOX_NAME}/" 2>&1 || echo 'not found')" >&"$_fd" + echo -e "${YELLOW}[DIAG]${NC} Registry: $(cat "$HOME/.nemoclaw/sandboxes.json" 2>&1 || echo 'not found')" >&"$_fd" + echo -e "${YELLOW}[DIAG]${NC} Registry lock: $(ls -la "$HOME/.nemoclaw/sandboxes.json.lock" 2>&1 || echo 'no lock')" >&"$_fd" + echo -e "${YELLOW}[DIAG]${NC} Config dir: $(ls -la "$HOME/.nemoclaw/" 2>&1 || echo 'not found')" >&"$_fd" + echo -e "${YELLOW}[DIAG]${NC} Docker ps: $(docker ps --format '{{.Names}} {{.Status}}' 2>&1 || echo 'unavailable')" >&"$_fd" + echo -e "${YELLOW}[DIAG]${NC} --- End diagnostics ---" >&"$_fd" +} + +fail() { + echo -e "${RED}[FAIL]${NC} $1" >&2 + dump_diagnostics 2 + exit 1 +} +info() { echo -e "${YELLOW}[INFO]${NC} $1"; } + +# Run a command, capture its output and exit code without set -e killing us. +# Usage: run_capture VAR_NAME command [args...] +# Sets $VAR_NAME to the combined stdout+stderr and $_CAPTURE_RC to the exit code. +_CAPTURE_RC=0 +run_capture() { + local _var_name="$1" + shift + _CAPTURE_RC=0 + local _output + _output=$("$@" 2>&1) || _CAPTURE_RC=$? + printf -v "$_var_name" '%s' "$_output" +} + +# ── Preflight ─────────────────────────────────────────────────────── +[ -n "${NVIDIA_API_KEY:-}" ] || fail "NVIDIA_API_KEY is required" +[ "${NEMOCLAW_NON_INTERACTIVE:-}" = "1" ] || fail "NEMOCLAW_NON_INTERACTIVE=1 is required" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" + +info "Snapshot commands E2E (sandbox: ${SANDBOX_NAME})" + +# ── Phase 1: Install NemoClaw ─────────────────────────────────────── +info "Phase 1: Installing NemoClaw via install.sh..." + +export NEMOCLAW_NON_INTERACTIVE=1 +export NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 +export NEMOCLAW_SANDBOX_NAME="${SANDBOX_NAME}" +export NEMOCLAW_RECREATE_SANDBOX=1 + +INSTALL_LOG="/tmp/nemoclaw-e2e-install.log" +if ! bash "${REPO_ROOT}/install.sh" --non-interactive >"$INSTALL_LOG" 2>&1; then + info "install.sh exited non-zero (may be expected on re-install). Checking for nemoclaw..." +fi + +# Source shell profile to pick up nvm/PATH changes +if [ -f "$HOME/.bashrc" ]; then + # shellcheck source=/dev/null + source "$HOME/.bashrc" 2>/dev/null || true +fi +export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" +if [ -s "$NVM_DIR/nvm.sh" ]; then + # shellcheck source=/dev/null + . "$NVM_DIR/nvm.sh" +fi +if [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]]; then + export PATH="$HOME/.local/bin:$PATH" +fi + +command -v nemoclaw >/dev/null 2>&1 || fail "nemoclaw not found on PATH after install" +command -v openshell >/dev/null 2>&1 || fail "openshell not found on PATH after install" +pass "NemoClaw installed" + +# ── Phase 2: Write marker files ──────────────────────────────────── +info "Phase 2: Writing marker files into sandbox..." + +openshell sandbox exec --name "${SANDBOX_NAME}" -- \ + sh -c "mkdir -p /sandbox/.openclaw/workspace && echo '${MARKER_CONTENT}' > ${MARKER_FILE}" \ + || fail "Failed to write marker file" + +VERIFY=$(openshell sandbox exec --name "${SANDBOX_NAME}" -- cat "${MARKER_FILE}" 2>/dev/null || true) +[ "$VERIFY" = "${MARKER_CONTENT}" ] || fail "Marker verification failed: got '${VERIFY}'" + +pass "Marker file written" + +# ── Phase 2b: Pre-snapshot diagnostics ───────────────────────────── +# Collect state that helps diagnose Phase 3 failures (see #2350). +info "Phase 2b: Pre-snapshot diagnostics..." +dump_diagnostics 1 # stdout — informational, not a failure + +# ── Phase 3: snapshot create ──────────────────────────────────────── +info "Phase 3: Creating snapshot..." + +# Use run_capture to prevent set -e from swallowing error output. +# Previously, $(nemoclaw ... 2>&1) would exit the script immediately on +# failure, hiding the actual error message. See #2350. +run_capture SNAPSHOT_OUTPUT nemoclaw "${SANDBOX_NAME}" snapshot create +echo "$SNAPSHOT_OUTPUT" + +if [ "$_CAPTURE_RC" -ne 0 ]; then + fail "snapshot create exited with code $_CAPTURE_RC: ${SNAPSHOT_OUTPUT}" +fi + +# The success marker is `Snapshot v created ( directories)` — the +# version token between "Snapshot" and "created" broke the old literal grep +# for "Snapshot created". Use a regex that tolerates the version field. +if echo "$SNAPSHOT_OUTPUT" | grep -qE "Snapshot v[0-9]+.*created"; then + pass "snapshot create succeeded" +else + fail "snapshot create did not report success: ${SNAPSHOT_OUTPUT}" +fi + +# Extract the snapshot path from output +SNAPSHOT_PATH=$(echo "$SNAPSHOT_OUTPUT" | grep -oE "/[^ ]*rebuild-backups/[^ ]+" || true) +info "Snapshot path: ${SNAPSHOT_PATH:-unknown}" + +# ── Phase 4: snapshot list ────────────────────────────────────────── +info "Phase 4: Listing snapshots..." + +run_capture LIST_OUTPUT nemoclaw "${SANDBOX_NAME}" snapshot list +echo "$LIST_OUTPUT" + +if [ "$_CAPTURE_RC" -ne 0 ]; then + fail "snapshot list exited with code $_CAPTURE_RC: ${LIST_OUTPUT}" +fi + +if echo "$LIST_OUTPUT" | grep -q "snapshot(s)"; then + pass "snapshot list shows snapshots" +else + fail "snapshot list shows no snapshots: ${LIST_OUTPUT}" +fi + +# Extract the timestamp from list output for targeted restore later +SNAPSHOT_TIMESTAMP=$(echo "$LIST_OUTPUT" | grep -oE "[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}-[0-9]{2}-[0-9]{2}-[0-9]+Z" | head -1 || true) +[ -n "${SNAPSHOT_TIMESTAMP}" ] || fail "Failed to parse a snapshot timestamp from list output: ${LIST_OUTPUT}" +info "Snapshot timestamp: ${SNAPSHOT_TIMESTAMP}" + +# ── Phase 5: Delete marker + write second marker, create 2nd snapshot +info "Phase 5: Modifying sandbox state and creating second snapshot..." + +openshell sandbox exec --name "${SANDBOX_NAME}" -- \ + sh -c "rm -f ${MARKER_FILE} && echo '${SECOND_CONTENT}' > ${SECOND_MARKER}" \ + || fail "Failed to modify sandbox state" + +# Verify first marker is gone +GONE=$(openshell sandbox exec --name "${SANDBOX_NAME}" -- cat "${MARKER_FILE}" 2>/dev/null || echo "GONE") +[ "$GONE" = "GONE" ] || fail "First marker should be deleted but got: ${GONE}" + +run_capture _SECOND_SNAP nemoclaw "${SANDBOX_NAME}" snapshot create +if [ "$_CAPTURE_RC" -ne 0 ]; then + fail "Second snapshot create failed (code $_CAPTURE_RC): ${_SECOND_SNAP}" +fi +pass "State modified, second snapshot created" + +# Perturb workspace so restore has to do real work +openshell sandbox exec --name "${SANDBOX_NAME}" -- \ + sh -c "rm -f ${SECOND_MARKER} && echo 'BROKEN' > ${MARKER_FILE}" \ + || fail "Failed to perturb sandbox before latest restore" + +# ── Phase 6: snapshot restore (latest) ────────────────────────────── +info "Phase 6: Restoring latest snapshot..." + +run_capture RESTORE_OUTPUT nemoclaw "${SANDBOX_NAME}" snapshot restore +echo "$RESTORE_OUTPUT" + +if [ "$_CAPTURE_RC" -ne 0 ]; then + fail "snapshot restore exited with code $_CAPTURE_RC: ${RESTORE_OUTPUT}" +fi + +if ! echo "$RESTORE_OUTPUT" | grep -q "Restored"; then + fail "snapshot restore did not report success: ${RESTORE_OUTPUT}" +fi + +SECOND_CHECK=$(openshell sandbox exec --name "${SANDBOX_NAME}" -- cat "${SECOND_MARKER}" 2>/dev/null || echo "MISSING") +[ "$SECOND_CHECK" = "${SECOND_CONTENT}" ] || fail "Latest restore did not recover the second marker: ${SECOND_CHECK}" +pass "Latest snapshot restored expected state" + +# ── Phase 7: snapshot restore with timestamp (first snapshot) ─────── +info "Phase 7: Restoring first snapshot by timestamp..." + +run_capture TARGETED_OUTPUT nemoclaw "${SANDBOX_NAME}" snapshot restore "${SNAPSHOT_TIMESTAMP}" +echo "$TARGETED_OUTPUT" + +if [ "$_CAPTURE_RC" -ne 0 ]; then + fail "targeted snapshot restore exited with code $_CAPTURE_RC: ${TARGETED_OUTPUT}" +fi + +if ! echo "$TARGETED_OUTPUT" | grep -q "Restored"; then + fail "targeted snapshot restore did not report success: ${TARGETED_OUTPUT}" +fi + +FIRST_CHECK=$(openshell sandbox exec --name "${SANDBOX_NAME}" -- cat "${MARKER_FILE}" 2>/dev/null || echo "MISSING") +[ "$FIRST_CHECK" = "${MARKER_CONTENT}" ] || fail "First snapshot did not restore the original marker: ${FIRST_CHECK}" +SECOND_AFTER_TARGETED=$(openshell sandbox exec --name "${SANDBOX_NAME}" -- cat "${SECOND_MARKER}" 2>/dev/null || echo "MISSING") +[ "$SECOND_AFTER_TARGETED" = "MISSING" ] || fail "First snapshot should not contain the second marker" +pass "First snapshot restored expected state" + +# ── Phase 8: No credentials in snapshots ──────────────────────────── +info "Phase 8: Checking snapshots for leaked credentials..." + +BACKUP_DIR="$HOME/.nemoclaw/rebuild-backups/${SANDBOX_NAME}" +if [ -d "$BACKUP_DIR" ]; then + CRED_LEAKS=$(find "$BACKUP_DIR" \ + \( -name "*.json" -o -name "*.env" -o -name ".env" \) \ + ! -name "package-lock.json" \ + ! -name "npm-shrinkwrap.json" \ + ! -name "yarn.lock" \ + ! -name "pnpm-lock.yaml" \ + ! -name "pnpm-lock.yml" \ + -exec grep -l "nvapi-\|sk-\|Bearer " {} \; 2>/dev/null || true) + if [ -z "$CRED_LEAKS" ]; then + pass "No credentials in snapshot directories" + else + fail "Credentials found: $CRED_LEAKS" + fi +else + fail "Backup directory missing: $BACKUP_DIR" +fi + +# ── Phase 9: snapshot help ────────────────────────────────────────── +info "Phase 9: Verifying snapshot help output..." + +run_capture HELP_OUTPUT nemoclaw "${SANDBOX_NAME}" snapshot +if [ "$_CAPTURE_RC" -ne 0 ]; then + fail "snapshot help exited with code $_CAPTURE_RC: ${HELP_OUTPUT}" +fi +if echo "$HELP_OUTPUT" | grep -q "snapshot create" \ + && echo "$HELP_OUTPUT" | grep -q "snapshot list" \ + && echo "$HELP_OUTPUT" | grep -q "snapshot restore"; then + pass "snapshot help shows create/list/restore" +else + fail "snapshot help incomplete: ${HELP_OUTPUT}" +fi + +# ── Cleanup ───────────────────────────────────────────────────────── +info "Cleaning up..." +[[ "${NEMOCLAW_E2E_KEEP_SANDBOX:-}" = "1" ]] || nemoclaw "${SANDBOX_NAME}" destroy --yes 2>/dev/null || true + +echo "" +echo -e "${GREEN}Snapshot commands E2E passed.${NC}" diff --git a/test/e2e-vpn/test-spark-install.sh b/test/e2e-vpn/test-spark-install.sh new file mode 100755 index 00000000000..7f8385ad024 --- /dev/null +++ b/test/e2e-vpn/test-spark-install.sh @@ -0,0 +1,157 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# DGX Spark install smoke: standard install.sh path on a Spark-class Linux host. +# +# Prerequisites: +# - Linux (DGX Spark or similar); other OS exits immediately (fail) +# - Docker running +# - Same env your non-interactive install needs (e.g. NEMOCLAW_NON_INTERACTIVE=1, NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1, API keys, …) +# +# Environment: +# NEMOCLAW_NON_INTERACTIVE=1 — required (matches full-e2e install phase) +# NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 — required for non-interactive install/onboard +# NEMOCLAW_E2E_PUBLIC_INSTALL=1 — use curl|bash instead of repo install.sh +# NEMOCLAW_INSTALL_SCRIPT_URL — URL when using public install (default: nemoclaw.sh) +# INSTALL_LOG — log file (default: /tmp/nemoclaw-e2e-spark-install.log) +# +# Usage: +# NEMOCLAW_NON_INTERACTIVE=1 NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 bash test/e2e-vpn/test-spark-install.sh +# +# See: spark-install.md + +set -uo pipefail + +PASS=0 +FAIL=0 +TOTAL=0 + +pass() { + ((PASS++)) + ((TOTAL++)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} +fail() { + ((FAIL++)) + ((TOTAL++)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } + +if [ -d /workspace ] && [ -f /workspace/install.sh ]; then + REPO="/workspace" +elif [ -f "$(cd "$(dirname "$0")/../.." && pwd)/install.sh" ]; then + REPO="$(cd "$(dirname "$0")/../.." && pwd)" +else + echo "ERROR: Cannot find repo root (install.sh)." + exit 1 +fi + +INSTALL_LOG="${INSTALL_LOG:-/tmp/nemoclaw-e2e-spark-install.log}" + +section "Phase 0: Platform" +if [ "$(uname -s)" = "Linux" ]; then + pass "Running on Linux" +else + fail "This script is for DGX Spark (Linux). On other OS use Vitest: NEMOCLAW_E2E_SPARK_INSTALL=1 --project spark-install-cli (skipped there on non-Linux)." + exit 1 +fi + +section "Phase 1: Prerequisites" +if docker info >/dev/null 2>&1; then + pass "Docker is running" +else + fail "Docker is not running" + exit 1 +fi + +if [ "${NEMOCLAW_NON_INTERACTIVE:-}" = "1" ]; then + pass "NEMOCLAW_NON_INTERACTIVE=1" +else + fail "NEMOCLAW_NON_INTERACTIVE=1 is required" + exit 1 +fi + +if [ "${NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE:-}" = "1" ]; then + pass "NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1" +else + fail "NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 is required for non-interactive install" + exit 1 +fi + +section "Phase 2: Standard installer path" +cd "$REPO" || { + fail "cd to repo: $REPO" + exit 1 +} + +pass "Using generic installer flow without Spark-specific setup" + +section "Phase 3: Install NemoClaw (non-interactive)" +info "Log: $INSTALL_LOG" +if [ "${NEMOCLAW_E2E_PUBLIC_INSTALL:-0}" = "1" ]; then + url="${NEMOCLAW_INSTALL_SCRIPT_URL:-https://www.nvidia.com/nemoclaw.sh}" + info "Running: curl -fsSL ... | bash (url=$url)" + curl -fsSL "$url" | NEMOCLAW_NON_INTERACTIVE=1 NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 bash >"$INSTALL_LOG" 2>&1 & +else + info "Running: bash install.sh --non-interactive" + NEMOCLAW_NON_INTERACTIVE=1 NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 bash install.sh --non-interactive >"$INSTALL_LOG" 2>&1 & +fi +install_pid=$! +tail -f "$INSTALL_LOG" --pid=$install_pid 2>/dev/null & +tail_pid=$! +wait "$install_pid" +install_exit=$? +kill "$tail_pid" 2>/dev/null || true +wait "$tail_pid" 2>/dev/null || true + +if [ "$install_exit" -ne 0 ]; then + fail "install failed (exit $install_exit); last 80 lines of log:" + tail -n 80 "$INSTALL_LOG" >&2 || true + exit 1 +fi +pass "install completed (exit 0)" + +if [ -f "$HOME/.bashrc" ]; then + # shellcheck source=/dev/null + source "$HOME/.bashrc" 2>/dev/null || true +fi +export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" +if [ -s "$NVM_DIR/nvm.sh" ]; then + # shellcheck source=/dev/null + . "$NVM_DIR/nvm.sh" +fi +if [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]]; then + export PATH="$HOME/.local/bin:$PATH" +fi + +section "Phase 4: Verify CLI" +if command -v nemoclaw >/dev/null 2>&1; then + pass "nemoclaw on PATH ($(command -v nemoclaw))" +else + fail "nemoclaw not on PATH" + exit 1 +fi + +if command -v openshell >/dev/null 2>&1; then + pass "openshell on PATH" +else + fail "openshell not on PATH" + exit 1 +fi + +if nemoclaw --help >/dev/null 2>&1; then + pass "nemoclaw --help exits 0" +else + fail "nemoclaw --help failed" + exit 1 +fi + +section "Summary" +printf '\033[1;32mOK: spark-install bash smoke (%d checks passed)\033[0m\n' "$PASS" +echo " Log: $INSTALL_LOG" diff --git a/test/e2e-vpn/test-state-backup-restore.sh b/test/e2e-vpn/test-state-backup-restore.sh new file mode 100755 index 00000000000..dadeb528585 --- /dev/null +++ b/test/e2e-vpn/test-state-backup-restore.sh @@ -0,0 +1,379 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# ============================================================================= +# test-state-backup-restore.sh +# NemoClaw Workspace Backup & Restore E2E Tests +# +# Covers: +# TC-STATE-01: backup-workspace.sh backup → destroy → recreate → restore +# +# Prerequisites: +# - Docker running +# - NVIDIA_API_KEY set +# - Network access to inference.nvidia.com +# ============================================================================= + +set -euo pipefail + +# ── Overall timeout ────────────────────────────────────────────────────────── +export NEMOCLAW_E2E_DEFAULT_TIMEOUT=3600 +SCRIPT_DIR_TIMEOUT="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +# shellcheck source=test/e2e-vpn/e2e-timeout.sh +source "${SCRIPT_DIR_TIMEOUT}/e2e-timeout.sh" +# shellcheck source=test/e2e-vpn/lib/install-path-refresh.sh +source "${SCRIPT_DIR_TIMEOUT}/lib/install-path-refresh.sh" + +# ── Colors ─────────────────────────────────────────────────────────────────── +GREEN='\033[0;32m' +RED='\033[0;31m' +YELLOW='\033[1;33m' +CYAN='\033[0;36m' +NC='\033[0m' + +PASS=0 +FAIL=0 +SKIP=0 +TOTAL=0 + +# Log a timestamped message. +log() { echo -e "${CYAN}[$(date +%H:%M:%S)]${NC} $*" | tee -a "$LOG_FILE"; } +# Record a passing assertion. +pass() { + ((PASS += 1)) + ((TOTAL += 1)) + echo -e "${GREEN} PASS${NC} $1" | tee -a "$LOG_FILE" +} +# Record a failing assertion. +fail() { + ((FAIL += 1)) + ((TOTAL += 1)) + echo -e "${RED} FAIL${NC} $1 — $2" | tee -a "$LOG_FILE" +} +# Record a skipped test. +# shellcheck disable=SC2317,SC2329 # Retained for manual triage paths not hit in every E2E run. +skip() { + ((SKIP += 1)) + ((TOTAL += 1)) + echo -e "${YELLOW} SKIP${NC} $1 — $2" | tee -a "$LOG_FILE" +} + +# ── Config ─────────────────────────────────────────────────────────────────── +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-state-backup}" +LOG_FILE="test-state-backup-restore-$(date +%Y%m%d-%H%M%S).log" + +# ── Resolve repo root ──────────────────────────────────────────────────────── +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" + +# ── Install NemoClaw if not present ────────────────────────────────────────── +install_nemoclaw() { + export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" + if [ -s "$NVM_DIR/nvm.sh" ]; then + # shellcheck source=/dev/null + . "$NVM_DIR/nvm.sh" + fi + nemoclaw_ensure_local_bin_on_path + + if command -v nemoclaw >/dev/null 2>&1; then + log "nemoclaw already installed: $(nemoclaw --version 2>/dev/null || echo unknown)" + return + fi + log "=== Installing NemoClaw via install.sh ===" + NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" \ + NVIDIA_API_KEY="${NVIDIA_API_KEY:-nvapi-DUMMY-FOR-INSTALL}" \ + NEMOCLAW_NON_INTERACTIVE=1 \ + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ + bash "$REPO_ROOT/install.sh" --non-interactive --yes-i-accept-third-party-software \ + 2>&1 | tee -a "$LOG_FILE" + nemoclaw_refresh_install_env + if ! command -v nemoclaw >/dev/null 2>&1; then + log "ERROR: install.sh failed — nemoclaw not found" + exit 1 + fi +} + +# ── Pre-flight ─────────────────────────────────────────────────────────────── +preflight() { + log "=== Pre-flight checks ===" + if ! docker info >/dev/null 2>&1; then + log "ERROR: Docker is not running." + exit 1 + fi + log "Docker is running" + + local api_key="${NVIDIA_API_KEY:-}" + if [[ -z "$api_key" ]]; then + log "ERROR: NVIDIA_API_KEY not set" + exit 1 + fi + + install_nemoclaw + + log "nemoclaw: $(nemoclaw --version 2>/dev/null || echo unknown)" + log "Pre-flight complete" +} + +# Execute a command inside the sandbox via SSH. +sandbox_exec() { + local cmd="$1" + local ssh_cfg + ssh_cfg="$(mktemp)" + if ! openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_cfg" 2>/dev/null; then + rm -f "$ssh_cfg" + echo "" + return 1 + fi + local result ssh_exit=0 + result=$(run_with_timeout 120 ssh -F "$ssh_cfg" \ + -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" "$cmd" 2>&1) || ssh_exit=$? + rm -f "$ssh_cfg" + echo "$result" + return $ssh_exit +} + +# ── Onboard helper ─────────────────────────────────────────────────────────── +onboard_sandbox() { + local name="$1" + log " Onboarding sandbox '$name'..." + NEMOCLAW_SANDBOX_NAME="$name" \ + NEMOCLAW_NON_INTERACTIVE=1 \ + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ + NEMOCLAW_POLICY_TIER="open" \ + run_with_timeout 1800 nemoclaw onboard --non-interactive --yes-i-accept-third-party-software \ + 2>&1 | tee -a "$LOG_FILE" || { + log "FATAL: Onboard failed for '$name'" + return 1 + } + log " Sandbox '$name' onboarded" +} + +# Print full restore output to help triage directory-restore failures. +print_restore_output_for_diag() { + local restore_output="$1" + log " --- Full restore output (for diagnostic) ---" + printf '%s\n' "$restore_output" | sed 's/^/ /' | tee -a "$LOG_FILE" || true + log " --- end restore output ---" +} + +# ============================================================================= +# TC-STATE-01: backup-workspace.sh lifecycle +# ============================================================================= +test_backup_restore_lifecycle() { + log "=== TC-STATE-01: Backup-Workspace Lifecycle ===" + + local workspace_path="/sandbox/.openclaw/workspace" + local marker_content + marker_content="E2E_BACKUP_TEST_$(date +%s)" + + log " Step 1: Writing marker content into workspace files..." + local files_written=0 + # Write the marker content into the workspace files + for f in SOUL.md USER.md IDENTITY.md AGENTS.md MEMORY.md; do + if sandbox_exec "mkdir -p $workspace_path && echo '${marker_content}_${f}' > ${workspace_path}/${f}" 2>/dev/null; then + files_written=$((files_written + 1)) + fi + done + # Write the marker content into the workspace memory directory + local memory_written=0 + if sandbox_exec "mkdir -p ${workspace_path}/memory && echo '${marker_content}_daily' > ${workspace_path}/memory/2026-04-20.md" 2>/dev/null; then + memory_written=1 + fi + + if [[ $files_written -ne 5 || $memory_written -ne 1 ]]; then + fail "TC-STATE-01: Setup" "Could not write workspace files (files_written=$files_written/5, memory_written=$memory_written/1)" + return + fi + log " Wrote marker content to $files_written/5 workspace files + $memory_written/1 memory directory" + + log " Step 2: Running backup-workspace.sh backup..." + local backup_output backup_rc=0 + backup_output=$(bash "$REPO_ROOT/scripts/backup-workspace.sh" backup "$SANDBOX_NAME" 2>&1) || backup_rc=$? + log " Backup output: ${backup_output}" + + if [[ $backup_rc -eq 0 ]] && echo "$backup_output" | grep -q "Backup saved"; then + pass "TC-STATE-01: Backup completed successfully" + else + fail "TC-STATE-01: Backup" "backup-workspace.sh backup failed (exit=$backup_rc) or did not report success" + return + fi + + local backup_dir + backup_dir=$(find "$HOME/.nemoclaw/backups" -mindepth 1 -maxdepth 1 -type d -printf '%T@ %p\n' 2>/dev/null \ + | sort -nr | awk 'NR==1 {print $2}') + if [[ -z "$backup_dir" || ! -d "$backup_dir" ]]; then + fail "TC-STATE-01: Backup dir" "No backup directory found" + return + fi + log " Backup dir found: $backup_dir" + + # Verify backup captured all 6 items on host (5 .md files + memory/ dir) BEFORE + # destroy, so a silent drop in the download chain doesn't surface as an + # ambiguous restore failure later. + log " Step 2b: Verifying backup captured all 5 .md files on host..." + local backup_files_ok=0 + for f in SOUL.md USER.md IDENTITY.md AGENTS.md MEMORY.md; do + if [[ -f "${backup_dir}/${f}" ]] && grep -Fq -- "${marker_content}_${f}" "${backup_dir}/${f}" 2>/dev/null; then + backup_files_ok=$((backup_files_ok + 1)) + else + log " WARNING: ${backup_dir}/${f} missing or content mismatch" + fi + done + if [[ $backup_files_ok -ne 5 ]]; then + fail "TC-STATE-01: BackupCaptureFiles" "Only $backup_files_ok/5 .md files captured correctly in host backup (docs say all 5 must be present — partial capture is a real bug in backup-workspace.sh FILES loop or 'openshell sandbox download')" + return + fi + pass "TC-STATE-01: BackupCaptureFiles — 5/5 .md files captured in host backup" + + log " Step 2c: Verifying backup captured memory directory on host..." + if [[ ! -f "${backup_dir}/memory/2026-04-20.md" ]]; then + fail "TC-STATE-01: BackupCaptureDir" "backup-workspace.sh reported success but '${backup_dir}/memory/2026-04-20.md' does NOT exist on host — backup did NOT capture memory directory (likely 'openshell sandbox download' directory bug)" + return + fi + if ! grep -Fq -- "${marker_content}_daily" "${backup_dir}/memory/2026-04-20.md" 2>/dev/null; then + fail "TC-STATE-01: BackupCaptureDir" "'${backup_dir}/memory/2026-04-20.md' exists on host but content does NOT contain expected marker — backup captured wrong content" + return + fi + pass "TC-STATE-01: BackupCaptureDir — memory directory captured in host backup" + + log " Step 3: Destroying sandbox..." + local destroy_ok=0 + for destroy_attempt in 1 2 3; do + nemoclaw "$SANDBOX_NAME" destroy --yes 2>&1 | tee -a "$LOG_FILE" || true + local list_output list_rc=0 + list_output=$(nemoclaw list 2>&1) || list_rc=$? + if [[ $list_rc -eq 0 ]]; then + if ! printf '%s\n' "$list_output" | grep -Fq -- "$SANDBOX_NAME"; then + destroy_ok=1 + break + fi + else + log " Destroy attempt $destroy_attempt: unable to read sandbox list (exit $list_rc), retrying..." + fi + if [[ $destroy_attempt -lt 3 ]]; then + log " Destroy attempt $destroy_attempt failed (sandbox still listed), retrying in 10s..." + sleep 10 + fi + done + + if [[ $destroy_ok -eq 0 ]]; then + fail "TC-STATE-01: Destroy" "Sandbox still exists after 3 destroy attempts" + return + fi + pass "TC-STATE-01: Sandbox destroyed" + + log " Step 4: Re-onboarding sandbox..." + if ! onboard_sandbox "$SANDBOX_NAME"; then + fail "TC-STATE-01: Re-onboard" "Could not recreate sandbox" + return + fi + pass "TC-STATE-01: Sandbox re-onboarded" + + log " Step 5: Running backup-workspace.sh restore..." + local restore_output restore_rc=0 + restore_output=$(bash "$REPO_ROOT/scripts/backup-workspace.sh" restore "$SANDBOX_NAME" 2>&1) || restore_rc=$? + log " Restore output: ${restore_output}" + + if [[ $restore_rc -eq 0 ]] && echo "$restore_output" | grep -q "Restored"; then + pass "TC-STATE-01: Restore completed successfully" + else + fail "TC-STATE-01: Restore" "backup-workspace.sh restore failed (exit=$restore_rc) or did not report success" + return + fi + + log " Step 6: Verifying workspace files restored..." + local files_restored=0 + for f in SOUL.md USER.md IDENTITY.md AGENTS.md MEMORY.md; do + local restored_content + restored_content=$(sandbox_exec "cat ${workspace_path}/${f} 2>/dev/null") || true + if echo "$restored_content" | grep -Fq -- "${marker_content}_${f}"; then + files_restored=$((files_restored + 1)) + else + log " WARNING: ${f} content mismatch: ${restored_content:0:100}" + fi + done + + if [[ $files_restored -eq 5 ]]; then + pass "TC-STATE-01: FilesRestore — ${files_restored}/5 workspace files restored correctly" + else + fail "TC-STATE-01: FilesRestore" "Only ${files_restored}/5 workspace files restored correctly (expected 5/5 — backup-workspace.sh contract is FILES=(SOUL,USER,IDENTITY,AGENTS,MEMORY); partial restore is a real bug, not tolerance)" + fi + + # Probe emits 'STATE=EXISTS' + content, or 'STATE=MISSING'. SSH errors fall through to the catch-all branch. + log " Verifying memory directory restored on sandbox..." + local memory_probe memory_probe_rc=0 + memory_probe=$(sandbox_exec "if [ -f '${workspace_path}/memory/2026-04-20.md' ]; then printf 'STATE=EXISTS\\n'; cat '${workspace_path}/memory/2026-04-20.md'; else printf 'STATE=MISSING\\n'; fi") || memory_probe_rc=$? + + if grep -Fq -- "STATE=EXISTS" <<<"$memory_probe" \ + && grep -Fq -- "${marker_content}_daily" <<<"$memory_probe"; then + pass "TC-STATE-01: MemoryDirRestore — memory directory contents restored correctly" + elif grep -q "^STATE=MISSING" <<<"$memory_probe"; then + print_restore_output_for_diag "$restore_output" + fail "TC-STATE-01: MemoryDirRestore" "memory/2026-04-20.md does NOT exist on sandbox after restore — backup captured it (BackupCaptureDir passed above) but restore chain dropped the directory (likely 'openshell sandbox upload' directory bug)" + else + log " Memory probe (rc=$memory_probe_rc, first 200B): ${memory_probe:0:200}" + print_restore_output_for_diag "$restore_output" + fail "TC-STATE-01: MemoryDirRestore" "memory/2026-04-20.md marker not found on sandbox — either SSH error (rc=$memory_probe_rc) or restore put wrong content. See probe output above." + fi +} + +# Clean up sandbox and services on exit. +teardown() { + # Do not unlink ~/.nemoclaw/onboard.lock: see rationale in + # test/e2e-vpn/lib/sandbox-teardown.sh — the lock is PID-ownership-aware + # and onboard cleans up stale locks itself. + set +e + nemoclaw stop 2>/dev/null || true + nemoclaw "$SANDBOX_NAME" destroy --yes 2>/dev/null || true + set -e +} + +# Print final PASS/FAIL/SKIP counts and exit. +summary() { + echo "" + echo "============================================================" + echo " Workspace Backup & Restore E2E Results" + echo "============================================================" + echo -e " ${GREEN}PASS: $PASS${NC}" + echo -e " ${RED}FAIL: $FAIL${NC}" + echo -e " ${YELLOW}SKIP: $SKIP${NC}" + echo " TOTAL: $TOTAL" + echo "============================================================" + echo " Log: $LOG_FILE" + echo "============================================================" + echo "" + + if [[ $FAIL -gt 0 ]]; then + exit 1 + fi + exit 0 +} + +# Entry point: preflight → onboard → tests → summary. +main() { + echo "" + echo "============================================================" + echo " NemoClaw Workspace Backup & Restore E2E Tests" + echo " $(date)" + echo "============================================================" + echo "" + + preflight + + log "=== Onboarding sandbox ===" + if ! onboard_sandbox "$SANDBOX_NAME"; then + log "FATAL: Could not onboard sandbox" + exit 1 + fi + + test_backup_restore_lifecycle + + teardown + trap - EXIT + summary +} + +trap teardown EXIT +main "$@" diff --git a/test/e2e-vpn/test-telegram-injection.sh b/test/e2e-vpn/test-telegram-injection.sh new file mode 100755 index 00000000000..0a1e000c0d1 --- /dev/null +++ b/test/e2e-vpn/test-telegram-injection.sh @@ -0,0 +1,476 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# shellcheck disable=SC2016,SC2034,SC2317,SC2329 +# SC2016: Single-quoted strings are intentional — these are injection payloads +# that must NOT be expanded by the shell. +# SC2034: Some variables are used indirectly or reserved for future test cases. +# SC2317: ShellCheck cannot see EXIT trap invocations of cleanup helpers in this E2E script. +# SC2329: Helper functions may be invoked conditionally or in later test phases. + +# Telegram Bridge Command Injection E2E Tests +# +# Validates that PR #119's fix prevents shell command injection through +# the Telegram bridge. Tests the runAgentInSandbox() code path by +# invoking the bridge's message-handling logic directly against a real +# sandbox, without requiring a live Telegram bot token. +# +# Attack surface: +# Before the fix, user messages were interpolated into a shell command +# string passed over SSH. $(cmd), `cmd`, and ${VAR} expansions inside +# user messages would execute in the sandbox, allowing credential +# exfiltration and arbitrary code execution. +# +# Prerequisites: +# - Docker running +# - NemoClaw installed and sandbox running (test-full-e2e.sh Phase 0-3) +# - NVIDIA_API_KEY set +# - openshell on PATH +# +# Environment variables: +# NEMOCLAW_SANDBOX_NAME — sandbox name (default: e2e-test) +# NVIDIA_API_KEY — required +# +# Usage: +# NEMOCLAW_NON_INTERACTIVE=1 NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 NVIDIA_API_KEY=nvapi-... bash test/e2e-vpn/test-telegram-injection.sh +# +# See: https://github.com/NVIDIA/NemoClaw/issues/118 +# https://github.com/NVIDIA/NemoClaw/pull/119 + +set -uo pipefail + +PASS=0 +FAIL=0 +SKIP=0 +TOTAL=0 + +pass() { + ((PASS++)) + ((TOTAL++)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} +fail() { + ((FAIL++)) + ((TOTAL++)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} +skip() { + ((SKIP++)) + ((TOTAL++)) + printf '\033[33m SKIP: %s\033[0m\n' "$1" +} +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } + +# Determine repo root +if [ -d /workspace ] && [ -f /workspace/install.sh ]; then + REPO="/workspace" +elif [ -f "$(cd "$(dirname "$0")/../.." && pwd)/install.sh" ]; then + REPO="$(cd "$(dirname "$0")/../.." && pwd)" +else + echo "ERROR: Cannot find repo root." + exit 1 +fi + +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-test}" + +# shellcheck source=test/e2e-vpn/lib/sandbox-teardown.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib/sandbox-teardown.sh" +register_sandbox_for_teardown "$SANDBOX_NAME" + +# ══════════════════════════════════════════════════════════════════ +# Helper: send a message to the agent inside the sandbox using the +# same mechanism as the Telegram bridge (SSH + nemoclaw-start). +# +# This exercises the exact code path that was vulnerable: user message +# → shell command → SSH → sandbox execution. +# +# We use the bridge's actual shellQuote + execFileSync approach from +# the fixed code on main. The test validates that the message content +# is treated as literal data, not shell commands. +# ══════════════════════════════════════════════════════════════════ + +send_message_to_sandbox() { + local message="$1" + local session_id="${2:-e2e-injection-test}" + + local ssh_config + ssh_config="$(mktemp)" + openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config" 2>/dev/null + + # Use the same mechanism as the bridge: pass message as an argument + # via SSH. The key security property is that the message must NOT be + # interpreted as shell code on the remote side. + local result + result=$(timeout 90 ssh -F "$ssh_config" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" \ + "echo 'INJECTION_PROBE_START' && echo $(printf '%q' "$message") && echo 'INJECTION_PROBE_END'" \ + 2>&1) || true + + rm -f "$ssh_config" + echo "$result" +} + +# Run a command inside the sandbox and capture output +sandbox_exec() { + local cmd="$1" + local ssh_config + ssh_config="$(mktemp)" + openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config" 2>/dev/null + + local result + result=$(timeout 60 ssh -F "$ssh_config" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=10 \ + -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" \ + "$cmd" \ + 2>&1) || true + + rm -f "$ssh_config" + echo "$result" +} + +# ══════════════════════════════════════════════════════════════════ +# Phase 0: Prerequisites +# ══════════════════════════════════════════════════════════════════ +section "Phase 0: Prerequisites" + +if [ -z "${NVIDIA_API_KEY:-}" ]; then + fail "NVIDIA_API_KEY not set" + exit 1 +fi +pass "NVIDIA_API_KEY is set" + +if ! command -v openshell >/dev/null 2>&1; then + fail "openshell not found on PATH" + exit 1 +fi +pass "openshell found" + +if ! command -v nemoclaw >/dev/null 2>&1; then + fail "nemoclaw not found on PATH" + exit 1 +fi +pass "nemoclaw found" + +# Verify sandbox is running +if status_output=$(nemoclaw "$SANDBOX_NAME" status 2>&1); then + pass "Sandbox '${SANDBOX_NAME}' is running" +else + fail "Sandbox '${SANDBOX_NAME}' not running — run test-full-e2e.sh first" + exit 1 +fi + +# Get the actual hostname inside the sandbox for comparison +SANDBOX_HOSTNAME=$(sandbox_exec "hostname" 2>/dev/null) || SANDBOX_HOSTNAME="" +SANDBOX_WHOAMI=$(sandbox_exec "whoami" 2>/dev/null) || SANDBOX_WHOAMI="" +info "Sandbox hostname: ${SANDBOX_HOSTNAME:-unknown}, user: ${SANDBOX_WHOAMI:-unknown}" + +# ══════════════════════════════════════════════════════════════════ +# Phase 1: Command Substitution Injection — $(command) +# ══════════════════════════════════════════════════════════════════ +section "Phase 1: Command Substitution Injection" + +# T1: $(whoami) must be treated as literal text, not executed +info "T1: Testing \$(whoami) injection..." +MARKER_FILE="/tmp/nemoclaw-injection-t1-$(date +%s)" +PAYLOAD='$(touch /tmp/injection-proof-t1 && echo INJECTED)' + +# Write the payload as a file in sandbox, then check if /tmp/injection-proof-t1 was created +sandbox_exec "rm -f /tmp/injection-proof-t1" >/dev/null 2>&1 + +# Use printf %q to safely pass the payload through SSH without local expansion +# This simulates what shellQuote does in the bridge +ssh_config_t1="$(mktemp)" +openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config_t1" 2>/dev/null + +# The critical test: pass a payload that would create a file if command +# substitution is executed. Use stdin to pass the message (like the fixed bridge). +timeout 30 ssh -F "$ssh_config_t1" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" \ + 'MSG=$(cat) && echo "Received: $MSG"' \ + <<<"$PAYLOAD" >/dev/null 2>&1 || true +rm -f "$ssh_config_t1" + +# Check if the injection file was created +injection_check=$(sandbox_exec "test -f /tmp/injection-proof-t1 && echo EXPLOITED || echo SAFE") +if echo "$injection_check" | grep -q "SAFE"; then + pass "T1: \$(command) substitution was NOT executed" +else + fail "T1: \$(command) substitution was EXECUTED — injection successful!" +fi + +# T2: Backtick injection — `command` +info "T2: Testing backtick injection..." +sandbox_exec "rm -f /tmp/injection-proof-t2" >/dev/null 2>&1 + +ssh_config_t2="$(mktemp)" +openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config_t2" 2>/dev/null +PAYLOAD_BT='`touch /tmp/injection-proof-t2`' + +timeout 30 ssh -F "$ssh_config_t2" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" \ + 'MSG=$(cat) && echo "Received: $MSG"' \ + <<<"$PAYLOAD_BT" >/dev/null 2>&1 || true +rm -f "$ssh_config_t2" + +injection_check_t2=$(sandbox_exec "test -f /tmp/injection-proof-t2 && echo EXPLOITED || echo SAFE") +if echo "$injection_check_t2" | grep -q "SAFE"; then + pass "T2: Backtick command substitution was NOT executed" +else + fail "T2: Backtick command substitution was EXECUTED — injection successful!" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 2: Quote Breakout Injection +# ══════════════════════════════════════════════════════════════════ +section "Phase 2: Quote Breakout Injection" + +# T3: Classic single-quote breakout +info "T3: Testing single-quote breakout..." +sandbox_exec "rm -f /tmp/injection-proof-t3" >/dev/null 2>&1 + +ssh_config_t3="$(mktemp)" +openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config_t3" 2>/dev/null +PAYLOAD_QUOTE="'; touch /tmp/injection-proof-t3; echo '" + +timeout 30 ssh -F "$ssh_config_t3" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" \ + 'MSG=$(cat) && echo "Received: $MSG"' \ + <<<"$PAYLOAD_QUOTE" >/dev/null 2>&1 || true +rm -f "$ssh_config_t3" + +injection_check_t3=$(sandbox_exec "test -f /tmp/injection-proof-t3 && echo EXPLOITED || echo SAFE") +if echo "$injection_check_t3" | grep -q "SAFE"; then + pass "T3: Single-quote breakout was NOT exploitable" +else + fail "T3: Single-quote breakout was EXECUTED — injection successful!" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 3: Environment Variable / Parameter Expansion +# ══════════════════════════════════════════════════════════════════ +section "Phase 3: Parameter Expansion" + +# T4: ${NVIDIA_API_KEY} must not expand to the actual key value +info "T4: Testing \${NVIDIA_API_KEY} expansion..." + +ssh_config_t4="$(mktemp)" +openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config_t4" 2>/dev/null +PAYLOAD_ENV='${NVIDIA_API_KEY}' + +t4_result=$(timeout 30 ssh -F "$ssh_config_t4" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" \ + 'MSG=$(cat) && echo "$MSG"' \ + <<<"$PAYLOAD_ENV" 2>&1) || true +rm -f "$ssh_config_t4" + +# The result should contain the literal string ${NVIDIA_API_KEY}, not a nvapi- value +if echo "$t4_result" | grep -q "nvapi-"; then + fail "T4: \${NVIDIA_API_KEY} expanded to actual key value — secret leaked!" +elif echo "$t4_result" | grep -qF '${NVIDIA_API_KEY}'; then + pass "T4: \${NVIDIA_API_KEY} treated as literal string (not expanded)" +else + # Empty or other result — still safe as long as key not leaked + pass "T4: \${NVIDIA_API_KEY} did not expand to key value (result: ${t4_result:0:100})" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 4: API Key Not in Process Table +# ══════════════════════════════════════════════════════════════════ +section "Phase 4: Process Table Leak Check" + +# T5: NVIDIA_API_KEY must not appear in ps aux output +info "T5: Checking process table for API key leaks..." + +# Get truncated key for a safe comparison (first 15 chars of key value) +API_KEY_PREFIX="${NVIDIA_API_KEY:0:15}" + +# Check both the Brev host and inside the sandbox +host_ps=$(ps aux 2>/dev/null || true) +sandbox_ps=$(sandbox_exec "ps aux" 2>/dev/null || true) + +HOST_LEAK=false +SANDBOX_LEAK=false + +if echo "$host_ps" | grep -qF "$API_KEY_PREFIX"; then + # Filter out our own grep and this test script + leaky_lines=$(echo "$host_ps" | grep -F "$API_KEY_PREFIX" | grep -v "grep" | grep -v "test-telegram-injection" || true) + if [ -n "$leaky_lines" ]; then + HOST_LEAK=true + fi +fi + +if echo "$sandbox_ps" | grep -qF "$API_KEY_PREFIX"; then + leaky_sandbox=$(echo "$sandbox_ps" | grep -F "$API_KEY_PREFIX" | grep -v "grep" || true) + if [ -n "$leaky_sandbox" ]; then + SANDBOX_LEAK=true + fi +fi + +if [ "$HOST_LEAK" = true ]; then + fail "T5: NVIDIA_API_KEY found in HOST process table" +elif [ "$SANDBOX_LEAK" = true ]; then + fail "T5: NVIDIA_API_KEY found in SANDBOX process table" +else + pass "T5: API key not visible in process tables (host or sandbox)" +fi + +# ══════════════════════════════════════════════════════════════════ +# Phase 5: SANDBOX_NAME Validation +# ══════════════════════════════════════════════════════════════════ +section "Phase 5: SANDBOX_NAME Validation" + +# T6: Invalid SANDBOX_NAME with shell metacharacters must be rejected +info "T6: Testing SANDBOX_NAME with shell metacharacters..." + +# The validateName() function in runner.js enforces RFC 1123: lowercase +# alphanumeric with optional internal hyphens, max 63 chars. +# Test by running the validation directly via node. +t6_result=$(cd "$REPO" && node -e " + const { validateName } = require('./dist/lib/runner'); + try { + validateName('foo;rm -rf /', 'SANDBOX_NAME'); + console.log('ACCEPTED'); + } catch (e) { + console.log('REJECTED: ' + e.message); + } +" 2>&1) + +if echo "$t6_result" | grep -q "REJECTED"; then + pass "T6: SANDBOX_NAME 'foo;rm -rf /' rejected by validateName()" +else + fail "T6: SANDBOX_NAME 'foo;rm -rf /' was ACCEPTED — validation bypass!" +fi + +# T7: Leading-hyphen option injection must be rejected +info "T7: Testing SANDBOX_NAME with leading hyphen (option injection)..." + +t7_result=$(cd "$REPO" && node -e " + const { validateName } = require('./dist/lib/runner'); + try { + validateName('--help', 'SANDBOX_NAME'); + console.log('ACCEPTED'); + } catch (e) { + console.log('REJECTED: ' + e.message); + } +" 2>&1) + +if echo "$t7_result" | grep -q "REJECTED"; then + pass "T7: SANDBOX_NAME '--help' rejected (option injection prevented)" +else + fail "T7: SANDBOX_NAME '--help' was ACCEPTED — option injection possible!" +fi + +# Additional invalid names — pass via process.argv to avoid shell expansion of +# backticks and $() in double-quoted node -e strings. +for invalid_name in '$(whoami)' '`id`' 'foo bar' '../etc/passwd' 'UPPERCASE'; do + t_result=$(cd "$REPO" && node -e " + const { validateName } = require('./dist/lib/runner'); + try { + validateName(process.argv[1], 'SANDBOX_NAME'); + console.log('ACCEPTED'); + } catch (e) { + console.log('REJECTED'); + } + " -- "$invalid_name" 2>&1) + + if echo "$t_result" | grep -q "REJECTED"; then + pass "T6/T7 extra: SANDBOX_NAME '${invalid_name}' correctly rejected" + else + fail "T6/T7 extra: SANDBOX_NAME '${invalid_name}' was ACCEPTED" + fi +done + +# ══════════════════════════════════════════════════════════════════ +# Phase 6: Regression — Normal Messages Still Work +# ══════════════════════════════════════════════════════════════════ +section "Phase 6: Normal Message Regression" + +# T8: A normal message should be passed through correctly +info "T8: Testing normal message passthrough..." + +ssh_config_t8="$(mktemp)" +openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config_t8" 2>/dev/null +NORMAL_MSG="Hello, what is two plus two?" + +t8_result=$(timeout 30 ssh -F "$ssh_config_t8" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" \ + 'MSG=$(cat) && echo "Received: $MSG"' \ + <<<"$NORMAL_MSG" 2>&1) || true +rm -f "$ssh_config_t8" + +if echo "$t8_result" | grep -qF "Hello, what is two plus two?"; then + pass "T8: Normal message passed through correctly" +else + fail "T8: Normal message was not echoed back correctly (got: ${t8_result:0:200})" +fi + +# T8b: Test message with special characters that should be treated as literal +info "T8b: Testing message with safe special characters..." + +ssh_config_t8b="$(mktemp)" +openshell sandbox ssh-config "$SANDBOX_NAME" >"$ssh_config_t8b" 2>/dev/null +SPECIAL_MSG="What's the meaning of life? It costs \$5 & is 100% free!" + +t8b_result=$(timeout 30 ssh -F "$ssh_config_t8b" \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o LogLevel=ERROR \ + "openshell-${SANDBOX_NAME}" \ + 'MSG=$(cat) && echo "$MSG"' \ + <<<"$SPECIAL_MSG" 2>&1) || true +rm -f "$ssh_config_t8b" + +# Check the message was received (may be slightly different due to shell, but +# the key test is that $ and & didn't cause errors or unexpected behavior) +if [ -n "$t8b_result" ]; then + pass "T8b: Message with special characters processed without error" +else + fail "T8b: Message with special characters caused empty/error response" +fi + +# ══════════════════════════════════════════════════════════════════ +# Summary +# ══════════════════════════════════════════════════════════════════ +echo "" +echo "========================================" +echo " Telegram Injection Test Results:" +echo " Passed: $PASS" +echo " Failed: $FAIL" +echo " Skipped: $SKIP" +echo " Total: $TOTAL" +echo "========================================" + +if [ "$FAIL" -eq 0 ]; then + printf '\n\033[1;32m Telegram injection tests PASSED — no injection vectors found.\033[0m\n' + exit 0 +else + printf '\n\033[1;31m %d test(s) failed — INJECTION VULNERABILITIES DETECTED.\033[0m\n' "$FAIL" + exit 1 +fi diff --git a/test/e2e-vpn/test-token-rotation.sh b/test/e2e-vpn/test-token-rotation.sh new file mode 100755 index 00000000000..c19c3278398 --- /dev/null +++ b/test/e2e-vpn/test-token-rotation.sh @@ -0,0 +1,607 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Token rotation E2E test (issue #1903): +# - prove that rotating a messaging token and re-running onboard propagates +# the new credential to the sandbox (sandbox is rebuilt automatically) +# - prove that re-running onboard with the same token reuses the sandbox +# - prove that rotating each provider in isolation only re-builds for that +# provider's bridge (no cross-talk between Telegram, Discord, and Slack +# detection) +# +# Uses two distinct fake tokens per provider. The test validates that NemoClaw +# detects the rotation and triggers a sandbox rebuild — it does not validate +# the Telegram, Discord, or Slack API responses. +# +# Prerequisites: +# - Docker running +# - NVIDIA_API_KEY set (or CI-compatible inference env) +# - TELEGRAM_BOT_TOKEN_A and TELEGRAM_BOT_TOKEN_B set (can be fake) +# - DISCORD_BOT_TOKEN_A and DISCORD_BOT_TOKEN_B set (can be fake) +# - SLACK_BOT_TOKEN_A and SLACK_BOT_TOKEN_B set (can be fake; xoxb- prefix) +# - SLACK_APP_TOKEN_A and SLACK_APP_TOKEN_B set (can be fake; xapp- prefix) +# +# Usage: +# NEMOCLAW_NON_INTERACTIVE=1 NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ +# NVIDIA_API_KEY=nvapi-... \ +# TELEGRAM_BOT_TOKEN_A=fake-a TELEGRAM_BOT_TOKEN_B=fake-b \ +# DISCORD_BOT_TOKEN_A=fake-c DISCORD_BOT_TOKEN_B=fake-d \ +# SLACK_BOT_TOKEN_A=xoxb-fake-a SLACK_BOT_TOKEN_B=xoxb-fake-b \ +# SLACK_APP_TOKEN_A=xapp-fake-a SLACK_APP_TOKEN_B=xapp-fake-b \ +# bash test/e2e-vpn/test-token-rotation.sh + +set -uo pipefail + +export NEMOCLAW_E2E_DEFAULT_TIMEOUT=2400 +SCRIPT_DIR_TIMEOUT="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +# shellcheck source=test/e2e-vpn/e2e-timeout.sh +source "${SCRIPT_DIR_TIMEOUT}/e2e-timeout.sh" +# shellcheck source=test/e2e-vpn/lib/ci-compatible-inference.sh +. "${SCRIPT_DIR_TIMEOUT}/lib/ci-compatible-inference.sh" + +PASS=0 +FAIL=0 +SKIP=0 +TOTAL=0 +INSTALL_OK=1 +PREREQS_OK=1 + +pass() { + ((PASS++)) + ((TOTAL++)) + printf '\033[32m PASS: %s\033[0m\n' "$1" +} +fail() { + ((FAIL++)) + ((TOTAL++)) + printf '\033[31m FAIL: %s\033[0m\n' "$1" +} +skip() { + ((SKIP++)) + ((TOTAL++)) + printf '\033[33m SKIP: %s\033[0m\n' "$1" +} +section() { + echo "" + printf '\033[1;36m=== %s ===\033[0m\n' "$1" +} +info() { printf '\033[1;34m [info]\033[0m %s\n' "$1"; } +print_summary() { + section "Summary" + echo " Total: $TOTAL Pass: $PASS Fail: $FAIL Skip: $SKIP" + if [ "$FAIL" -gt 0 ]; then + echo "" + echo "FAILED" + exit 1 + fi + echo "" + if [ "$SKIP" -gt 0 ]; then + echo "PASSED (with $SKIP skipped)" + else + echo "ALL PASSED" + fi +} + +# Determine repo root +if [ -d /workspace ] && [ -f /workspace/install.sh ]; then + REPO="/workspace" +elif [ -f "$(cd "$(dirname "$0")/../.." && pwd)/install.sh" ]; then + REPO="$(cd "$(dirname "$0")/../.." && pwd)" +else + echo "ERROR: Cannot find repo root." + exit 1 +fi + +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-token-rotation}" +REGISTRY="$HOME/.nemoclaw/sandboxes.json" +INSTALL_LOG="/tmp/nemoclaw-e2e-install.log" + +nemoclaw_e2e_configure_compatible_inference + +# ── Prerequisite checks ────────────────────────────────────────── + +if [ -z "${TELEGRAM_BOT_TOKEN_A:-}" ] || [ -z "${TELEGRAM_BOT_TOKEN_B:-}" ]; then + skip "TELEGRAM_BOT_TOKEN_A and TELEGRAM_BOT_TOKEN_B must both be set" + PREREQS_OK=0 +fi + +if [ -z "${DISCORD_BOT_TOKEN_A:-}" ] || [ -z "${DISCORD_BOT_TOKEN_B:-}" ]; then + skip "DISCORD_BOT_TOKEN_A and DISCORD_BOT_TOKEN_B must both be set" + PREREQS_OK=0 +fi + +if [ -n "${TELEGRAM_BOT_TOKEN_A:-}" ] && [ "${TELEGRAM_BOT_TOKEN_A}" = "${TELEGRAM_BOT_TOKEN_B:-}" ]; then + skip "TELEGRAM_BOT_TOKEN_A and TELEGRAM_BOT_TOKEN_B must be different" + PREREQS_OK=0 +fi + +if [ -n "${DISCORD_BOT_TOKEN_A:-}" ] && [ "${DISCORD_BOT_TOKEN_A}" = "${DISCORD_BOT_TOKEN_B:-}" ]; then + skip "DISCORD_BOT_TOKEN_A and DISCORD_BOT_TOKEN_B must be different" + PREREQS_OK=0 +fi + +if [ -z "${SLACK_BOT_TOKEN_A:-}" ] || [ -z "${SLACK_BOT_TOKEN_B:-}" ]; then + skip "SLACK_BOT_TOKEN_A and SLACK_BOT_TOKEN_B must both be set" + PREREQS_OK=0 +fi + +if [ -z "${SLACK_APP_TOKEN_A:-}" ] || [ -z "${SLACK_APP_TOKEN_B:-}" ]; then + skip "SLACK_APP_TOKEN_A and SLACK_APP_TOKEN_B must both be set" + PREREQS_OK=0 +fi + +if [ -n "${SLACK_BOT_TOKEN_A:-}" ] && [ "${SLACK_BOT_TOKEN_A}" = "${SLACK_BOT_TOKEN_B:-}" ]; then + skip "SLACK_BOT_TOKEN_A and SLACK_BOT_TOKEN_B must be different" + PREREQS_OK=0 +fi + +if [ -n "${SLACK_APP_TOKEN_A:-}" ] && [ "${SLACK_APP_TOKEN_A}" = "${SLACK_APP_TOKEN_B:-}" ]; then + skip "SLACK_APP_TOKEN_A and SLACK_APP_TOKEN_B must be different" + PREREQS_OK=0 +fi + +# Bail to summary if any prereq failed (no phases run, but Summary still prints) +if [ "$PREREQS_OK" != "1" ]; then + print_summary + exit 0 +fi + +# ── Helpers ─────────────────────────────────────────────────────── + +cleanup() { + openshell sandbox delete "$SANDBOX_NAME" 2>/dev/null || true +} +trap cleanup EXIT + +is_fake_telegram_token() { + case "${1:-}" in + *fake*) return 0 ;; + *) return 1 ;; + esac +} +is_fake_slack_token() { + case "${1:-}" in + xoxb-fake-* | xoxb-test-* | xapp-fake-* | xapp-test-*) return 0 ;; + *) return 1 ;; + esac +} + +registry_has_messaging_credential_hash() { + local env_key="$1" + [ -f "$REGISTRY" ] && node -e " +const r = JSON.parse(require('fs').readFileSync(process.argv[1], 'utf8')); +const sandbox = (r.sandboxes || {})[process.argv[2]]; +const bindings = sandbox?.messaging?.plan?.credentialBindings; +if (!Array.isArray(bindings)) process.exit(1); +const found = bindings.some((entry) => + entry?.providerEnvKey === process.argv[3] && + typeof entry.credentialHash === 'string' && + entry.credentialHash.length > 0, +); +process.exit(found ? 0 : 1); +" "$REGISTRY" "$SANDBOX_NAME" "$env_key" 2>/dev/null +} + +# ── Phase 0: Install NemoClaw with token A ──────────────────────── + +section "Phase 0: Install NemoClaw and first onboard with token A" + +# Pre-clean +openshell sandbox delete "$SANDBOX_NAME" 2>/dev/null || true +openshell gateway destroy -g nemoclaw 2>/dev/null || true + +if [ -z "${NEMOCLAW_SKIP_TELEGRAM_REACHABILITY:-}" ] \ + && { is_fake_telegram_token "$TELEGRAM_BOT_TOKEN_A" || is_fake_telegram_token "$TELEGRAM_BOT_TOKEN_B"; }; then + # This E2E normally uses fake tokens to exercise rotation plumbing, not the + # live Telegram API. Remove once onboard has a hermetic fake Telegram API. + export NEMOCLAW_SKIP_TELEGRAM_REACHABILITY=1 + info "Skipping onboarding Telegram reachability probe for fake-token E2E" +fi +if [ -z "${NEMOCLAW_SKIP_SLACK_AUTH_VALIDATION:-}" ] \ + && { is_fake_slack_token "$SLACK_BOT_TOKEN_A" || is_fake_slack_token "$SLACK_BOT_TOKEN_B" || is_fake_slack_token "$SLACK_APP_TOKEN_A" || is_fake_slack_token "$SLACK_APP_TOKEN_B"; }; then + # This E2E normally uses fake Slack tokens to exercise rotation plumbing, not + # the live Slack API. + export NEMOCLAW_SKIP_SLACK_AUTH_VALIDATION=1 + info "Skipping onboarding Slack auth validation for fake-token E2E" +fi + +export TELEGRAM_BOT_TOKEN="$TELEGRAM_BOT_TOKEN_A" +export DISCORD_BOT_TOKEN="$DISCORD_BOT_TOKEN_A" +export SLACK_BOT_TOKEN="$SLACK_BOT_TOKEN_A" +export SLACK_APP_TOKEN="$SLACK_APP_TOKEN_A" +export NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" +export NEMOCLAW_POLICY_TIER="open" +export NEMOCLAW_RECREATE_SANDBOX=1 + +info "Running install.sh --non-interactive (includes first onboard)..." +cd "$REPO" || exit 1 +touch "$INSTALL_LOG" +bash install.sh --non-interactive >"$INSTALL_LOG" 2>&1 & +install_pid=$! +tail -f "$INSTALL_LOG" --pid=$install_pid 2>/dev/null & +tail_pid=$! +wait $install_pid +install_exit=$? +kill $tail_pid 2>/dev/null || true +wait $tail_pid 2>/dev/null || true + +# Source shell profile to pick up nvm/PATH changes from install.sh +if [ -f "$HOME/.bashrc" ]; then + # shellcheck source=/dev/null + source "$HOME/.bashrc" 2>/dev/null || true +fi +export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" +if [ -s "$NVM_DIR/nvm.sh" ]; then + # shellcheck source=/dev/null + . "$NVM_DIR/nvm.sh" +fi +if [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]]; then + export PATH="$HOME/.local/bin:$PATH" +fi + +if [ $install_exit -eq 0 ]; then + pass "install.sh completed (exit 0)" +else + INSTALL_OK=0 + if grep -qE "(Telegram|Discord) network reachability failure" "$INSTALL_LOG" 2>/dev/null; then + skip "install.sh aborted: messaging API unreachable (likely VPN / corporate proxy)" + info "Detected ' network reachability failure' in install log." + else + fail "install.sh failed (exit $install_exit)" + fi + info "Last 30 lines of install log:" + tail -30 "$INSTALL_LOG" 2>/dev/null || true +fi + +# Verify tools are on PATH +if [ "$INSTALL_OK" = "1" ]; then + if ! command -v openshell >/dev/null 2>&1; then + fail "openshell not found on PATH after install" + exit 1 + fi + pass "openshell installed ($(openshell --version 2>&1 || echo unknown))" + + if ! command -v nemoclaw >/dev/null 2>&1; then + fail "nemoclaw not found on PATH after install" + exit 1 + fi + pass "nemoclaw installed at $(command -v nemoclaw)" +fi + +if [ "$INSTALL_OK" != "1" ]; then + section "Skipping verification phases — initial install did not complete" + skip "Phase 1: Verify first onboard results" + skip "Phase 2: Re-onboard with rotated TELEGRAM_BOT_TOKEN_B" + skip "Phase 3: Re-onboard with same tokens (after Telegram rotation)" + skip "Phase 4: Re-onboard with rotated DISCORD_BOT_TOKEN_B" + skip "Phase 5: Re-onboard with same tokens (after Discord rotation)" + skip "Phase 6: Re-onboard with rotated SLACK_BOT_TOKEN_B and SLACK_APP_TOKEN_B" + skip "Phase 7: Re-onboard with same tokens (after Slack rotation)" +else + # ── Phase 1: Verify first onboard with token A ────────────────── + + section "Phase 1: Verify first onboard results" + + if openshell sandbox list 2>/dev/null | grep -q "$SANDBOX_NAME"; then + pass "Sandbox $SANDBOX_NAME created and running" + else + fail "Sandbox $SANDBOX_NAME not running after first onboard" + fi + + if openshell provider get "${SANDBOX_NAME}-telegram-bridge" >/dev/null 2>&1; then + pass "Provider ${SANDBOX_NAME}-telegram-bridge exists" + else + fail "Provider ${SANDBOX_NAME}-telegram-bridge not found" + fi + + if openshell provider get "${SANDBOX_NAME}-discord-bridge" >/dev/null 2>&1; then + pass "Provider ${SANDBOX_NAME}-discord-bridge exists" + else + fail "Provider ${SANDBOX_NAME}-discord-bridge not found" + fi + + if openshell provider get "${SANDBOX_NAME}-slack-bridge" >/dev/null 2>&1; then + pass "Provider ${SANDBOX_NAME}-slack-bridge exists" + else + fail "Provider ${SANDBOX_NAME}-slack-bridge not found" + fi + + if openshell provider get "${SANDBOX_NAME}-slack-app" >/dev/null 2>&1; then + pass "Provider ${SANDBOX_NAME}-slack-app exists" + else + fail "Provider ${SANDBOX_NAME}-slack-app not found" + fi + + # Verify credential hashes are stored in the persisted messaging plan. + if registry_has_messaging_credential_hash "TELEGRAM_BOT_TOKEN"; then + pass "Telegram credential hash stored in messaging plan for $SANDBOX_NAME" + else + fail "Telegram credential hash not found in messaging plan for $SANDBOX_NAME" + fi + + if registry_has_messaging_credential_hash "DISCORD_BOT_TOKEN"; then + pass "Discord credential hash stored in messaging plan for $SANDBOX_NAME" + else + fail "Discord credential hash not found in messaging plan for $SANDBOX_NAME" + fi + + if registry_has_messaging_credential_hash "SLACK_BOT_TOKEN"; then + pass "Slack bot credential hash stored in messaging plan for $SANDBOX_NAME" + else + fail "Slack bot credential hash not found in messaging plan for $SANDBOX_NAME" + fi + + if registry_has_messaging_credential_hash "SLACK_APP_TOKEN"; then + pass "Slack app credential hash stored in messaging plan for $SANDBOX_NAME" + else + fail "Slack app credential hash not found in messaging plan for $SANDBOX_NAME" + fi + + # ── Phase 2: Rotate Telegram token only (re-onboard with token B) ─ + + section "Phase 2: Re-onboard with rotated TELEGRAM_BOT_TOKEN_B (Discord unchanged)" + + export TELEGRAM_BOT_TOKEN="$TELEGRAM_BOT_TOKEN_B" + export DISCORD_BOT_TOKEN="$DISCORD_BOT_TOKEN_A" + export SLACK_BOT_TOKEN="$SLACK_BOT_TOKEN_A" + export SLACK_APP_TOKEN="$SLACK_APP_TOKEN_A" + unset NEMOCLAW_RECREATE_SANDBOX + + ONBOARD_OUTPUT=$(nemoclaw onboard --non-interactive 2>&1) + onboard_exit=$? + + if [ $onboard_exit -ne 0 ]; then + fail "Phase 2 onboard failed (exit $onboard_exit)" + echo "$ONBOARD_OUTPUT" | tail -30 + fi + + if grep -q "credential(s) rotated" <<<"$ONBOARD_OUTPUT"; then + pass "Credential rotation detected" + else + fail "Credential rotation not detected in onboard output" + info "Onboard output:" + echo "$ONBOARD_OUTPUT" | tail -20 + fi + + # Rotation message must name only the telegram-bridge provider — Discord + # token is unchanged, so a stray discord-bridge entry would indicate a + # false-positive in detectMessagingCredentialRotation. + if grep -q "credential(s) rotated:.*telegram-bridge" <<<"$ONBOARD_OUTPUT"; then + pass "Rotation message identifies telegram-bridge" + else + fail "Rotation message did not identify telegram-bridge" + info "Onboard output:" + grep "credential(s) rotated" <<<"$ONBOARD_OUTPUT" || true + fi + + if grep -q "credential(s) rotated:.*discord-bridge" <<<"$ONBOARD_OUTPUT"; then + fail "Rotation message unexpectedly named discord-bridge (Discord token did not change)" + info "Onboard output:" + grep "credential(s) rotated" <<<"$ONBOARD_OUTPUT" || true + else + pass "Rotation message did not name discord-bridge (Discord unchanged)" + fi + + if grep -qE "credential\(s\) rotated:.*slack-(bridge|app)" <<<"$ONBOARD_OUTPUT"; then + fail "Rotation message unexpectedly named slack-bridge/slack-app (Slack tokens did not change)" + info "Onboard output:" + grep "credential(s) rotated" <<<"$ONBOARD_OUTPUT" || true + else + pass "Rotation message did not name slack-bridge or slack-app (Slack unchanged)" + fi + + if grep -q "Rebuilding sandbox" <<<"$ONBOARD_OUTPUT"; then + pass "Sandbox rebuild triggered by rotation" + else + fail "Sandbox rebuild not triggered" + info "Onboard output:" + echo "$ONBOARD_OUTPUT" | tail -20 + fi + + if openshell sandbox list 2>/dev/null | grep -q "$SANDBOX_NAME"; then + pass "Sandbox running after Telegram rotation" + else + fail "Sandbox not running after Telegram rotation" + fi + + # ── Phase 3: Re-onboard with same tokens (no change) ───────────── + + section "Phase 3: Re-onboard with same tokens (no rotation expected)" + + ONBOARD_OUTPUT=$(nemoclaw onboard --non-interactive 2>&1) + onboard_exit=$? + + if [ $onboard_exit -ne 0 ]; then + fail "Phase 3 onboard failed (exit $onboard_exit)" + echo "$ONBOARD_OUTPUT" | tail -30 + fi + + if grep -q "reusing it" <<<"$ONBOARD_OUTPUT"; then + pass "Sandbox reused when tokens unchanged" + else + fail "Sandbox was not reused (unexpected rebuild)" + info "Onboard output:" + echo "$ONBOARD_OUTPUT" | tail -20 + fi + + # ── Phase 4: Rotate Discord token only (re-onboard with token B) ─ + + section "Phase 4: Re-onboard with rotated DISCORD_BOT_TOKEN_B (Telegram unchanged)" + + export TELEGRAM_BOT_TOKEN="$TELEGRAM_BOT_TOKEN_B" + export DISCORD_BOT_TOKEN="$DISCORD_BOT_TOKEN_B" + export SLACK_BOT_TOKEN="$SLACK_BOT_TOKEN_A" + export SLACK_APP_TOKEN="$SLACK_APP_TOKEN_A" + + ONBOARD_OUTPUT=$(nemoclaw onboard --non-interactive 2>&1) + onboard_exit=$? + + if [ $onboard_exit -ne 0 ]; then + fail "Phase 4 onboard failed (exit $onboard_exit)" + echo "$ONBOARD_OUTPUT" | tail -30 + fi + + if grep -q "credential(s) rotated" <<<"$ONBOARD_OUTPUT"; then + pass "Credential rotation detected" + else + fail "Credential rotation not detected in onboard output" + info "Onboard output:" + echo "$ONBOARD_OUTPUT" | tail -20 + fi + + # Symmetric assertion to Phase 2: only the discord-bridge entry should appear. + if grep -q "credential(s) rotated:.*discord-bridge" <<<"$ONBOARD_OUTPUT"; then + pass "Rotation message identifies discord-bridge" + else + fail "Rotation message did not identify discord-bridge" + info "Onboard output:" + grep "credential(s) rotated" <<<"$ONBOARD_OUTPUT" || true + fi + + if grep -q "credential(s) rotated:.*telegram-bridge" <<<"$ONBOARD_OUTPUT"; then + fail "Rotation message unexpectedly named telegram-bridge (Telegram token did not change)" + info "Onboard output:" + grep "credential(s) rotated" <<<"$ONBOARD_OUTPUT" || true + else + pass "Rotation message did not name telegram-bridge (Telegram unchanged)" + fi + + if grep -qE "credential\(s\) rotated:.*slack-(bridge|app)" <<<"$ONBOARD_OUTPUT"; then + fail "Rotation message unexpectedly named slack-bridge/slack-app (Slack tokens did not change)" + info "Onboard output:" + grep "credential(s) rotated" <<<"$ONBOARD_OUTPUT" || true + else + pass "Rotation message did not name slack-bridge or slack-app (Slack unchanged)" + fi + + if grep -q "Rebuilding sandbox" <<<"$ONBOARD_OUTPUT"; then + pass "Sandbox rebuild triggered by rotation" + else + fail "Sandbox rebuild not triggered" + info "Onboard output:" + echo "$ONBOARD_OUTPUT" | tail -20 + fi + + if openshell sandbox list 2>/dev/null | grep -q "$SANDBOX_NAME"; then + pass "Sandbox running after Discord rotation" + else + fail "Sandbox not running after Discord rotation" + fi + + # ── Phase 5: Re-onboard with same tokens (no change) ───────────── + + section "Phase 5: Re-onboard with same tokens (no rotation expected)" + + ONBOARD_OUTPUT=$(nemoclaw onboard --non-interactive 2>&1) + onboard_exit=$? + + if [ $onboard_exit -ne 0 ]; then + fail "Phase 5 onboard failed (exit $onboard_exit)" + echo "$ONBOARD_OUTPUT" | tail -30 + fi + + if grep -q "reusing it" <<<"$ONBOARD_OUTPUT"; then + pass "Sandbox reused when tokens unchanged" + else + fail "Sandbox was not reused (unexpected rebuild)" + info "Onboard output:" + echo "$ONBOARD_OUTPUT" | tail -20 + fi + + # ── Phase 6: Rotate Slack tokens (re-onboard with token B) ─────── + + section "Phase 6: Re-onboard with rotated SLACK_BOT_TOKEN_B and SLACK_APP_TOKEN_B (Telegram + Discord unchanged)" + + export TELEGRAM_BOT_TOKEN="$TELEGRAM_BOT_TOKEN_B" + export DISCORD_BOT_TOKEN="$DISCORD_BOT_TOKEN_B" + export SLACK_BOT_TOKEN="$SLACK_BOT_TOKEN_B" + export SLACK_APP_TOKEN="$SLACK_APP_TOKEN_B" + + ONBOARD_OUTPUT=$(nemoclaw onboard --non-interactive 2>&1) + onboard_exit=$? + + if [ $onboard_exit -ne 0 ]; then + fail "Phase 6 onboard failed (exit $onboard_exit)" + echo "$ONBOARD_OUTPUT" | tail -30 + fi + + if grep -q "credential(s) rotated" <<<"$ONBOARD_OUTPUT"; then + pass "Credential rotation detected" + else + fail "Credential rotation not detected in onboard output" + info "Onboard output:" + echo "$ONBOARD_OUTPUT" | tail -20 + fi + + # Both slack-bridge (bot token) and slack-app (app token) should rotate. + if grep -q "credential(s) rotated:.*slack-bridge" <<<"$ONBOARD_OUTPUT"; then + pass "Rotation message identifies slack-bridge" + else + fail "Rotation message did not identify slack-bridge" + info "Onboard output:" + grep "credential(s) rotated" <<<"$ONBOARD_OUTPUT" || true + fi + + if grep -q "credential(s) rotated:.*slack-app" <<<"$ONBOARD_OUTPUT"; then + pass "Rotation message identifies slack-app" + else + fail "Rotation message did not identify slack-app" + info "Onboard output:" + grep "credential(s) rotated" <<<"$ONBOARD_OUTPUT" || true + fi + + if grep -q "credential(s) rotated:.*telegram-bridge" <<<"$ONBOARD_OUTPUT"; then + fail "Rotation message unexpectedly named telegram-bridge (Telegram token did not change)" + info "Onboard output:" + grep "credential(s) rotated" <<<"$ONBOARD_OUTPUT" || true + else + pass "Rotation message did not name telegram-bridge (Telegram unchanged)" + fi + + if grep -q "credential(s) rotated:.*discord-bridge" <<<"$ONBOARD_OUTPUT"; then + fail "Rotation message unexpectedly named discord-bridge (Discord token did not change)" + info "Onboard output:" + grep "credential(s) rotated" <<<"$ONBOARD_OUTPUT" || true + else + pass "Rotation message did not name discord-bridge (Discord unchanged)" + fi + + if grep -q "Rebuilding sandbox" <<<"$ONBOARD_OUTPUT"; then + pass "Sandbox rebuild triggered by Slack rotation" + else + fail "Sandbox rebuild not triggered" + info "Onboard output:" + echo "$ONBOARD_OUTPUT" | tail -20 + fi + + if openshell sandbox list 2>/dev/null | grep -q "$SANDBOX_NAME"; then + pass "Sandbox running after Slack rotation" + else + fail "Sandbox not running after Slack rotation" + fi + + # ── Phase 7: Re-onboard with same tokens (no change) ───────────── + + section "Phase 7: Re-onboard with same tokens (no rotation expected)" + + ONBOARD_OUTPUT=$(nemoclaw onboard --non-interactive 2>&1) + onboard_exit=$? + + if [ $onboard_exit -ne 0 ]; then + fail "Phase 7 onboard failed (exit $onboard_exit)" + echo "$ONBOARD_OUTPUT" | tail -30 + fi + + if grep -q "reusing it" <<<"$ONBOARD_OUTPUT"; then + pass "Sandbox reused when tokens unchanged" + else + fail "Sandbox was not reused (unexpected rebuild)" + info "Onboard output:" + echo "$ONBOARD_OUTPUT" | tail -20 + fi +fi + +# ── Summary ─────────────────────────────────────────────────────── + +print_summary diff --git a/test/e2e-vpn/test-tunnel-lifecycle.sh b/test/e2e-vpn/test-tunnel-lifecycle.sh new file mode 100755 index 00000000000..aa060c54c1a --- /dev/null +++ b/test/e2e-vpn/test-tunnel-lifecycle.sh @@ -0,0 +1,516 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# ============================================================================= +# test-tunnel-lifecycle.sh +# NemoClaw Tunnel Lifecycle E2E Tests +# +# Covers: +# TC-DEPLOY-01a: nemoclaw tunnel start (cloudflared tunnel) +# TC-DEPLOY-01b: tunnel URL serves the OpenClaw dashboard +# TC-DEPLOY-01c: nemoclaw tunnel stop removes URL from status +# +# Prerequisites: +# - Docker running +# - NVIDIA_API_KEY set +# - Network access to inference.nvidia.com +# ============================================================================= + +set -euo pipefail + +# ── Overall timeout ────────────────────────────────────────────────────────── +export NEMOCLAW_E2E_DEFAULT_TIMEOUT=3600 +SCRIPT_DIR_TIMEOUT="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +# shellcheck source=test/e2e-vpn/e2e-timeout.sh +source "${SCRIPT_DIR_TIMEOUT}/e2e-timeout.sh" +# shellcheck source=test/e2e-vpn/lib/install-path-refresh.sh +source "${SCRIPT_DIR_TIMEOUT}/lib/install-path-refresh.sh" +# shellcheck source=test/e2e-vpn/lib/cloudflared-version-resolver.sh +source "${SCRIPT_DIR_TIMEOUT}/lib/cloudflared-version-resolver.sh" + +# ── Colors ─────────────────────────────────────────────────────────────────── +GREEN='\033[0;32m' +RED='\033[0;31m' +YELLOW='\033[1;33m' +CYAN='\033[0;36m' +NC='\033[0m' + +PASS=0 +FAIL=0 +SKIP=0 +TOTAL=0 + +# Log a timestamped message. +log() { echo -e "${CYAN}[$(date +%H:%M:%S)]${NC} $*" | tee -a "$LOG_FILE"; } +# Record a passing assertion. +pass() { + ((PASS += 1)) + ((TOTAL += 1)) + echo -e "${GREEN} PASS${NC} $1" | tee -a "$LOG_FILE" +} +# Record a failing assertion. +fail() { + ((FAIL += 1)) + ((TOTAL += 1)) + echo -e "${RED} FAIL${NC} $1 — $2" | tee -a "$LOG_FILE" +} +# Record a skipped test. +skip() { + ((SKIP += 1)) + ((TOTAL += 1)) + echo -e "${YELLOW} SKIP${NC} $1 — $2" | tee -a "$LOG_FILE" +} + +# ── Config ─────────────────────────────────────────────────────────────────── +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-tunnel-lifecycle}" +LOG_FILE="test-tunnel-lifecycle-$(date +%Y%m%d-%H%M%S).log" +# Local dashboard port mirrors nemoclaw/src/lib/ports.ts DASHBOARD_PORT default. +LOCAL_DASHBOARD_PORT="${NEMOCLAW_DASHBOARD_PORT:-18789}" + +# ── Resolve repo root ──────────────────────────────────────────────────────── +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" + +# ── Install NemoClaw if not present ────────────────────────────────────────── +install_nemoclaw() { + export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" + if [ -s "$NVM_DIR/nvm.sh" ]; then + # shellcheck source=/dev/null + . "$NVM_DIR/nvm.sh" + fi + nemoclaw_ensure_local_bin_on_path + + if command -v nemoclaw >/dev/null 2>&1; then + log "nemoclaw already installed: $(nemoclaw --version 2>/dev/null || echo unknown)" + return + fi + log "=== Installing NemoClaw via install.sh ===" + NEMOCLAW_SANDBOX_NAME="$SANDBOX_NAME" \ + NVIDIA_API_KEY="${NVIDIA_API_KEY:-nvapi-DUMMY-FOR-INSTALL}" \ + NEMOCLAW_NON_INTERACTIVE=1 \ + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ + bash "$REPO_ROOT/install.sh" --non-interactive --yes-i-accept-third-party-software \ + 2>&1 | tee -a "$LOG_FILE" + nemoclaw_refresh_install_env + if ! command -v nemoclaw >/dev/null 2>&1; then + log "ERROR: install.sh failed — nemoclaw not found" + exit 1 + fi +} + +# ── Pre-flight ─────────────────────────────────────────────────────────────── +preflight() { + log "=== Pre-flight checks ===" + if ! docker info >/dev/null 2>&1; then + log "ERROR: Docker is not running." + exit 1 + fi + log "Docker is running" + + local api_key="${NVIDIA_API_KEY:-}" + if [[ -z "$api_key" ]]; then + log "ERROR: NVIDIA_API_KEY not set" + exit 1 + fi + + install_nemoclaw + + if ! command -v cloudflared >/dev/null 2>&1; then + # Install via Cloudflare's GPG-signed APT repo — trust anchor for secret-bearing + # CI; APT verifies GPG-signed Release → package SHA256 (no per-version SHA pin). + sudo mkdir -p --mode=0755 /usr/share/keyrings + curl -fsSL https://pkg.cloudflare.com/cloudflare-main.gpg \ + | sudo tee /usr/share/keyrings/cloudflare-main.gpg >/dev/null + echo "deb [signed-by=/usr/share/keyrings/cloudflare-main.gpg] https://pkg.cloudflare.com/cloudflared $(lsb_release -cs) main" \ + | sudo tee /etc/apt/sources.list.d/cloudflared.list >/dev/null + sudo apt-get update -qq + + local available_versions + available_versions="$(apt-cache madison cloudflared | awk '{print $3}')" + + local cf_version cf_min_version + cf_min_version="${CLOUDFLARED_MIN_VERSION:-$CLOUDFLARED_DEFAULT_MIN_VERSION}" + if [[ -n "${CLOUDFLARED_VERSION:-}" ]]; then + cf_version="$(cloudflared_resolve_package_version "$available_versions" "$cf_min_version" "$CLOUDFLARED_VERSION")" + log "Using explicit cloudflared version override: ${cf_version}" + else + if ! cf_version="$(cloudflared_resolve_package_version "$available_versions" "$cf_min_version" 2>&1)"; then + log "$cf_version" + log "Available versions:" + log "${available_versions:-}" + exit 1 + fi + log "Resolved cloudflared ${cf_version} from Cloudflare APT repo (minimum ${cf_min_version})" + fi + + log "Installing cloudflared ${cf_version} via Cloudflare APT repo..." + sudo apt-get install -y "cloudflared=${cf_version}" \ + || { + log "ERROR: cloudflared ${cf_version} not available in Cloudflare APT repo" + log "Available versions:" + log "$available_versions" + exit 1 + } + log "cloudflared ${cf_version} installed (GPG verified via Cloudflare APT repo)" + fi + + log "nemoclaw: $(nemoclaw --version 2>/dev/null || echo unknown)" + log "cloudflared: $(cloudflared --version 2>/dev/null || echo 'not available')" + log "Pre-flight complete" +} + +# ── Onboard helper ─────────────────────────────────────────────────────────── +onboard_sandbox() { + local name="$1" + log " Onboarding sandbox '$name'..." + NEMOCLAW_SANDBOX_NAME="$name" \ + NEMOCLAW_NON_INTERACTIVE=1 \ + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ + NEMOCLAW_POLICY_TIER="open" \ + run_with_timeout 1800 nemoclaw onboard --non-interactive --yes-i-accept-third-party-software \ + 2>&1 | tee -a "$LOG_FILE" || { + log "FATAL: Onboard failed for '$name'" + return 1 + } + log " Sandbox '$name' onboarded" +} + +# Resolve /tmp/nemoclaw-services-/cloudflared.log; fall back to the +# most recently modified one if SANDBOX_NAME wasn't propagated to NemoClaw. +get_cloudflared_log_path() { + local log="/tmp/nemoclaw-services-${SANDBOX_NAME}/cloudflared.log" + if [[ -f "$log" ]]; then + printf '%s\n' "$log" + return 0 + fi + # shellcheck disable=SC2012 + log="$(ls -t /tmp/nemoclaw-services-*/cloudflared.log 2>/dev/null | head -1 || true)" + if [[ -n "$log" && -f "$log" ]]; then + printf '%s\n' "$log" + fi + return 0 +} + +is_cloudflare_transient_text() { + grep -qiE 'failed to unmarshal quick Tunnel|quick tunnels? (are )?(temporarily )?disabled|failed to (dial|register)|tunnel server.*error|i/o timeout|EOF.*tunnel|couldn.?t start tunnel|tunnel creation failed|bad gateway|\b50[234]\b' <<<"$1" +} + +is_cloudflare_transient_http_code() { + case "${1:-}" in + 000 | 502 | 503 | 504) return 0 ;; + *) return 1 ;; + esac +} + +# Classify failure cause from cloudflared.log. Echoes one of: +# nemoclaw_no_spawn / nemoclaw_capture_bug / nemoclaw_local / cloudflare / unknown +classify_cloudflared_log() { + local cf_log + cf_log=$(get_cloudflared_log_path) + if [[ -z "$cf_log" ]]; then + echo "nemoclaw_no_spawn" + return + fi + if grep -qE 'https://[a-z0-9-]+\.trycloudflare\.com' "$cf_log" 2>/dev/null; then + echo "nemoclaw_capture_bug" + return + fi + if grep -qiE 'unable to reach the origin|connection refused.*127\.0\.0\.1|connection refused.*localhost|dial tcp.*127\.0\.0\.1.*refused' "$cf_log" 2>/dev/null; then + echo "nemoclaw_local" + return + fi + if is_cloudflare_transient_text "$(cat "$cf_log" 2>/dev/null)"; then + echo "cloudflare" + return + fi + echo "unknown" +} + +# Print the tail of cloudflared.log to the test log for human triage. +show_cloudflared_log() { + local cf_log tail_lines=40 + cf_log=$(get_cloudflared_log_path) + if [[ -z "$cf_log" ]]; then + log " (no cloudflared.log found under /tmp/nemoclaw-services-*/)" + return + fi + log " --- cloudflared.log ($cf_log, last ${tail_lines} lines) ---" + tail -n "$tail_lines" "$cf_log" 2>/dev/null | sed 's/^/ /' | tee -a "$LOG_FILE" || true + log " --- end cloudflared.log ---" +} + +# Probe local dashboard: any HTTP response (incl. 401/403) = up; "000" = down. +# Mirrors src/lib/verify-deployment.ts:128. +probe_local_dashboard() { + local code + code="$(curl -sS -o /dev/null -w '%{http_code}' \ + --max-time 5 "http://localhost:${LOCAL_DASHBOARD_PORT}/" 2>/dev/null || true)" + [[ -z "$code" ]] && code="000" + [[ "$code" != "000" ]] +} + +# Wait up to N seconds for local dashboard to become reachable. +# Returns 0 if reachable within timeout, 1 if not. +wait_local_dashboard_ready() { + local max_tries="${1:-30}" + for i in $(seq 1 "$max_tries"); do + if probe_local_dashboard; then + log " ✓ Local dashboard reachable on localhost:${LOCAL_DASHBOARD_PORT} after ${i}s" + return 0 + fi + [[ $((i % 5)) -eq 0 ]] && log " ... still waiting for localhost:${LOCAL_DASHBOARD_PORT} (${i}/${max_tries}s)" + sleep 1 + done + return 1 +} + +# ============================================================================= +# TC-DEPLOY-01a: nemoclaw tunnel start (cloudflared tunnel) +# TC-DEPLOY-01b: tunnel URL serves the OpenClaw dashboard +# TC-DEPLOY-01c: nemoclaw tunnel stop removes tunnel URL from status +# ============================================================================= +test_tunnel_lifecycle() { + log "=== TC-DEPLOY-01a/b/c: Start / Probe / Stop ===" + + # Fail closed: skip would let a broken install path silently pass. + if ! command -v cloudflared >/dev/null 2>&1; then + fail "TC-DEPLOY-01a / TC-DEPLOY-01b / TC-DEPLOY-01c" \ + "cloudflared not available — required for tunnel validation. Preflight install should have run; check earlier log." + return + fi + + # Cascade guard: skip if a prior step left the sandbox missing. + if ! nemoclaw list 2>/dev/null | grep -Fq -- "$SANDBOX_NAME"; then + skip "TC-DEPLOY-01a / TC-DEPLOY-01b / TC-DEPLOY-01c" \ + "Sandbox '$SANDBOX_NAME' not present" + return + fi + + # ── Local dashboard pre-check (BEFORE tunnel start) ─────────────────────── + # Catch local-not-ready before tunnel start to avoid 502s blamed on Cloudflare. + log " Pre-check: Waiting for local dashboard at localhost:${LOCAL_DASHBOARD_PORT}..." + if ! wait_local_dashboard_ready 30; then + fail "TC-DEPLOY-01a: LocalReadiness" \ + "[NemoClaw fault] Local OpenClaw dashboard not reachable on localhost:${LOCAL_DASHBOARD_PORT} after 30s. Tunnel cannot proxy a dead origin — this is NOT a Cloudflare issue." + return + fi + pass "TC-DEPLOY-01a: Local dashboard reachable (pre-check passed)" + + # ── TC-DEPLOY-01a: Start tunnel + verify URL surfaces ─────────────────────────────────── + log " Step 1: Running nemoclaw tunnel start..." + local start_output start_rc=0 + start_output=$(nemoclaw tunnel start 2>&1) || start_rc=$? + log " Start output:" + log " ---" + log "$start_output" + log " ---" + if [[ $start_rc -ne 0 ]]; then + show_cloudflared_log + if is_cloudflare_transient_text "$start_output" || [[ "$(classify_cloudflared_log)" == "cloudflare" ]]; then + skip "TC-DEPLOY-01a: CloudflareRegister" \ + "[Cloudflare fault] 'nemoclaw tunnel start' exited with code $start_rc because quick-tunnel registration returned a transient external error." + log " Stopping tunnel after Cloudflare start failure..." + nemoclaw tunnel stop 2>/dev/null || true + return + fi + fail "TC-DEPLOY-01a: Start" "[NemoClaw fault] 'nemoclaw tunnel start' exited with code $start_rc — start command itself failed." + return + fi + + log " Step 2: Reading nemoclaw status (polling for tunnel URL)..." + local status_output tunnel_url + for i in $(seq 1 15); do + status_output=$(nemoclaw status 2>&1) || true + tunnel_url=$(printf '%s\n' "$status_output" | grep -oE "https://[a-z0-9-]+\.trycloudflare\.com" | head -1) || true + [[ -n "$tunnel_url" ]] && break + sleep 1 + done + + if [[ -n "$tunnel_url" ]]; then + pass "TC-DEPLOY-01a: Tunnel URL found in status ($tunnel_url)" + else + # Classify failure cause from cloudflared.log to attribute fault accurately. + # Print log tail first so the diagnostic is visible above the fail line in CI logs. + show_cloudflared_log + local cf_class + cf_class=$(classify_cloudflared_log) + case "$cf_class" in + nemoclaw_no_spawn) + fail "TC-DEPLOY-01a: NoSpawn" \ + "[NemoClaw fault] cloudflared.log missing — NemoClaw failed to spawn the cloudflared process. Check tunnel start impl." + ;; + nemoclaw_capture_bug) + fail "TC-DEPLOY-01a: CaptureBug" \ + "[NemoClaw fault] cloudflared.log HAS trycloudflare URL but 'nemoclaw status' did not surface it. Status capture bug in NemoClaw." + ;; + nemoclaw_local) + fail "TC-DEPLOY-01a: LocalOrigin" \ + "[NemoClaw fault] cloudflared log reports it cannot reach localhost:${LOCAL_DASHBOARD_PORT} (origin not serving). Pre-check should have caught this — review pre-check timeout." + ;; + cloudflare) + skip "TC-DEPLOY-01a: CloudflareRegister" \ + "[Cloudflare fault] cloudflared failed to register with Cloudflare." + ;; + *) + fail "TC-DEPLOY-01a: Start" \ + "[Unclassified] Tunnel URL did not surface and cloudflared.log did not match any known pattern. See log tail above." + ;; + esac + # Stop the tunnel even no tunnel URL was found + log " Stopping tunnel..." + nemoclaw tunnel stop 2>/dev/null || true + log " Tunnel stopped" + return + fi + + # ── TC-DEPLOY-01b: Tunnel serves the OpenClaw dashboard ──────────────────────── + if [[ -n "$tunnel_url" ]]; then + log " Step 3: Probing tunnel URL (exponential backoff + local re-verify)..." + local http_code="000" body_file backoff=2 max_retries=15 + body_file=$(mktemp) + for i in $(seq 1 "$max_retries"); do + # curl -w '%{http_code}' always writes the 3-char status (writes "000" on + # connection failure), so do NOT chain `|| echo "000"` — that would append + # a second "000" to whatever curl already wrote, producing "000000". + http_code=$(curl -sS -o "$body_file" -w '%{http_code}' \ + --max-time 30 "$tunnel_url" 2>/dev/null) || true + [[ -z "$http_code" ]] && http_code="000" + if [[ "$http_code" == "200" ]]; then + break + fi + + # Re-verify local BEFORE attributing the failure to Cloudflare — fact-find + # first so the log message reflects truth at this moment (avoid lying logs). + if ! probe_local_dashboard; then + fail "TC-DEPLOY-01b: LocalRegression" \ + "[NemoClaw fault] Tunnel returned $http_code AND local dashboard regressed during retry loop (was healthy at pre-check). Likely sandbox/dashboard crash — NOT a Cloudflare issue." + rm -f "$body_file" + return + fi + + log " [$i/$max_retries] Tunnel not yet reachable ('$http_code'); LOCAL is healthy → Cloudflare quick-tunnel not ready (DNS propagation or edge instability); backoff ${backoff}s..." + sleep "$backoff" + backoff=$((backoff * 2)) + ((backoff > 30)) && backoff=30 + done + + if [[ "$http_code" == "200" ]]; then + if grep -qE 'OpenClaw Control|/dev/null)"; then + skip "TC-DEPLOY-01b: CloudflareEdge" \ + "[Cloudflare fault] Tunnel URL never became reachable after $max_retries retries (last status '$http_code') while local stayed healthy throughout — Cloudflare quick-tunnel did not become reachable in time (slow DNS propagation or edge instability)." + else + fail "TC-DEPLOY-01b: UnexpectedStatus" \ + "[NemoClaw fault] Tunnel returned unexpected HTTP $http_code while local stayed healthy; not classified as external Cloudflare flake (first 200B: $(head -c 200 "$body_file" | tr -d '\n'))." + fi + fi + rm -f "$body_file" + else + skip "TC-DEPLOY-01b" "Tunnel URL not available" + fi + + log " Step 4: Running nemoclaw tunnel stop..." + local stop_output stop_rc=0 + stop_output=$(nemoclaw tunnel stop 2>&1) || stop_rc=$? + log " Tunnel stop output:" + printf '%s\n' "$stop_output" | sed 's/^/ /' | tee -a "$LOG_FILE" || true + if [[ $stop_rc -ne 0 ]]; then + fail "TC-DEPLOY-01c: Stop command" "nemoclaw tunnel stop failed (exit $stop_rc)" + return + fi + + # ── TC-DEPLOY-01c: Tunnel URL absent after stop ───────────────────────────── + log " Step 5: Verifying tunnel stopped (polling for URL removal)..." + if [[ -z "$tunnel_url" ]]; then + skip "TC-DEPLOY-01c" "Tunnel URL was never confirmed in status" + else + local post_status post_url status_rc=0 status_ok=0 + for i in $(seq 1 10); do + status_rc=0 + post_status=$(nemoclaw status 2>&1) || status_rc=$? + if [[ $status_rc -ne 0 ]]; then + log " [$i] nemoclaw status failed (exit $status_rc), retrying in 1s..." + sleep 1 + continue + fi + status_ok=1 + post_url=$(printf '%s\n' "$post_status" | grep -oE "https://[a-z0-9-]+\.trycloudflare\.com" | head -1) || true + [[ -z "$post_url" ]] && break + sleep 1 + done + if [[ $status_ok -eq 0 ]]; then + fail "TC-DEPLOY-01c: Stop" "Could not read nemoclaw status after stop" + elif [[ -z "$post_url" ]]; then + pass "TC-DEPLOY-01c: Tunnel URL absent after stop" + else + fail "TC-DEPLOY-01c: Stop" "Tunnel URL still present after stop ($post_url)" + fi + fi +} + +# Clean up sandbox and services on exit. +teardown() { + # Do not unlink ~/.nemoclaw/onboard.lock: see rationale in + # test/e2e-vpn/lib/sandbox-teardown.sh — the lock is PID-ownership-aware + # and onboard cleans up stale locks itself. + set +e + nemoclaw stop 2>/dev/null || true + nemoclaw "$SANDBOX_NAME" destroy --yes 2>/dev/null || true + set -e +} + +# Print final PASS/FAIL/SKIP counts and exit. +summary() { + echo "" + echo "============================================================" + echo " Tunnel Lifecycle E2E Results" + echo "============================================================" + echo -e " ${GREEN}PASS: $PASS${NC}" + echo -e " ${RED}FAIL: $FAIL${NC}" + echo -e " ${YELLOW}SKIP: $SKIP${NC}" + echo " TOTAL: $TOTAL" + echo "============================================================" + echo " Log: $LOG_FILE" + echo "============================================================" + echo "" + + if [[ $FAIL -gt 0 ]]; then + exit 1 + fi + exit 0 +} + +# Entry point: preflight → onboard → tests → summary. +main() { + echo "" + echo "============================================================" + echo " NemoClaw Tunnel Lifecycle E2E Tests" + echo " $(date)" + echo "============================================================" + echo "" + + preflight + + log "=== Onboarding sandbox ===" + if ! onboard_sandbox "$SANDBOX_NAME"; then + log "FATAL: Could not onboard sandbox" + exit 1 + fi + + test_tunnel_lifecycle + + teardown + trap - EXIT + summary +} + +trap teardown EXIT +main "$@" diff --git a/test/e2e-vpn/test-upgrade-stale-sandbox.sh b/test/e2e-vpn/test-upgrade-stale-sandbox.sh new file mode 100755 index 00000000000..0e4326b218c --- /dev/null +++ b/test/e2e-vpn/test-upgrade-stale-sandbox.sh @@ -0,0 +1,251 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Issue #1904 reproduction — "sandbox OpenClaw version is not upgraded +# after NemoClaw upgrade". +# +# 1. Install current NemoClaw via install.sh (sets up gateway + OpenShell) +# 2. Delete the sandbox install.sh created (keep the gateway) +# 3. Build a base image with an OLDER OpenClaw version (2026.3.11) +# 4. Create a sandbox from that old image via openshell directly +# 5. Register it in NemoClaw's registry with the old agentVersion +# 6. Run `nemoclaw upgrade-sandboxes --check` +# 7. Verify it detects the sandbox as stale +# 8. Run `nemoclaw rebuild --yes` to upgrade +# 9. Verify the sandbox now runs the current OpenClaw version +# 10. Verify `upgrade-sandboxes --check` reports clean +# +# Prerequisites: +# - Docker running +# - NVIDIA_API_KEY set (real key, starts with nvapi-) + +set -euo pipefail + +OLD_OPENCLAW_VERSION="2026.3.11" +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-upgrade-stale}" + +# shellcheck source=test/e2e-vpn/lib/sandbox-teardown.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib/sandbox-teardown.sh" +register_sandbox_for_teardown "$SANDBOX_NAME" + +REGISTRY_FILE="$HOME/.nemoclaw/sandboxes.json" +SESSION_FILE="$HOME/.nemoclaw/onboard-session.json" + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +pass() { echo -e "${GREEN}[PASS]${NC} $1"; } +fail() { + echo -e "${RED}[FAIL]${NC} $1" >&2 + echo -e "${YELLOW}[DIAG]${NC} --- Failure diagnostics ---" >&2 + echo -e "${YELLOW}[DIAG]${NC} Registry: $(cat "${REGISTRY_FILE}" 2>/dev/null || echo 'not found')" >&2 + echo -e "${YELLOW}[DIAG]${NC} Sandboxes: $(openshell sandbox list 2>&1 || echo 'openshell unavailable')" >&2 + echo -e "${YELLOW}[DIAG]${NC} Docker images: $(docker images --format '{{.Repository}}:{{.Tag}} {{.ID}}' | grep -Ei 'sandbox|nemoclaw|openclaw' | head -10 || true)" >&2 + echo -e "${YELLOW}[DIAG]${NC} --- End diagnostics ---" >&2 + exit 1 +} +info() { echo -e "${YELLOW}[INFO]${NC} $1"; } +diag() { echo -e "${YELLOW}[DIAG]${NC} $1"; } + +# ── Preflight ─────────────────────────────────────────────────────── +[ -n "${NVIDIA_API_KEY:-}" ] || fail "NVIDIA_API_KEY is required" +[ "${NEMOCLAW_NON_INTERACTIVE:-}" = "1" ] || fail "NEMOCLAW_NON_INTERACTIVE=1 is required" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" + +export NEMOCLAW_REBUILD_VERBOSE=1 + +info "Issue #1904 reproduction (old OpenClaw: ${OLD_OPENCLAW_VERSION}, sandbox: ${SANDBOX_NAME})" + +# ── Phase 1: Install current NemoClaw ──────────────────────────────── +info "Phase 1: Installing current NemoClaw via install.sh..." + +export NEMOCLAW_NON_INTERACTIVE=1 +export NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 +export NEMOCLAW_SANDBOX_NAME="${SANDBOX_NAME}" +export NEMOCLAW_RECREATE_SANDBOX=1 + +INSTALL_LOG="/tmp/nemoclaw-e2e-upgrade-install.log" +if ! bash "${REPO_ROOT}/install.sh" --non-interactive >"$INSTALL_LOG" 2>&1; then + info "install.sh exited non-zero (may be expected). Checking..." +fi + +# Source shell profile to pick up nvm/PATH changes +if [ -f "$HOME/.bashrc" ]; then + # shellcheck source=/dev/null + source "$HOME/.bashrc" 2>/dev/null || true +fi +export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" +if [ -s "$NVM_DIR/nvm.sh" ]; then + # shellcheck source=/dev/null + . "$NVM_DIR/nvm.sh" +fi +if [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]]; then + export PATH="$HOME/.local/bin:$PATH" +fi + +command -v nemoclaw >/dev/null 2>&1 || fail "nemoclaw not found on PATH after install" +command -v openshell >/dev/null 2>&1 || fail "openshell not found on PATH after install" +pass "NemoClaw installed" + +# ── Phase 2: Delete sandbox, build old base image ──────────────────── +info "Phase 2: Replacing sandbox with old OpenClaw ${OLD_OPENCLAW_VERSION}..." + +# Delete the sandbox that install.sh created — we'll make our own old one. +openshell sandbox delete "${SANDBOX_NAME}" 2>/dev/null || true +diag "Deleted Phase 1 sandbox, gateway preserved" + +OLD_BASE_TAG="nemoclaw-old-base:e2e-upgrade-stale" +BLUEPRINT="${REPO_ROOT}/nemoclaw-blueprint/blueprint.yaml" +BLUEPRINT_BAK="${BLUEPRINT}.bak" + +# Temporarily lower min_openclaw_version so the old version builds. +cp "${BLUEPRINT}" "${BLUEPRINT_BAK}" +sed "s/min_openclaw_version:.*/min_openclaw_version: \"${OLD_OPENCLAW_VERSION}\"/" "${BLUEPRINT}" >"${BLUEPRINT}.tmp" +mv "${BLUEPRINT}.tmp" "${BLUEPRINT}" + +docker build \ + --build-arg "OPENCLAW_VERSION=${OLD_OPENCLAW_VERSION}" \ + -f "${REPO_ROOT}/Dockerfile.base" \ + -t "${OLD_BASE_TAG}" \ + "${REPO_ROOT}" +BUILD_RC=$? + +mv "${BLUEPRINT_BAK}" "${BLUEPRINT}" +[ "$BUILD_RC" -eq 0 ] || fail "Failed to build old base image" + +pass "Old base image built (OpenClaw ${OLD_OPENCLAW_VERSION})" + +# ── Phase 3: Create old sandbox via openshell ──────────────────────── +info "Phase 3: Creating sandbox with old OpenClaw..." + +TESTDIR=$(mktemp -d) +cat >"${TESTDIR}/Dockerfile" < /sandbox/.openclaw/openclaw.json +CMD ["/bin/bash"] +DOCKERFILE + +openshell sandbox create --name "${SANDBOX_NAME}" --from "${TESTDIR}/Dockerfile" --gateway nemoclaw --no-tty -- true +rm -rf "${TESTDIR}" + +# Wait for Ready +for _i in $(seq 1 30); do + if openshell sandbox list 2>/dev/null | grep -q "${SANDBOX_NAME}.*Ready"; then + break + fi + sleep 5 +done +openshell sandbox list 2>/dev/null | grep -q "${SANDBOX_NAME}.*Ready" \ + || fail "Sandbox did not become Ready" + +SANDBOX_VERSION=$(openshell sandbox exec --name "${SANDBOX_NAME}" -- openclaw --version 2>&1) \ + || fail "Failed to read OpenClaw version from old sandbox" +info "Old sandbox OpenClaw version: ${SANDBOX_VERSION}" + +pass "Old sandbox created (OpenClaw ${OLD_OPENCLAW_VERSION})" + +# ── Phase 4: Register with old agentVersion ────────────────────────── +info "Phase 4: Registering sandbox with old agentVersion..." + +python3 -c " +import json, os +sess_path = '${SESSION_FILE}' +try: + with open(sess_path) as f: + sess = json.load(f) +except Exception: + sess = {} +env_provider = (os.environ.get('NEMOCLAW_PROVIDER') or '').strip() +if env_provider == 'custom': + env_provider = 'compatible-endpoint' +provider = sess.get('provider') or env_provider or 'compatible-endpoint' +model = ( + sess.get('model') + or os.environ.get('NEMOCLAW_MODEL') + or os.environ.get('NEMOCLAW_COMPAT_MODEL') + or 'nvidia/nvidia/nemotron-3-super-v3' +) +reg = {'sandboxes': {'${SANDBOX_NAME}': { + 'name': '${SANDBOX_NAME}', + 'createdAt': '$(date -u +%Y-%m-%dT%H:%M:%SZ)', + 'model': model, + 'provider': provider, + 'gpuEnabled': False, + 'policies': [], + 'policyTier': None, + 'agent': None, + 'agentVersion': '${OLD_OPENCLAW_VERSION}' +}}, 'defaultSandbox': '${SANDBOX_NAME}'} +with open('${REGISTRY_FILE}', 'w') as f: + json.dump(reg, f, indent=2) + +sess['sandboxName'] = '${SANDBOX_NAME}' +sess['status'] = 'complete' +with open(sess_path, 'w') as f: + json.dump(sess, f, indent=2) +print('Registry and session updated') +" + +pass "Sandbox registered with agentVersion=${OLD_OPENCLAW_VERSION}" + +# ── Phase 5: Verify upgrade-sandboxes detects the stale sandbox ────── +info "Phase 5: Running upgrade-sandboxes --check..." + +CHECK_OUTPUT=$(nemoclaw upgrade-sandboxes --check 2>&1 || true) +echo "$CHECK_OUTPUT" + +if echo "$CHECK_OUTPUT" | grep -qi "stale\|need upgrading"; then + pass "Phase 5: upgrade-sandboxes --check detected stale sandbox" +elif echo "$CHECK_OUTPUT" | grep -qi "up to date"; then + fail "upgrade-sandboxes --check says all up to date — stale sandbox NOT detected (#1904)" +else + fail "upgrade-sandboxes --check produced unexpected output" +fi + +# ── Phase 6: Rebuild and verify new version ────────────────────────── +info "Phase 6: Rebuilding sandbox..." + +nemoclaw "${SANDBOX_NAME}" rebuild --yes 2>&1 || fail "Sandbox rebuild failed" + +for _i in $(seq 1 30); do + if openshell sandbox list 2>/dev/null | grep -q "${SANDBOX_NAME}.*Ready"; then + break + fi + sleep 5 +done + +NEW_OPENCLAW_VERSION=$(openshell sandbox exec --name "${SANDBOX_NAME}" -- openclaw --version 2>&1) \ + || fail "Failed to read OpenClaw version after rebuild" +info "New sandbox OpenClaw version: ${NEW_OPENCLAW_VERSION}" + +if echo "${NEW_OPENCLAW_VERSION}" | grep -q "${OLD_OPENCLAW_VERSION}"; then + fail "Sandbox still running old OpenClaw ${OLD_OPENCLAW_VERSION} after rebuild — #1904 NOT fixed" +fi + +pass "Phase 6: Sandbox upgraded from OpenClaw ${OLD_OPENCLAW_VERSION} to ${NEW_OPENCLAW_VERSION}" + +# ── Phase 7: Verify clean ──────────────────────────────────────────── +info "Phase 7: Verifying upgrade-sandboxes --check is clean..." + +RECHECK_OUTPUT=$(nemoclaw upgrade-sandboxes --check 2>&1 || true) +echo "$RECHECK_OUTPUT" + +if echo "$RECHECK_OUTPUT" | grep -qi "up to date"; then + pass "Phase 7: All sandboxes up to date after rebuild" +else + fail "Phase 7: upgrade-sandboxes --check did not report 'up to date' after rebuild" +fi + +echo "" +echo -e "${GREEN}═══════════════════════════════════════════════════════════${NC}" +echo -e "${GREEN} Issue #1904 E2E PASSED${NC}" +echo -e "${GREEN} Old: OpenClaw ${OLD_OPENCLAW_VERSION}${NC}" +echo -e "${GREEN} New: OpenClaw ${NEW_OPENCLAW_VERSION}${NC}" +echo -e "${GREEN}═══════════════════════════════════════════════════════════${NC}" diff --git a/test/e2e-vpn/test-vm-driver-privileged-exec-routing.sh b/test/e2e-vpn/test-vm-driver-privileged-exec-routing.sh new file mode 100755 index 00000000000..cf0ed344238 --- /dev/null +++ b/test/e2e-vpn/test-vm-driver-privileged-exec-routing.sh @@ -0,0 +1,142 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# VM/Docker privileged-exec routing regression for #4245. +# +# This is a hermetic host-side check: it builds the CLI, writes a fake +# NemoClaw sandbox registry, puts a fake docker binary first in PATH, and +# imports the built privileged-exec helper directly. It verifies VM and +# Docker-driver sandboxes route only through their direct sandbox containers. + +set -euo pipefail + +_script_dir="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" +REPO="$(cd "${_script_dir}/../.." && pwd)" +TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/nemoclaw-vm-driver-privexec.XXXXXX")" +trap 'rm -rf "$TMP_DIR"' EXIT + +FAKE_BIN="$TMP_DIR/bin" +XDG_NEMOCLAW_FAKE_DOCKER_PS_FILE="$TMP_DIR/docker-ps.txt" +XDG_NEMOCLAW_FAKE_DOCKER_LOG="$TMP_DIR/docker.log" +mkdir -p "$FAKE_BIN" +: >"$XDG_NEMOCLAW_FAKE_DOCKER_PS_FILE" +: >"$XDG_NEMOCLAW_FAKE_DOCKER_LOG" + +cat >"$FAKE_BIN/docker" <<'SH' +#!/bin/bash +set -euo pipefail +printf '%s\n' "$*" >>"${XDG_NEMOCLAW_FAKE_DOCKER_LOG:?}" +if [ "${1:-}" = "ps" ]; then + cat "${XDG_NEMOCLAW_FAKE_DOCKER_PS_FILE:?}" + exit 0 +fi +echo "unexpected fake docker invocation: $*" >&2 +exit 64 +SH +chmod 755 "$FAKE_BIN/docker" + +cd "$REPO" +BUILD_LOG="/tmp/nemoclaw-vm-driver-privileged-exec-routing-build.log" +if [ ! -d node_modules/@types/node ]; then + echo "[vm-driver-privileged-exec-routing] Installing npm dependencies" + { + echo "Installing npm dependencies" + npm ci --ignore-scripts + } >"$BUILD_LOG" 2>&1 +else + echo "npm dependencies already present" >"$BUILD_LOG" +fi +echo "[vm-driver-privileged-exec-routing] Building CLI" +npm run build:cli >>"$BUILD_LOG" 2>&1 + +export XDG_NEMOCLAW_FAKE_DOCKER_PS_FILE +export XDG_NEMOCLAW_FAKE_DOCKER_LOG +export HOME="$TMP_DIR/home" +export PATH="$FAKE_BIN:$PATH" +mkdir -p "$HOME/.nemoclaw" + +node <<'NODE' +const assert = require("node:assert/strict"); +const fs = require("node:fs"); +const path = require("node:path"); + +const repo = process.cwd(); +const registryPath = path.join(process.env.HOME, ".nemoclaw", "sandboxes.json"); +const psFile = process.env.XDG_NEMOCLAW_FAKE_DOCKER_PS_FILE; + +function writeRegistry(entries) { + const sandboxes = {}; + for (const entry of entries) sandboxes[entry.name] = entry; + fs.writeFileSync( + registryPath, + JSON.stringify({ defaultSandbox: entries[0]?.name ?? null, sandboxes }, null, 2), + ); +} + +function writeDockerPs(names) { + fs.writeFileSync(psFile, `${names.join("\n")}\n`); +} + +function assertDirect(args, expectedContainer, label) { + assert.deepEqual( + args, + ["exec", "--user", "root", expectedContainer, "stat", "-c", "%a", "/sandbox/.openclaw/openclaw.json"], + `${label} should route to the direct sandbox container`, + ); + assert.equal( + args.includes("openshell-gateway-nemoclaw"), + false, + `${label} unexpectedly routed through a non-sandbox gateway container`, + ); +} + +writeRegistry([ + { name: "alpha", openshellDriver: "vm" }, + { name: "alpha-child", openshellDriver: "vm" }, + { name: "dockerbox", openshellDriver: "docker" }, + { name: "unknown-driver", openshellDriver: null }, +]); + +writeDockerPs([ + "openshell-gateway-nemoclaw", + "openshell-alpha-child", + "openshell-alpha-child-2026", + "openshell-alpha-abc123", + "openshell-dockerbox-987", + "openshell-unknown-driver", +]); + +const helper = require(path.join(repo, "dist", "lib", "sandbox", "privileged-exec.js")); +const cmd = ["stat", "-c", "%a", "/sandbox/.openclaw/openclaw.json"]; + +assertDirect( + helper.privilegedSandboxExecArgv("alpha", cmd), + "openshell-alpha-abc123", + "VM driver with prefix collision", +); +assertDirect( + helper.privilegedSandboxExecArgv("alpha-child", cmd), + "openshell-alpha-child", + "VM driver with exact container", +); +assertDirect( + helper.privilegedSandboxExecArgv("dockerbox", cmd), + "openshell-dockerbox-987", + "Docker driver", +); +assertDirect( + helper.privilegedSandboxExecArgv("unknown-driver", cmd), + "openshell-unknown-driver", + "registry entry without a recorded driver", +); + +writeDockerPs(["openshell-gateway-nemoclaw", "openshell-other"]); +assert.throws( + () => helper.privilegedSandboxExecArgv("alpha", ["id"]), + /No running direct OpenShell sandbox container found for 'alpha'.*driver: vm/, + "missing VM direct container should fail clearly", +); + +console.log("PASS: VM and Docker privileged exec routing uses direct sandbox containers"); +NODE diff --git a/test/nightly-e2e-vpn-workflow.test.ts b/test/nightly-e2e-vpn-workflow.test.ts new file mode 100644 index 00000000000..5cd3372f649 --- /dev/null +++ b/test/nightly-e2e-vpn-workflow.test.ts @@ -0,0 +1,175 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { existsSync, readFileSync, readdirSync, statSync } from "node:fs"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { + readYaml, + type NightlyWorkflow, + type RunnerWorkflow, + type WorkflowJob, +} from "./helpers/e2e-workflow-contract"; + +const CPU_RUNNER = "linux-amd64-cpu4"; +const VPN_WORKFLOW = ".github/workflows/nightly-e2e-vpn.yaml"; +const BASE_WORKFLOW = ".github/workflows/nightly-e2e.yaml"; +const VPN_RUNNER = ".github/workflows/e2e-script-vpn.yaml"; +const VPN_ACTION = ".github/actions/run-e2e-script-vpn"; +const VPN_E2E_TREE = "test/e2e-vpn"; +const INFRA_JOBS = new Set(["notify-on-failure", "report-to-pr", "scorecard"]); +const SELF_HOSTED_GPU_JOBS = new Set(["gpu-e2e", "gpu-double-onboard-e2e", "gpu-jetson-nvmap-e2e"]); + +function e2eJobNames(workflow: NightlyWorkflow): string[] { + return Object.keys(workflow.jobs).filter((name) => !INFRA_JOBS.has(name)); +} + +function reusableJobs(workflow: NightlyWorkflow): Array<[string, WorkflowJob]> { + return Object.entries(workflow.jobs).filter(([, job]) => job.uses !== undefined); +} + +function needsOf(job: WorkflowJob | undefined): string[] { + const needs = (job as Record | undefined)?.needs; + if (typeof needs === "string") return [needs]; + if (Array.isArray(needs)) return needs.filter((name): name is string => typeof name === "string"); + return []; +} + +function collectStrings(value: unknown): string[] { + if (typeof value === "string") return [value]; + if (Array.isArray(value)) return value.flatMap(collectStrings); + if (!value || typeof value !== "object") return []; + return Object.values(value).flatMap(collectStrings); +} + +function listFiles(path: string): string[] { + if (!existsSync(path)) return []; + if (statSync(path).isFile()) return [path]; + return readdirSync(path).flatMap((entry) => listFiles(join(path, entry))); +} + +function workflowCall(runner: RunnerWorkflow): Record { + return runner.on?.workflow_call ?? runner.true?.workflow_call ?? {}; +} + +describe("VPN nightly E2E workflow validation", () => { + const baseWorkflow = readYaml(BASE_WORKFLOW); + const vpnWorkflow = readYaml(VPN_WORKFLOW); + const vpnRunner = readYaml(VPN_RUNNER); + const baseE2eJobs = e2eJobNames(baseWorkflow); + const vpnE2eJobs = e2eJobNames(vpnWorkflow); + + it("keeps the VPN nightly job graph parallel to the current nightly", () => { + expect(vpnE2eJobs).toEqual(baseE2eJobs); + + for (const aggregateJob of INFRA_JOBS) { + expect(vpnWorkflow.jobs[aggregateJob], aggregateJob).toBeDefined(); + expect(new Set(needsOf(vpnWorkflow.jobs[aggregateJob]))).toEqual(new Set(vpnE2eJobs)); + } + }); + + it("uses the VPN reusable runner for every reusable E2E job", () => { + const baseReusableNames = reusableJobs(baseWorkflow).map(([name]) => name); + const vpnReusable = reusableJobs(vpnWorkflow); + + expect(vpnReusable.map(([name]) => name)).toEqual(baseReusableNames); + for (const [name, job] of vpnReusable) { + expect(job.uses, name).toBe("./.github/workflows/e2e-script-vpn.yaml"); + expect(job.with?.script, name).toMatch(/^test\/e2e-vpn\/test-.*\.sh$/); + expect(existsSync(job.with?.script ?? ""), name).toBe(true); + } + }); + + it("runs CPU jobs on the VPN CPU runner and preserves GPU runner labels", () => { + const call = workflowCall(vpnRunner); + expect(call.inputs?.runner?.default).toBe(CPU_RUNNER); + + for (const [name, job] of Object.entries(vpnWorkflow.jobs)) { + if (job.uses) { + expect(job.with?.runner ?? CPU_RUNNER, name).toBe(CPU_RUNNER); + continue; + } + + if (SELF_HOSTED_GPU_JOBS.has(name)) { + expect(job["runs-on"], name).toBe(baseWorkflow.jobs[name]?.["runs-on"]); + } else { + expect(job["runs-on"], name).toBe(CPU_RUNNER); + } + } + }); + + it("points direct E2E jobs at copied VPN shell scripts only", () => { + const missing: string[] = []; + for (const [name, job] of Object.entries(vpnWorkflow.jobs)) { + if (INFRA_JOBS.has(name) || job.uses) continue; + + const strings = collectStrings(job); + const oldPaths = strings.filter( + (value) => value.includes("test/e2e/") || value.includes("test/e2e-scenario/"), + ); + expect(oldPaths, name).toEqual([]); + + const scriptRefs = new Set( + strings.flatMap((value) => + [...value.matchAll(/test\/e2e-vpn\/test-[A-Za-z0-9_.-]+\.sh/g)].map((match) => match[0]), + ), + ); + for (const script of scriptRefs) { + if (!existsSync(script)) missing.push(`${name}:${script}`); + } + } + + expect(missing).toEqual([]); + }); + + it("keeps VPN files free of the legacy hosted inference contract", () => { + const files = [VPN_WORKFLOW, VPN_RUNNER, ...listFiles(VPN_ACTION), ...listFiles(VPN_E2E_TREE)]; + const forbidden = [ + /NVIDIA_INFERENCE_API_KEY/u, + /inference-api\.nvidia\.com/u, + /build\.nvidia\.com/u, + /\bNVCF\b|\bnvcf\b/u, + /nvidia-prod/u, + /NEMOCLAW_PROVIDER=(?:cloud|build)/u, + /NEMOCLAW_PROVIDER="build"/u, + /NEMOCLAW_PROVIDER: "build"/u, + /public NVIDIA/u, + /NVIDIA Endpoints/u, + /test\/e2e-scenario/u, + /test\/e2e-vpn-scenario/u, + /--project e2e-scenarios-live/u, + ]; + const violations: string[] = []; + + for (const file of files) { + const text = readFileSync(file, "utf8"); + for (const pattern of forbidden) { + if (pattern.test(text)) violations.push(`${file}: ${pattern}`); + } + } + + expect(violations).toEqual([]); + }); + + it("sources only NVIDIA_API_KEY as the VPN inference secret", () => { + const call = workflowCall(vpnRunner); + const secretNames = Object.keys(call.secrets ?? {}).filter((name) => name.includes("NVIDIA")); + const exportStep = vpnRunner.jobs.run.steps.find( + (step) => step.name === "Export hosted CI inference environment", + ); + const runStep = vpnRunner.jobs.run.steps.find((step) => step.name === "Run E2E script"); + + expect(secretNames).toEqual(["NVIDIA_API_KEY"]); + expect(exportStep?.env).toEqual({ NVIDIA_API_KEY: "${{ secrets.NVIDIA_API_KEY }}" }); + expect(exportStep?.run).toContain("NEMOCLAW_PROVIDER=custom"); + expect(exportStep?.run).toContain("NEMOCLAW_ENDPOINT_URL=https://inference.nvidia.com/v1"); + expect(exportStep?.run).toContain("COMPATIBLE_API_KEY=%s\\n"); + expect(exportStep?.run).toContain('"${NVIDIA_API_KEY}"'); + expect(runStep?.uses).toBe("./workflow-actions/.github/actions/run-e2e-script-vpn"); + expect(runStep?.env?.NVIDIA_API_KEY).toBe( + "${{ inputs.nvidia_api_key && secrets.NVIDIA_API_KEY || '' }}", + ); + }); +});