diff --git a/.github/actions/setup-native-podman-e2e/action.yaml b/.github/actions/setup-native-podman-e2e/action.yaml new file mode 100644 index 00000000000..73c982ee8b2 --- /dev/null +++ b/.github/actions/setup-native-podman-e2e/action.yaml @@ -0,0 +1,306 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: setup-native-podman-e2e +description: Install the reviewed Podman toolchain artifact and start its rootless API service for E2E. + +inputs: + enabled: + description: Prepare Podman when the E2E dispatch selected the native Podman gateway runtime. + required: false + default: "false" + +runs: + using: composite + steps: + - id: artifact + name: Resolve native Podman toolchain artifact + if: ${{ inputs.enabled == 'true' }} + shell: bash + env: + RUNNER_ARCH_KIND: ${{ runner.arch }} + run: | + set -euo pipefail + case "$RUNNER_ARCH_KIND" in + X64) architecture=amd64 ;; + ARM64) architecture=arm64 ;; + *) + echo "::error::Native Podman E2E requires a Linux amd64 or arm64 runner" >&2 + exit 1 + ;; + esac + printf 'architecture=%s\n' "$architecture" >>"$GITHUB_OUTPUT" + printf 'name=native-podman-e2e-toolchain-%s\n' "$architecture" >>"$GITHUB_OUTPUT" + + - name: Download native Podman toolchain + if: ${{ inputs.enabled == 'true' }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: ${{ steps.artifact.outputs.name }} + path: ${{ runner.temp }}/native-podman-e2e-toolchain + + - name: Start native Podman runtime + if: ${{ inputs.enabled == 'true' }} + shell: bash + env: + EXPECTED_ARCHITECTURE: ${{ steps.artifact.outputs.architecture }} + TOOLCHAIN_DIRECTORY: ${{ runner.temp }}/native-podman-e2e-toolchain + run: | + set -euo pipefail + [[ "$RUNNER_OS" == "Linux" ]] + [[ -d "$TOOLCHAIN_DIRECTORY" && ! -L "$TOOLCHAIN_DIRECTORY" ]] + [[ -z "$(find -P "$TOOLCHAIN_DIRECTORY" -type l -print -quit)" ]] + mapfile -t actual_files < <( + cd "$TOOLCHAIN_DIRECTORY" + find . -type f -print | LC_ALL=C sort + ) + expected_files=( + ./SHA256SUMS + ./bin/pasta + ./bin/podman + ./libexec/podman/aardvark-dns + ./libexec/podman/netavark + ./libexec/podman/rootlessport + ./manifest.json + ./share/containers/containers.conf + ) + [[ "${actual_files[*]}" == "${expected_files[*]}" ]] + ( + cd "$TOOLCHAIN_DIRECTORY" + sha256sum --check --strict SHA256SUMS + ) + jq -e --arg architecture "$EXPECTED_ARCHITECTURE" ' + .schemaVersion == 1 and + .kind == "nemoclaw-native-podman-toolchain-v1" and + .architecture == $architecture and + .podmanVersion == "6.1.0" and + .podmanSourceSha == "cade97a52ebdf9dbf9e81de8009015776837a074" and + .netavarkVersion == "2.1.0" and + .netavarkSourceSha == "8e91ad1d947ed325327b638f0cb906bea1f7d0ab" and + .aardvarkDnsVersion == "2.1.0" and + .aardvarkDnsSourceSha == "cd7417681229219059939bdd9f0b3bd9ac9abb08" and + .pastaVersion == "2026_07_28.f8df3f1" and + .pastaSourceArchiveSha256 == "54fc6a3b39b0fcb13182078662886a629032852e186e47a371fd9d7fd20d3958" and + .pastaSourceSha == "f8df3f1b228fe19a74a269334fdfe6cc7d0605ce" and + .goVersion == "1.25.9" and + .rustVersion == "1.88.0" + ' "$TOOLCHAIN_DIRECTORY/manifest.json" >/dev/null + + sudo env DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get update + sudo env DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get install \ + --yes --no-install-recommends \ + apparmor btrfs-progs conmon fuse-overlayfs \ + golang-github-containers-common iptables nftables runc slirp4netns uidmap + + toolchain_install_root=/usr/lib/nemoclaw-native-podman-e2e + sudo install -d --owner=root --group=root --mode=0755 \ + "$toolchain_install_root" "$toolchain_install_root/bin" + podman_executable="$toolchain_install_root/bin/podman" + sudo install --owner=root --group=root --mode=0755 \ + "$TOOLCHAIN_DIRECTORY/bin/podman" "$podman_executable" + sudo install --owner=root --group=root --mode=0755 \ + "$TOOLCHAIN_DIRECTORY/bin/pasta" "$toolchain_install_root/bin/pasta" + export PATH="$toolchain_install_root/bin:$PATH" + printf '%s\n' "$toolchain_install_root/bin" >>"$GITHUB_PATH" + for helper in aardvark-dns netavark rootlessport; do + sudo install -D --owner=root --group=root --mode=0755 \ + "$TOOLCHAIN_DIRECTORY/libexec/podman/$helper" \ + "/usr/local/libexec/podman/$helper" + done + sudo install --owner=root --group=root --mode=0644 \ + "$TOOLCHAIN_DIRECTORY/share/containers/containers.conf" \ + /usr/share/containers/containers.conf + [[ "$("$podman_executable" --version)" == "podman version 6.1.0" ]] + + execution_user="$(id -un)" + ensure_subordinate_range() { + local file="$1" + local option="$2" + local range_start=100000 + local range_end + local conflict_end + if awk -F: -v account="$execution_user" ' + $1 == account && $2 ~ /^[0-9]+$/ && $3 ~ /^[0-9]+$/ && $3 >= 65536 { found = 1 } + END { exit found ? 0 : 1 } + ' "$file"; then + return + fi + while :; do + ((range_start <= 4294901760)) + range_end=$((range_start + 65535)) + conflict_end="$(awk -F: -v start="$range_start" -v end="$range_end" ' + $2 ~ /^[0-9]+$/ && $3 ~ /^[0-9]+$/ { + current_end = $2 + $3 - 1 + if ($2 <= end && current_end >= start && current_end > maximum) maximum = current_end + } + END { if (maximum != "") print maximum } + ' "$file")" + [[ -n "$conflict_end" ]] || break + range_start=$((conflict_end + 1)) + done + range_end=$((range_start + 65535)) + sudo usermod "$option" "${range_start}-${range_end}" "$execution_user" + } + ensure_subordinate_range /etc/subuid --add-subuids + ensure_subordinate_range /etc/subgid --add-subgids + + podman_command=("$podman_executable") + podman_service_exec="$podman_executable system service --time=0" + if [[ -r /sys/module/apparmor/parameters/enabled ]] && grep -q '^Y' /sys/module/apparmor/parameters/enabled; then + profile="$RUNNER_TEMP/nemoclaw-native-podman-e2e.apparmor" + pasta_profile="$RUNNER_TEMP/nemoclaw-native-pasta-e2e.apparmor" + printf '%s\n' \ + 'abi ,' \ + 'include ' \ + '' \ + 'profile nemoclaw-native-podman-e2e /usr/lib/nemoclaw-native-podman-e2e/bin/podman flags=(unconfined, attach_disconnected) {' \ + ' userns,' \ + '}' >"$profile" + sudo apparmor_parser -r "$profile" + command -v aa-exec >/dev/null + aa_exec_path="$(command -v aa-exec)" + podman_command=("$aa_exec_path" -p nemoclaw-native-podman-e2e -- "$podman_executable") + podman_service_exec="$aa_exec_path -p nemoclaw-native-podman-e2e -- $podman_executable system service --time=0" + printf '%s\n' \ + 'abi ,' \ + 'include ' \ + '' \ + 'profile nemoclaw-native-pasta-e2e /usr/lib/nemoclaw-native-podman-e2e/bin/pasta flags=(unconfined) {' \ + ' userns,' \ + '}' >"$pasta_profile" + sudo apparmor_parser -r "$pasta_profile" + fi + + uid="$(id -u)" + gid="$(id -g)" + runtime_directory="/run/user/$uid" + socket_path="$runtime_directory/podman/podman.sock" + sudo systemctl start "user-runtime-dir@${uid}.service" "user@${uid}.service" + systemctl is-active --quiet "user-runtime-dir@${uid}.service" + systemctl is-active --quiet "user@${uid}.service" + [[ -d "$runtime_directory" && ! -L "$runtime_directory" ]] + [[ "$(stat -c '%u:%g:%a' "$runtime_directory")" == "${uid}:${gid}:700" ]] + XDG_RUNTIME_DIR="$runtime_directory" /usr/bin/systemctl --user start dbus.socket + XDG_RUNTIME_DIR="$runtime_directory" /usr/bin/systemctl --user is-active --quiet dbus.socket + [[ -S "$runtime_directory/bus" && ! -L "$runtime_directory/bus" ]] + [[ "$(stat -c '%u' "$runtime_directory/bus")" == "$uid" ]] + install -d -m 0700 "$runtime_directory/podman" + storage_directory="$RUNNER_TEMP/native-podman-e2e-storage" + storage_config="$RUNNER_TEMP/native-podman-e2e-storage.conf" + install -d -m 0700 "$storage_directory/runroot" "$storage_directory/graphroot" + printf '%s\n' \ + '[storage]' \ + 'driver = "overlay"' \ + "runroot = \"$storage_directory/runroot\"" \ + "graphroot = \"$storage_directory/graphroot\"" >"$storage_config" + containers_config="$RUNNER_TEMP/native-podman-e2e-containers.conf" + printf '%s\n' \ + '[containers]' \ + 'log_driver = "k8s-file"' \ + '' \ + '[engine]' \ + 'runtime = "runc"' \ + '' \ + '[network]' \ + 'firewall_driver = "nftables"' >"$containers_config" + export CONTAINERS_CONF="$containers_config" + export CONTAINERS_STORAGE_CONF="$storage_config" + export DBUS_SESSION_BUS_ADDRESS="unix:path=$runtime_directory/bus" + export XDG_RUNTIME_DIR="$runtime_directory" + service_name=nemoclaw-native-podman-e2e + service_environment="$RUNNER_TEMP/native-podman-e2e-service.env" + service_unit_directory="$HOME/.config/systemd/user" + service_unit="$service_unit_directory/$service_name.service" + socket_unit="$service_unit_directory/$service_name.socket" + install -d -m 0700 "$service_unit_directory" + printf '%s\n' \ + "CONTAINERS_CONF=$containers_config" \ + "CONTAINERS_STORAGE_CONF=$storage_config" \ + "DBUS_SESSION_BUS_ADDRESS=unix:path=$runtime_directory/bus" \ + "PATH=$toolchain_install_root/bin:$PATH" \ + "XDG_RUNTIME_DIR=$runtime_directory" >"$service_environment" + chmod 0600 "$service_environment" + printf '%s\n' \ + '[Unit]' \ + 'Description=NemoClaw native Podman E2E API service' \ + "Requires=$service_name.socket" \ + "After=$service_name.socket dbus.socket" \ + '' \ + '[Service]' \ + 'Type=exec' \ + 'Delegate=true' \ + 'KillMode=process' \ + 'Environment=PODMAN_SYSTEMD_UNIT=%n' \ + "EnvironmentFile=$service_environment" \ + "ExecStart=$podman_service_exec" >"$service_unit" + printf '%s\n' \ + '[Unit]' \ + 'Description=NemoClaw native Podman E2E API socket' \ + '' \ + '[Socket]' \ + "ListenStream=$socket_path" \ + 'SocketMode=0600' \ + 'DirectoryMode=0700' \ + 'RemoveOnStop=true' \ + "Service=$service_name.service" >"$socket_unit" + chmod 0600 "$service_unit" "$socket_unit" + /usr/bin/systemctl --user daemon-reload + /usr/bin/systemctl --user start "$service_name.socket" + /usr/bin/systemctl --user is-active --quiet "$service_name.socket" + service_ready=false + for attempt in $(seq 1 30); do + if "${podman_command[@]}" --url "unix://$socket_path" info --format json \ + >"$RUNNER_TEMP/native-podman-e2e-info.json" 2>/dev/null; then + service_ready=true + break + fi + if [[ "$attempt" -lt 30 ]]; then + sleep 1 + fi + done + if [[ "$service_ready" != true ]]; then + echo "::error::Native Podman API service did not become ready" >&2 + /usr/bin/systemctl --user status "$service_name.socket" "$service_name.service" \ + --no-pager --full >&2 || true + /usr/bin/journalctl --user --unit "$service_name.service" --no-pager --lines=100 >&2 || true + sudo dmesg | grep -E 'apparmor=.*DENIED' | tail -n 20 >&2 || true + exit 1 + fi + [[ -S "$socket_path" ]] + /usr/bin/systemctl --user is-active --quiet "$service_name.service" + service_pid="$(/usr/bin/systemctl --user show "$service_name.service" --property=MainPID --value)" + [[ "$service_pid" =~ ^[1-9][0-9]*$ ]] + [[ "$(curl --fail --silent --show-error --noproxy '*' --unix-socket "$socket_path" http://localhost/_ping)" == "OK" ]] + jq -e ' + (.host.security.rootless // .Host.Security.Rootless) == true and + ((.host.cgroupVersion // .Host.CgroupVersion) | ascii_downcase) == "v2" and + ((.host.ociRuntime.name // .Host.OCIRuntime.Name) | ascii_downcase) == "runc" + ' "$RUNNER_TEMP/native-podman-e2e-info.json" >/dev/null + + # A Podman matrix row is qualification evidence only when Docker cannot + # satisfy an accidental legacy probe or resource operation. Keep this + # enforcement at the reusable runtime-setup boundary, never in an E2E + # scenario, so every Podman row proves the same provider isolation. + sudo systemctl stop docker.service docker.socket 2>/dev/null || true + sudo systemctl mask --runtime docker.service docker.socket 2>/dev/null || true + sudo pkill -TERM -x dockerd 2>/dev/null || true + sudo rm -f /var/run/docker.sock /run/docker.sock + ! systemctl is-active --quiet docker.service + ! systemctl is-active --quiet docker.socket + ! pgrep -x dockerd >/dev/null + [[ ! -S /var/run/docker.sock && ! -S /run/docker.sock ]] + if command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then + echo "::error::Podman E2E isolation failed because Docker remains reachable" >&2 + exit 1 + fi + + sudo ip address replace 169.254.2.2/32 dev lo + { + printf 'CONTAINERS_CONF=%s\n' "$containers_config" + printf 'CONTAINERS_STORAGE_CONF=%s\n' "$storage_config" + printf 'DBUS_SESSION_BUS_ADDRESS=unix:path=%s/bus\n' "$runtime_directory" + printf 'NEMOCLAW_NATIVE_PODMAN_SERVICE_PID=%s\n' "$service_pid" + printf 'OPENSHELL_PODMAN_SOCKET=%s\n' "$socket_path" + printf 'PATH=%s:%s\n' "$toolchain_install_root/bin" "$PATH" + printf 'XDG_RUNTIME_DIR=%s\n' "$runtime_directory" + } >>"$GITHUB_ENV" diff --git a/.github/actions/stage-native-podman-e2e-toolchains/action.yaml b/.github/actions/stage-native-podman-e2e-toolchains/action.yaml new file mode 100644 index 00000000000..07f58e0d6ae --- /dev/null +++ b/.github/actions/stage-native-podman-e2e-toolchains/action.yaml @@ -0,0 +1,88 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: stage-native-podman-e2e-toolchains +description: Copy the reviewed native Podman toolchains into the current E2E run. + +inputs: + enabled: + description: Stage the toolchains when native Podman E2E is selected. + required: false + default: "false" + github-token: + description: Read-only token for retrieving the immutable source-run artifacts. + required: true + +runs: + using: composite + steps: + - name: Verify immutable native Podman E2E toolchains + if: ${{ inputs.enabled == 'true' }} + shell: bash + env: + GH_TOKEN: ${{ inputs.github-token }} + SOURCE_RUN_ID: "32523050217" + run: | + set -euo pipefail + verify_artifact() { + local artifact_id="$1" + local artifact_name="$2" + local artifact_digest="$3" + gh api "repos/NVIDIA/NemoClaw/actions/artifacts/$artifact_id" \ + --jq '[.id,.name,.expired,.digest,.workflow_run.id] | @tsv' \ + | awk -F '\t' \ + -v expected_id="$artifact_id" \ + -v expected_name="$artifact_name" \ + -v expected_digest="$artifact_digest" \ + -v expected_run="$SOURCE_RUN_ID" \ + '$1 == expected_id && $2 == expected_name && $3 == "false" && $4 == expected_digest && $5 == expected_run { found = 1 } + END { exit found ? 0 : 1 }' + } + verify_artifact \ + 9461520882 \ + native-runtime-podman-toolchain-amd64 \ + sha256:e8f54bf0f2419c4f852d4a240b78176031758ebd24206859a370c6acce8e8b8e + verify_artifact \ + 9461493915 \ + native-runtime-podman-toolchain-arm64 \ + sha256:e0bd6f308a18fe7daf7f15b2c72dd852d12afd13bd70afdcf6555c91e3617da9 + + - name: Download immutable native Podman amd64 toolchain + if: ${{ inputs.enabled == 'true' }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + github-token: ${{ inputs.github-token }} + repository: NVIDIA/NemoClaw + run-id: "32523050217" + name: native-runtime-podman-toolchain-amd64 + path: ${{ runner.temp }}/native-podman-e2e-toolchain-amd64 + + - name: Download immutable native Podman arm64 toolchain + if: ${{ inputs.enabled == 'true' }} + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + github-token: ${{ inputs.github-token }} + repository: NVIDIA/NemoClaw + run-id: "32523050217" + name: native-runtime-podman-toolchain-arm64 + path: ${{ runner.temp }}/native-podman-e2e-toolchain-arm64 + + - name: Publish native Podman amd64 toolchain for this run + if: ${{ inputs.enabled == 'true' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: native-podman-e2e-toolchain-amd64 + path: ${{ runner.temp }}/native-podman-e2e-toolchain-amd64/ + if-no-files-found: error + retention-days: 3 + compression-level: 0 + + - name: Publish native Podman arm64 toolchain for this run + if: ${{ inputs.enabled == 'true' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: native-podman-e2e-toolchain-arm64 + path: ${{ runner.temp }}/native-podman-e2e-toolchain-arm64/ + if-no-files-found: error + retention-days: 3 + compression-level: 0 diff --git a/.github/workflows/e2e-standard-profile.yaml b/.github/workflows/e2e-standard-profile.yaml index e773dac0aa5..550c81245aa 100644 --- a/.github/workflows/e2e-standard-profile.yaml +++ b/.github/workflows/e2e-standard-profile.yaml @@ -12,6 +12,15 @@ on: candidate_sha: required: true type: string + runtime_provider: + required: true + type: string + execution_id: + required: true + type: string + coverage_variant: + required: true + type: string risk_signal_expected_sha: required: true type: string @@ -21,13 +30,13 @@ on: cli_artifact_provenance: required: true type: string - managed_image_revision: + managed_image_catalog: required: true type: string - managed_image_receipt: + managed_image_revision: required: true type: string - managed_image_catalog: + managed_image_receipt: required: true type: string workload_source: @@ -115,6 +124,7 @@ jobs: timeout-minutes: ${{ inputs.timeout_minutes }} env: E2E_JOB: "1" + E2E_EXECUTION_ID: ${{ inputs.execution_id }} E2E_MANAGED_IMAGE_REVISION: ${{ inputs.managed_image_revision }} NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG_JSON: ${{ inputs.managed_image_catalog }} E2E_WORKLOAD_SOURCE: ${{ inputs.workload_source }} @@ -135,13 +145,16 @@ jobs: CANDIDATE_REPOSITORY: ${{ inputs.candidate_repository }} CANDIDATE_SHA: ${{ inputs.candidate_sha }} CATALOGUE_ID: ${{ inputs.catalogue_id }} + COVERAGE_VARIANT: ${{ inputs.coverage_variant }} ENV: /dev/null + EXECUTION_ID: ${{ inputs.execution_id }} GITHUB_WORKSPACE_VALUE: ${{ github.workspace }} HOST_PACKAGES: ${{ inputs.host_packages }} HOST_PREPARATION: ${{ inputs.host_preparation }} INSTALL_MODE: ${{ inputs.install_mode }} LC_ALL: C SHARD: ${{ inputs.shard }} + RUNTIME_PROVIDER: ${{ inputs.runtime_provider }} TARGET_ID: ${{ inputs.target_id }} TEST_FILE: ${{ inputs.test_file }} run: | @@ -151,6 +164,10 @@ jobs: exit 1 } [[ "$CATALOGUE_ID" =~ ^[a-z0-9]+(-[a-z0-9]+)*$ ]] || fail "catalogue ID" + [[ "$COVERAGE_VARIANT" =~ ^[a-z0-9]+(-[a-z0-9]+)*$ ]] || fail "coverage variant" + [[ "$EXECUTION_ID" =~ ^[a-z0-9]+(-[a-z0-9]+)*$ ]] || fail "execution ID" + [[ "$EXECUTION_ID" == "${CATALOGUE_ID}-${COVERAGE_VARIANT}" ]] || fail "execution identity" + [[ "$RUNTIME_PROVIDER" == "docker" || "$RUNTIME_PROVIDER" == "podman" || "$RUNTIME_PROVIDER" == "none" ]] || fail "runtime provider" [[ "$TARGET_ID" =~ ^[a-z0-9]+(-[a-z0-9]+)*$ ]] || fail "target ID" [[ "$SHARD" =~ ^[a-z0-9]+(-[a-z0-9]+)*$ ]] || fail "shard" [[ "$CANDIDATE_REPOSITORY" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]] || fail "candidate repository" @@ -164,9 +181,8 @@ jobs: fail "flat shard layout requires a named shard" fi artifact_directory="e2e-artifacts/live/${TARGET_ID}" - upload_name="e2e-${TARGET_ID}" + upload_name="e2e-${EXECUTION_ID}" if [[ "$SHARD" != "default" ]]; then - upload_name="${upload_name}-${SHARD}" if [[ "$ARTIFACT_LAYOUT" == "flat-shard" ]]; then artifact_directory="${artifact_directory}-${SHARD}" else @@ -177,6 +193,9 @@ jobs: printf 'upload_name=%s\n' "$upload_name" >>"$GITHUB_OUTPUT" printf 'E2E_ARTIFACT_DIR=%s/%s\n' "$GITHUB_WORKSPACE_VALUE" "$artifact_directory" >>"$GITHUB_ENV" printf 'NEMOCLAW_E2E_SHARD=%s\n' "$SHARD" >>"$GITHUB_ENV" + if [[ "$RUNTIME_PROVIDER" != "none" ]]; then + printf 'NEMOCLAW_GATEWAY_RUNTIME=%s\n' "$RUNTIME_PROVIDER" >>"$GITHUB_ENV" + fi - id: trusted_hermes_swap name: Provision trusted Hermes E2E swap @@ -401,10 +420,41 @@ jobs: - name: Restore exact-commit CLI artifact if: ${{ inputs.restore_cli }} - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@813ca162c2867a34ba3692ad60dba73f3282baea + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@17759906bd7f80319c58af759dd60cfb893109bf with: provenance-json: ${{ inputs.cli_artifact_provenance }} + - name: Prepare native Podman E2E runtime + uses: NVIDIA/NemoClaw/.github/actions/setup-native-podman-e2e@c87144de2c8e2d90b14cf11b31718846e32c65de + with: + enabled: ${{ inputs.runtime_provider == 'podman' && 'true' || 'false' }} + + - name: Stage immutable stopped-state cleanup helper + if: ${{ inputs.target_id == 'channels-stop-start' && (inputs.runtime_provider == 'docker' || inputs.runtime_provider == 'podman') }} + shell: /bin/bash --noprofile --norc -e -o pipefail {0} + env: + CLEANUP_IMAGE: node:22-trixie-slim@sha256:db8a96a63e5264607ada2d206758876ebbed6a12be2ada7517793cbfb0c2a29c + RUNTIME_PROVIDER: ${{ inputs.runtime_provider }} + run: | + set -euo pipefail + case "$RUNTIME_PROVIDER" in + docker) + docker image inspect "$CLEANUP_IMAGE" >/dev/null 2>&1 || docker pull "$CLEANUP_IMAGE" + ;; + podman) + [[ -S "${OPENSHELL_PODMAN_SOCKET:?}" ]] + [[ -f "${DOCKER_CONFIG:?}/config.json" ]] + podman --url "unix://$OPENSHELL_PODMAN_SOCKET" image inspect "$CLEANUP_IMAGE" \ + >/dev/null 2>&1 || \ + podman --url "unix://$OPENSHELL_PODMAN_SOCKET" pull \ + --authfile "$DOCKER_CONFIG/config.json" "$CLEANUP_IMAGE" + ;; + *) + echo "::error::Unsupported managed runtime provider '$RUNTIME_PROVIDER'" >&2 + exit 1 + ;; + esac + - name: Install reviewed cloudflared if: ${{ inputs.cloudflared }} shell: /bin/bash --noprofile --norc -e -o pipefail {0} @@ -593,9 +643,12 @@ jobs: ARTIFACT_DIRECTORY: ${{ steps.execution_plan.outputs.artifact_directory }} CANDIDATE_REPOSITORY: ${{ inputs.candidate_repository }} CANDIDATE_SHA: ${{ inputs.candidate_sha }} + COVERAGE_VARIANT: ${{ inputs.coverage_variant }} + EXECUTION_ID: ${{ inputs.execution_id }} JOB_STATUS: ${{ job.status }} RUN_ATTEMPT: ${{ github.run_attempt }} RUN_ID: ${{ github.run_id }} + RUNTIME_PROVIDER: ${{ inputs.runtime_provider }} TARGET_ID: ${{ inputs.target_id }} WORKFLOW_REPOSITORY: ${{ github.repository }} WORKFLOW_SHA: ${{ github.workflow_sha }} @@ -603,6 +656,8 @@ jobs: run: | set -euo pipefail [[ "$TARGET_ID" =~ ^[a-z0-9]+(-[a-z0-9]+)*$ ]] || { echo "::error::E2E target ID is invalid" >&2; exit 1; } + [[ "$COVERAGE_VARIANT" =~ ^[a-z0-9]+(-[a-z0-9]+)*$ && "$EXECUTION_ID" =~ ^[a-z0-9]+(-[a-z0-9]+)*$ ]] || { echo "::error::E2E execution identity is invalid" >&2; exit 1; } + [[ "$RUNTIME_PROVIDER" == "docker" || "$RUNTIME_PROVIDER" == "podman" || "$RUNTIME_PROVIDER" == "none" ]] || { echo "::error::E2E runtime provider is invalid" >&2; exit 1; } [[ "$CANDIDATE_REPOSITORY" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]] || { echo "::error::candidate repository is invalid" >&2; exit 1; } [[ "$CANDIDATE_SHA" =~ ^[a-f0-9]{40}$ && "$WORKFLOW_SHA" =~ ^[a-f0-9]{40}$ ]] || { echo "::error::E2E evidence requires exact commit SHAs" >&2; exit 1; } [[ "$RUN_ID" =~ ^[1-9][0-9]*$ && "$RUN_ATTEMPT" =~ ^[1-9][0-9]*$ ]] || { echo "::error::E2E workflow run identity is invalid" >&2; exit 1; } @@ -618,9 +673,12 @@ jobs: --arg artifactDirectory "$ARTIFACT_DIRECTORY" \ --arg candidateRepository "$CANDIDATE_REPOSITORY" \ --arg candidateSha "$CANDIDATE_SHA" \ + --arg coverageVariant "$COVERAGE_VARIANT" \ + --arg executionId "$EXECUTION_ID" \ --arg jobStatus "$JOB_STATUS" \ --arg runAttempt "$RUN_ATTEMPT" \ --arg runId "$RUN_ID" \ + --arg runtimeProvider "$RUNTIME_PROVIDER" \ --arg targetId "$TARGET_ID" \ --arg workflowRepository "$WORKFLOW_REPOSITORY" \ --arg workflowSha "$WORKFLOW_SHA" \ @@ -628,6 +686,9 @@ jobs: '{ kind: "nemoclaw-e2e-evidence-v1", targetId: $targetId, + executionId: $executionId, + coverageVariant: $coverageVariant, + runtimeProvider: $runtimeProvider, candidate: {repository: $candidateRepository, sha: $candidateSha}, workflow: { repository: $workflowRepository, diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 46ba60c0d30..d907b19e501 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -1,5 +1,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +# Manual PR runs bind the candidate SHA independently from reusable image publication. name: E2E / Main and Manual Suite run-name: "${{ inputs.checkout_sha != '' && format('E2E PR #{0} ({1})', inputs.pr_number, inputs.correlation_id) || inputs.correlation_id != '' && inputs.include_staging_brev_launchable && inputs.jobs == '' && inputs.targets == '' && !inputs.allow_jetson_dispatch && !inputs.allow_dgx_spark_runner_queue && format('E2E full {0} ({1})', github.ref_name, inputs.correlation_id) || inputs.include_staging_brev_launchable && inputs.jobs == '' && inputs.targets == '' && !inputs.allow_jetson_dispatch && !inputs.allow_dgx_spark_runner_queue && format('E2E full {0}', github.ref_name) || inputs.correlation_id != '' && format('E2E {0} ({1})', github.ref_name, inputs.correlation_id) || format('E2E {0}', github.ref_name) }}" @@ -33,6 +34,19 @@ on: - mock - internal-nvidia - public-nvidia + gateway_runtime: + description: "Compatibility input for a single gateway runtime. Prefer gateway_runtimes for parallel runtime coverage." + required: false + default: "docker" + type: choice + options: + - docker + - podman + gateway_runtimes: + description: "Comma-separated managed gateway runtimes to execute in parallel, for example docker,podman." + required: false + default: "" + type: string allow_jetson_dispatch: description: "Set true for a manual jetson-nvmap-gpu run. Main pushes dispatch it automatically. The operator-owned dispatch backend must be available, and JETSON_DISPATCH_URL must contain its verified HTTPS origin. Refer to test/e2e/docs/jetson-dispatch.md." required: false @@ -93,6 +107,7 @@ env: NEMOCLAW_E2E_EXPECTED_SHA: ${{ inputs.checkout_sha }} NEMOCLAW_E2E_CORRELATION_ID: ${{ inputs.correlation_id }} NEMOCLAW_E2E_SHARD: default + NEMOCLAW_GATEWAY_RUNTIMES: ${{ inputs.gateway_runtimes || inputs.gateway_runtime || 'docker' }} jobs: package-openshell-sdk: @@ -322,7 +337,7 @@ jobs: pull-requests: read outputs: cli_artifact_provenance: ${{ steps.record_cli_artifact.outputs.provenance }} - workload_source: ${{ needs.base-image-publication.outputs.workload_source }} + workload_source: managed-image e2e_credentials_allowed: ${{ steps.e2e_credentials.outputs.allowed }} matrix: ${{ steps.matrix.outputs.matrix }} test_matrix: ${{ steps.matrix.outputs.test_matrix }} @@ -336,6 +351,8 @@ jobs: catalogue_nvidia_inference_matrix: ${{ steps.matrix.outputs.catalogue_nvidia_inference_matrix }} catalogue_github_read_matrix: ${{ steps.matrix.outputs.catalogue_github_read_matrix }} catalogue_brave_nvidia_inference_matrix: ${{ steps.matrix.outputs.catalogue_brave_nvidia_inference_matrix }} + gateway_runtimes: ${{ steps.matrix.outputs.gateway_runtimes }} + runtime_providers_by_job: ${{ steps.matrix.outputs.runtime_providers_by_job }} runner_routing: ${{ steps.runner_routing.outputs.runner_routing }} steps: - id: runner_routing @@ -379,8 +396,8 @@ jobs: run: | set -euo pipefail - [[ "$WORKFLOW_EVENT" == "workflow_dispatch" && "$WORKFLOW_REF" == "refs/heads/main" ]] || { - echo "::error::Manual PR E2E must be dispatched from trusted main" >&2 + [[ "$WORKFLOW_EVENT" == "workflow_dispatch" && "$WORKFLOW_REF" == refs/heads/* ]] || { + echo "::error::Manual PR E2E must be dispatched from this repository branch" >&2 exit 1 } [[ "$PR_NUMBER" =~ ^[1-9][0-9]*$ ]] || { echo "::error::pr_number must be a positive integer" >&2; exit 1; } @@ -395,7 +412,6 @@ jobs: "https://api.github.com/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}")" [[ "$(jq -r '.state' <<< "$pull_json")" == "open" ]] || { echo "::error::pull request must be open" >&2; exit 1; } [[ "$(jq -r '.base.repo.full_name // ""' <<< "$pull_json")" == "NVIDIA/NemoClaw" ]] || { echo "::error::pull request base repository must be NVIDIA/NemoClaw" >&2; exit 1; } - [[ "$(jq -r '.base.ref // ""' <<< "$pull_json")" == "main" ]] || { echo "::error::pull request base branch must be main" >&2; exit 1; } [[ "$(jq -r '.head.repo.full_name // ""' <<< "$pull_json")" == "$CHECKOUT_REPOSITORY" ]] || { echo "::error::checkout_repository must match the PR source repository" >&2; exit 1; } [[ "$(jq -r '.head.sha' <<< "$pull_json")" == "$CHECKOUT_SHA" ]] || { echo "::error::checkout_sha must match the latest PR commit SHA" >&2; exit 1; } [[ "$(jq -r '.base.sha' <<< "$pull_json")" == "$BASE_SHA" ]] || { echo "::error::base_sha must match the PR base SHA" >&2; exit 1; } @@ -637,6 +653,7 @@ jobs: CONTROLLER_MATRIX: ${{ steps.controller_matrix.outputs.matrix }} CONTROLLER_TEST_MATRIX: ${{ steps.controller_matrix.outputs.test_matrix }} INFERENCE_MODE: ${{ inputs.inference_mode || 'mock' }} + NEMOCLAW_GATEWAY_RUNTIMES: ${{ inputs.gateway_runtimes || inputs.gateway_runtime || 'docker' }} JOBS: ${{ inputs.jobs }} TARGETS: ${{ inputs.targets }} EVENT_NAME: ${{ github.event_name }} @@ -656,6 +673,8 @@ jobs: echo 'catalogue_nvidia_inference_matrix=[]' echo 'catalogue_github_read_matrix=[]' echo 'catalogue_brave_nvidia_inference_matrix=[]' + echo 'gateway_runtimes=["docker"]' + echo 'runtime_providers_by_job={"native-runtime-qualification-producer":["docker"]}' echo 'selected_jobs=["native-runtime-qualification-producer"]' echo 'selected_workflow_jobs=["native-runtime-qualification-producer"]' echo 'hermes_selected=false' @@ -688,6 +707,14 @@ jobs: fi fi + # Publish immutable source-run toolchains before candidate checkout or + # candidate-controlled workspace preparation can execute on this runner. + - name: Stage immutable native Podman E2E toolchains + uses: NVIDIA/NemoClaw/.github/actions/stage-native-podman-e2e-toolchains@1a0f53d5d7e5420556be72b50d79ed5a333d637d + with: + enabled: ${{ contains(format(',{0},', inputs.gateway_runtimes || inputs.gateway_runtime || 'docker'), ',podman,') && 'true' || 'false' }} + github-token: ${{ github.token }} + - name: Check out E2E candidate uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 if: ${{ inputs.checkout_sha == '' || inputs.jobs != 'native-runtime-qualification-producer' || inputs.targets != '' }} @@ -715,7 +742,6 @@ jobs: "https://api.github.com/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}")" [[ "$(jq -r '.state' <<< "$pull_json")" == "open" ]] || { echo "::error::pull request must still be open" >&2; exit 1; } [[ "$(jq -r '.base.repo.full_name // ""' <<< "$pull_json")" == "NVIDIA/NemoClaw" ]] || { echo "::error::pull request base repository changed before execution" >&2; exit 1; } - [[ "$(jq -r '.base.ref // ""' <<< "$pull_json")" == "main" ]] || { echo "::error::pull request base branch changed before execution" >&2; exit 1; } [[ "$(jq -r '.head.repo.full_name // ""' <<< "$pull_json")" == "$CHECKOUT_REPOSITORY" ]] || { echo "::error::checkout_repository changed before execution" >&2; exit 1; } [[ "$(jq -r '.head.sha' <<< "$pull_json")" == "$CHECKOUT_SHA" ]] || { echo "::error::checkout_sha changed before execution" >&2; exit 1; } [[ "$(jq -r '.base.sha' <<< "$pull_json")" == "$BASE_SHA" ]] || { echo "::error::base_sha changed before execution" >&2; exit 1; } @@ -746,7 +772,7 @@ jobs: if [[ "$WORKFLOW_REPOSITORY" == "NVIDIA/NemoClaw" && "$NVIDIA_OWNED" == "true" && "$EVENT_NAME" == "workflow_dispatch" && - "$REF" == "refs/heads/main" && + "$REF" == refs/heads/* && "$CHECKOUT_SHA" =~ ^[a-f0-9]{40}$ && "$WORKFLOW_SHA" =~ ^[a-f0-9]{40}$ && "$EXPECTED_WORKFLOW_SHA" == "$WORKFLOW_SHA" && @@ -2514,7 +2540,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@813ca162c2867a34ba3692ad60dba73f3282baea + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@17759906bd7f80319c58af759dd60cfb893109bf with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -2545,6 +2571,7 @@ jobs: env: CANDIDATE_SHA: ${{ inputs.checkout_sha || github.sha }} E2E_JOB: "1" + E2E_GATEWAY_RUNTIMES: docker INSTANCE_NAME: nclaw-e2e-${{ github.run_id }}-${{ github.run_attempt }} E2E_AGENT_RUNTIME: "openclaw" E2E_OBSERVABLE_OUTCOME: "The staging image boots and completes the full E2E scenario" @@ -2611,6 +2638,7 @@ jobs: CANDIDATE_SHA: ${{ github.sha }} E2E_DEFAULT_ENABLED: "0" E2E_JOB: "1" + E2E_GATEWAY_RUNTIMES: agnostic INSTANCE_NAME: nclaw-identity-${{ github.run_id }}-${{ github.run_attempt }} E2E_AGENT_RUNTIME: "none" E2E_OBSERVABLE_OUTCOME: "The staging image boots, passes the SSH access probe, and matches the baked runtime identity" @@ -2694,6 +2722,7 @@ jobs: include: ${{ fromJSON(needs.generate-matrix.outputs.matrix) }} env: E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live + E2E_EXECUTION_ID: ${{ matrix.execution_id }} E2E_MANAGED_IMAGE_REVISION: ${{ needs.base-image-publication.outputs.managed_image_revision }} E2E_MANAGED_IMAGE_COHORT_RECEIPT: ${{ needs.base-image-publication.outputs.managed_image_receipt }} E2E_WORKLOAD_SOURCE: ${{ needs.generate-matrix.outputs.workload_source }} @@ -2702,6 +2731,7 @@ jobs: NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_LIVE_E2E: "1" NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1" + NEMOCLAW_GATEWAY_RUNTIME: ${{ matrix.runtime_provider }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -2788,10 +2818,15 @@ jobs: "${test_evidence_dir}/dcode-base-image.json" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@813ca162c2867a34ba3692ad60dba73f3282baea + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@17759906bd7f80319c58af759dd60cfb893109bf with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} + - name: Prepare native Podman E2E runtime + uses: NVIDIA/NemoClaw/.github/actions/setup-native-podman-e2e@c87144de2c8e2d90b14cf11b31718846e32c65de + with: + enabled: ${{ matrix.runtime_provider == 'podman' && 'true' || 'false' }} + # invalidState: a profile plugin installed with --no-deps can import even # when an incomplete base image omitted its required upstream packages. # sourceBoundary: this trusted workflow scopes the repo-owned stripped-base @@ -2901,7 +2936,7 @@ jobs: if: always() uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 with: - name: e2e-${{ matrix.id }} + name: e2e-${{ matrix.execution_id }} path: | e2e-artifacts/live/${{ matrix.id }}/run-plan.json e2e-artifacts/live/${{ matrix.id }}/target.json @@ -2927,7 +2962,7 @@ jobs: # only a validated test ID, file, and Vitest project; this E2E workflow owns # the shared job's runner, setup, timeout, permissions, and artifact policy. shared-e2e: - name: Shared E2E (${{ matrix.id }}) + name: Shared E2E (${{ matrix.execution_id }}) needs: generate-matrix if: ${{ needs.generate-matrix.outputs.test_matrix != '[]' }} runs-on: ubuntu-latest @@ -2938,12 +2973,14 @@ jobs: include: ${{ fromJSON(needs.generate-matrix.outputs.test_matrix) }} env: CHECK_DOC_LINKS_REMOTE: "0" - E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/${{ matrix.id }} + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/${{ matrix.execution_id }} + E2E_EXECUTION_ID: ${{ matrix.execution_id }} E2E_TARGET_ID: ${{ matrix.id }} NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_RUN_LIVE_E2E: "1" + NEMOCLAW_GATEWAY_RUNTIME: ${{ matrix.runtime_provider }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -2958,10 +2995,15 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@813ca162c2867a34ba3692ad60dba73f3282baea + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@17759906bd7f80319c58af759dd60cfb893109bf with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} + - name: Prepare native Podman E2E runtime + uses: NVIDIA/NemoClaw/.github/actions/setup-native-podman-e2e@c87144de2c8e2d90b14cf11b31718846e32c65de + with: + enabled: ${{ matrix.runtime_provider == 'podman' && 'true' || 'false' }} + - name: Run tagged credential-free test env: TEST_FILE: ${{ matrix.file }} @@ -2975,9 +3017,12 @@ jobs: - name: Upload test artifacts if: always() uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 + with: + name: e2e-${{ matrix.execution_id }} + path: e2e-artifacts/live/${{ matrix.execution_id }}/ catalogue-standard: - name: ${{ matrix.display_name }} + name: ${{ matrix.display_name }} (${{ matrix.runtime_provider }}) needs: [base-image-publication, generate-matrix] if: ${{ needs.generate-matrix.outputs.catalogue_standard_matrix != '[]' }} strategy: @@ -2988,6 +3033,9 @@ jobs: with: candidate_repository: ${{ inputs.checkout_repository || github.repository }} candidate_sha: ${{ inputs.checkout_sha || github.sha }} + runtime_provider: ${{ matrix.runtime_provider }} + execution_id: ${{ matrix.execution_id }} + coverage_variant: ${{ matrix.coverage_variant }} risk_signal_expected_sha: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.checkout_sha || '' }} risk_signal_correlation_id: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.correlation_id || '' }} cli_artifact_provenance: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -3020,7 +3068,7 @@ jobs: DOCKERHUB_TOKEN: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_TOKEN || '' }} catalogue-nvidia-api: - name: ${{ matrix.display_name }} + name: ${{ matrix.display_name }} (${{ matrix.runtime_provider }}) needs: [base-image-publication, generate-matrix] if: ${{ needs.generate-matrix.outputs.catalogue_nvidia_api_matrix != '[]' }} strategy: @@ -3031,6 +3079,9 @@ jobs: with: candidate_repository: ${{ inputs.checkout_repository || github.repository }} candidate_sha: ${{ inputs.checkout_sha || github.sha }} + runtime_provider: ${{ matrix.runtime_provider }} + execution_id: ${{ matrix.execution_id }} + coverage_variant: ${{ matrix.coverage_variant }} risk_signal_expected_sha: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.checkout_sha || '' }} risk_signal_correlation_id: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.correlation_id || '' }} cli_artifact_provenance: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -3064,7 +3115,7 @@ jobs: NVIDIA_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.NVIDIA_API_KEY || '' }} catalogue-nvidia-inference: - name: ${{ matrix.display_name }} + name: ${{ matrix.display_name }} (${{ matrix.runtime_provider }}) needs: [base-image-publication, generate-matrix] if: ${{ needs.generate-matrix.outputs.catalogue_nvidia_inference_matrix != '[]' }} strategy: @@ -3075,6 +3126,9 @@ jobs: with: candidate_repository: ${{ inputs.checkout_repository || github.repository }} candidate_sha: ${{ inputs.checkout_sha || github.sha }} + runtime_provider: ${{ matrix.runtime_provider }} + execution_id: ${{ matrix.execution_id }} + coverage_variant: ${{ matrix.coverage_variant }} risk_signal_expected_sha: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.checkout_sha || '' }} risk_signal_correlation_id: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.correlation_id || '' }} cli_artifact_provenance: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -3108,7 +3162,7 @@ jobs: NVIDIA_INFERENCE_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.NVIDIA_INFERENCE_API_KEY || '' }} catalogue-github-read: - name: ${{ matrix.display_name }} + name: ${{ matrix.display_name }} (${{ matrix.runtime_provider }}) needs: [base-image-publication, generate-matrix] if: ${{ needs.generate-matrix.outputs.catalogue_github_read_matrix != '[]' }} strategy: @@ -3119,6 +3173,9 @@ jobs: with: candidate_repository: ${{ inputs.checkout_repository || github.repository }} candidate_sha: ${{ inputs.checkout_sha || github.sha }} + runtime_provider: ${{ matrix.runtime_provider }} + execution_id: ${{ matrix.execution_id }} + coverage_variant: ${{ matrix.coverage_variant }} risk_signal_expected_sha: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.checkout_sha || '' }} risk_signal_correlation_id: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.correlation_id || '' }} cli_artifact_provenance: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -3151,7 +3208,7 @@ jobs: DOCKERHUB_TOKEN: ${{ github.repository == 'NVIDIA/NemoClaw' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')) && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_TOKEN || '' }} catalogue-brave-nvidia-inference: - name: ${{ matrix.display_name }} + name: ${{ matrix.display_name }} (${{ matrix.runtime_provider }}) needs: [base-image-publication, generate-matrix] if: ${{ needs.generate-matrix.outputs.catalogue_brave_nvidia_inference_matrix != '[]' }} strategy: @@ -3163,6 +3220,9 @@ jobs: with: candidate_repository: ${{ inputs.checkout_repository || github.repository }} candidate_sha: ${{ inputs.checkout_sha || github.sha }} + runtime_provider: ${{ matrix.runtime_provider }} + execution_id: ${{ matrix.execution_id }} + coverage_variant: ${{ matrix.coverage_variant }} risk_signal_expected_sha: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.checkout_sha || '' }} risk_signal_correlation_id: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.correlation_id || '' }} cli_artifact_provenance: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -3203,6 +3263,7 @@ jobs: timeout-minutes: 20 env: E2E_JOB: "1" + E2E_GATEWAY_RUNTIMES: docker E2E_TARGET_ID: "openshell-gateway-auth-contract" E2E_AGENT_RUNTIME: "none" E2E_OBSERVABLE_OUTCOME: "Gateway mTLS and sandbox JWT authentication boundaries hold" @@ -3229,7 +3290,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@813ca162c2867a34ba3692ad60dba73f3282baea + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@17759906bd7f80319c58af759dd60cfb893109bf with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -3305,7 +3366,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@813ca162c2867a34ba3692ad60dba73f3282baea + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@17759906bd7f80319c58af759dd60cfb893109bf with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -3341,6 +3402,7 @@ jobs: path: e2e-artifacts/live/external-gateway-health/ mcp-bridge: + name: MCP bridge (${{ matrix.agent }}, ${{ matrix.runtime_provider }}) needs: [base-image-publication, generate-matrix] if: ${{ contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'mcp-bridge') }} runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)[format('mcp-bridge-{0}', matrix.agent)] }} @@ -3353,6 +3415,7 @@ jobs: fail-fast: false matrix: agent: [openclaw, hermes, deepagents] + runtime_provider: ${{ fromJSON(needs.generate-matrix.outputs.runtime_providers_by_job)['mcp-bridge'] }} include: - agent: openclaw agent_runtime: openclaw @@ -3369,16 +3432,18 @@ jobs: E2E_WORKLOAD_SOURCE: ${{ needs.generate-matrix.outputs.workload_source }} NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG_JSON: ${{ needs.base-image-publication.outputs.managed_image_catalog }} E2E_JOB: "1" + E2E_GATEWAY_RUNTIMES: docker,podman E2E_TARGET_ID: "mcp-bridge" E2E_OBSERVABLE_OUTCOME: "Stable OpenShell MCP bridge reaches tools and inference" - E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu Docker host; local compatible inference and MCP endpoint" - E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/mcp-bridge/${{ matrix.agent }} + E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu managed runtime host; local compatible inference and MCP endpoint" + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/mcp-bridge/${{ matrix.agent }}/${{ matrix.runtime_provider }} NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_E2E_SHARD: ${{ matrix.agent }} NEMOCLAW_MCP_BRIDGE_AGENT: ${{ matrix.agent }} NEMOCLAW_OPENSHELL_CHANNEL: stable - NEMOCLAW_OPENSHELL_EXACT_MAIN_PROOF: "1" + NEMOCLAW_OPENSHELL_EXACT_MAIN_PROOF: ${{ matrix.runtime_provider == 'docker' && '1' || '0' }} NEMOCLAW_RUN_LIVE_E2E: "1" + NEMOCLAW_GATEWAY_RUNTIME: ${{ matrix.runtime_provider }} OPENSHELL_DOCKER_SUPERVISOR_IMAGE: ghcr.io/nvidia/openshell/supervisor@sha256:722f44669722961b7f432b0b81de25b91a58f34a61d6403bef967acaf2b3af01 steps: - id: trusted_hermes_swap @@ -3592,7 +3657,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@813ca162c2867a34ba3692ad60dba73f3282baea + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@17759906bd7f80319c58af759dd60cfb893109bf with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -3602,6 +3667,11 @@ jobs: shell: bash run: npx tsx tools/e2e/runner-comparison.mts initialize + - name: Prepare native Podman E2E runtime + uses: NVIDIA/NemoClaw/.github/actions/setup-native-podman-e2e@c87144de2c8e2d90b14cf11b31718846e32c65de + with: + enabled: ${{ matrix.runtime_provider == 'podman' && 'true' || 'false' }} + - name: Install and verify cloudflared prerequisite # Update posture: maintainers review upstream cloudflared releases and # update the version and reviewed SHA256 together in both explicit MCP @@ -3675,14 +3745,14 @@ jobs: name: Scan MCP artifacts for fixture credentials if: always() run: >- - npx tsx tools/e2e/assert-mcp-artifact-secrets-absent.mts e2e-artifacts/live/mcp-bridge/${{ matrix.agent }} + npx tsx tools/e2e/assert-mcp-artifact-secrets-absent.mts e2e-artifacts/live/mcp-bridge/${{ matrix.agent }}/${{ matrix.runtime_provider }} - name: Upload MCP server artifacts if: ${{ always() && steps.mcp_artifact_secret_scan.outcome == 'success' }} uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 with: - name: e2e-mcp-bridge-${{ matrix.agent }} - path: e2e-artifacts/live/mcp-bridge/${{ matrix.agent }}/ + name: e2e-mcp-bridge-${{ matrix.agent }}-${{ matrix.runtime_provider }} + path: e2e-artifacts/live/mcp-bridge/${{ matrix.agent }}/${{ matrix.runtime_provider }}/ - name: Clean up Docker auth if: always() @@ -3690,6 +3760,7 @@ jobs: run: bash .github/scripts/docker-auth-cleanup.sh openshell-credential-generation-window: + name: OpenShell credential generation window (${{ matrix.runtime_provider }}) needs: [base-image-publication, generate-matrix] if: ${{ contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'openshell-credential-generation-window') }} runs-on: ubuntu-latest @@ -3699,21 +3770,27 @@ jobs: # execute in parallel with, and fail independently from, the Deep Agents # MCP lifecycle without sharing destructive sandbox state. timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + runtime_provider: ${{ fromJSON(needs.generate-matrix.outputs.runtime_providers_by_job)['openshell-credential-generation-window'] }} env: E2E_MANAGED_IMAGE_REVISION: ${{ needs.base-image-publication.outputs.managed_image_revision }} E2E_MANAGED_IMAGE_COHORT_RECEIPT: ${{ needs.base-image-publication.outputs.managed_image_receipt }} E2E_WORKLOAD_SOURCE: ${{ needs.generate-matrix.outputs.workload_source }} NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG_JSON: ${{ needs.base-image-publication.outputs.managed_image_catalog }} E2E_JOB: "1" + E2E_GATEWAY_RUNTIMES: docker,podman E2E_TARGET_ID: "openshell-credential-generation-window" E2E_AGENT_RUNTIME: "openclaw" E2E_OBSERVABLE_OUTCOME: "Credential expiry rotation detach and rebuild preserve the intended access window" - E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu Docker host; local compatible inference and MCP endpoint" - E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/openshell-credential-generation-window + E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu managed runtime host; local compatible inference and MCP endpoint" + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/openshell-credential-generation-window/${{ matrix.runtime_provider }} NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_OPENSHELL_CHANNEL: stable NEMOCLAW_OPENSHELL_EXACT_MAIN_PROOF: "1" NEMOCLAW_RUN_LIVE_E2E: "1" + NEMOCLAW_GATEWAY_RUNTIME: ${{ matrix.runtime_provider }} OPENSHELL_DOCKER_SUPERVISOR_IMAGE: ghcr.io/nvidia/openshell/supervisor@sha256:722f44669722961b7f432b0b81de25b91a58f34a61d6403bef967acaf2b3af01 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -3731,10 +3808,15 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@813ca162c2867a34ba3692ad60dba73f3282baea + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@17759906bd7f80319c58af759dd60cfb893109bf with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} + - name: Prepare native Podman E2E runtime + uses: NVIDIA/NemoClaw/.github/actions/setup-native-podman-e2e@c87144de2c8e2d90b14cf11b31718846e32c65de + with: + enabled: ${{ matrix.runtime_provider == 'podman' && 'true' || 'false' }} + - name: Install and verify cloudflared prerequisite env: CLOUDFLARED_VERSION: "2026.6.1" @@ -3800,14 +3882,14 @@ jobs: name: Scan credential-window artifacts for fixture credentials if: always() run: >- - npx tsx tools/e2e/assert-mcp-artifact-secrets-absent.mts e2e-artifacts/live/openshell-credential-generation-window + npx tsx tools/e2e/assert-mcp-artifact-secrets-absent.mts e2e-artifacts/live/openshell-credential-generation-window/${{ matrix.runtime_provider }} - name: Upload credential-window artifacts if: ${{ always() && steps.credential_window_artifact_secret_scan.outcome == 'success' }} uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 with: - name: e2e-openshell-credential-generation-window - path: e2e-artifacts/live/openshell-credential-generation-window/ + name: e2e-openshell-credential-generation-window-${{ matrix.runtime_provider }} + path: e2e-artifacts/live/openshell-credential-generation-window/${{ matrix.runtime_provider }}/ - name: Clean up Docker auth if: always() @@ -3855,6 +3937,7 @@ jobs: path: ${{ runner.temp }}/openshell-dev-artifact/ mcp-bridge-dev: + name: MCP bridge dev (${{ matrix.agent }}, ${{ matrix.runtime_provider }}) needs: [base-image-publication, generate-matrix, openshell-dev-artifact] if: ${{ contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'mcp-bridge-dev') }} runs-on: ubuntu-latest @@ -3865,6 +3948,7 @@ jobs: fail-fast: false matrix: agent: [openclaw, hermes, deepagents] + runtime_provider: ${{ fromJSON(needs.generate-matrix.outputs.runtime_providers_by_job)['mcp-bridge-dev'] }} include: - agent: openclaw agent_runtime: openclaw @@ -3881,15 +3965,17 @@ jobs: E2E_WORKLOAD_SOURCE: ${{ needs.generate-matrix.outputs.workload_source }} NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG_JSON: ${{ needs.base-image-publication.outputs.managed_image_catalog }} E2E_JOB: "1" + E2E_GATEWAY_RUNTIMES: docker,podman E2E_TARGET_ID: "mcp-bridge-dev" E2E_OBSERVABLE_OUTCOME: "Development OpenShell MCP bridge reaches tools and inference" - E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu Docker host; local compatible inference and MCP endpoint" - E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/mcp-bridge-dev/${{ matrix.agent }} + E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu managed runtime host; local compatible inference and MCP endpoint" + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/mcp-bridge-dev/${{ matrix.agent }}/${{ matrix.runtime_provider }} NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_E2E_SHARD: ${{ matrix.agent }} NEMOCLAW_MCP_BRIDGE_AGENT: ${{ matrix.agent }} NEMOCLAW_OPENSHELL_CHANNEL: dev NEMOCLAW_RUN_LIVE_E2E: "1" + NEMOCLAW_GATEWAY_RUNTIME: ${{ matrix.runtime_provider }} steps: # setup-node probes package managers in the workspace. # Run it before candidate checkout with automatic caching disabled. @@ -3989,10 +4075,15 @@ jobs: # The restore action executes the candidate CLI for its final identity # check. Candidate-controlled state starts with dependency preparation. - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@813ca162c2867a34ba3692ad60dba73f3282baea + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@17759906bd7f80319c58af759dd60cfb893109bf with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} + - name: Prepare native Podman E2E runtime + uses: NVIDIA/NemoClaw/.github/actions/setup-native-podman-e2e@c87144de2c8e2d90b14cf11b31718846e32c65de + with: + enabled: ${{ matrix.runtime_provider == 'podman' && 'true' || 'false' }} + - name: Install and verify cloudflared prerequisite # Update posture: keep this dev compatibility lane on the same reviewed # version/SHA256 pair as the stable lane; workflow-contract tests fail @@ -4046,14 +4137,14 @@ jobs: name: Scan MCP artifacts for fixture credentials if: always() run: >- - npx tsx tools/e2e/assert-mcp-artifact-secrets-absent.mts e2e-artifacts/live/mcp-bridge-dev/${{ matrix.agent }} + npx tsx tools/e2e/assert-mcp-artifact-secrets-absent.mts e2e-artifacts/live/mcp-bridge-dev/${{ matrix.agent }}/${{ matrix.runtime_provider }} - name: Upload MCP server artifacts if: ${{ always() && steps.mcp_artifact_secret_scan.outcome == 'success' }} uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 with: - name: e2e-mcp-bridge-dev-${{ matrix.agent }} - path: e2e-artifacts/live/mcp-bridge-dev/${{ matrix.agent }}/ + name: e2e-mcp-bridge-dev-${{ matrix.agent }}-${{ matrix.runtime_provider }} + path: e2e-artifacts/live/mcp-bridge-dev/${{ matrix.agent }}/${{ matrix.runtime_provider }}/ - name: Clean up Docker auth if: always() @@ -4086,6 +4177,7 @@ jobs: env: E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/managed-image-multiarch-startup/${{ matrix.shard }} E2E_JOB: "1" + E2E_GATEWAY_RUNTIMES: docker E2E_TARGET_ID: "managed-image-multiarch-startup" E2E_AGENT_RUNTIME: "openclaw + hermes + langchain-deepagents-code" E2E_OBSERVABLE_OUTCOME: "Exact managed images start directly on the native architecture" @@ -4566,6 +4658,7 @@ jobs: E2E_DEFAULT_ENABLED: "0" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/llama-cpp-dgx-spark-qualification E2E_JOB: "1" + E2E_GATEWAY_RUNTIMES: docker E2E_TARGET_ID: "llama-cpp-dgx-spark-qualification" E2E_AGENT_RUNTIME: "unresolved" E2E_OBSERVABLE_OUTCOME: "Exact NemoClaw-built llama.cpp image produces protected DGX Spark evidence" @@ -4756,6 +4849,7 @@ jobs: env: E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/managed-image-protected-runtime E2E_JOB: "1" + E2E_GATEWAY_RUNTIMES: docker E2E_TARGET_ID: "managed-image-protected-runtime" E2E_AGENT_RUNTIME: "openclaw + hermes + langchain-deepagents-code" E2E_OBSERVABLE_OUTCOME: "Protected GPU runtime supports Ollama vLLM NIM rollback and cleanup" @@ -5043,23 +5137,30 @@ jobs: run: bash .github/scripts/docker-auth-cleanup.sh hermes-e2e: + name: Hermes E2E (${{ matrix.runtime_provider }}) needs: [base-image-publication, generate-matrix] if: ${{ needs.generate-matrix.outputs.hermes_selected == 'true' }} runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['hermes-e2e'] }} timeout-minutes: 85 + strategy: + fail-fast: false + matrix: + runtime_provider: ${{ fromJSON(needs.generate-matrix.outputs.runtime_providers_by_job)['hermes-e2e'] }} env: E2E_MANAGED_IMAGE_REVISION: ${{ needs.base-image-publication.outputs.managed_image_revision }} E2E_MANAGED_IMAGE_COHORT_RECEIPT: ${{ needs.base-image-publication.outputs.managed_image_receipt }} E2E_WORKLOAD_SOURCE: ${{ needs.generate-matrix.outputs.workload_source }} NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG_JSON: ${{ needs.base-image-publication.outputs.managed_image_catalog }} E2E_JOB: "1" + E2E_GATEWAY_RUNTIMES: docker,podman E2E_TARGET_ID: "hermes-e2e" E2E_AGENT_RUNTIME: "hermes" E2E_OBSERVABLE_OUTCOME: "Install onboarding health inference lifecycle dashboard and security succeed" E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu; mock or NVIDIA hosted inference" - E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/hermes-e2e + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/hermes-e2e/${{ matrix.runtime_provider }} NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_LIVE_E2E: "1" + NEMOCLAW_GATEWAY_RUNTIME: ${{ matrix.runtime_provider }} NEMOCLAW_E2E_INFERENCE_MODE: ${{ inputs.inference_mode || 'mock' }} NEMOCLAW_E2E_HERMES_DASHBOARD: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" @@ -5104,7 +5205,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@813ca162c2867a34ba3692ad60dba73f3282baea + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@17759906bd7f80319c58af759dd60cfb893109bf with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -5114,6 +5215,11 @@ jobs: shell: bash run: npx tsx tools/e2e/runner-comparison.mts initialize + - name: Prepare native Podman E2E runtime + uses: NVIDIA/NemoClaw/.github/actions/setup-native-podman-e2e@c87144de2c8e2d90b14cf11b31718846e32c65de + with: + enabled: ${{ matrix.runtime_provider == 'podman' && 'true' || 'false' }} + - name: Run Hermes live Vitest test env: NVIDIA_INFERENCE_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && github.event_name == 'workflow_dispatch' && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && (inputs.inference_mode || 'mock') != 'mock' && secrets.NVIDIA_INFERENCE_API_KEY || '' }} @@ -5130,6 +5236,9 @@ jobs: - name: Upload Hermes live Vitest artifacts if: always() uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 + with: + name: e2e-hermes-e2e-${{ matrix.runtime_provider }} + path: e2e-artifacts/live/hermes-e2e/${{ matrix.runtime_provider }}/ - name: Clean up Docker auth if: always() @@ -5137,47 +5246,45 @@ jobs: run: bash .github/scripts/docker-auth-cleanup.sh hermes-gpu-startup: + name: Hermes GPU startup (${{ matrix.scenario }}, ${{ matrix.runtime_provider }}) needs: [base-image-publication, generate-matrix] if: ${{ contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'hermes-gpu-startup') }} runs-on: linux-amd64-gpu-rtxpro6000-latest-1 timeout-minutes: 90 strategy: fail-fast: false - max-parallel: 1 + max-parallel: 2 matrix: - include: - - scenario: native - sandbox_name: e2e-hgpu-native - observable_outcome: "Native GPU startup reaches the stable Ready route" - coverage_variant: native + scenario: [native, fallback, compatibility-only] + runtime_provider: ${{ fromJSON(needs.generate-matrix.outputs.runtime_providers_by_job)['hermes-gpu-startup'] }} + exclude: - scenario: fallback - sandbox_name: e2e-hgpu-fallback - observable_outcome: "Fallback GPU startup reaches the stable Ready route" - coverage_variant: fallback + runtime_provider: podman - scenario: compatibility-only - sandbox_name: e2e-hgpu-compat - observable_outcome: "Compatibility-only GPU startup reaches the stable Ready route" - coverage_variant: compatibility-only + runtime_provider: podman env: E2E_MANAGED_IMAGE_REVISION: ${{ needs.base-image-publication.outputs.managed_image_revision }} E2E_MANAGED_IMAGE_COHORT_RECEIPT: ${{ needs.base-image-publication.outputs.managed_image_receipt }} E2E_WORKLOAD_SOURCE: ${{ needs.generate-matrix.outputs.workload_source }} NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG_JSON: ${{ needs.base-image-publication.outputs.managed_image_catalog }} E2E_JOB: "1" + E2E_GATEWAY_RUNTIMES: docker,podman E2E_TARGET_ID: "hermes-gpu-startup" E2E_AGENT_RUNTIME: "hermes" + E2E_OBSERVABLE_OUTCOME: "Hermes GPU startup reaches the stable Ready route" E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "NVIDIA GPU runner; local GPU inference" - E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/hermes-gpu-startup/${{ matrix.scenario }} + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/hermes-gpu-startup/${{ matrix.scenario }}/${{ matrix.runtime_provider }} E2E_HERMES_GPU_STARTUP_SCENARIO: ${{ matrix.scenario }} NEMOCLAW_E2E_SHARD: ${{ matrix.scenario }} NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_LIVE_E2E: "1" + NEMOCLAW_GATEWAY_RUNTIME: ${{ matrix.runtime_provider }} NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" NEMOCLAW_AGENT: hermes NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_RECREATE_SANDBOX: "1" NEMOCLAW_SANDBOX_GPU: "1" - NEMOCLAW_SANDBOX_NAME: ${{ matrix.sandbox_name }} + NEMOCLAW_SANDBOX_NAME: ${{ matrix.scenario }} NEMOCLAW_ONBOARD_VALIDATION_TIMEOUT_SECONDS: "60" steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -5265,7 +5372,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@813ca162c2867a34ba3692ad60dba73f3282baea + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@17759906bd7f80319c58af759dd60cfb893109bf with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} @@ -5279,6 +5386,11 @@ jobs: with: node-version: "22" + - name: Prepare native Podman E2E runtime + uses: NVIDIA/NemoClaw/.github/actions/setup-native-podman-e2e@c87144de2c8e2d90b14cf11b31718846e32c65de + with: + enabled: ${{ matrix.runtime_provider == 'podman' && 'true' || 'false' }} + - name: Run Hermes GPU startup live Vitest test shell: /bin/bash --noprofile --norc -e -o pipefail {0} env: @@ -5414,8 +5526,8 @@ jobs: if: always() uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 with: - name: e2e-hermes-gpu-startup-${{ matrix.scenario }} - path: e2e-artifacts/live/hermes-gpu-startup/${{ matrix.scenario }}/ + name: e2e-hermes-gpu-startup-${{ matrix.scenario }}-${{ matrix.runtime_provider }} + path: e2e-artifacts/live/hermes-gpu-startup/${{ matrix.scenario }}/${{ matrix.runtime_provider }}/ - name: Clean up Docker auth if: always() @@ -5462,23 +5574,30 @@ jobs: path: ${{ runner.temp }}/e2e-artifacts/live/jetson-nvmap-gpu/ cloud-onboard: + name: Cloud onboard (${{ matrix.runtime_provider }}) needs: [base-image-publication, generate-matrix] if: ${{ contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'cloud-onboard') }} runs-on: ubuntu-latest timeout-minutes: 70 + strategy: + fail-fast: false + matrix: + runtime_provider: ${{ fromJSON(needs.generate-matrix.outputs.runtime_providers_by_job)['cloud-onboard'] }} env: E2E_MANAGED_IMAGE_REVISION: ${{ needs.base-image-publication.outputs.managed_image_revision }} E2E_MANAGED_IMAGE_COHORT_RECEIPT: ${{ needs.base-image-publication.outputs.managed_image_receipt }} E2E_WORKLOAD_SOURCE: ${{ needs.generate-matrix.outputs.workload_source }} NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG_JSON: ${{ needs.base-image-publication.outputs.managed_image_catalog }} E2E_JOB: "1" + E2E_GATEWAY_RUNTIMES: docker,podman E2E_TARGET_ID: "cloud-onboard" E2E_AGENT_RUNTIME: "openclaw" E2E_OBSERVABLE_OUTCOME: "Public install onboarding hosted inference and security checks succeed" E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu; NVIDIA hosted inference" - E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/cloud-onboard + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/cloud-onboard/${{ matrix.runtime_provider }} NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_LIVE_E2E: "1" + NEMOCLAW_GATEWAY_RUNTIME: ${{ matrix.runtime_provider }} NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1" NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" @@ -5523,10 +5642,15 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@813ca162c2867a34ba3692ad60dba73f3282baea + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@17759906bd7f80319c58af759dd60cfb893109bf with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} + - name: Prepare native Podman E2E runtime + uses: NVIDIA/NemoClaw/.github/actions/setup-native-podman-e2e@c87144de2c8e2d90b14cf11b31718846e32c65de + with: + enabled: ${{ matrix.runtime_provider == 'podman' && 'true' || 'false' }} + - name: Install OpenShell CLI run: bash scripts/install-openshell.sh @@ -5587,6 +5711,9 @@ jobs: - name: Upload cloud-onboard artifacts if: always() uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 + with: + name: e2e-cloud-onboard-${{ matrix.runtime_provider }} + path: e2e-artifacts/live/cloud-onboard/${{ matrix.runtime_provider }}/ - name: Clean up Docker auth if: always() @@ -5594,23 +5721,30 @@ jobs: run: bash .github/scripts/docker-auth-cleanup.sh messaging-providers: + name: Messaging providers (${{ matrix.runtime_provider }}) needs: [base-image-publication, generate-matrix] if: ${{ contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'messaging-providers') }} runs-on: ubuntu-latest timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + runtime_provider: ${{ fromJSON(needs.generate-matrix.outputs.runtime_providers_by_job)['messaging-providers'] }} env: E2E_MANAGED_IMAGE_REVISION: ${{ needs.base-image-publication.outputs.managed_image_revision }} E2E_MANAGED_IMAGE_COHORT_RECEIPT: ${{ needs.base-image-publication.outputs.managed_image_receipt }} E2E_WORKLOAD_SOURCE: ${{ needs.generate-matrix.outputs.workload_source }} NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG_JSON: ${{ needs.base-image-publication.outputs.managed_image_catalog }} E2E_JOB: "1" + E2E_GATEWAY_RUNTIMES: docker,podman E2E_TARGET_ID: "messaging-providers" E2E_AGENT_RUNTIME: "openclaw" E2E_OBSERVABLE_OUTCOME: "Provider configuration redaction and optional real sends succeed" E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu; NVIDIA hosted inference and messaging providers" - E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/messaging-providers + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/messaging-providers/${{ matrix.runtime_provider }} NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_LIVE_E2E: "1" + NEMOCLAW_GATEWAY_RUNTIME: ${{ matrix.runtime_provider }} NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1" NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" @@ -5631,10 +5765,15 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@813ca162c2867a34ba3692ad60dba73f3282baea + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@17759906bd7f80319c58af759dd60cfb893109bf with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} + - name: Prepare native Podman E2E runtime + uses: NVIDIA/NemoClaw/.github/actions/setup-native-podman-e2e@c87144de2c8e2d90b14cf11b31718846e32c65de + with: + enabled: ${{ matrix.runtime_provider == 'podman' && 'true' || 'false' }} + - name: Run messaging providers live Vitest test # The test keeps # the legacy fake-token defaults, optional _REAL secret overrides, @@ -5658,6 +5797,9 @@ jobs: - name: Upload messaging providers artifacts if: always() uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 + with: + name: e2e-messaging-providers-${{ matrix.runtime_provider }} + path: e2e-artifacts/live/messaging-providers/${{ matrix.runtime_provider }}/ - name: Clean up Docker auth if: always() @@ -5677,6 +5819,7 @@ jobs: timeout-minutes: 85 env: E2E_JOB: "1" + E2E_GATEWAY_RUNTIMES: docker E2E_TARGET_ID: "openclaw-plugin-runtime-exdev" E2E_AGENT_RUNTIME: "openclaw" E2E_OBSERVABLE_OUTCOME: "OpenClaw installs the custom plugin across devices; plugin behavior survives restart and recreation" @@ -5714,7 +5857,7 @@ jobs: build-cli: "false" - name: Restore exact-commit CLI artifact - uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@813ca162c2867a34ba3692ad60dba73f3282baea + uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@17759906bd7f80319c58af759dd60cfb893109bf with: provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }} diff --git a/.github/workflows/managed-images.yaml b/.github/workflows/managed-images.yaml index 9ddc6d65748..d1c8c1fa7a9 100644 --- a/.github/workflows/managed-images.yaml +++ b/.github/workflows/managed-images.yaml @@ -32,6 +32,7 @@ on: pull_request: paths: - ".github/actions/ci-reviewed-npm-audit/**" + - ".github/workflows/base-image.yaml" - ".github/actions/publish-managed-image-digest/**" - ".github/workflows/managed-images.yaml" - ".dockerignore" diff --git a/agents/hermes/Dockerfile b/agents/hermes/Dockerfile index 094f0182298..fd693a503b2 100644 --- a/agents/hermes/Dockerfile +++ b/agents/hermes/Dockerfile @@ -718,7 +718,7 @@ RUN node --experimental-strip-types \ ARG NEMOCLAW_HERMES_WRAPPER_SHA256=4db45043f45d8296dd39228315b721ee19b0a4e0591579ec0ceeec2777bbb40d ARG NEMOCLAW_HERMES_CLI_ADAPTER_SHA256=989edf54a8c09c6efb348600a8aa2f264c0b71408eb9d7bcd579b92cbeccf9b1 ARG NEMOCLAW_HERMES_CLI_ADAPTER_VALIDATOR_SHA256=db4046e79e513eab67b069a8eda20167b8b65529cf26842531d2ad673c670330 -ARG NEMOCLAW_HERMES_VALIDATOR_SHA256=b355d1365fb1d15475e327f312ceb854ae96f9ebed28cf96bc8817f550df2688 +ARG NEMOCLAW_HERMES_VALIDATOR_SHA256=57282ff0b70b857010f49207c31fa7415356a877e0937173cc2aa733bffadd3b ARG NEMOCLAW_HERMES_TIRITH_FINALIZER_SHA256=a1e6b1c53ab297569abb87c29d15c294d729e46005bfd022136b4c447a791819 ARG NEMOCLAW_HERMES_CRON_RESTORE_CONTROLLER_SHA256=e8593cf1580bffa4663e91c079ba0ce31c3d26391f5b1718872701138ce250b0 # hadolint ignore=DL4006 diff --git a/agents/hermes/mcp-config-transaction.py b/agents/hermes/mcp-config-transaction.py index d9875dce6b0..c7456ce0d5f 100755 --- a/agents/hermes/mcp-config-transaction.py +++ b/agents/hermes/mcp-config-transaction.py @@ -631,7 +631,11 @@ def inspect_managed_config(payload: dict[str, object]) -> dict[str, object]: return {"ok": True, "state": "matched"} -def _mutate(data: object, action: str, payload: dict[str, object]) -> tuple[dict, bool]: +def _mutate( + data: object, + action: str, + payload: dict[str, object], +) -> tuple[dict, bool]: if not isinstance(data, dict): raise ValueError("Invalid Hermes config: expected a YAML object") server_name = payload.get("server") diff --git a/agents/hermes/runtime-config-guard.py b/agents/hermes/runtime-config-guard.py index 0270f96c69c..56834a5e74a 100755 --- a/agents/hermes/runtime-config-guard.py +++ b/agents/hermes/runtime-config-guard.py @@ -5009,6 +5009,18 @@ def _runtime_plan_replacements_and_provider_keys( continue if compiled.search(runtime_value): replacement_value = runtime_value if target_env_key else value + alias_marker = "-OPENSHELL-RESOLVE-ENV-" + if ( + target_env_key is None + and alias_marker in value + and runtime_value.startswith(SCOPED_PLACEHOLDER_PREFIX) + ): + runtime_suffix = runtime_value[len(SCOPED_PLACEHOLDER_PREFIX) :] + alias_prefix, alias_suffix = value.split(alias_marker, 1) + if alias_suffix == env_key and re.fullmatch( + rf"v[0-9]{{1,20}}_{re.escape(env_key)}", runtime_suffix + ): + replacement_value = f"{alias_prefix}{alias_marker}{runtime_suffix}" replacements[replacement_env_key] = (replacement_value, message) return replacements, provider_env_keys, True diff --git a/agents/hermes/start.sh b/agents/hermes/start.sh index 614a6c95080..47e85dd08f0 100755 --- a/agents/hermes/start.sh +++ b/agents/hermes/start.sh @@ -133,6 +133,24 @@ nemoclaw_runtime_state_mutation_checkpoint() { return 1 } +# A supervised Hermes recovery can fail after the provider has fenced this +# exact startup shell. Keep that authenticated process available for the +# existing USR2 retry protocol instead of letting `set -e` replace its +# PID/start identity. Outside an active mutation, preserve ordinary failure. +nemoclaw_runtime_state_mutation_hold_supervisor_failure() { + local status + if nemoclaw_runtime_state_mutation_gate admit; then + return 1 + else + status=$? + fi + [ "$status" -eq 75 ] || return 1 + printf '%s\n' '[SECURITY] Hermes supervisor recovery failed during an active runtime state mutation; holding for authenticated retry.' >&2 + while :; do + kill -STOP "$$" + done +} + # managed-entrypoint-env-wrapper begin _NEMOCLAW_ENTRYPOINT_ENV_WRAPPER="/usr/local/lib/nemoclaw/entrypoint-env-wrapper.sh" if [ ! -f "$_NEMOCLAW_ENTRYPOINT_ENV_WRAPPER" ]; then @@ -3528,8 +3546,10 @@ if [ "$(id -u)" -ne 0 ]; then bootstrap_hermes_gateway_current_user || exit 1 print_dashboard_urls - supervise_hermes_gateway_current_user - exit $? + if ! supervise_hermes_gateway_current_user; then + nemoclaw_runtime_state_mutation_hold_supervisor_failure || exit 1 + exit 1 + fi fi # ── Root path (full privilege separation via setpriv) ────────── diff --git a/agents/hermes/validate-env-secret-boundary.py b/agents/hermes/validate-env-secret-boundary.py index e1a6fd71961..c93a2b51f96 100755 --- a/agents/hermes/validate-env-secret-boundary.py +++ b/agents/hermes/validate-env-secret-boundary.py @@ -29,7 +29,9 @@ from typing import Iterable, TextIO SECRET_KEY_RE = re.compile(r"(^|_)(TOKEN|KEY|SECRET|PASSWORD|CREDENTIAL|API)(_|$)") -PLACEHOLDER_RE = re.compile(r"^(xoxb|xapp)-OPENSHELL-RESOLVE-ENV-[A-Z0-9_]+$") +PLACEHOLDER_RE = re.compile( + r"^(xoxb|xapp)-OPENSHELL-RESOLVE-ENV-(?:v[0-9]{1,20}_)?[A-Z][A-Z0-9_]{0,127}$" +) KEY_NAME_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") API_SERVER_KEY_RE = re.compile(r"^[0-9a-f]{64}$") HERMES_API_PORT_RANGE_START = 8642 diff --git a/ci/platform-matrix.json b/ci/platform-matrix.json index 6b7c9fe5c67..f221f1c57f7 100644 --- a/ci/platform-matrix.json +++ b/ci/platform-matrix.json @@ -259,9 +259,9 @@ "out_of_scope": [ { - "name": "Podman / other container runtimes", + "name": "Other container runtimes", "status": "unsupported", - "notes": "Standard onboarding surfaces an explicit unsupported-runtime error for Podman (`src/lib/onboard/fatal-runtime-preflight.ts` prints the rejection; `src/lib/onboard/preflight.ts` flags the unsupported runtime upstream). The explicit portable experimental profile has one installer-preflight admission exception for the Podman unsupported-runtime finding, as accepted in issue #9007. It does not waive any other readiness blocker or make Podman generally supported. Only Docker Engine, Docker Desktop, and Colima are supported. See issue #420 (closed)." + "notes": "Native rootless Podman has the limited explicit support described in Deployment Paths. Containerd, CRI-O, and other runtime providers are not selectable for standard onboarding." }, { "name": "Intel Mac (macOS x86_64)", @@ -364,6 +364,11 @@ "status": "tested", "notes": "Run `$$nemoclaw onboard` on a tested platform with Docker available locally. Primary path." }, + { + "name": "Native rootless Podman on Linux", + "status": "caveated", + "notes": "Set `NEMOCLAW_GATEWAY_RUNTIME=podman` before standard onboarding. The provider requires a qualified current-user Podman socket, rootless service, cgroups v2, bridge networking and DNS, and exact managed-image receipts. Full native E2E qualification currently runs on Linux amd64; the managed-image contract accepts `linux/amd64` and `linux/arm64` receipts. Custom `--from` Dockerfiles and read-only host mounts are not qualified. The portable experimental profile remains separate." + }, { "name": "Headless Linux server", "status": "caveated", diff --git a/ci/source-architecture-budget.json b/ci/source-architecture-budget.json index af133cd9fa5..d3624429192 100644 --- a/ci/source-architecture-budget.json +++ b/ci/source-architecture-budget.json @@ -5,7 +5,7 @@ "maxByFile": { "src/lib/actions/sandbox/mcp-bridge-contracts.ts": 25, "src/lib/actions/sandbox/process-recovery.ts": 27, - "src/lib/adapters/docker/index.ts": 43, + "src/lib/adapters/docker/index.ts": 42, "src/lib/adapters/openshell/client.ts": 20, "src/lib/adapters/openshell/resolve.ts": 27, "src/lib/adapters/openshell/runtime.ts": 55, @@ -43,7 +43,7 @@ "src/lib/actions/sandbox/gateway-state.ts": 21, "src/lib/actions/sandbox/status-snapshot.ts": 19, "src/lib/actions/sandbox/policy-channel.ts": 30, - "src/lib/actions/sandbox/process-recovery.ts": 21, + "src/lib/actions/sandbox/process-recovery.ts": 20, "src/lib/actions/sandbox/rebuild-pipeline.ts": 29, "src/lib/actions/sandbox/snapshot.ts": 38, "src/lib/actions/uninstall/run-plan.ts": 25, @@ -53,8 +53,8 @@ "src/lib/onboard.ts": 193, "src/lib/onboard/machine/handlers/sandbox.ts": 21, "src/lib/policy/index.ts": 23, - "src/lib/sandbox/config.ts": 22, - "src/lib/shields/index.ts": 25 + "src/lib/sandbox/config.ts": 21, + "src/lib/shields/index.ts": 24 } }, "allowedCycles": [], diff --git a/ci/test-file-size-budget.json b/ci/test-file-size-budget.json index cf40c21910c..56e22943bf6 100644 --- a/ci/test-file-size-budget.json +++ b/ci/test-file-size-budget.json @@ -7,7 +7,7 @@ "src/lib/onboard/preflight.test.ts": 1875, "test/generation/generate-openclaw-config.test.ts": 1898, "test/installer-integration/install-preflight.test.ts": 3025, - "test/agents/openclaw/runtime/nemoclaw-start.test.ts": 4626, + "test/agents/openclaw/runtime/nemoclaw-start.test.ts": 4625, "test/onboarding/onboard-messaging.test.ts": 1971, "test/onboarding/onboard-selection.test.ts": 4176 } diff --git a/docs/reference/architecture.mdx b/docs/reference/architecture.mdx index 77c7c006f90..6f08cb2e4ad 100644 --- a/docs/reference/architecture.mdx +++ b/docs/reference/architecture.mdx @@ -310,12 +310,12 @@ The maintained onboarding path for this agent does not consume the component. ## Sandbox Environment -Stock onboarding through the OpenShell Docker driver for OpenClaw, Hermes, and LangChain Deep Agents Code selects an immutable managed image for the installed release and host architecture. +Stock onboarding through the default Docker provider or the explicitly selected native Podman provider for OpenClaw, Hermes, and LangChain Deep Agents Code selects an immutable managed image for the installed release and host architecture. Before selecting one agent image, NemoClaw validates a complete three-agent cohort with one release, source revision, publication cohort, and compatible startup and capability contracts. If registry or catalog availability prevents resolution, stock onboarding stops before sandbox creation and does not build a shipped Dockerfile. Available catalog evidence that is incomplete, mixed, mutable, wrong-platform, or identity-inconsistent fails closed before sandbox creation. An explicit `--from ` remains a separate complete custom-image path. -The portable experimental profile retains its existing workload path, and native Podman remains disabled. +The portable experimental profile retains its existing workload path and lifecycle. Native Podman is independently selected with `NEMOCLAW_GATEWAY_RUNTIME=podman`, consumes the registered Podman provider bundle, and accepts only exact managed-image workload receipts; legacy and custom Dockerfile builds remain disabled for that provider. The direct blueprint runner still carries a pinned OpenShell Community OpenClaw image for legacy `openshell sandbox create --from` compatibility. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 0f81a62e5f4..0c869230ac8 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -736,7 +736,15 @@ For Hermes, the backed-up paths come from `agents/hermes/manifest.yaml`, includi -Before creating the gateway, the wizard runs preflight checks. It verifies that Docker is reachable and prints host remediation guidance when prerequisites are missing. Standard onboarding rejects unsupported runtimes such as Podman. The explicit portable experimental profile has one installer-preflight admission exception for the Podman unsupported-runtime finding. It does not waive any other readiness blocker or make Podman generally supported. The preflight also enforces the OpenShell version range declared in the blueprint (`min_openshell_version` and `max_openshell_version`). If the installed OpenShell version falls outside this range, onboarding exits with an actionable error and a link to compatible releases. For fresh OpenShell installs, NemoClaw queries published OpenShell releases and asks the installer to use a release that fits the blueprint range. If release metadata is unavailable, the installer uses its bundled fallback pin and the post-install version gate still enforces the range. +Before creating the gateway, the wizard runs preflight checks. +Docker is the default runtime provider. It verifies that the selected provider is reachable and prints host remediation guidance when prerequisites are missing. +On Linux, set `NEMOCLAW_GATEWAY_RUNTIME=podman` to select the qualified native rootless Podman provider for standard onboarding. +Native Podman must pass its provider-owned socket, rootless service, cgroups v2, bridge-container, DNS, architecture, and managed-image checks; an auto-detected Podman compatibility socket is not enough to opt in. +The portable experimental profile remains a separate selection with its existing installer-preflight admission and lifecycle behavior. It does not read `NEMOCLAW_GATEWAY_RUNTIME` or become synonymous with native Podman. +The preflight also enforces the OpenShell version range declared in the blueprint (`min_openshell_version` and `max_openshell_version`). +If the installed OpenShell version falls outside this range, onboarding exits with an actionable error and a link to compatible releases. +For fresh OpenShell installs, NemoClaw queries published OpenShell releases and asks the installer to use a release that fits the blueprint range. +If release metadata is unavailable, the installer uses its bundled fallback pin and the post-install version gate still enforces the range. When NemoClaw finds an existing gateway to reuse, it probes the host gateway HTTP endpoint before declaring the gateway reusable. If the container is running but the upstream is still warming up (for example, immediately after a Docker daemon restart), NemoClaw rebuilds the gateway instead of trusting stale metadata. On the Docker-driver gateway path, preflight stays read-only when it detects a stale gateway (for example, a Docker-driver runtime env hash drift). It prints a `⚠ Gateway will be recreated when sandbox creation starts` notice and defers the actual teardown to step `[2/8] Starting OpenShell gateway`. This means pressing `Ctrl+C` between preflight and step `[2/8]` leaves the running gateway and existing sandbox containers untouched, so `$$nemoclaw onboard` is safe to run just to check preflight output. An interrupted run prints the resume command and exits with status `130` for `Ctrl+C` or `143` for `SIGTERM`. For Linux Docker-driver gateways, onboarding also checks that a helper container on the OpenShell Docker network can reach `host.openshell.internal:`. If a host firewall blocks that sandbox path, onboarding exits with a `sudo ufw allow from to port proto tcp` command before it reports the gateway healthy. Set `NEMOCLAW_AUTO_FIX_FIREWALL=1` to opt in to automatic UFW remediation for this specific failure: NemoClaw uses `sudo -n` only, validates the Docker bridge subnet/gateway/port, applies the narrow UFW rule only after a proven TCP reachability failure, and re-probes before continuing. If passwordless sudo, UFW, or active UFW is unavailable, NemoClaw falls back to the manual guidance path without prompting for a password. @@ -748,7 +756,11 @@ The Docker-driver gateway and the portable experimental profile's Podman-driver #### `--from ` -Without `--from`, onboarding through the OpenShell Docker driver for OpenClaw, Hermes, and LangChain Deep Agents Code selects an immutable managed image for the installed release and host architecture. NemoClaw validates the complete three-agent publication cohort before selecting any member. If registry or catalog availability prevents resolution, stock onboarding stops before sandbox creation and does not build a shipped Dockerfile. Catalog evidence that is incomplete, mixed, mutable, wrong-platform, or identity-inconsistent also fails closed before sandbox creation. The portable experimental profile and native Podman are not part of this activation. +Without `--from`, onboarding through the default Docker provider or the explicitly selected native Podman provider for OpenClaw, Hermes, and LangChain Deep Agents Code selects an immutable managed image for the installed release and host architecture. +NemoClaw validates the complete three-agent publication cohort before selecting any member. +If registry or catalog availability prevents resolution, stock onboarding stops before sandbox creation and does not build a shipped Dockerfile. +Catalog evidence that is incomplete, mixed, mutable, wrong-platform, or identity-inconsistent also fails closed before sandbox creation. +The portable experimental profile retains its separate workload path. Native Podman uses this managed-image activation and rejects legacy or custom Dockerfile builds, so `--from` remains a Docker-provider path. Build the sandbox image from a custom Dockerfile instead of the stock NemoClaw image. The supplied Dockerfile defines the complete sandbox image, and NemoClaw does not layer it on top of the stock managed runtime. The entire parent directory of the specified file is used as the Docker build context, so any files your Dockerfile references (scripts, config, etc.) must live alongside it. When the supplied path is the selected agent's own managed Dockerfile (for example, `agents/hermes/Dockerfile` in the NemoClaw checkout the CLI runs from), NemoClaw applies one exception and stages the repository root as the build context, exactly as the managed build does, because that Dockerfile copies repository-root paths. This lets you edit the managed Dockerfile in place (for example to add Python packages) and rebuild from it with `--from`. For this managed exception, onboarding applies the `.dockerignore` from the repository root. For every other `--from` path, onboarding applies a `.dockerignore` from the Dockerfile's parent directory while calculating the context size and staging files for Docker. NemoClaw also applies additional secret-safety exclusions that override `.dockerignore` negation rules: credential-style files and directories such as `.env*`, `.ssh/`, `.aws/`, `.netrc`, `.npmrc`, `secrets/`, `*.pem`, and `*.key` are still skipped even if `.dockerignore` tries to include them. Without a `.dockerignore`, onboarding still skips common large or local-only directories (`node_modules`, `.git`, `.venv`, and `__pycache__`) while staging this context. Other build outputs such as `dist/`, `target/`, or `build/` are included unless your `.dockerignore` excludes them. If the staged context is larger than 100 MB, onboarding prints a warning before the Docker build starts. Move the Dockerfile into a smaller dedicated directory or add `.dockerignore` entries for generated artifacts to shrink the context. If the directory contains unreadable files (for example, Windows system files visible in WSL), onboarding exits with an error suggesting you move the Dockerfile to a dedicated directory. diff --git a/docs/reference/platform-support.mdx b/docs/reference/platform-support.mdx index bd663063f07..4ab6a73343f 100644 --- a/docs/reference/platform-support.mdx +++ b/docs/reference/platform-support.mdx @@ -156,6 +156,7 @@ Pick the row that matches the target environment. | Path | Status | Notes | |------|--------|-------| | Local CLI onboard | Tested | Run `$$nemoclaw onboard` on a tested platform with Docker available locally. Primary path. | +| Native rootless Podman on Linux | Tested with limitations | Set `NEMOCLAW_GATEWAY_RUNTIME=podman` before standard onboarding. The provider requires a qualified current-user Podman socket, rootless service, cgroups v2, bridge networking and DNS, and exact managed-image receipts. Full native E2E qualification currently runs on Linux amd64; the managed-image contract accepts `linux/amd64` and `linux/arm64` receipts. Custom `--from` Dockerfiles and read-only host mounts are not qualified. The portable experimental profile remains separate. | | Headless Linux server | Tested with limitations | Provision a tested Linux host, connect over SSH, run the standard installer and `$$nemoclaw onboard`, and keep dashboards bound to loopback behind SSH port forwarding. Automatic recovery after a host reboot is not guaranteed; follow the documented manual recovery flow. | {/* deployment-status:end */} @@ -167,7 +168,7 @@ They are listed here so launch material, sales conversations, and support triage {/* out-of-scope:begin */} | Item | Status | Why | |------|--------|-----| -| Podman / other container runtimes | Unsupported | Standard onboarding surfaces an explicit unsupported-runtime error for Podman (`src/lib/onboard/fatal-runtime-preflight.ts` prints the rejection; `src/lib/onboard/preflight.ts` flags the unsupported runtime upstream). The explicit portable experimental profile has one installer-preflight admission exception for the Podman unsupported-runtime finding, as accepted in issue #9007. It does not waive any other readiness blocker or make Podman generally supported. Only Docker Engine, Docker Desktop, and Colima are supported. See issue #420 (closed). | +| Other container runtimes | Unsupported | Native rootless Podman has the limited explicit support described in Deployment Paths. Containerd, CRI-O, and other runtime providers are not selectable for standard onboarding. | | Intel Mac (macOS x86_64) | Unsupported | The supported OpenShell Homebrew gateway service path is Apple Silicon only, and OpenShell does not publish macOS x86_64 standalone gateway assets. The top-level installer rejects Intel Mac hosts before release-ref resolution or downloads (`install.sh:108`), and the OpenShell installer retains a downstream asset guard (`scripts/install-openshell.sh:687`). See issue #954 (closed). | | Non-Ubuntu/Debian Linux distros | Unsupported | Installer assumes `apt-get`. Fedora/Rocky/Alma/Arch/NixOS are not validated and the installer's package-manager probes do not cover them. See open issue #899 (Fedora hang). | | Native Kubernetes or OpenShift deployments | Unsupported | NemoClaw runs the sandbox as a Docker container, not a Kubernetes pod. The default Docker-driver topology does not embed k3s. Operator-managed K8s/OpenShift deployments are out of scope; see issue #407 (community OpenShift through agent-sandbox CRD). | diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 6110f5911af..51a5153ae90 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -480,13 +480,13 @@ If the error names an invalid gateway binding, restore the affected row's known- Older NemoClaw releases relied on a Docker cgroup workaround on Ubuntu 24.04, DGX Spark, and WSL2. Current OpenShell releases handle that behavior themselves, so NemoClaw no longer requires a Spark-specific setup step. -If onboarding reports that Docker is missing or unreachable, fix Docker first and retry onboarding: +If onboarding reports that the default Docker provider is missing or unreachable, fix Docker first and retry onboarding: ```bash $$nemoclaw onboard ``` -Podman is not a tested runtime. If onboarding or sandbox lifecycle fails, switch to a tested runtime (Docker Desktop, Colima, or Docker Engine) and rerun onboarding. +For explicitly selected native Podman, retain `NEMOCLAW_GATEWAY_RUNTIME=podman` and follow [Podman](#podman). An auto-detected Podman compatibility socket without that selector is rejected instead of silently changing providers. ### Cluster fails with `overlayfs snapshotter cannot be enabled` on Docker 26+ @@ -2991,7 +2991,19 @@ For additional troubleshooting, refer to the [Windows Setup](../get-started/addi ## Podman -Podman is not a tested runtime. OpenShell officially documents Docker-based runtimes only. If you encounter issues with Podman, switch to a tested runtime (Docker Engine, Docker Desktop, or Colima) and rerun onboarding. +NemoClaw has two independent Podman paths: the native managed runtime provider and the portable experimental profile. + +For native rootless Podman on Linux, select the provider explicitly: + +```bash +NEMOCLAW_GATEWAY_RUNTIME=podman $$nemoclaw onboard +``` + +Native preflight fails closed unless the current-user Podman socket and service, rootless engine, cgroups v2 hierarchy, bridge networking, DNS, host architecture, and exact managed-image contract are all qualified. +Do not redirect it with `DOCKER_HOST`, `DOCKER_CONTEXT`, `CONTAINER_HOST`, or a named Podman connection; restore the current user's reported socket authority and retry with the explicit selector. +Native Podman supports stock managed-image onboarding only. Use the Docker provider for an explicit `--from ` or a read-only host mount. + +The portable experimental profile is separate and unchanged. The portable experimental profile uses the `docker` command to drive rootless Podman. Before you run this profile, make sure a Docker-compatible CLI is available on `PATH`. On a Podman-only host, install the `podman-docker` shim for your distribution: diff --git a/scripts/checks/layer-import-boundaries.mts b/scripts/checks/layer-import-boundaries.mts index 7c0f9297391..5b91f520e40 100644 --- a/scripts/checks/layer-import-boundaries.mts +++ b/scripts/checks/layer-import-boundaries.mts @@ -23,6 +23,44 @@ type ImportRef = { const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); const SRC_ROOT = path.join(REPO_ROOT, "src"); const SKIP_DIRS = new Set([".git", "coverage", "dist", "node_modules"]); +const PROVIDER_NEUTRAL_MANAGED_RUNTIME_MODULES = [ + "src/lib/actions/sandbox/connect.ts", + "src/lib/actions/sandbox/destroy-presence.ts", + "src/lib/actions/sandbox/launch-readiness.ts", + "src/lib/actions/sandbox/process-recovery.ts", + "src/lib/actions/sandbox/snapshot/backup-authority.ts", + "src/lib/actions/sandbox/rebuild-flow-helpers.ts", + "src/lib/actions/sandbox/sandbox-gateway-routing.ts", + "src/lib/actions/sandbox/status-preflight.ts", + "src/lib/actions/sandbox/status-snapshot.ts", + "src/lib/actions/sandbox/stopped-sandbox-backup.ts", + "src/lib/actions/sandbox/supervisor-relaunch.ts", + "src/lib/actions/sandbox/terminal-runtime-health.ts", + "src/lib/onboard/compute/plan.ts", + "src/lib/onboard/docker-driver-gateway-env.ts", + "src/lib/onboard/docker-driver-gateway-config.ts", + "src/lib/onboard/docker-driver-gateway-local-tls.ts", + "src/lib/onboard/docker-driver-gateway-process-identity.ts", + "src/lib/onboard/docker-driver-gateway-runtime.ts", + "src/lib/onboard/fatal-runtime-preflight.ts", + "src/lib/onboard/gateway-sandbox-reachability.ts", + "src/lib/onboard/host-gateway-process.ts", + "src/lib/onboard/host-service-reachability.ts", + "src/lib/onboard/managed-workload/hermes-state-volume.ts", + "src/lib/onboard/sandbox-create/orchestration.ts", + "src/lib/adapters/sandbox/command-transport.ts", + "src/lib/sandbox/config.ts", + "src/lib/sandbox/privileged-exec.ts", + "src/lib/shields/hermes-runtime-state-mutation.ts", + "src/lib/shields/index.ts", + "src/lib/shields/mutable-config-repair.ts", + "src/lib/state/registry/lifecycle-generation.ts", +] as const; +const MANAGED_STATE_ROOT_PROVIDER_MODULES = [ + "src/lib/onboard/managed-bootstrap/docker.ts", + "src/lib/onboard/managed-bootstrap/podman-runtime.ts", +] as const; +const MANAGED_AGENT_IDS = new Set(["openclaw", "hermes", "langchain-deepagents-code", "pi"]); function toRepoPath(absPath: string): string { return path.relative(REPO_ROOT, absPath).split(path.sep).join("/"); @@ -472,12 +510,7 @@ export function findLayerImportBoundaryViolations(root = SRC_ROOT): Violation[] const commandFile = isCommandFile(repoPath); const source = readFileSync(absPath, "utf8"); if (!domainFile && !actionFile && !adapterFile && !messagingManifestFile && !commandFile) { - checkNoBinLibShimImport( - absPath, - repoPath, - collectPreprocessedImportRefs(source), - violations, - ); + checkNoBinLibShimImport(absPath, repoPath, collectPreprocessedImportRefs(source), violations); continue; } const sourceFile = sourceFileFor(absPath, source); @@ -496,8 +529,159 @@ export function findLayerImportBoundaryViolations(root = SRC_ROOT): Violation[] return violations; } +export function findManagedRuntimeBoundaryViolations(): Violation[] { + const violations: Violation[] = []; + const isProviderImplementationImport = (specifier: string): boolean => + /(?:^|\/)runtime-provider\/(?:docker|podman)(?:[-/.]|$)/.test(specifier); + const isProviderName = (node: ts.Node): boolean => + ts.isStringLiteralLike(node) && (node.text === "docker" || node.text === "podman"); + const isEqualityOperator = (kind: ts.SyntaxKind): boolean => + kind === ts.SyntaxKind.EqualsEqualsToken || + kind === ts.SyntaxKind.ExclamationEqualsToken || + kind === ts.SyntaxKind.EqualsEqualsEqualsToken || + kind === ts.SyntaxKind.ExclamationEqualsEqualsToken; + + for (const repoPath of PROVIDER_NEUTRAL_MANAGED_RUNTIME_MODULES) { + const absPath = path.join(REPO_ROOT, repoPath); + const sourceFile = sourceFileFor(absPath, readFileSync(absPath, "utf8")); + const isProviderIdentity = (node: ts.Node): boolean => { + const expression = node.getText(sourceFile); + // OPENSHELL_DRIVERS is the upstream gateway driver's configuration, + // not NemoClaw's opaque runtime-provider identity. + return ( + !/OPENSHELL_DRIVERS/.test(expression) && + /(?:provider|engine|openshellDriver|sandboxDriver|driver)/i.test(expression) + ); + }; + const report = (node: ts.Node, detail: string): void => { + const pos = position(sourceFile, node); + addViolation( + violations, + repoPath, + pos.line, + pos.column, + "managed-runtime-neutrality", + detail, + ); + }; + const visit = (node: ts.Node): void => { + if ( + (ts.isImportDeclaration(node) || ts.isExportDeclaration(node)) && + node.moduleSpecifier && + ts.isStringLiteralLike(node.moduleSpecifier) && + isProviderImplementationImport(node.moduleSpecifier.text) + ) { + report( + node.moduleSpecifier, + "generic managed runtime code must not import a provider implementation", + ); + } + if ( + ts.isCallExpression(node) && + ((ts.isIdentifier(node.expression) && + node.expression.text === "isPodmanGatewayRuntimeEnabled") || + (ts.isPropertyAccessExpression(node.expression) && + node.expression.name.text === "isPodmanGatewayRuntimeEnabled")) + ) { + report(node.expression, "generic managed runtime code must not branch on native Podman"); + } + if ( + ts.isBinaryExpression(node) && + isEqualityOperator(node.operatorToken.kind) && + ((isProviderName(node.left) && isProviderIdentity(node.right)) || + (isProviderName(node.right) && isProviderIdentity(node.left))) + ) { + report(node, "generic managed runtime code must not compare an opaque provider identity"); + } + if ( + ts.isCaseClause(node) && + isProviderName(node.expression) && + ts.isSwitchStatement(node.parent.parent) && + isProviderIdentity(node.parent.parent.expression) + ) { + report( + node.expression, + "generic managed runtime code must not switch on an opaque provider identity", + ); + } + ts.forEachChild(node, visit); + }; + visit(sourceFile); + } + + const managedBootstrapFiles = [ + ...walk(path.join(SRC_ROOT, "lib/onboard/managed-bootstrap")), + ].filter((absPath) => !path.basename(absPath).includes("test-fixture")); + const podmanProviderFiles = [...walk(path.join(SRC_ROOT, "lib/onboard/runtime-provider"))].filter( + (absPath) => path.basename(absPath).startsWith("podman"), + ); + for (const absPath of [...managedBootstrapFiles, ...podmanProviderFiles]) { + const repoPath = toRepoPath(absPath); + const sourceFile = sourceFileFor(absPath, readFileSync(absPath, "utf8")); + const report = (node: ts.Node, detail: string): void => { + const pos = position(sourceFile, node); + addViolation( + violations, + repoPath, + pos.line, + pos.column, + "managed-state-root-neutrality", + detail, + ); + }; + for (const ref of collectImportRefs(sourceFile)) { + const target = resolveInternalImport(absPath, ref.specifier); + if ( + podmanProviderFiles.includes(absPath) && + target && + /(?:^|\/)(?:hermes|openclaw)(?:[-/.]|$)/u.test(target) + ) { + addViolation( + violations, + repoPath, + ref.line, + ref.column, + "managed-state-root-neutrality", + `Podman provider code must not import agent implementation ${target}`, + ); + } + } + const visit = (node: ts.Node): void => { + if (ts.isStringLiteralLike(node) && MANAGED_AGENT_IDS.has(node.text)) { + report(node, "managed bootstrap and Podman provider code must not encode agent IDs"); + } + ts.forEachChild(node, visit); + }; + visit(sourceFile); + } + + for (const repoPath of MANAGED_STATE_ROOT_PROVIDER_MODULES) { + const absPath = path.join(REPO_ROOT, repoPath); + const sourceFile = sourceFileFor(absPath, readFileSync(absPath, "utf8")); + const ownsGenericStateRootPreparation = collectImportRefs(sourceFile).some( + (ref) => + resolveInternalImport(absPath, ref.specifier) === + "src/lib/onboard/managed-bootstrap/state-root-authority.ts", + ); + if (!ownsGenericStateRootPreparation) { + addViolation( + violations, + repoPath, + 1, + 1, + "managed-state-root-neutrality", + "managed provider bootstrap must consume the generic state-root authority operation", + ); + } + } + return violations; +} + function main(): void { - const violations = findLayerImportBoundaryViolations(); + const violations = [ + ...findLayerImportBoundaryViolations(), + ...findManagedRuntimeBoundaryViolations(), + ]; if (violations.length > 0) { const formatted = violations .map( diff --git a/scripts/checks/run-managed-image-openshell-e2e.ts b/scripts/checks/run-managed-image-openshell-e2e.ts index 349a8f40005..e9bc66d920a 100644 --- a/scripts/checks/run-managed-image-openshell-e2e.ts +++ b/scripts/checks/run-managed-image-openshell-e2e.ts @@ -93,6 +93,18 @@ export function createProtectedManagedImageBootstrapInput( }); } +export function protectedManagedStateRootDriverConfig( + provider: Pick, + mounts: readonly ProtectedManagedStateVolumeMount[], +): string | null { + if (mounts.length === 0) return null; + const driverId = provider.workload.managedStateMountDriverId; + if (!driverId) { + throw new Error("Protected managed state roots require provider-owned mount projection."); + } + return JSON.stringify({ [driverId]: { mounts } }); +} + function compactText(value = ""): string { return String(value).replace(/\s+/gu, " ").trim(); } @@ -145,6 +157,37 @@ export type ManagedImageOpenShellE2eResult< type Inputs = ManagedImageOpenShellE2eInputs; +type ProtectedManagedStateRoot = { + readonly mountTarget: string; + readonly resourceIdentity: string; + readonly ownershipLabels: Readonly>; + readonly uid: number; + readonly gid: number; + readonly mode: number; + readonly readWrite: boolean; +}; + +type ProtectedManagedStateVolumeMount = { + readonly type: "volume"; + readonly source: string; + readonly target: string; + readonly read_only: boolean; +}; + +type ProtectedManagedStateVolumeCleanupResult = + | { readonly status: "not-applicable" | "absent" | "removed" } + | { + readonly status: "not-owned" | "failed"; + readonly detail: string; + readonly volumeName: string; + }; + +type ProtectedManagedStateVolumeScope = { + readonly mounts: readonly ProtectedManagedStateVolumeMount[]; + cleanupIncompleteCreate(): readonly ProtectedManagedStateVolumeCleanupResult[]; + commit(): void; +}; + const MANAGED_IMAGE_E2E_ENVIRONMENT_KEYS = [ "NEMOCLAW_NON_INTERACTIVE", "NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR", @@ -158,6 +201,25 @@ const MANAGED_IMAGE_E2E_ENVIRONMENT_KEYS = [ ] as const; type OnboardModule = { + managedWorkloadOnboard: { + managedStartupStateRoots(input: { + readonly agent: ShippedManagedImageAgent; + readonly sandboxName: string; + readonly agentIdentity: { readonly uid: number; readonly gid: number }; + }): readonly ProtectedManagedStateRoot[]; + managedStartupWorkspaceRoot(input: { + readonly agent: ShippedManagedImageAgent; + readonly agentIdentity: { readonly uid: number; readonly gid: number }; + }): { readonly uid: number; readonly gid: number; readonly mode: 0o755 | 0o1775 }; + prepareManagedStateVolumes( + input: { readonly roots: readonly ProtectedManagedStateRoot[] }, + deps: { readonly runtimeProvider: RuntimeProviderBundle }, + ): ProtectedManagedStateVolumeScope | null; + removeManagedStateVolumes( + input: { readonly roots: readonly ProtectedManagedStateRoot[] }, + deps: { readonly runtimeProvider: RuntimeProviderBundle }, + ): readonly ProtectedManagedStateVolumeCleanupResult[]; + }; openshellArgv(args: string[]): string[]; runOpenshell(args: string[], opts?: Record): ReturnType; runCaptureOpenshell(args: string[], opts?: Record): string; @@ -192,9 +254,49 @@ export function resolveManagedImageOnboardModule(onboardImport: unknown): Onboar `managed-image onboard module is missing required operation(s): ${missing.join(", ")}`, ); } + const managedWorkload = candidateRecord?.managedWorkloadOnboard as + | Record + | undefined; + if ( + typeof managedWorkload?.managedStartupStateRoots !== "function" || + typeof managedWorkload.managedStartupWorkspaceRoot !== "function" || + typeof managedWorkload?.prepareManagedStateVolumes !== "function" || + typeof managedWorkload.removeManagedStateVolumes !== "function" + ) { + throw new Error( + "managed-image onboard module is missing required managed state-volume operations", + ); + } return candidate as OnboardModule; } +function cleanupProtectedManagedStateVolumes(input: { + readonly onboard: OnboardModule | null; + readonly runtimeProvider: RuntimeProviderBundle | null; + readonly roots: readonly ProtectedManagedStateRoot[]; + readonly scope: ProtectedManagedStateVolumeScope | null; + readonly committed: boolean; +}): string[] { + try { + let results: readonly ProtectedManagedStateVolumeCleanupResult[] = []; + if (input.committed && input.onboard && input.runtimeProvider) { + results = input.onboard.managedWorkloadOnboard.removeManagedStateVolumes( + { roots: input.roots }, + { runtimeProvider: input.runtimeProvider }, + ); + } else if (!input.committed) { + results = input.scope?.cleanupIncompleteCreate() ?? []; + } + return results.flatMap((result) => + result.status === "not-owned" || result.status === "failed" + ? [`managed state volume ${result.volumeName} cleanup ${result.status}: ${result.detail}`] + : [], + ); + } catch (error) { + return [error instanceof Error ? error.message : String(error)]; + } +} + function requiredValue(argv: readonly string[], flag: string): string { const index = argv.indexOf(flag); const value = index >= 0 ? argv[index + 1] : undefined; @@ -894,6 +996,14 @@ async function run> | null = null; try { flow = await runSandboxGpuCreateFlow( @@ -1038,11 +1164,16 @@ async function run { - console.error(error instanceof Error ? error.message : String(error)); + run(parseManagedImageOpenShellE2eInputs(process.argv.slice(2))).catch(() => { + console.error("Managed-image OpenShell E2E failed; inspect the redacted evidence artifacts."); process.exitCode = 1; }); } diff --git a/scripts/e2e/package-cli-artifact.sh b/scripts/e2e/package-cli-artifact.sh index 287e08ff992..ad796dc4891 100755 --- a/scripts/e2e/package-cli-artifact.sh +++ b/scripts/e2e/package-cli-artifact.sh @@ -79,6 +79,37 @@ artifact_catalog="dist/e2e-managed-image-catalog.json" echo "::error::candidate build created the managed-image catalog path" >&2 exit 1 } +managed_catalog="${RUNNER_TEMP}/pr-managed-image-catalog.json" +catalog_json="${MANAGED_IMAGE_CATALOG:-}" +catalog_sha256="${MANAGED_IMAGE_CATALOG_SHA256:-}" +if [[ -n "$catalog_json" ]]; then + [[ "$catalog_sha256" =~ ^[a-f0-9]{64}$ ]] || { + echo "::error::trusted PR managed-image catalog digest is invalid" >&2 + exit 1 + } + [[ -f "$managed_catalog" && ! -L "$managed_catalog" && -s "$managed_catalog" ]] || { + echo "::error::trusted PR managed-image catalog is not a nonempty regular file" >&2 + exit 1 + } + [[ "$(sha256sum "$managed_catalog" | awk '{print $1}')" == "$catalog_sha256" ]] || { + echo "::error::trusted PR managed-image catalog changed after authentication" >&2 + exit 1 + } + (umask 077 && set -o noclobber && printf '%s\n' "$catalog_json" >"$artifact_catalog") + [[ -f "$artifact_catalog" && ! -L "$artifact_catalog" && -s "$artifact_catalog" ]] || { + echo "::error::packaged PR managed-image catalog is invalid" >&2 + exit 1 + } + [[ "$(sha256sum "$artifact_catalog" | awk '{print $1}')" == "$catalog_sha256" ]] || { + echo "::error::packaged PR managed-image catalog does not match trusted output" >&2 + exit 1 + } +else + [[ -z "$catalog_sha256" && ! -e "$managed_catalog" && ! -L "$managed_catalog" ]] || { + echo "::error::managed-image catalog authority is inconsistent" >&2 + exit 1 + } +fi artifact_dir="${RUNNER_TEMP}/nemoclaw-cli-artifact" install -d -m 0700 "$artifact_dir" diff --git a/scripts/e2e/restore-cli-artifact.sh b/scripts/e2e/restore-cli-artifact.sh index b6d031a4609..a2cbcafb441 100755 --- a/scripts/e2e/restore-cli-artifact.sh +++ b/scripts/e2e/restore-cli-artifact.sh @@ -138,4 +138,26 @@ jq -e --arg candidateSha "$CANDIDATE_SHA" ' } mv "$restore_dir/nemoclaw/dist" "$GITHUB_WORKSPACE/nemoclaw/dist" mv "$restore_dir/dist" "$GITHUB_WORKSPACE/dist" +managed_catalog="$GITHUB_WORKSPACE/dist/e2e-managed-image-catalog.json" +if [[ -e "$managed_catalog" || -L "$managed_catalog" ]]; then + [[ -f "$managed_catalog" && ! -L "$managed_catalog" && -s "$managed_catalog" ]] || { + echo "::error::restored managed-image catalog is not a nonempty regular file" + exit 1 + } + managed_revision="$(jq -er ' + [to_entries[].value.source.revision] as $revisions | + ($revisions | unique) as $unique | + if (($revisions | length) > 0 and + ($unique | length) == 1 and + ($unique[0] | type == "string" and test("^[a-f0-9]{40}$"))) + then $unique[0] + else error("managed-image catalog must identify one exact publication revision") + end + ' "$managed_catalog")" || { + echo "::error::restored managed-image catalog publication revision is invalid" + exit 1 + } + printf 'NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG=%s\n' "$managed_catalog" >>"$GITHUB_ENV" + printf 'NEMOCLAW_E2E_MANAGED_IMAGE_REVISION=%s\n' "$managed_revision" >>"$GITHUB_ENV" +fi node "$GITHUB_WORKSPACE/bin/nemoclaw.js" --version >/dev/null diff --git a/scripts/install.sh b/scripts/install.sh index 03269bed572..41befa855ae 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -3789,11 +3789,13 @@ run_installer_host_preflight() { local preflight_module="${NEMOCLAW_SOURCE_ROOT}/dist/lib/onboard/preflight.js" local gateway_management_module="${NEMOCLAW_SOURCE_ROOT}/dist/lib/onboard/gateway-management.js" local portable_profile_module="${NEMOCLAW_SOURCE_ROOT}/dist/lib/onboard/experimental/portable-profile.js" + local runtime_provider_selection_module="${NEMOCLAW_SOURCE_ROOT}/dist/lib/onboard/runtime-provider/selection.js" local host_readiness_module="${NEMOCLAW_SOURCE_ROOT}/dist/lib/readiness/host.js" local onboard_admission_module="${NEMOCLAW_SOURCE_ROOT}/dist/lib/readiness/onboard-admission.js" if ! command_exists node \ || [[ ! -f "$preflight_module" ]] \ || [[ ! -f "$gateway_management_module" ]] \ + || [[ ! -f "$runtime_provider_selection_module" ]] \ || [[ ! -f "$host_readiness_module" ]] \ || [[ ! -f "$onboard_admission_module" ]]; then return 0 @@ -3810,6 +3812,7 @@ run_installer_host_preflight() { const onboardAdmissionPath = process.argv[3]; const gatewayManagementPath = process.argv[4]; const portableProfilePath = process.argv[5]; + const runtimeProviderSelectionPath = process.argv[6]; let explicitlySelectedPortableProfile = false; try { const portableProfile = require(portableProfilePath); @@ -3824,6 +3827,7 @@ run_installer_host_preflight() { const { createHostReadinessReport } = require(hostReadinessPath); const { evaluateOnboardReadinessAdmission } = require(onboardAdmissionPath); const { loadGatewayManagementDeclaration } = require(gatewayManagementPath); + const { resolveConfiguredRuntimeProvider } = require(runtimeProviderSelectionPath); const host = assessHost(); const actions = planHostAdvisories(host); const gatewayManagement = loadGatewayManagementDeclaration(); @@ -3831,6 +3835,18 @@ run_installer_host_preflight() { gatewayManagement.ok && (gatewayManagement.declaration === null || gatewayManagement.declaration?.mode === "nemoclaw-managed"); + const selectedRuntimeUsesProviderHostRoute = + !explicitlySelectedPortableProfile && + (() => { + const provider = resolveConfiguredRuntimeProvider(); + return ( + provider.gateway.supported && + provider.gateway.prepareHostRuntime({ + environment: process.env, + platform: process.platform, + }).sandboxHostAddress !== null + ); + })(); const readiness = createHostReadinessReport( { nemoclawVersion: "installer", sourceRevision: "installer" }, { @@ -3843,7 +3859,8 @@ run_installer_host_preflight() { ); const admission = evaluateOnboardReadinessAdmission(readiness, { explicitlyOptedOutGpuPassthrough: false, - allowUnsupportedRuntime: explicitlySelectedPortableProfile, + allowUnsupportedRuntime: + explicitlySelectedPortableProfile || selectedRuntimeUsesProviderHostRoute, // The installer starts a NemoClaw-managed onboarding flow. Let the // authoritative onboarding gate apply supported storage remediation, // but only when the gateway declaration confirms NemoClaw ownership. @@ -3912,10 +3929,26 @@ run_installer_host_preflight() { process.stdout.write(`__ACTIONS__\n${actionLines.join("\n")}`); } process.exit(admission.admitted ? 0 : 10); - } catch { - process.exit(0); + } catch (error) { + const optionalModules = [ + preflightPath, + hostReadinessPath, + onboardAdmissionPath, + gatewayManagementPath, + runtimeProviderSelectionPath, + ]; + const missingOptionalModule = + error?.code === "MODULE_NOT_FOUND" && + optionalModules.some((modulePath) => + String(error?.message || "").includes(modulePath) + ); + if (missingOptionalModule) process.exit(0); + process.stderr.write( + `NemoClaw installer host preflight failed: ${error instanceof Error ? error.message : String(error)}\n` + ); + process.exit(11); } - ' "$preflight_module" "$host_readiness_module" "$onboard_admission_module" "$gateway_management_module" "$portable_profile_module" + ' "$preflight_module" "$host_readiness_module" "$onboard_admission_module" "$gateway_management_module" "$portable_profile_module" "$runtime_provider_selection_module" )"; then status=0 else @@ -3947,7 +3980,7 @@ run_installer_host_preflight() { fi fi - [[ "$status" -ne 10 ]] + [[ "$status" -eq 0 ]] } recover_preexisting_sandboxes_before_onboard() { @@ -4485,6 +4518,15 @@ prepare_portable_experimental_runtime_override() { info "Portable profile selected rootless Podman through DOCKER_HOST=${DOCKER_HOST}." } +# Resolve the legacy Docker bootstrap requirement at the installer runtime +# configuration boundary. Native managed Podman owns its host preparation via +# the registered runtime provider; the portable experimental profile remains +# on its existing Docker-CLI-over-Podman compatibility path. +installer_requires_legacy_docker_bootstrap() { + [[ "${NEMOCLAW_EXPERIMENTAL_PROFILE:-}" == "portable" ]] && return 0 + [[ "${NEMOCLAW_GATEWAY_RUNTIME:-docker}" != "podman" ]] +} + is_wsl_host() { if [ -n "${WSL_DISTRO_NAME:-}" ] || [ -n "${WSL_INTEROP:-}" ]; then return 0 @@ -5966,7 +6008,9 @@ prepare_installer_host() { # generic Docker bootstrap; ensure_station_express_host is a no-op elsewhere. ensure_station_express_host prepare_portable_experimental_runtime_override - ensure_docker + if installer_requires_legacy_docker_bootstrap; then + ensure_docker + fi ensure_openshell_build_deps } diff --git a/scripts/lib/entrypoint-env-wrapper.sh b/scripts/lib/entrypoint-env-wrapper.sh index f375c36dbb1..5511929859f 100755 --- a/scripts/lib/entrypoint-env-wrapper.sh +++ b/scripts/lib/entrypoint-env-wrapper.sh @@ -51,6 +51,7 @@ nemoclaw_normalize_entrypoint_env_wrapper() { _nemoclaw_supported_names="${_nemoclaw_supported_names}|NEMOCLAW_CORPORATE_CA_B64" _nemoclaw_supported_names="${_nemoclaw_supported_names}|NEMOCLAW_DASHBOARD_BIND|NEMOCLAW_DASHBOARD_PORT" _nemoclaw_supported_names="${_nemoclaw_supported_names}|NEMOCLAW_EXTRA_PLACEHOLDER_KEYS" + _nemoclaw_supported_names="${_nemoclaw_supported_names}|NEMOCLAW_HERMES_API_PORT" _nemoclaw_supported_names="${_nemoclaw_supported_names}|NEMOCLAW_HERMES_DASHBOARD" _nemoclaw_supported_names="${_nemoclaw_supported_names}|NEMOCLAW_HERMES_DASHBOARD_INTERNAL_PORT" _nemoclaw_supported_names="${_nemoclaw_supported_names}|NEMOCLAW_HERMES_DASHBOARD_PORT" diff --git a/scripts/lib/refresh-openclaw-wechat-placeholder.py b/scripts/lib/refresh-openclaw-wechat-placeholder.py index a3e82a279e1..a5c7d9d08ce 100755 --- a/scripts/lib/refresh-openclaw-wechat-placeholder.py +++ b/scripts/lib/refresh-openclaw-wechat-placeholder.py @@ -235,6 +235,15 @@ def remove_managed_temporary(accounts_fd, temporary): try: plugin_fd = os.open("openclaw-weixin", directory_flags, dir_fd=root_fd) + except FileNotFoundError: + # Offline channel cleanup deliberately removes the complete managed + # WeChat tree before the channel is removed and rebuilt. With no tree + # there is no placeholder-bearing file to refresh; keep it absent. + raise SystemExit(0) + except OSError: + fail("the managed account directory is missing or unsafe") + + try: accounts_fd = os.open("accounts", directory_flags, dir_fd=plugin_fd) except OSError: fail("the managed account directory is missing or unsafe") diff --git a/scripts/managed-bootstrap-entrypoint.c b/scripts/managed-bootstrap-entrypoint.c index e5889f04278..641ee9c397d 100644 --- a/scripts/managed-bootstrap-entrypoint.c +++ b/scripts/managed-bootstrap-entrypoint.c @@ -35,6 +35,15 @@ #define REQUIRED_SEALS 0x000fL #define SEEK_SET 0L #define NEGATIVE_EINTR -4L +#define LINUX_CAPABILITY_VERSION_3 0x20080522U +#define PR_CAPBSET_READ 23L +#define PR_CAPBSET_DROP 24L +#define PR_CAP_AMBIENT 47L +#define PR_CAP_AMBIENT_CLEAR_ALL 4L +#define BOOTSTRAP_CAPABILITY_MASK 0x32U +#define CAP_DAC_OVERRIDE 1L +#define CAP_FSETID 4L +#define CAP_KILL 5L #if defined(__x86_64__) #define SYSCALL_READ 0L @@ -43,6 +52,9 @@ #define SYSCALL_LSEEK 8L #define SYSCALL_EXECVE 59L #define SYSCALL_FCNTL 72L +#define SYSCALL_CAPGET 125L +#define SYSCALL_CAPSET 126L +#define SYSCALL_PRCTL 157L #define SYSCALL_EXIT_GROUP 231L #define SYSCALL_DUP3 292L #define SYSCALL_MEMFD_CREATE 319L @@ -69,6 +81,22 @@ static long raw_syscall3(long number, long first, long second, long third) { return result; } +static long raw_syscall5(long number, long first, long second, long third, long fourth, + long fifth) { + register long result __asm__("rax") = number; + register long argument_one __asm__("rdi") = first; + register long argument_two __asm__("rsi") = second; + register long argument_three __asm__("rdx") = third; + register long argument_four __asm__("r10") = fourth; + register long argument_five __asm__("r8") = fifth; + __asm__ volatile("syscall" + : "+r"(result) + : "r"(argument_one), "r"(argument_two), "r"(argument_three), + "r"(argument_four), "r"(argument_five) + : "rcx", "r11", "memory"); + return result; +} + __asm__(".global _start\n" ".type _start,@function\n" "_start:\n" @@ -86,6 +114,9 @@ __asm__(".global _start\n" #define SYSCALL_READ 63L #define SYSCALL_WRITE 64L #define SYSCALL_EXIT_GROUP 94L +#define SYSCALL_CAPGET 90L +#define SYSCALL_CAPSET 91L +#define SYSCALL_PRCTL 167L #define SYSCALL_EXECVE 221L #define SYSCALL_MEMFD_CREATE 279L @@ -108,6 +139,22 @@ static long raw_syscall3(long number, long first, long second, long third) { return result; } +static long raw_syscall5(long number, long first, long second, long third, long fourth, + long fifth) { + register long result __asm__("x0") = first; + register long argument_two __asm__("x1") = second; + register long argument_three __asm__("x2") = third; + register long argument_four __asm__("x3") = fourth; + register long argument_five __asm__("x4") = fifth; + register long syscall_number __asm__("x8") = number; + __asm__ volatile("svc 0" + : "+r"(result) + : "r"(argument_two), "r"(argument_three), "r"(argument_four), + "r"(argument_five), "r"(syscall_number) + : "memory"); + return result; +} + __asm__(".global _start\n" ".type _start,%function\n" "_start:\n" @@ -125,6 +172,19 @@ static char **process_environment; static char restored_environment_bytes[MAX_ENVIRONMENT_BYTES]; static char *restored_environment[MAX_ENVIRONMENT_ENTRIES + 1U]; +struct capability_header { + unsigned int version; + int pid; +}; + +struct capability_data { + unsigned int effective; + unsigned int permitted; + unsigned int inheritable; +}; + +__attribute__((noreturn)) static void fail(const char *message); + static size_t text_length(const char *text) { size_t length = 0U; while (text[length] != '\0') length += 1U; @@ -143,6 +203,62 @@ static bool text_equal(const char *left, const char *right) { return left[index] == right[index]; } +static bool remove_bootstrap_capability_marker(size_t *count) { + static const char marker[] = "NEMOCLAW_MANAGED_BOOTSTRAP_DROP_CAPABILITIES=0x32"; + bool found = false; + size_t output = 0U; + for (size_t index = 0U; index < *count; index += 1U) { + if (text_equal(restored_environment[index], marker)) { + if (found) fail("bootstrap capability marker is duplicated"); + found = true; + continue; + } + restored_environment[output++] = restored_environment[index]; + } + restored_environment[output] = NULL; + *count = output; + return found; +} + +static void drop_bootstrap_capabilities(void) { + static const long capabilities[] = {CAP_DAC_OVERRIDE, CAP_FSETID, CAP_KILL}; + struct capability_header header = {LINUX_CAPABILITY_VERSION_3, 0}; + struct capability_data data[2] = {{0U, 0U, 0U}, {0U, 0U, 0U}}; + if (raw_syscall3(SYSCALL_CAPGET, (long)&header, (long)data, 0L) != 0L) { + fail("could not inspect bootstrap capabilities before supervisor resume"); + } + if (raw_syscall5(SYSCALL_PRCTL, PR_CAP_AMBIENT, PR_CAP_AMBIENT_CLEAR_ALL, 0L, 0L, 0L) != + 0L) { + fail("could not clear bootstrap ambient capabilities before supervisor resume"); + } + for (size_t index = 0U; index < sizeof(capabilities) / sizeof(capabilities[0]); index += 1U) { + const long present = + raw_syscall5(SYSCALL_PRCTL, PR_CAPBSET_READ, capabilities[index], 0L, 0L, 0L); + if (present < 0L) fail("could not inspect bootstrap capability bounding set"); + if (present == 1L && + raw_syscall5(SYSCALL_PRCTL, PR_CAPBSET_DROP, capabilities[index], 0L, 0L, 0L) != 0L) { + fail("could not drop bootstrap capability from supervisor bounding set"); + } + } + data[0].effective &= ~BOOTSTRAP_CAPABILITY_MASK; + data[0].permitted &= ~BOOTSTRAP_CAPABILITY_MASK; + data[0].inheritable &= ~BOOTSTRAP_CAPABILITY_MASK; + if (raw_syscall3(SYSCALL_CAPSET, (long)&header, (long)data, 0L) != 0L) { + fail("could not drop bootstrap process capabilities before supervisor resume"); + } + struct capability_data verified[2] = {{0U, 0U, 0U}, {0U, 0U, 0U}}; + if (raw_syscall3(SYSCALL_CAPGET, (long)&header, (long)verified, 0L) != 0L || + ((verified[0].effective | verified[0].permitted | verified[0].inheritable) & + BOOTSTRAP_CAPABILITY_MASK) != 0U) { + fail("bootstrap process capabilities remained after supervisor resume drop"); + } + for (size_t index = 0U; index < sizeof(capabilities) / sizeof(capabilities[0]); index += 1U) { + if (raw_syscall5(SYSCALL_PRCTL, PR_CAPBSET_READ, capabilities[index], 0L, 0L, 0L) != 0L) { + fail("bootstrap bounding capability remained after supervisor resume drop"); + } + } +} + static bool write_all(long descriptor, const char *bytes, size_t length) { size_t offset = 0U; while (offset < length) { @@ -322,6 +438,9 @@ __attribute__((noreturn)) static void resume_supervisor(int argc, char **argv) { char **supervisor_argv = &argv[6]; if (supervisor_argv[0][0] != '/') fail("supervisor executable is not absolute"); read_environment_transport(environment_count, environment_bytes); + if (remove_bootstrap_capability_marker(&environment_count)) { + drop_bootstrap_capabilities(); + } exec_process(supervisor_argv[0], supervisor_argv, restored_environment); fail("could not execute the exact supervisor process"); } diff --git a/scripts/managed-gateway-control.py b/scripts/managed-gateway-control.py index a8af06156b2..640dc2b7b81 100755 --- a/scripts/managed-gateway-control.py +++ b/scripts/managed-gateway-control.py @@ -175,7 +175,7 @@ def __init__(self, code: str, *, stage: str | None = None): @contextmanager def _control_stage(stage: str) -> Iterator[None]: - """Attach one fixed lifecycle stage to health or supervisor loss.""" + """Attach one fixed lifecycle stage to a managed-control failure.""" if stage not in CONTROL_STAGES: raise AssertionError(f"unknown managed-control stage: {stage}") @@ -183,11 +183,14 @@ def _control_stage(stage: str) -> Iterator[None]: yield except ControlError as error: if ( - error.code in ("GATEWAY_HEALTH_TIMEOUT", "SUPERVISOR_UNAVAILABLE") + error.code + in ("GATEWAY_FAILED", "GATEWAY_HEALTH_TIMEOUT", "SUPERVISOR_UNAVAILABLE") and error.stage is None ): error.stage = stage raise + except (OSError, subprocess.SubprocessError) as error: + raise ControlError("GATEWAY_FAILED", stage=stage) from error @dataclass(frozen=True) @@ -1417,8 +1420,17 @@ def _http_healthy( return False finally: if response is not None: - response.close() - connection.close() + try: + response.close() + except OSError: + # The response has already been bounded and fully read. A + # transport-close race must not escape the health probe and + # abort the entire managed restart as a generic failure. + pass + try: + connection.close() + except OSError: + pass def _http_healthy_in_gateway_namespace( @@ -1454,6 +1466,16 @@ def _http_healthy_in_gateway_namespace( return False if _recovery_deadline_reached(recovery_deadline): return False + current_namespace_stat = os.fstat(current_namespace) + target_namespace_stat = os.fstat(target_namespace) + if ( + current_namespace_stat.st_dev == target_namespace_stat.st_dev + and current_namespace_stat.st_ino == target_namespace_stat.st_ino + ): + return bool( + _http_healthy(port, path, recovery_deadline) + and not _recovery_deadline_reached(recovery_deadline) + ) setns(target_namespace, getattr(os, "CLONE_NEWNET", 0x40000000)) switched = True healthy = _http_healthy(port, path, recovery_deadline) @@ -2252,7 +2274,11 @@ def main(argv: list[str]) -> int: print(error.code, file=sys.stderr) if error.stage is not None: print(f"NEMOCLAW_CONTROL_STAGE={error.stage}", file=sys.stderr) - if error.code in ("GATEWAY_HEALTH_TIMEOUT", "SUPERVISOR_UNAVAILABLE"): + if error.code in ( + "GATEWAY_FAILED", + "GATEWAY_HEALTH_TIMEOUT", + "SUPERVISOR_UNAVAILABLE", + ): try: diagnostics = _managed_failure_diagnostics() except Exception: # diagnostics must not hide the original failure diff --git a/scripts/nemoclaw-start.sh b/scripts/nemoclaw-start.sh index e9b1b3c1be5..3da0aca348c 100755 --- a/scripts/nemoclaw-start.sh +++ b/scripts/nemoclaw-start.sh @@ -1759,6 +1759,27 @@ def rewrite(value): if count: refreshed.add(key) value = updated + alias_index = value.find(alias_marker) + if alias_index > 0: + alias_suffix = value[alias_index + len(alias_marker) :] + for env_key in keys: + if alias_suffix != env_key and not re.fullmatch( + rf"v[0-9]{{1,20}}_{re.escape(env_key)}", alias_suffix + ): + continue + runtime_value = os.environ.get(env_key, "") + if not runtime_value.startswith(prefix): + continue + runtime_suffix = runtime_value[len(prefix) :] + if runtime_suffix != env_key and not re.fullmatch( + rf"v[0-9]{{1,20}}_{re.escape(env_key)}", runtime_suffix + ): + continue + updated = value[: alias_index + len(alias_marker)] + runtime_suffix + if updated != value: + refreshed.add(env_key) + value = updated + break return value if isinstance(value, list): return [rewrite(item) for item in value] @@ -2112,11 +2133,22 @@ import sys with open(sys.argv[1], encoding="utf-8") as handle: plan = json.load(handle) for alias in plan.get("envAliases", []): - if not re.search(alias["match"], os.environ.get(alias["envKey"], "")): + env_key = alias["envKey"] + runtime_value = os.environ.get(env_key, "") + if not re.search(alias["match"], runtime_value): continue + value = alias["value"] + marker = "-OPENSHELL-RESOLVE-ENV-" + placeholder_prefix = "openshell:resolve:env:" + if marker in value and runtime_value.startswith(placeholder_prefix): + runtime_suffix = runtime_value[len(placeholder_prefix) :] + if re.fullmatch(rf"v[0-9]{{1,20}}_{re.escape(env_key)}", runtime_suffix): + alias_suffix = value.split(marker, 1)[1] + if alias_suffix == env_key: + value = value.split(marker, 1)[0] + marker + runtime_suffix print("\t".join([ alias.get("targetEnvKey", alias["envKey"]), - alias["value"], + value, alias.get("message", ""), ])) PYMESSAGINGALIASES diff --git a/scripts/openclaw-config-guard.py b/scripts/openclaw-config-guard.py index 9e57df15f27..9db6fff7ce7 100755 --- a/scripts/openclaw-config-guard.py +++ b/scripts/openclaw-config-guard.py @@ -1728,6 +1728,14 @@ def _clear_secondary_journal(identity: Identity) -> None: def _open_config(config_path: str) -> OpenConfig: + """Pin the exact OpenClaw state root, including its managed-volume form. + + SOURCE_OF_TRUTH_REVIEW + The host runtime validates and mounts the declared ``/sandbox/.openclaw`` + state root. A different device at that exact root is therefore expected; + descriptor-relative traversal below it remains bound to ``config_stat`` + and continues to reject symlinks, races, hardlinks, and nested devices. + """ normalized = posixpath.normpath(config_path) parent_path = posixpath.dirname(normalized) config_name = posixpath.basename(normalized) @@ -1755,7 +1763,10 @@ def _open_config(config_path: str) -> OpenConfig: raise GuardError( "entry-raced", normalized, "config directory changed while opening" ) - if config_stat.st_dev != parent_stat.st_dev: + if ( + config_stat.st_dev != parent_stat.st_dev + and normalized != PRODUCTION_CONFIG_DIR + ): os.close(config_fd) raise GuardError( "cross-device-entry", @@ -1810,6 +1821,7 @@ def _open_config_for_lock(config_path: str, identity: Identity) -> OpenConfig: before is not None and stat.S_ISDIR(before.st_mode) and before.st_dev != original_parent.st_dev + and normalized != PRODUCTION_CONFIG_DIR ): if not already_protected: os.fchown(parent_fd, identity.root_uid, identity.sandbox_gid) @@ -1834,6 +1846,7 @@ def _open_config_for_lock(config_path: str, identity: Identity) -> OpenConfig: before is not None and stat.S_ISDIR(before.st_mode) and before.st_dev != original_parent.st_dev + and normalized != PRODUCTION_CONFIG_DIR ): raise GuardError( "cross-device-entry", diff --git a/scripts/runtime_state_mutation_hermes_publisher.py b/scripts/runtime_state_mutation_hermes_publisher.py index 1efc3b5e842..a5459f70f70 100755 --- a/scripts/runtime_state_mutation_hermes_publisher.py +++ b/scripts/runtime_state_mutation_hermes_publisher.py @@ -882,6 +882,10 @@ def _verify_state_posture(posture: str, plan_json: str) -> None: plan = guard.parse_agent_state_lock_plan(plan_json) identity = guard._production_identity() if posture == "mutable": + try: + gateway_uid = pwd.getpwnam("gateway").pw_uid + except KeyError: + _fail("publisher-state-posture-invalid") result = guard.run_guard( "verify-mutable", HERMES_DIR, @@ -890,6 +894,7 @@ def _verify_state_posture(posture: str, plan_json: str) -> None: mutable_top_level_files=tuple( os.path.join(HERMES_DIR, name) for name in TOP_SELECTORS ), + mutable_service_uids=(gateway_uid,), ) if not result.ok: _fail("publisher-state-posture-invalid") diff --git a/scripts/state-dir-guard.py b/scripts/state-dir-guard.py index 56c563cf68a..15aff86216e 100755 --- a/scripts/state-dir-guard.py +++ b/scripts/state-dir-guard.py @@ -84,7 +84,15 @@ FS_IOC_GETFLAGS = 0x80086601 FS_IOC_SETFLAGS = 0x40086602 -Action = Literal["preflight", "lock", "unlock", "verify-mutable", "startup"] +Action = Literal[ + "preflight", + "lock", + "unlock", + "verify-lock", + "verify-unlock", + "verify-mutable", + "startup", +] Policy = Literal["high-risk", "confidentiality"] @@ -1833,7 +1841,59 @@ def _verify_metadata( identity: Identity, is_confidentiality_root: bool = False, allow_openclaw_native_mutable: bool = False, + mutable_service_uids: frozenset[int] = frozenset(), ) -> Issue | None: + if action == "unlock" and policy == "confidentiality" and mutable_service_uids: + if st.st_uid != identity.sandbox_uid or st.st_gid != identity.sandbox_gid: + return Issue( + "verification-owner-mismatch", + path, + f"owner is {st.st_uid}:{st.st_gid}, " + f"expected {identity.sandbox_uid}:{identity.sandbox_gid}", + ) + if entry_type == "symlink": + return None + mode = stat.S_IMODE(st.st_mode) + accepted_modes = (0o700, 0o2770) if entry_type == "directory" else (0o600, 0o660) + if mode not in accepted_modes: + expected = " or ".join(f"{accepted:04o}" for accepted in accepted_modes) + return Issue( + "verification-mode-mismatch", + path, + f"confidential service-mutable {entry_type} mode is {mode:04o}, " + f"expected {expected}", + ) + return None + if action == "unlock" and policy == "high-risk" and mutable_service_uids: + allowed_uids = frozenset((identity.sandbox_uid, *mutable_service_uids)) + if st.st_uid not in allowed_uids or st.st_gid != identity.sandbox_gid: + expected_uids = " or ".join(str(uid) for uid in sorted(allowed_uids)) + return Issue( + "verification-owner-mismatch", + path, + f"owner is {st.st_uid}:{st.st_gid}, expected uid {expected_uids} " + f"with gid {identity.sandbox_gid}", + ) + if entry_type == "symlink": + return None + mode = stat.S_IMODE(st.st_mode) + if entry_type == "directory": + if mode & 0o4000 or mode & 0o002 or mode & 0o700 != 0o700: + return Issue( + "verification-mode-mismatch", + path, + f"service-mutable directory must be owner-accessible and not " + f"world-writable: {mode:04o}", + ) + return None + if mode & 0o7000 or mode & 0o002 or mode & 0o400 != 0o400: + return Issue( + "verification-mode-mismatch", + path, + f"service-mutable file must be owner-readable, free of special bits, " + f"and not world-writable: {mode:04o}", + ) + return None expected_uid, expected_gid = _expected_ids( policy, action, identity, is_confidentiality_root ) @@ -2133,6 +2193,7 @@ def _verify_dir( depth: int, is_root: bool = False, verify_mutation_flags: bool = False, + mutable_service_uids: frozenset[int] = frozenset(), ) -> None: if depth > MAX_TRAVERSAL_DEPTH: issues.append( @@ -2165,6 +2226,7 @@ def _verify_dir( action == "unlock" and policy == "high-risk" and context.is_openclaw_native_mutable_path(relative_dir), + mutable_service_uids, ) if dir_issue is not None: issues.append(dir_issue) @@ -2242,6 +2304,7 @@ def _verify_dir( issues, depth + 1, verify_mutation_flags=verify_mutation_flags, + mutable_service_uids=mutable_service_uids, ) if verify_mutation_flags: after = os.stat(name, dir_fd=dir_fd, follow_symlinks=False) @@ -2276,6 +2339,7 @@ def _verify_dir( and policy == "high-risk" and context.is_openclaw_native_mutable_path(relative_path) ), + mutable_service_uids=mutable_service_uids, ) if metadata_issue is not None: issues.append(metadata_issue) @@ -2461,6 +2525,7 @@ def _run_guard_unserialized( identity: Identity, plan: AgentStateLockPlan, mutable_top_level_files: tuple[str, ...] = (), + mutable_service_uids: frozenset[int] = frozenset(), ) -> GuardResult: """Run one guard action. ``identity`` is explicit for focused tests.""" @@ -2478,6 +2543,14 @@ def _run_guard_unserialized( return _restore_empty_credentials_startup_access( normalized_config, identity, deadline, plan ) + posture_action: Action = ( + "lock" + if action == "verify-lock" + else "unlock" + if action in ("verify-unlock", "verify-mutable") + else action + ) + verify_only = action in ("verify-lock", "verify-unlock", "verify-mutable") fail_closed_config_root = action == "lock" and ( normalized_config in PRODUCTION_FAIL_CLOSED_CONFIG_DIRS or os.environ.get("NEMOCLAW_TEST_OPENCLAW_FAIL_CLOSED") == "1" @@ -2520,7 +2593,7 @@ def _run_guard_unserialized( normalized_config, config_st.st_dev, deadline, - action, + posture_action, plan, ) result.roots = len(roots) @@ -2555,13 +2628,11 @@ def _run_guard_unserialized( plan.writable_subpaths, ) replaced_inodes: dict[str, int] = {} - if action != "verify-mutable": + if not verify_only: for root in roots: path = context.display(root.name) try: - root_lstat = os.stat( - root.name, dir_fd=config_fd, follow_symlinks=False - ) + root_lstat = os.stat(root.name, dir_fd=config_fd, follow_symlinks=False) context.budget.observe_entry(path, root_lstat) if root_lstat.st_dev != root.dev or root_lstat.st_ino != root.ino: raise GuardOperationError( @@ -2578,7 +2649,7 @@ def _run_guard_unserialized( root_fd, root.name, root.policy, - action, + posture_action, identity, result, replaced_inodes, @@ -2614,9 +2685,6 @@ def _run_guard_unserialized( ) return result context.budget = WorkBudget(deadline) - verification_action: Action = ( - "unlock" if action == "verify-mutable" else action - ) for root in verify_roots: path = context.display(root.name) try: @@ -2637,13 +2705,14 @@ def _run_guard_unserialized( root_fd, root.name, root.policy, - verification_action, + posture_action, identity, replaced_inodes, result.issues, 1, is_root=True, verify_mutation_flags=action == "verify-mutable", + mutable_service_uids=mutable_service_uids, ) if action == "verify-mutable": root_after = os.stat( @@ -2839,10 +2908,34 @@ def run_guard( *, transition_lock_fd: int | None = None, mutable_top_level_files: tuple[str, ...] = (), + mutable_service_uids: tuple[int, ...] = (), ) -> GuardResult: """Serialize production OpenClaw recursive transitions with its top guard.""" normalized_config = posixpath.normpath(config_dir) + normalized_service_uids = frozenset(mutable_service_uids) + if len(normalized_service_uids) != len(mutable_service_uids) or any( + type(uid) is not int or uid <= 0 for uid in normalized_service_uids + ): + result = GuardResult(action=action) + result.issues.append( + Issue( + "invalid-mutable-service-owner", + normalized_config, + "mutable service owner UIDs must be distinct positive integers", + ) + ) + return result + if normalized_service_uids and action != "verify-mutable": + result = GuardResult(action=action) + result.issues.append( + Issue( + "invalid-mutable-service-owner", + normalized_config, + "mutable service owners are valid only for verify-mutable", + ) + ) + return result if action == "verify-mutable": return _run_guard_unserialized( action, @@ -2850,6 +2943,7 @@ def run_guard( identity, plan, mutable_top_level_files, + normalized_service_uids, ) lock_path = _transition_lock_path(normalized_config) if transition_lock_fd is not None: @@ -2925,7 +3019,15 @@ def _parse_args(argv: list[str]) -> argparse.Namespace: ) parser.add_argument( "action", - choices=("preflight", "lock", "unlock", "verify-mutable", "startup"), + choices=( + "preflight", + "lock", + "unlock", + "verify-lock", + "verify-unlock", + "verify-mutable", + "startup", + ), ) parser.add_argument("--config-dir", required=True) plan_source = parser.add_mutually_exclusive_group() @@ -2933,6 +3035,7 @@ def _parse_args(argv: list[str]) -> argparse.Namespace: plan_source.add_argument("--plan-file") parser.add_argument("--transition-lock-fd", type=int, help=argparse.SUPPRESS) parser.add_argument("--mutable-top-level-file", action="append", default=[]) + parser.add_argument("--mutable-service-user", action="append", default=[]) return parser.parse_args(argv) @@ -2998,14 +3101,30 @@ def main(argv: list[str] | None = None) -> int: ) ) else: - result = run_guard( - args.action, - args.config_dir, - identity, - plan, - transition_lock_fd=args.transition_lock_fd, - mutable_top_level_files=tuple(args.mutable_top_level_file), - ) + mutable_service_uids: tuple[int, ...] = () + try: + mutable_service_uids = tuple( + pwd.getpwnam(name).pw_uid for name in args.mutable_service_user + ) + except KeyError as exc: + result = GuardResult(action=args.action) + result.issues.append( + Issue( + "identity-unavailable", + args.config_dir, + f"mutable service account is unavailable: {exc}", + ) + ) + else: + result = run_guard( + args.action, + args.config_dir, + identity, + plan, + transition_lock_fd=args.transition_lock_fd, + mutable_top_level_files=tuple(args.mutable_top_level_file), + mutable_service_uids=mutable_service_uids, + ) for issue in result.issues: print(json.dumps(issue.as_json(), sort_keys=True, separators=(",", ":"))) diff --git a/src/lib/actions/inference-set-compatible-provider.test.ts b/src/lib/actions/inference-set-compatible-provider.test.ts index d6bc4befe96..ba727132ca3 100644 --- a/src/lib/actions/inference-set-compatible-provider.test.ts +++ b/src/lib/actions/inference-set-compatible-provider.test.ts @@ -1151,7 +1151,12 @@ describe("runInferenceSet compatible providers", () => { // rewriteConfigUrlsWithDnsPinning, so its real SSRF preflight is // exercised here too, with the same injected DNS lookup. ensureHttpsPinRuntimeAdapter: (adapterOptions) => - realEnsureHttpsPinRuntimeAdapter({ ...adapterOptions, lookup }), + realEnsureHttpsPinRuntimeAdapter({ + ...adapterOptions, + lookup, + discoverAllowedSourceCidrs: + adapterOptions.discoverAllowedSourceCidrs ?? (() => ["172.18.0.0/16"]), + }), }); await expect( diff --git a/src/lib/actions/inference-set-provider.ts b/src/lib/actions/inference-set-provider.ts index 2d1a8cc855b..b98cf916b18 100644 --- a/src/lib/actions/inference-set-provider.ts +++ b/src/lib/actions/inference-set-provider.ts @@ -14,6 +14,7 @@ import { import { assertHermesPortableCommandUnavailable } from "../onboard/experimental/portable-agent-lifecycle"; import { CURRENT_RUNTIME_PROVIDER_BUNDLES, + type RuntimeProviderBundle, type RuntimeProviderBundleRegistry, RuntimeProviderSelectionError, requireRuntimeProviderBundleForSandbox, @@ -106,9 +107,10 @@ export async function probeInferenceSetSandboxRouteUntilConverged( export function requireInferenceSetRuntimeAuthority( entry: SandboxEntry, providers: RuntimeProviderBundleRegistry = CURRENT_RUNTIME_PROVIDER_BUNDLES, -): void { +): RuntimeProviderBundle { const runtimeProvider = requireRuntimeProviderBundleForSandbox(entry, providers); requireRuntimeProviderMutationAuthority(runtimeProvider, "inference-set"); + return runtimeProvider; } export function assertInferenceSetCommandAvailable(sandboxName: string): void { diff --git a/src/lib/actions/inference-set-route-containment.ts b/src/lib/actions/inference-set-route-containment.ts index 2b13bd9ea5a..1396339c8a2 100644 --- a/src/lib/actions/inference-set-route-containment.ts +++ b/src/lib/actions/inference-set-route-containment.ts @@ -60,6 +60,7 @@ export interface EnsureHttpsPinRuntimeAdapterOptions { endpointUrl: string; providerType: HttpsPinCredentialProviderType; credentialValue: string; + discoverAllowedSourceCidrs?: () => readonly string[]; } export type EnsureHttpsPinRuntimeAdapterFn = ( options: EnsureHttpsPinRuntimeAdapterOptions, diff --git a/src/lib/actions/inference-set.ts b/src/lib/actions/inference-set.ts index edb26b9056f..e246e566230 100644 --- a/src/lib/actions/inference-set.ts +++ b/src/lib/actions/inference-set.ts @@ -276,7 +276,15 @@ function defaultDeps(): InferenceSetDeps { resolveContextWindowForModel, rewriteConfigUrlsWithDnsPinning, resolveCredentialValue: (credentialEnv) => process.env[credentialEnv] ?? "", - ensureHttpsPinRuntimeAdapter, + ensureHttpsPinRuntimeAdapter: (options) => { + if (!options.discoverAllowedSourceCidrs) { + throw new Error("HTTPS Pin Runtime adapter is missing runtime-provider network authority."); + } + return ensureHttpsPinRuntimeAdapter({ + ...options, + discoverAllowedSourceCidrs: options.discoverAllowedSourceCidrs, + }); + }, revokeHttpsPinRuntimeAdapterRoute, probeSandboxRoute: probeInferenceSetSandboxRoute, sleep: sleepInferenceSetRouteConvergence, @@ -307,9 +315,9 @@ function assertSupportedProvider(provider: string, model: string): void { function assertInferenceSetRuntimeAuthority( entry: SandboxEntry, providers: RuntimeProviderBundleRegistry | undefined, -): void { +): ReturnType { try { - requireInferenceSetRuntimeAuthority(entry, providers); + return requireInferenceSetRuntimeAuthority(entry, providers); } catch (error) { if (!(error instanceof RuntimeProviderSelectionError)) throw error; throw new InferenceSetError(error.message, 2); @@ -786,6 +794,7 @@ async function runInferenceSetWithoutHostLock( options: InferenceSetOptions, deps: InferenceSetDeps, expectedGatewayName: string, + runtimeProvider: ReturnType, ): Promise> { // #6321: accept the installer-style provider name onboard uses (e.g. // `anthropicCompatible`) as well as the OpenShell provider name, by @@ -938,7 +947,14 @@ async function runInferenceSetWithoutHostLock( getSandboxes: () => deps.listSandboxes().sandboxes, rewriteUrlWithDnsPinning: deps.rewriteConfigUrlsWithDnsPinning, resolveCredentialValue: deps.resolveCredentialValue, - ensureHttpsPinRuntimeAdapter: deps.ensureHttpsPinRuntimeAdapter, + ensureHttpsPinRuntimeAdapter: (adapterOptions) => + deps.ensureHttpsPinRuntimeAdapter({ + ...adapterOptions, + discoverAllowedSourceCidrs: () => + runtimeProvider.gateway + .prepareHostRuntime({ environment: process.env, platform: process.platform }) + .network.sandboxSourceCidrs(), + }), effectiveInferenceApi: preparedRoute.preliminaryExplicitMetadata?.preferredInferenceApi ?? null, }); @@ -1521,7 +1537,10 @@ export async function runInferenceSet( return withSandboxMutationLock(selected.sandboxName, async () => { assertInferenceSetCommandAvailable(selected.sandboxName); const lockedSelection = resolveTargetSandbox(selected.sandboxName, deps); - assertInferenceSetRuntimeAuthority(lockedSelection.entry, deps.runtimeProviders); + const runtimeProvider = assertInferenceSetRuntimeAuthority( + lockedSelection.entry, + deps.runtimeProviders, + ); const gatewayName = resolveSandboxGatewayName(lockedSelection.entry); const mutation = await deps.withGatewayRouteMutationLock(gatewayName, () => withTimerBoundShieldsMutationLockAsync(selected.sandboxName, "inference set", () => @@ -1529,6 +1548,7 @@ export async function runInferenceSet( { ...options, sandboxName: selected.sandboxName }, deps, gatewayName, + runtimeProvider, ), ), ); diff --git a/src/lib/actions/maintenance.test.ts b/src/lib/actions/maintenance.test.ts index c73ccee09d4..b374831b353 100644 --- a/src/lib/actions/maintenance.test.ts +++ b/src/lib/actions/maintenance.test.ts @@ -318,6 +318,7 @@ describe("backupAll", () => { readySandboxNames = new Set(); mocks.startStoppedSandboxContainerForBackup.mockReturnValue({ containerName: "openshell-sb-stopped-abc", + runtimeProviderId: "docker", }); mocks.withSandboxMutationLock.mockRejectedValueOnce( new Error("Timed out waiting for the sandbox mutation lock"), @@ -696,7 +697,9 @@ describe("backupAll", () => { manifest: { backupPath: "/backups/sb-good/timestamp" }, }); mocks.startStoppedSandboxContainerForBackup.mockImplementation((name: string) => - name === "sb-stopped" ? { containerName: "openshell-sb-stopped-abc" } : null, + name === "sb-stopped" + ? { containerName: "openshell-sb-stopped-abc", runtimeProviderId: "docker" } + : null, ); mocks.backupStartedSandboxState.mockResolvedValue({ success: true, @@ -717,7 +720,10 @@ describe("backupAll", () => { expect(exitSpy).not.toHaveBeenCalled(); expect(mocks.backupStartedSandboxState).toHaveBeenCalledWith("sb-stopped"); expect(mocks.backupSandboxState).toHaveBeenCalledWith("sb-good"); - expect(mocks.returnSandboxContainerToStopped).toHaveBeenCalledWith("openshell-sb-stopped-abc"); + expect(mocks.returnSandboxContainerToStopped).toHaveBeenCalledWith({ + containerName: "openshell-sb-stopped-abc", + runtimeProviderId: "docker", + }); expect(mocks.relockBackupShieldsWindow.mock.invocationCallOrder.at(-1)!).toBeLessThan( mocks.returnSandboxContainerToStopped.mock.invocationCallOrder.at(-1)!, ); @@ -752,7 +758,7 @@ describe("backupAll", () => { mocks.startStoppedSandboxContainerForBackup.mockImplementation((name: string) => { expect(lockActive).toBe(true); events.push(`start:${name}`); - return { containerName: "openshell-sb-stopped-abc" }; + return { containerName: "openshell-sb-stopped-abc", runtimeProviderId: "docker" }; }); mocks.openBackupShieldsWindow.mockImplementation((name: string) => { expect(lockActive).toBe(true); @@ -807,6 +813,7 @@ describe("backupAll", () => { readySandboxNames = new Set(); mocks.startStoppedSandboxContainerForBackup.mockReturnValue({ containerName: "openshell-sb-stopped-abc", + runtimeProviderId: "docker", }); mocks.backupStartedSandboxState.mockResolvedValue({ success: false, @@ -825,7 +832,10 @@ describe("backupAll", () => { await expect(backupAll()).rejects.toThrow("exit:1"); - expect(mocks.returnSandboxContainerToStopped).toHaveBeenCalledWith("openshell-sb-stopped-abc"); + expect(mocks.returnSandboxContainerToStopped).toHaveBeenCalledWith({ + containerName: "openshell-sb-stopped-abc", + runtimeProviderId: "docker", + }); expect(logSpy.mock.calls.flat().join("\n")).toContain("0 backed up, 1 failed, 0 skipped"); expect(errorSpy.mock.calls.flat().join("\n")).toContain( "backup failed (identity (permission denied))", @@ -840,6 +850,7 @@ describe("backupAll", () => { readySandboxNames = new Set(); mocks.startStoppedSandboxContainerForBackup.mockReturnValue({ containerName: "openshell-sb-stopped-abc", + runtimeProviderId: "docker", }); mocks.backupStartedSandboxState.mockResolvedValue({ success: true, @@ -881,6 +892,7 @@ describe("backupAll", () => { ); mocks.startStoppedSandboxContainerForBackup.mockReturnValue({ containerName: "openshell-sb-stopped-abc", + runtimeProviderId: "docker", }); mocks.openBackupShieldsWindow.mockReturnValue({ relocked: false, wasLocked: true }); mocks.backupStartedSandboxState.mockResolvedValue({ @@ -913,7 +925,10 @@ describe("backupAll", () => { expect(lockActive).toBe(false); expect(mocks.withSandboxMutationLock).toHaveBeenCalledOnce(); expect(mocks.relockBackupShieldsWindow).toHaveBeenCalledOnce(); - expect(mocks.returnSandboxContainerToStopped).toHaveBeenCalledWith("openshell-sb-stopped-abc"); + expect(mocks.returnSandboxContainerToStopped).toHaveBeenCalledWith({ + containerName: "openshell-sb-stopped-abc", + runtimeProviderId: "docker", + }); expect(mocks.openBackupShieldsWindow).toHaveBeenCalledOnce(); }); @@ -925,6 +940,7 @@ describe("backupAll", () => { readySandboxNames = new Set(); mocks.startStoppedSandboxContainerForBackup.mockReturnValue({ containerName: "openshell-sb-stopped-abc", + runtimeProviderId: "docker", }); mocks.backupStartedSandboxState.mockRejectedValue( new Error("Agent 'sb-stopped' not found: /path/to/manifest.yaml"), @@ -933,7 +949,10 @@ describe("backupAll", () => { await backupAll(); - expect(mocks.returnSandboxContainerToStopped).toHaveBeenCalledWith("openshell-sb-stopped-abc"); + expect(mocks.returnSandboxContainerToStopped).toHaveBeenCalledWith({ + containerName: "openshell-sb-stopped-abc", + runtimeProviderId: "docker", + }); const output = logSpy.mock.calls.flat().join("\n"); expect(output).toContain("Returned 'sb-stopped' to its stopped state"); expect(output).toContain("Skipped 'sb-stopped' (orphan manifest)"); diff --git a/src/lib/actions/maintenance.ts b/src/lib/actions/maintenance.ts index 5cb471ffa40..bf2229fa98c 100644 --- a/src/lib/actions/maintenance.ts +++ b/src/lib/actions/maintenance.ts @@ -108,7 +108,7 @@ function returnStartedSandboxToStopped( "could not return its container to the stopped state; the container was left running"; const failureMessage = `Backup cleanup failed for '${sandboxName}': ${failureDetail}.`; try { - if (returnSandboxContainerToStopped(startedForBackup.containerName)) { + if (returnSandboxContainerToStopped(startedForBackup)) { console.log(` ${D}Returned '${sandboxName}' to its stopped state.${R}`); return null; } diff --git a/src/lib/actions/sandbox/connect-flow.test.ts b/src/lib/actions/sandbox/connect-flow.test.ts index b22400b8faf..cafdf0ba1a0 100644 --- a/src/lib/actions/sandbox/connect-flow.test.ts +++ b/src/lib/actions/sandbox/connect-flow.test.ts @@ -320,7 +320,7 @@ describe("connectSandbox flow", () => { expect(exitSpy).toHaveBeenCalledWith(255); }); - it("prints the terminal launch command in the connect hint for terminal agents", async () => { + it("prints a credential-safe connect hint for terminal agents", async () => { const harness = createConnectHarness({ agentName: "langchain-deepagents-code", sessionAgent: { @@ -332,8 +332,9 @@ describe("connectSandbox flow", () => { await expect(harness.connectSandbox("alpha")).rejects.toThrow("process.exit(0)"); const output = harness.logSpy.mock.calls.map((call) => String(call[0])).join("\n"); - expect(output).toContain("Inside the sandbox, run `dcode`"); - expect(output).not.toContain("Inside the sandbox, run `langchain-deepagents-code`"); + expect(output).toContain("Inside the sandbox, run the configured command to start chatting."); + expect(output).not.toContain("dcode"); + expect(output).not.toContain("langchain-deepagents-code"); expect(exitSpy).toHaveBeenCalledWith(0); }); diff --git a/src/lib/actions/sandbox/connect-inference-gateway.ts b/src/lib/actions/sandbox/connect-inference-gateway.ts index d55042cea5c..0834b10ea7a 100644 --- a/src/lib/actions/sandbox/connect-inference-gateway.ts +++ b/src/lib/actions/sandbox/connect-inference-gateway.ts @@ -7,9 +7,19 @@ import { isAdvisoryProviderModelRouteConflict, } from "../../inference/gateway-route-compatibility"; import { LOCAL_INFERENCE_TIMEOUT_SECS } from "../../onboard/env"; +import { resolveRegisteredRuntimeProvider } from "../../onboard/runtime-provider/selection"; import type { SandboxEntry } from "../../state/registry"; import * as registry from "../../state/registry"; +/** Identify the legacy cluster gateway without branching on managed provider IDs. */ +export function sandboxUsesLegacyClusterGateway(sandbox: SandboxEntry | null): boolean { + const driver = sandbox?.openshellDriver; + if (!driver) return true; + const provider = resolveRegisteredRuntimeProvider(driver); + if (provider) return provider.gateway.launcher !== "nemoclaw"; + return driver !== "vm"; +} + function sandboxGatewayRouteCompatibility( sandboxName: string, sb: SandboxEntry, diff --git a/src/lib/actions/sandbox/connect-probe-observe.test.ts b/src/lib/actions/sandbox/connect-probe-observe.test.ts index 86e06b81aa1..ee7e990cc48 100644 --- a/src/lib/actions/sandbox/connect-probe-observe.test.ts +++ b/src/lib/actions/sandbox/connect-probe-observe.test.ts @@ -104,7 +104,7 @@ describe("connectSandbox probe-only observe mode", () => { ); }, ); - expect(listInvocations).toHaveLength(2); + expect(listInvocations).toHaveLength(3); const listArgs = harness.captureOpenshellSpy.mock.calls .map((call) => call[0]) .filter( @@ -114,6 +114,7 @@ describe("connectSandbox probe-only observe mode", () => { expect(listArgs).toEqual([ ["sandbox", "list", "-g", "nemoclaw-8091"], ["sandbox", "list", "-g", "nemoclaw-8091"], + ["sandbox", "list", "-g", "nemoclaw-8091"], ]); const liveLookupOrder = harness.ensureLiveSandboxSpy.mock.invocationCallOrder; expect(liveLookupOrder).toHaveLength(2); @@ -121,6 +122,10 @@ describe("connectSandbox probe-only observe mode", () => { expect(recoveryOrder).toHaveLength(1); expect(listInvocations[1]).toBeLessThan(liveLookupOrder[1]); expect(liveLookupOrder[1]).toBeLessThan(recoveryOrder[0]); + expect(recoveryOrder[0]).toBeLessThan(listInvocations[2]!); + expect(listInvocations[2]).toBeLessThan( + harness.publishLaunchReadinessSpy.mock.invocationCallOrder[0]!, + ); expect(harness.logSpy).toHaveBeenCalledWith( expect.stringContaining("restored dashboard port forward"), ); @@ -167,7 +172,7 @@ describe("connectSandbox probe-only observe mode", () => { expect(harness.dockerStartSpy.mock.invocationCallOrder[0]!).toBeLessThan( listInvocations[0]!.order, ); - expect(listInvocations).toHaveLength(3); + expect(listInvocations).toHaveLength(4); expect(exitSpy).not.toHaveBeenCalled(); }); diff --git a/src/lib/actions/sandbox/connect-route-repair.test.ts b/src/lib/actions/sandbox/connect-route-repair.test.ts index e0dd43801ff..3137eabfd7d 100644 --- a/src/lib/actions/sandbox/connect-route-repair.test.ts +++ b/src/lib/actions/sandbox/connect-route-repair.test.ts @@ -185,22 +185,25 @@ describe("sandbox connect route repair unit flow", () => { expect(calls.reapplications).toEqual([]); }); - it("uses inference route reapply instead of legacy DNS repair for docker sandboxes", () => { - const { calls, deps } = makeRepairDeps([broken(), healthy()]); - - const result = repairSandboxInferenceRouteWithDeps( - "docker-box", - sandbox({ openshellDriver: "docker" }), - {}, - deps, - ); - - expect(result.healthy).toBe(true); - expect(result.repairAttempted).toBe(true); - expect(calls.legacyRepairs).toEqual([]); - expect(calls.reapplications).toEqual(["docker-box"]); - expect(calls.logs).toContain(" inference.local route repaired."); - }); + it.each(["docker", "podman"])( + "uses inference route reapply instead of legacy DNS repair for %s sandboxes", + (driver) => { + const { calls, deps } = makeRepairDeps([broken(), healthy()]); + + const result = repairSandboxInferenceRouteWithDeps( + `${driver}-box`, + sandbox({ openshellDriver: driver }), + {}, + deps, + ); + + expect(result.healthy).toBe(true); + expect(result.repairAttempted).toBe(true); + expect(calls.legacyRepairs).toEqual([]); + expect(calls.reapplications).toEqual([`${driver}-box`]); + expect(calls.logs).toContain(" inference.local route repaired."); + }, + ); it("lets the VM monkeypatch satisfy the route before inference reapply", () => { const { calls, deps } = makeRepairDeps([broken(), healthy()], { diff --git a/src/lib/actions/sandbox/connect.ts b/src/lib/actions/sandbox/connect.ts index 89ad53e897d..c964c21abe8 100644 --- a/src/lib/actions/sandbox/connect.ts +++ b/src/lib/actions/sandbox/connect.ts @@ -69,6 +69,7 @@ import { prepareHermesLightTerminalSkin } from "./connect-hermes-light-skin"; import { assertSandboxGatewayRouteCompatible, buildGatewayInferenceSetArgs, + sandboxUsesLegacyClusterGateway, } from "./connect-inference-gateway"; import { buildSandboxInferenceRouteProbeArgs, @@ -1148,10 +1149,8 @@ function shouldUseLegacyDnsProxyRepair(sb: SandboxEntry | null): boolean { // runs the gateway as `nemoclaw-openshell-gateway` with host networking, and // the vm driver has no cluster container either, so both recover the route via // `openshell inference set` instead of the cluster CoreDNS patch. Mirrors - // usesGatewayMetadataProbe (snapshot.ts) and the `!== "docker"` guard on the - // snapshot DNS-proxy step. (#3403) - const driver = sb?.openshellDriver; - return driver !== "vm" && driver !== "docker"; + // usesGatewayMetadataProbe (snapshot.ts). (#3403) + return sandboxUsesLegacyClusterGateway(sb); } function reapplyVmInferenceRoute( @@ -2679,6 +2678,16 @@ async function prepareConnectSandboxWithinLifecycleFence( probeOnly: true, probeTiming, }); + if (!hermesPortable) { + probeTiming!.measure("gateway", () => + waitForSandboxReadyOrExit(sandboxName, { + allowInitialErrorAfterStart: true, + allowDockerRuntimeInspection: false, + defaultTimeoutSec: SANDBOX_REPAIR_READY_TIMEOUT_SEC, + retryCommand: "connect --probe-only", + }), + ); + } requalify(); }, }); @@ -2797,11 +2806,9 @@ async function prepareConnectSandboxWithinLifecycleFence( console.log(""); // Same resolver `launch` uses, so the hint cannot drift from the command // that `nemoclaw launch ` actually runs (#6006). - const agentCmd = agentRuntime.getInteractiveAgentCommand(agent, sb?.agent); + void agentRuntime.getInteractiveAgentCommand(agent, sb?.agent); console.log(` ${G}✓${R} Connecting to sandbox '${sandboxName}'`); - console.log( - ` ${D}Inside the sandbox, run \`${agentCmd}\` to start chatting with the agent.${R}`, - ); + console.log(` ${D}Inside the sandbox, run the configured command to start chatting.${R}`); console.log( ` ${D}Type \`/exit\` to leave the chat, then \`exit\` to return to the host shell.${R}`, ); diff --git a/src/lib/actions/sandbox/destroy-container-identity.test.ts b/src/lib/actions/sandbox/destroy-container-identity.test.ts index a0d3b524557..4c1db1a033f 100644 --- a/src/lib/actions/sandbox/destroy-container-identity.test.ts +++ b/src/lib/actions/sandbox/destroy-container-identity.test.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it, vi } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { fingerprintOpenShellSandboxId } from "../../adapters/openshell/sandbox-identity"; import { @@ -52,6 +52,10 @@ function expectAmbiguous( } describe("classifyDestroyContainerIdentity", () => { + afterEach(() => { + vi.unstubAllEnvs(); + }); + it("is clear when no container carries the sandbox-name label", () => { expect(classifyDestroyContainerIdentity("destroytest", observeRows([]))).toEqual({ status: "clear", @@ -66,6 +70,15 @@ describe("classifyDestroyContainerIdentity", () => { }); }); + it("does not treat native Podman ownership as Docker compatibility", () => { + const podmanManaged = { ...MANAGED, managedBy: "true" }; + + expect( + expectAmbiguous(classifyDestroyContainerIdentity("destroytest", observeRows([podmanManaged]))) + .foreign, + ).toEqual([podmanManaged]); + }); + it("refuses when a foreign container shares the sandbox-name label (#8999)", () => { // The exact repro: a real managed sandbox plus a busybox that borrows the // sandbox-name label with a different workspace and no managed marker. diff --git a/src/lib/actions/sandbox/destroy-execution.ts b/src/lib/actions/sandbox/destroy-execution.ts index e2d454681d6..2a1ef90c100 100644 --- a/src/lib/actions/sandbox/destroy-execution.ts +++ b/src/lib/actions/sandbox/destroy-execution.ts @@ -16,7 +16,9 @@ import { type RuntimeProviderBundle, type RuntimeProviderBundleRegistry, requireRuntimeProviderDestructiveCleanupAuthority, + resolveRuntimeProviderBundle, } from "../../onboard/runtime-provider/access"; +import type { RuntimeProviderDestroyIdentityReceipt } from "../../onboard/runtime-provider/contract"; import { type HostLocalInferenceLifecycleOptions, type PreparedHostLocalInferenceAuthority, @@ -75,6 +77,7 @@ type SandboxDestroyExecutionInput = { // Docker IDs qualified before destroy preparation. expectedContainerIdentities?: readonly SandboxNameLabeledContainer[]; expectedContainerIdentityFingerprint?: string; + expectedRuntimeProviderIdentity?: RuntimeProviderDestroyIdentityReceipt; portableContainerAuthority?: PreparedPortableDemoSandboxDestroyAuthority; stopInferenceResources: () => void; runtimeProviders?: RuntimeProviderBundleRegistry; @@ -307,6 +310,7 @@ export async function executeSandboxDestroy({ sandboxName, expectedContainerIdentities, expectedContainerIdentityFingerprint, + expectedRuntimeProviderIdentity, portableContainerAuthority, stopInferenceResources, runtimeProviders = CURRENT_RUNTIME_PROVIDER_BUNDLES, @@ -318,9 +322,16 @@ export async function executeSandboxDestroy({ | { status: "changed"; subject?: string } | { status: "ambiguous"; detail: string; subject?: string } | { status: "probe-failed"; detail: string; subject?: string }; + const identityProvider = resolveRuntimeProviderBundle( + sandbox?.openshellDriver ?? expectedRuntimeProviderIdentity?.providerId, + runtimeProviders, + ); const pendingCreateIdentity = sandbox?.pendingCreateIdentity; - const expectedContainerProof: DestroyContainerIdentityProof = - expectedContainerIdentities === undefined ? {} : { identities: expectedContainerIdentities }; + const expectedContainerProof: DestroyContainerIdentityProof = expectedRuntimeProviderIdentity + ? { identities: undefined, providerIdentity: expectedRuntimeProviderIdentity } + : expectedContainerIdentities === undefined + ? { identities: undefined } + : { identities: expectedContainerIdentities }; const proofFromVerdict = ( verdict: ReturnType, ): DestroyContainerIdentityProof | null => { @@ -389,6 +400,33 @@ export async function executeSandboxDestroy({ return { status: "probe-failed", detail: redactDestroyError(error) }; } } + if (expectedRuntimeProviderIdentity) { + if (identityProvider?.cleanup.supported !== true) { + return { + status: "probe-failed", + detail: "the selected runtime provider has no destroy identity observer", + }; + } + try { + const actual = sandbox + ? identityProvider.cleanup.captureDestroyIdentity?.({ sandbox, sandboxName }) + : identityProvider.cleanup.captureDestroyIdentityByName?.(sandboxName); + if (!actual) { + return { + status: "probe-failed", + detail: "the selected runtime provider has no destroy identity observer", + }; + } + return actual.schemaVersion === expectedRuntimeProviderIdentity.schemaVersion && + actual.providerId === expectedRuntimeProviderIdentity.providerId && + actual.resourceHandle === expectedRuntimeProviderIdentity.resourceHandle && + actual.ownershipSha256 === expectedRuntimeProviderIdentity.ownershipSha256 + ? { status: "match" } + : { status: "changed" }; + } catch (error) { + return { status: "probe-failed", detail: redactDestroyError(error) }; + } + } if (expectedContainerIdentities === undefined) return { status: "match" }; const verdict = classifyDestroyContainerIdentity( sandboxName, diff --git a/src/lib/actions/sandbox/destroy-flow.test.ts b/src/lib/actions/sandbox/destroy-flow.test.ts index 3cd7d7f513d..43632074051 100644 --- a/src/lib/actions/sandbox/destroy-flow.test.ts +++ b/src/lib/actions/sandbox/destroy-flow.test.ts @@ -549,18 +549,21 @@ describe("destroySandbox flow", () => { agent: "hermes", openshellDriver: "docker", workload: managedHermesWorkload, - managedHermesStateVolumeCleanupResult: { status: "removed" }, + managedAgentStateVolumeCleanupResults: [{ status: "removed" }], }); await expect(harness.destroySandbox("alpha", { yes: true })).resolves.toBeUndefined(); - expect(harness.removeManagedHermesStateVolumeSpy).toHaveBeenCalledWith({ - agentName: "hermes", - runtimeProviderId: "docker", - sandboxName: "alpha", - workloadKind: "managed-image", - }); - expect(harness.removeManagedHermesStateVolumeSpy.mock.invocationCallOrder[0]).toBeLessThan( + expect(harness.removeManagedAgentStateVolumesSpy).toHaveBeenCalledWith( + { + agentName: "hermes", + runtimeProviderId: "docker", + sandboxName: "alpha", + workloadKind: "managed-image", + }, + { runtimeProviders: expect.any(Object) }, + ); + expect(harness.removeManagedAgentStateVolumesSpy.mock.invocationCallOrder[0]).toBeLessThan( harness.removeSandboxSpy.mock.invocationCallOrder[0], ); }, @@ -571,11 +574,13 @@ describe("destroySandbox flow", () => { agent: "hermes", openshellDriver: "docker", workload: managedHermesWorkload, - managedHermesStateVolumeCleanupResult: { - status: "failed", - detail: "volume is still in use", - volumeName: "nemoclaw-hermes-state-v1-alpha", - }, + managedAgentStateVolumeCleanupResults: [ + { + status: "failed", + detail: "volume is still in use", + volumeName: "nemoclaw-hermes-state-v1-alpha", + }, + ], }); await expect(harness.destroySandbox("alpha", { yes: true })).rejects.toThrow("process.exit(1)"); @@ -591,17 +596,19 @@ describe("destroySandbox flow", () => { agent: "hermes", openshellDriver: "docker", workload: managedHermesWorkload, - managedHermesStateVolumeCleanupResult: { - status: "not-owned", - detail: "the exact NemoClaw ownership labels are absent or changed", - volumeName: "nemoclaw-hermes-state-v1-alpha", - }, + managedAgentStateVolumeCleanupResults: [ + { + status: "not-owned", + detail: "the exact NemoClaw ownership labels are absent or changed", + volumeName: "nemoclaw-hermes-state-v1-alpha", + }, + ], }); await expect(harness.destroySandbox("alpha", { yes: true })).resolves.toBeUndefined(); expect(harness.warnSpy.mock.calls.map((call) => String(call[0])).join("\n")).toContain( - "Left Docker volume 'nemoclaw-hermes-state-v1-alpha' untouched", + "Left managed state volume 'nemoclaw-hermes-state-v1-alpha' untouched", ); expect(harness.removeSandboxSpy).toHaveBeenCalledWith("alpha"); }); diff --git a/src/lib/actions/sandbox/destroy-presence.ts b/src/lib/actions/sandbox/destroy-presence.ts index 78ef7b10175..cb25a56b178 100644 --- a/src/lib/actions/sandbox/destroy-presence.ts +++ b/src/lib/actions/sandbox/destroy-presence.ts @@ -2,18 +2,23 @@ // SPDX-License-Identifier: Apache-2.0 import { - OPENSHELL_MANAGED_BY_LABEL, - OPENSHELL_MANAGED_BY_VALUE, OPENSHELL_SANDBOX_ID_LABEL, OPENSHELL_SANDBOX_NAME_LABEL, OPENSHELL_SANDBOX_WORKSPACE_LABEL, inspectDockerSandboxNameLabeledContainers, + resolveOpenShellSandboxOwnershipLabel, } from "../../onboard/openshell-docker-sandbox-containers"; import { fingerprintOpenShellSandboxId } from "../../adapters/openshell/sandbox-identity"; import { sanitizeReadinessText } from "../../readiness/sanitize"; +import type { SandboxEntry } from "../../state/registry"; +import type { RuntimeProviderDestroyIdentityReceipt } from "../../onboard/runtime-provider/contract"; import { type DockerSandboxIdentityObservation, } from "../../adapters/docker/inspect"; +import { + registeredRuntimeProviderSupportsContainerEngineOperation, + resolveRegisteredRuntimeProvider, +} from "../../onboard/runtime-provider/selection"; import { classifyOpenShellSandboxPresence, type OpenShellSandboxPresence, @@ -46,6 +51,12 @@ export type DestroyContainerIdentityVerdict = export type AssertUnambiguousDestroyIdentityDeps = { providerId: string; redact: (detail: string) => string; + sandbox?: SandboxEntry | null; + captureProviderIdentity?: ( + sandbox: SandboxEntry, + sandboxName: string, + ) => RuntimeProviderDestroyIdentityReceipt; + captureProviderIdentityByName?: (sandboxName: string) => RuntimeProviderDestroyIdentityReceipt; retainedSandboxIdentityFingerprint?: string; cliName?: string; classify?: ( @@ -60,8 +71,8 @@ export type DestroyContainerIdentityProof = { // array records confirmed Docker absence; other arrays contain the exact // immutable Docker identities qualified for this destroy operation. identities?: readonly SandboxNameLabeledContainer[]; + providerIdentity?: RuntimeProviderDestroyIdentityReceipt; }; - /** Read the host observation consumed by the pure identity classifier. */ export function observeDestroyContainerIdentity( sandboxName: string, @@ -89,8 +100,9 @@ export function classifyDestroyContainerIdentity( } const { malformedRows, rows } = observation; - const managed = rows.filter((row) => row.managedBy === OPENSHELL_MANAGED_BY_VALUE); - const foreign = rows.filter((row) => row.managedBy !== OPENSHELL_MANAGED_BY_VALUE); + const ownership = resolveOpenShellSandboxOwnershipLabel(); + const managed = rows.filter((row) => row.managedBy === ownership.value); + const foreign = rows.filter((row) => row.managedBy !== ownership.value); if (malformedRows > 0) { return { @@ -108,8 +120,7 @@ export function classifyDestroyContainerIdentity( sandboxName, reason: `${String(foreign.length)} container(s) carry the '${OPENSHELL_SANDBOX_NAME_LABEL}=` + - `${sandboxName}' label without the '${OPENSHELL_MANAGED_BY_LABEL}=` + - `${OPENSHELL_MANAGED_BY_VALUE}' marker`, + `${sandboxName}' label without the '${ownership.label}=${ownership.value}' marker`, foreign, managed, }; @@ -172,11 +183,12 @@ export function formatAmbiguousDestroyIdentity( verdict: Extract, cliName: string, ): string[] { + const ownership = resolveOpenShellSandboxOwnershipLabel(); const display = (value: string, fallback = ""): string => sanitizeReadinessText(value || fallback, IDENTITY_VALUE_MAX_LENGTH); const displayLabel = (value: string): string => JSON.stringify(display(value)); const describe = (row: SandboxNameLabeledContainer): string => - `${display(row.id).slice(0, 12)} (${OPENSHELL_MANAGED_BY_LABEL}=${displayLabel(row.managedBy)}, ` + + `${display(row.id).slice(0, 12)} (${ownership.label}=${displayLabel(row.managedBy)}, ` + `${OPENSHELL_SANDBOX_WORKSPACE_LABEL}=${displayLabel(row.workspace)}, ` + `${OPENSHELL_SANDBOX_ID_LABEL}=${displayLabel(row.sandboxId)})`; const sandboxName = display(verdict.sandboxName); @@ -205,6 +217,58 @@ export function assertUnambiguousDestroyContainerIdentity( sandboxName: string, deps: AssertUnambiguousDestroyIdentityDeps, ): DestroyContainerIdentityProof | false { + const provider = resolveRegisteredRuntimeProvider(deps.providerId); + const providerCapture = + provider?.cleanup.supported === true ? provider.cleanup.captureDestroyIdentity : undefined; + const captureProviderIdentity = + deps.captureProviderIdentity ?? + (providerCapture + ? (sandbox: SandboxEntry, name: string) => providerCapture({ sandbox, sandboxName: name }) + : undefined); + const captureProviderIdentityByName = + deps.captureProviderIdentityByName ?? + (provider?.cleanup.supported === true + ? provider.cleanup.captureDestroyIdentityByName + : undefined); + if ( + !provider || + !registeredRuntimeProviderSupportsContainerEngineOperation( + deps.providerId, + "gateway-inspection", + ) + ) { + return { identities: undefined }; + } + let providerOwnsIdentity = + deps.captureProviderIdentity !== undefined || deps.captureProviderIdentityByName !== undefined; + try { + providerOwnsIdentity ||= Boolean( + provider.gateway.prepareHostRuntime({ + environment: process.env, + platform: process.platform, + }).socketPath !== null, + ); + } catch { + return { identities: undefined }; + } + const error = deps.error ?? ((message: string) => console.error(` ${message}`)); + if (providerOwnsIdentity) { + try { + const providerIdentity = + deps.sandbox && captureProviderIdentity + ? captureProviderIdentity(deps.sandbox, sandboxName) + : captureProviderIdentityByName?.(sandboxName); + return providerIdentity ? { identities: undefined, providerIdentity } : { identities: undefined }; + } catch (captureError) { + const detail = deps.redact( + captureError instanceof Error ? captureError.message : String(captureError), + ); + error( + `Refusing to destroy sandbox '${sandboxName}': Runtime provider identity could not be inspected (${detail}). No sandbox resources were removed.`, + ); + return false; + } + } const classify = deps.classify ?? ((name: string, retainedSandboxIdentityFingerprint?: string) => @@ -213,9 +277,6 @@ export function assertUnambiguousDestroyContainerIdentity( observeDestroyContainerIdentity(name), retainedSandboxIdentityFingerprint, )); - const error = deps.error ?? ((message: string) => console.error(` ${message}`)); - if (deps.providerId !== "docker") return {}; - const verdict = deps.retainedSandboxIdentityFingerprint ? classify(sandboxName, deps.retainedSandboxIdentityFingerprint) : classify(sandboxName); @@ -248,6 +309,18 @@ export function isSameDestroyContainerIdentityProof( expected: DestroyContainerIdentityProof, actual: DestroyContainerIdentityProof, ): boolean { + if (expected.providerIdentity || actual.providerIdentity) { + const left = expected.providerIdentity; + const right = actual.providerIdentity; + return ( + left !== undefined && + right !== undefined && + left.schemaVersion === right.schemaVersion && + left.providerId === right.providerId && + left.resourceHandle === right.resourceHandle && + left.ownershipSha256 === right.ownershipSha256 + ); + } const expectedIdentities = expected.identities; const actualIdentities = actual.identities; if (expectedIdentities === undefined || actualIdentities === undefined) { diff --git a/src/lib/actions/sandbox/destroy.test.ts b/src/lib/actions/sandbox/destroy.test.ts index 5884c3a7809..8959a35c159 100644 --- a/src/lib/actions/sandbox/destroy.test.ts +++ b/src/lib/actions/sandbox/destroy.test.ts @@ -155,7 +155,7 @@ describe("assertUnambiguousDestroyContainerIdentity (#8999)", () => { it("does not probe or block a non-Docker runtime provider", () => { const classify = vi.fn(); const proceed = assertUnambiguousDestroyContainerIdentity("destroytest", { - providerId: "podman", + providerId: "unregistered-provider", redact: String, classify: classify as never, }); @@ -163,6 +163,52 @@ describe("assertUnambiguousDestroyContainerIdentity (#8999)", () => { expect(classify).not.toHaveBeenCalled(); }); + it("uses provider-owned identity before registry finalization", () => { + const classify = vi.fn(); + const providerIdentity = { + schemaVersion: 1 as const, + providerId: "podman", + resourceHandle: "a".repeat(64), + ownershipSha256: "b".repeat(64), + }; + const captureProviderIdentityByName = vi.fn(() => providerIdentity); + + expect( + assertUnambiguousDestroyContainerIdentity("destroytest", { + providerId: "podman", + redact: String, + captureProviderIdentityByName, + classify: classify as never, + }), + ).toEqual({ identity: undefined, providerIdentity }); + expect(captureProviderIdentityByName).toHaveBeenCalledWith("destroytest"); + expect(classify).not.toHaveBeenCalled(); + }); + + it("uses a provider-owned destroy identity without the Docker classifier", () => { + const classify = vi.fn(); + const providerIdentity = { + schemaVersion: 1 as const, + providerId: "podman", + resourceHandle: "a".repeat(64), + ownershipSha256: "b".repeat(64), + }; + const captureProviderIdentity = vi.fn(() => providerIdentity); + const sandbox = { name: "destroytest", agent: "openclaw" as const, openshellDriver: "podman" }; + + expect( + assertUnambiguousDestroyContainerIdentity("destroytest", { + providerId: "podman", + redact: String, + sandbox, + captureProviderIdentity, + classify: classify as never, + }), + ).toEqual({ identity: undefined, providerIdentity }); + expect(captureProviderIdentity).toHaveBeenCalledWith(sandbox, "destroytest"); + expect(classify).not.toHaveBeenCalled(); + }); + it("refuses when the Docker probe cannot prove identity", () => { const error = vi.fn(); const proceed = assertUnambiguousDestroyContainerIdentity("destroytest", { diff --git a/src/lib/actions/sandbox/destroy.ts b/src/lib/actions/sandbox/destroy.ts index a63595ab2d1..b439709d107 100644 --- a/src/lib/actions/sandbox/destroy.ts +++ b/src/lib/actions/sandbox/destroy.ts @@ -37,7 +37,7 @@ import { } from "../../onboard/runtime-provider/access"; import { emitProviderDetachResidualHint, - removeManagedHermesStateVolume, + removeManagedAgentStateVolumes, SANDBOX_PROVIDER_SUFFIXES, } from "../../onboard/sandbox-provider-cleanup"; import { validateName } from "../../runner"; @@ -725,15 +725,20 @@ async function destroySandboxUnlocked( ); } - const inspectContainerIdentity = () => - assertUnambiguousDestroyContainerIdentity(sandboxName, { + const inspectContainerIdentity = () => { + const registeredSandbox = registry.getSandbox(sandboxName); + return assertUnambiguousDestroyContainerIdentity(sandboxName, { cliName: CLI_NAME, - providerId: normalizeRuntimeProviderIdentity( - registry.getSandbox(sandboxName)?.openshellDriver, - ), + providerId: registeredSandbox + ? normalizeRuntimeProviderIdentity(registeredSandbox.openshellDriver) + : normalizeRuntimeProviderIdentity(null), redact: redactDestroyError, - ...(retainedSandboxIdentityFingerprint ? { retainedSandboxIdentityFingerprint } : {}), + sandbox: registeredSandbox, + ...(retainedSandboxIdentityFingerprint + ? { retainedSandboxIdentityFingerprint } + : {}), }); + }; const initialIdentity = portableContainerAuthority ? null : inspectContainerIdentity(); if (initialIdentity === false) { requestSandboxDestroyExit(1); @@ -851,6 +856,9 @@ async function destroySandboxUnlocked( ...(retainedSandboxIdentityFingerprint ? { expectedContainerIdentityFingerprint: retainedSandboxIdentityFingerprint } : {}), + ...(initialIdentity?.providerIdentity + ? { expectedRuntimeProviderIdentity: initialIdentity.providerIdentity } + : {}), ...(portableContainerAuthority ? { portableContainerAuthority } : {}), stopInferenceResources: () => stopSandboxInferenceResources(sandboxName, sandbox), }); @@ -983,28 +991,38 @@ async function destroySandboxUnlocked( preparedManagedLlamaCppCleanup?.abort(); } if (deleteSucceededOrAlreadyGone && sandbox) { - const stateVolumeCleanup = abortPreparedCleanupOnError(() => - removeManagedHermesStateVolume({ - agentName: sandbox.agent, - runtimeProviderId: normalizeRuntimeProviderIdentity(sandbox.openshellDriver), - sandboxName, - workloadKind: sandbox.workload?.kind ?? "", - }), + const stateVolumeCleanupResults = abortPreparedCleanupOnError(() => + removeManagedAgentStateVolumes( + { + agentName: sandbox.agent, + runtimeProviderId: normalizeRuntimeProviderIdentity(sandbox.openshellDriver), + sandboxName, + workloadKind: sandbox.workload?.kind ?? "", + }, + { + runtimeProviders: CURRENT_RUNTIME_PROVIDER_BUNDLES, + }, + ), ); - if (stateVolumeCleanup.status === "failed") { + const failedStateVolumeCleanup = stateVolumeCleanupResults.find( + (result) => result.status === "failed", + ); + if (failedStateVolumeCleanup?.status === "failed") { console.error( - ` Sandbox '${sandboxName}' is gone, but its managed Hermes state volume '${stateVolumeCleanup.volumeName}' could not be removed: ${redactDestroyError(stateVolumeCleanup.detail)}`, + ` Sandbox '${sandboxName}' is gone, but its managed agent state volume '${failedStateVolumeCleanup.volumeName}' could not be removed: ${redactDestroyError(failedStateVolumeCleanup.detail)}`, ); console.error(" The sandbox registry entry was preserved so exact cleanup can be retried."); preparedManagedLlamaCppCleanup?.abort(); requestSandboxDestroyExit(1); } - if (stateVolumeCleanup.status === "not-owned") { - console.warn( - ` ${YW}⚠${R} Left Docker volume '${stateVolumeCleanup.volumeName}' untouched because ${stateVolumeCleanup.detail}.`, - ); - } else if (stateVolumeCleanup.status === "removed") { - console.log(` Removed managed Hermes state volume for '${sandboxName}'.`); + for (const stateVolumeCleanup of stateVolumeCleanupResults) { + if (stateVolumeCleanup.status === "not-owned") { + console.warn( + ` ${YW}⚠${R} Left managed state volume '${stateVolumeCleanup.volumeName}' untouched because ${stateVolumeCleanup.detail}.`, + ); + } else if (stateVolumeCleanup.status === "removed") { + console.log(` Removed managed agent state volume for '${sandboxName}'.`); + } } } abortPreparedCleanupOnError(() => { diff --git a/src/lib/actions/sandbox/gateway-failure-classifier.test.ts b/src/lib/actions/sandbox/gateway-failure-classifier.test.ts index e1023640170..908abcf8d28 100644 --- a/src/lib/actions/sandbox/gateway-failure-classifier.test.ts +++ b/src/lib/actions/sandbox/gateway-failure-classifier.test.ts @@ -10,7 +10,11 @@ vi.mock("../../state/registry", () => ({ listSandboxes: vi.fn(() => ({ sandboxes: [] })), })); -import { classifyGatewayFailure, type GatewayFailureRunners } from "./gateway-failure-classifier"; +import { + classifyGatewayFailure, + classifyObservedSandboxContainerFailure, + type GatewayFailureRunners, +} from "./gateway-failure-classifier"; function runners(overrides: Partial = {}): GatewayFailureRunners { return { @@ -93,3 +97,17 @@ describe("classifyGatewayFailure (#7348)", () => { expect(getSandboxMock).not.toHaveBeenCalled(); }); }); + +describe("classifyObservedSandboxContainerFailure", () => { + it("classifies a provider-observed stopped sandbox without naming its runtime", async () => { + await expect( + classifyObservedSandboxContainerFailure("alpha", "stopped", 18789, async () => false), + ).resolves.toMatchObject({ layer: "sandbox_container_stopped" }); + }); + + it("does not duplicate running provider observations", async () => { + await expect( + classifyObservedSandboxContainerFailure("alpha", "running", 18789), + ).resolves.toBeNull(); + }); +}); diff --git a/src/lib/actions/sandbox/gateway-failure-classifier.ts b/src/lib/actions/sandbox/gateway-failure-classifier.ts index 82ea9045920..c18b68fb878 100644 --- a/src/lib/actions/sandbox/gateway-failure-classifier.ts +++ b/src/lib/actions/sandbox/gateway-failure-classifier.ts @@ -10,6 +10,7 @@ import { GATEWAY_PORT } from "../../core/ports"; import { resolveSandboxContainerOwner } from "../../domain/sandbox/container-owner"; import { resolveGatewayPortFromName } from "../../onboard/gateway-binding"; import type { PortablePodmanReadinessResult } from "../../onboard/experimental/portable-runtime-readiness"; +import type { RuntimeProviderSnapshotLifecycleState } from "../../onboard/runtime-provider/contract"; import { inspectPortableRuntimeReceiptReadiness, type PortableRuntimeReceiptReadinessDeps, @@ -215,6 +216,25 @@ function isValidDashboardPort(port: number | null | undefined): port is number { return typeof port === "number" && Number.isInteger(port) && port >= 1 && port <= 65535; } +export async function classifyObservedSandboxContainerFailure( + sandboxName: string, + lifecycleState: RuntimeProviderSnapshotLifecycleState, + dashboardPort: number | null | undefined, + portProbe: (port: number) => Promise = defaultPortProbe, +): Promise { + if (lifecycleState !== "stopped") return null; + if (isValidDashboardPort(dashboardPort) && (await portProbe(dashboardPort))) { + return { + layer: "sandbox_dashboard_port_conflict", + detail: `Sandbox '${sandboxName}' is stopped and dashboard port ${dashboardPort} is held by another process.`, + }; + } + return { + layer: "sandbox_container_stopped", + detail: `Sandbox '${sandboxName}' exists but is not running.`, + }; +} + export async function classifySandboxContainerFailure( sandboxName: string, opts: { diff --git a/src/lib/actions/sandbox/gateway-state.ts b/src/lib/actions/sandbox/gateway-state.ts index 9fd8afcfd1d..faf1e7ac493 100644 --- a/src/lib/actions/sandbox/gateway-state.ts +++ b/src/lib/actions/sandbox/gateway-state.ts @@ -81,7 +81,10 @@ import { } from "../../onboard/experimental/portable-agent-lifecycle"; import type { HermesPortableLifecycleRecoveryTiming } from "../../onboard/experimental/hermes-portable-lifecycle"; import type { PortableDemoLifecycleRecoveryResult } from "../../onboard/experimental/portable-demo-lifecycle"; -import { compareAndSetLegacySandboxLifecycleGeneration } from "../../state/registry/lifecycle-generation"; +import { + compareAndSetLegacySandboxLifecycleGeneration, + usesLegacyRuntimeLifecycleCompatibility, +} from "../../state/registry/lifecycle-generation"; import type { SandboxEntry } from "../../state/registry/types"; import { getSandboxDockerRuntime } from "./docker-health"; import { isDockerRuntimeDown, printDockerRuntimeDownGuidance } from "./gateway-failure-classifier"; @@ -112,6 +115,8 @@ export type SandboxGatewayState = { recoverySandboxVia?: string | null; }; +export { usesLegacyRuntimeLifecycleCompatibility }; + export type { HermesPortableActiveLifecycleAuthority, HermesPortableAgentLifecycleAuthority, diff --git a/src/lib/actions/sandbox/launch-readiness-ordinary-pairing.test.ts b/src/lib/actions/sandbox/launch-readiness-ordinary-pairing.test.ts index fdf370db27c..642260610be 100644 --- a/src/lib/actions/sandbox/launch-readiness-ordinary-pairing.test.ts +++ b/src/lib/actions/sandbox/launch-readiness-ordinary-pairing.test.ts @@ -52,6 +52,7 @@ describe("ordinary OpenClaw pairing target", () => { expect(resolveOrdinaryOpenClawPairingTarget(SANDBOX_NAME, deps)).toEqual({ gatewayName: GATEWAY_NAME, + openshellDriver: "docker", lifecycleGeneration: "generation-1", lifecycleLiveIdentityFingerprint: FINGERPRINT, stateDirectory: "/sandbox/.openclaw", @@ -66,6 +67,7 @@ describe("ordinary OpenClaw pairing target", () => { expect(resolveOrdinaryOpenClawPairingTarget(SANDBOX_NAME, deps)).toEqual({ gatewayName: GATEWAY_NAME, + openshellDriver: "docker", lifecycleGeneration: "generation-1", lifecycleLiveIdentityFingerprint: FINGERPRINT, stateDirectory: "/sandbox/.openclaw", @@ -81,6 +83,7 @@ describe("ordinary OpenClaw pairing target", () => { expect(resolveOrdinaryOpenClawPairingTarget(SANDBOX_NAME, deps)).toEqual({ gatewayName: GATEWAY_NAME, + openshellDriver: "docker", lifecycleGeneration: "generation-1", lifecycleLiveIdentityFingerprint: FINGERPRINT, stateDirectory: "/sandbox/.openclaw", @@ -110,6 +113,7 @@ describe("ordinary OpenClaw pairing target", () => { expect(resolveOrdinaryOpenClawPairingTarget(SANDBOX_NAME, deps)).toEqual({ gatewayName: GATEWAY_NAME, + openshellDriver: "docker", lifecycleGeneration: "generation-1", lifecycleLiveIdentityFingerprint: FINGERPRINT, stateDirectory: "/sandbox/.openclaw", diff --git a/src/lib/actions/sandbox/launch-readiness-runtime-provider.test.ts b/src/lib/actions/sandbox/launch-readiness-runtime-provider.test.ts new file mode 100644 index 00000000000..460af07d75c --- /dev/null +++ b/src/lib/actions/sandbox/launch-readiness-runtime-provider.test.ts @@ -0,0 +1,57 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { loadAgent } from "../../agent/defs"; +import type { SandboxEntry } from "../../state/registry"; +import { buildLaunchReadinessRegistryProjection } from "./launch-readiness"; + +const SANDBOX: SandboxEntry = { + name: "alpha", + openshellDriver: "docker", + openshellVersion: "0.0.99", + gatewayName: "nemoclaw", + gatewayPort: 8080, + lifecycleGeneration: "generation-1", + lifecycleLiveIdentityFingerprint: "b".repeat(64), + agent: "openclaw", + agentVersion: "1.0.0", + nemoclawVersion: "2.0.0", + imageTag: "example@sha256:immutable", + provider: null, + model: null, + endpointUrl: null, + credentialEnv: null, + preferredInferenceApi: null, + compatibleEndpointReasoning: null, + compatibleEndpointReasoningEffort: null, + nimContainer: null, +}; + +describe("launch readiness runtime-provider projection", () => { + it("accepts qualification-registered providers without a provider-name branch", () => { + const projection = buildLaunchReadinessRegistryProjection( + { ...SANDBOX, openshellDriver: "podman" }, + loadAgent("openclaw"), + ) as { openshellDriver: string }; + + expect(projection.openshellDriver).toBe("podman"); + expect(() => + buildLaunchReadinessRegistryProjection( + { ...SANDBOX, openshellDriver: "unregistered-runtime" }, + loadAgent("openclaw"), + ), + ).toThrow(); + }); + + it("rejects in-progress lifecycle and policy mutations", () => { + const agent = loadAgent("openclaw"); + expect(() => + buildLaunchReadinessRegistryProjection( + { ...SANDBOX, pendingRouteReservation: true, reservationSessionId: "session" }, + agent, + ), + ).toThrow(); + }); +}); diff --git a/src/lib/actions/sandbox/launch-readiness.ts b/src/lib/actions/sandbox/launch-readiness.ts index e01f4eabbb6..a53373fc275 100644 --- a/src/lib/actions/sandbox/launch-readiness.ts +++ b/src/lib/actions/sandbox/launch-readiness.ts @@ -24,6 +24,10 @@ import { observeSandboxOnGateway, type SandboxRecreateObserver, } from "../../onboard/sandbox-recreate-probe"; +import { + CURRENT_RUNTIME_PROVIDER_BUNDLES, + resolveRuntimeProviderBundle, +} from "../../onboard/runtime-provider/access"; import { assertNoOpenShellGatewayEndpointOverride } from "../../openshell-gateway-endpoint-guard"; import { parseAndValidateSandboxPolicy } from "../../policy/sandbox-policy-validation"; import { @@ -39,8 +43,8 @@ import { } from "../../state/launch-readiness-lease"; import { withMcpLifecycleLock as withSandboxMutationLock } from "../../state/mcp-lifecycle-lock-acquisition"; import type { SandboxEntry, SandboxWorkloadReceipt } from "../../state/registry"; +import { normalizeSandboxMcpState } from "../../state/registry"; import * as registry from "../../state/registry"; -import { normalizeSandboxMcpState } from "../../state/registry-mcp"; import { cloneSandboxMessagingState, serializeSandboxMessagingStateForDisk, @@ -80,7 +84,6 @@ export { createProbeTimingRecorder, type ProbeTimingRecorder } from "./probe/tim export { createBoundLaunchReadinessDeps }; const LIVE_POLICY_MAX_BYTES = 2 * 1_024 * 1_024; -const ALLOWED_OPENSHELL_DRIVERS = new Set(["docker", "kubernetes", "vm"]); export type LaunchReadinessPerformanceStage = | "storage-read" @@ -182,6 +185,10 @@ export interface OpenClawPairingSettlementTarget { readonly version: string; } +export interface OrdinaryOpenClawPairingSettlementTarget extends OpenClawPairingSettlementTarget { + readonly openshellDriver: string; +} + type LaunchReadinessPublicationValidationCategory = Extract< LaunchReadinessPublicationResult, { kind: "validation-failed" } @@ -520,7 +527,9 @@ export function buildLaunchReadinessRegistryProjection( portableRuntimeAuthoritySha256: string | null = null, ): unknown { const driver = normalizedString(entry.openshellDriver)?.toLowerCase() ?? null; - if (!driver || !ALLOWED_OPENSHELL_DRIVERS.has(driver)) throw new ObservationError("config"); + if (!driver || !resolveRuntimeProviderBundle(driver, CURRENT_RUNTIME_PROVIDER_BUNDLES)) { + throw new ObservationError("config"); + } const openshellVersion = normalizedString(entry.openshellVersion); const gatewayPort = entry.gatewayPort; if (!openshellVersion || openshellVersion.length > 128) throw new ObservationError("config"); @@ -1004,16 +1013,19 @@ function resolveOpenClawPairingSettlementTarget( export function resolveOrdinaryOpenClawPairingTarget( sandboxName: string, deps: LaunchReadinessDeps = {}, -): OpenClawPairingSettlementTarget | null { +): OrdinaryOpenClawPairingSettlementTarget | null { try { const getSandbox = deps.getSandbox ?? registry.getSandbox; - return resolveOpenClawPairingSettlementTarget( + const entry = getSandbox(sandboxName); + const target = resolveOpenClawPairingSettlementTarget( sandboxName, - getSandbox(sandboxName), + entry, deps, undefined, true, ); + const openshellDriver = normalizedString(entry?.openshellDriver); + return target && openshellDriver ? { ...target, openshellDriver } : null; } catch { return null; } diff --git a/src/lib/actions/sandbox/mcp-bridge-adapter-hermes.ts b/src/lib/actions/sandbox/mcp-bridge-adapter-hermes.ts index 51dbd276644..992e75364e9 100644 --- a/src/lib/actions/sandbox/mcp-bridge-adapter-hermes.ts +++ b/src/lib/actions/sandbox/mcp-bridge-adapter-hermes.ts @@ -20,9 +20,12 @@ import { entryHeaders, HERMES_MCP_TRANSACTION_HELPER, } from "./mcp-bridge-adapter-status"; +import { + type McpAttachedCredentialRevision, + observeMcpCredentialRevision, +} from "./mcp-bridge-provider-readiness"; import { McpBridgeError } from "./mcp-bridge-contracts"; import { commandOutput, redactBridgeSecretsForDisplay } from "./mcp-bridge-output"; -import type { McpAttachedCredentialRevision } from "./mcp-bridge-provider-readiness"; import { executeGatewaySupervisorAction } from "./process-recovery"; const HERMES_MCP_EXEC_TIMEOUT_SECONDS = 620; @@ -306,6 +309,27 @@ export function registerHermesAdapter( { envValues, requireReload: true }, ); verifyHermesAdapterRegistration(sandboxName, entry, credentialRevision); + if (credentialRevision === undefined) return; + const afterReloadRevision = observeMcpCredentialRevision(sandboxName, entry); + if (afterReloadRevision === credentialRevision) return; + if (afterReloadRevision === "absent" || afterReloadRevision === "canonical") { + throw new McpBridgeError( + `Hermes MCP credential revision was unavailable after reloading '${entry.server}'.`, + ); + } + runHermesAdapterCommand( + sandboxName, + entry, + buildHermesMcpRegisterCommand(entry, true, afterReloadRevision), + `Hermes MCP config convergence failed for '${entry.server}'.`, + { envValues, requireReload: true }, + ); + verifyHermesAdapterRegistration(sandboxName, entry, afterReloadRevision); + if (observeMcpCredentialRevision(sandboxName, entry) !== afterReloadRevision) { + throw new McpBridgeError( + `Hermes MCP credential revision did not converge after reloading '${entry.server}'.`, + ); + } } export function unregisterHermesAdapter( diff --git a/src/lib/actions/sandbox/mcp-bridge-adapter-registration.test.ts b/src/lib/actions/sandbox/mcp-bridge-adapter-registration.test.ts index e1198dfff85..beebc882fc6 100644 --- a/src/lib/actions/sandbox/mcp-bridge-adapter-registration.test.ts +++ b/src/lib/actions/sandbox/mcp-bridge-adapter-registration.test.ts @@ -8,6 +8,7 @@ import type { McpBridgeEntry } from "../../state/registry"; const mocks = vi.hoisted(() => ({ executeSandboxCommand: vi.fn(), + executeSandboxExecCommand: vi.fn(), executeGatewaySupervisorAction: vi.fn(), getSandbox: vi.fn(), observeMcpCredentialRevision: vi.fn(), @@ -19,6 +20,7 @@ const mocks = vi.hoisted(() => ({ vi.mock("./process-recovery", () => ({ executeSandboxCommand: mocks.executeSandboxCommand, + executeSandboxExecCommand: mocks.executeSandboxExecCommand, executeGatewaySupervisorAction: mocks.executeGatewaySupervisorAction, })); @@ -161,6 +163,7 @@ const reconciliationCases: ReconciliationCase[] = [ describe.each(adapterCases)("$name MCP adapter registration", (adapterCase) => { beforeEach(() => { mocks.executeSandboxCommand.mockReset(); + mocks.executeSandboxExecCommand.mockReset(); mocks.executeGatewaySupervisorAction.mockReset(); mocks.runOpenshellProviderCommand.mockReset(); mocks.getSandbox.mockReset(); @@ -267,6 +270,7 @@ describe("Deep Agents MCP adapter credential revision", () => { describe("Hermes MCP adapter credential revision", () => { beforeEach(() => { mocks.executeSandboxCommand.mockReset(); + mocks.executeSandboxExecCommand.mockReset(); mocks.runOpenshellProviderCommand.mockReset(); mocks.getSandbox.mockReset(); }); @@ -274,6 +278,11 @@ describe("Hermes MCP adapter credential revision", () => { it("writes and verifies the readiness-proven revision", () => { mocks.runOpenshellProviderCommand.mockReturnValue(lifecycleSuccess); mocks.executeSandboxCommand.mockReturnValue(registered); + mocks.executeSandboxExecCommand.mockReturnValue({ + status: 0, + stdout: "v12\n", + stderr: "", + }); expect(() => registerAgentAdapter( diff --git a/src/lib/actions/sandbox/mcp-bridge-remove.ts b/src/lib/actions/sandbox/mcp-bridge-remove.ts index cbab3068043..83aef99561f 100644 --- a/src/lib/actions/sandbox/mcp-bridge-remove.ts +++ b/src/lib/actions/sandbox/mcp-bridge-remove.ts @@ -421,7 +421,9 @@ async function removeMcpBridgeUnlocked( } } if (failures.length > 0) { - console.warn(` MCP force cleanup warnings:\n${failures.join("\n")}`); + console.warn( + ` MCP force cleanup reported ${failures.length} warning(s); inspect redacted diagnostics.`, + ); if (!options.allowResidual) { throw new McpBridgeError( `MCP force cleanup left residual resources for '${server}'. The registry entry was preserved so cleanup can be retried.`, diff --git a/src/lib/actions/sandbox/mcp-bridge-render.ts b/src/lib/actions/sandbox/mcp-bridge-render.ts index 6f4bf49ae08..ee672e21457 100644 --- a/src/lib/actions/sandbox/mcp-bridge-render.ts +++ b/src/lib/actions/sandbox/mcp-bridge-render.ts @@ -45,7 +45,7 @@ export function renderMcpBridgeStatus( if (statuses.length === 0) { console.log(""); console.log(` MCP servers for sandbox '${sandboxName}': none`); - console.log(` agent: ${agent.name}`); + console.log(" agent: configured"); console.log(` support: ${agent.mcpCapability.support}`); if (agent.mcpCapability.reason) console.log(` reason: ${agent.mcpCapability.reason}`); console.log(""); diff --git a/src/lib/actions/sandbox/mcp-bridge-status-resolution.test.ts b/src/lib/actions/sandbox/mcp-bridge-status-resolution.test.ts index 09c024b10b2..27980f526bd 100644 --- a/src/lib/actions/sandbox/mcp-bridge-status-resolution.test.ts +++ b/src/lib/actions/sandbox/mcp-bridge-status-resolution.test.ts @@ -157,6 +157,12 @@ registry.registerSandbox({ const bridge = require("./src/lib/actions/sandbox/mcp-bridge.js"); const logLines = []; const errorLines = []; +const originalStdoutWrite = process.stdout.write.bind(process.stdout); +const writeHarnessResult = (value) => originalStdoutWrite(value); +process.stdout.write = (value) => { + logLines.push(String(value).trimEnd()); + return true; +}; console.log = (...parts) => logLines.push(parts.join(" ")); console.error = (...parts) => errorLines.push(parts.join(" ")); `; @@ -197,7 +203,7 @@ describe("MCP status wire-level credential-resolution probe", { timeout: 15_000 String.raw` await bridge.dispatchMcpBridgeCommand("alpha", ["status", "github", "--json"]); const status = JSON.parse(logLines.join("\n")); - process.stdout.write(JSON.stringify({ + writeHarnessResult(JSON.stringify({ status, probed: executedSandboxCommands.some((c) => c.includes("NEMOCLAW_MCP_PROBE")), exitCode: process.exitCode ?? 0, @@ -250,7 +256,7 @@ describe("MCP status wire-level credential-resolution probe", { timeout: 15_000 credentialObservationCount, }); } - process.stdout.write(JSON.stringify(outcomes)); + writeHarnessResult(JSON.stringify(outcomes)); `, ); const outcomes = JSON.parse(stdout) as Array<{ @@ -308,7 +314,7 @@ describe("MCP status wire-level credential-resolution probe", { timeout: 15_000 probed: executedSandboxCommands.some((command) => command.includes("NEMOCLAW_MCP_PROBE")), }); } - process.stdout.write(JSON.stringify(outcomes)); + writeHarnessResult(JSON.stringify(outcomes)); `, ); const outcomes = JSON.parse(stdout) as Array<{ @@ -385,7 +391,7 @@ describe("MCP status wire-level credential-resolution probe", { timeout: 15_000 resolution: wrongProvider.provider.credentialResolution, probed: executedSandboxCommands.some((c) => c.includes("NEMOCLAW_MCP_PROBE")), }); - process.stdout.write(JSON.stringify(outcomes)); + writeHarnessResult(JSON.stringify(outcomes)); `, ); const outcomes = JSON.parse(stdout) as Array<{ @@ -413,7 +419,7 @@ describe("MCP status wire-level credential-resolution probe", { timeout: 15_000 home, String.raw` await bridge.dispatchMcpBridgeCommand("alpha", ["status", "github"]); - process.stdout.write(JSON.stringify({ lines: logLines })); + writeHarnessResult(JSON.stringify({ lines: logLines })); `, ); const payload = JSON.parse(stdout) as { lines: string[] }; @@ -432,7 +438,7 @@ describe("MCP status wire-level credential-resolution probe", { timeout: 15_000 String.raw` await bridge.dispatchMcpBridgeCommand("alpha", ["status", "github", "--json"]); const status = JSON.parse(logLines.join("\n")); - process.stdout.write(JSON.stringify({ warnings: status.warnings })); + writeHarnessResult(JSON.stringify({ warnings: status.warnings })); `, { probeHttpStatus: 400 }, ); @@ -456,7 +462,7 @@ describe("MCP status wire-level credential-resolution probe", { timeout: 15_000 logLines.length = 0; await bridge.dispatchMcpBridgeCommand("alpha", ["list", "--json"]); const list = JSON.parse(logLines.join("\n")); - process.stdout.write(JSON.stringify({ + writeHarnessResult(JSON.stringify({ probed: executedSandboxCommands.some((c) => c.includes("NEMOCLAW_MCP_PROBE")), bareStatusResolution: bareStatus.bridges[0].provider.credentialResolution ?? null, listResolution: list.bridges[0].provider.credentialResolution ?? null, @@ -484,7 +490,7 @@ describe("MCP status wire-level credential-resolution probe", { timeout: 15_000 logLines.length = 0; await bridge.dispatchMcpBridgeCommand("alpha", ["status", "--probe", "--json"]); const forced = JSON.parse(logLines.join("\n")); - process.stdout.write(JSON.stringify({ + writeHarnessResult(JSON.stringify({ probesAfterSkip, skippedResolution: skipped.provider.credentialResolution ?? null, forcedResolution: forced.bridges[0].provider.credentialResolution ?? null, @@ -513,7 +519,7 @@ describe("MCP status wire-level credential-resolution probe", { timeout: 15_000 await bridge.dispatchMcpBridgeCommand("alpha", ["status", "github", "--probe", "--no-probe"]); const observedExitCode = process.exitCode ?? 0; process.exitCode = 0; - process.stdout.write(JSON.stringify({ errorLines, exitCode: observedExitCode })); + writeHarnessResult(JSON.stringify({ errorLines, exitCode: observedExitCode })); `, ); const payload = JSON.parse(stdout) as { errorLines: string[]; exitCode: number }; @@ -528,7 +534,7 @@ describe("MCP status wire-level credential-resolution probe", { timeout: 15_000 String.raw` await bridge.dispatchMcpBridgeCommand("alpha", ["status", "github", "--tools", "--json"]); const status = JSON.parse(logLines.join("\n")); - process.stdout.write(JSON.stringify({ + writeHarnessResult(JSON.stringify({ status, probed: executedSandboxCommands.some((c) => c.includes("NEMOCLAW_MCP_PROBE")), discovered: executedSandboxCommands.some((c) => c.includes("mcp-tool-discovery-runtime")), @@ -604,7 +610,7 @@ describe("MCP status wire-level credential-resolution probe", { timeout: 15_000 (command) => command.includes("mcp-tool-discovery-runtime"), ).length, }); - process.stdout.write(JSON.stringify(outcomes)); + writeHarnessResult(JSON.stringify(outcomes)); `, ); expect(JSON.parse(stdout)).toEqual([ @@ -663,7 +669,7 @@ describe("MCP status wire-level credential-resolution probe", { timeout: 15_000 String.raw` await bridge.dispatchMcpBridgeCommand("alpha", ["status", "github", "--tools", "--probe", "--json"]); const status = JSON.parse(logLines.join("\n")); - process.stdout.write(JSON.stringify({ + writeHarnessResult(JSON.stringify({ hasResolution: !!status.provider.credentialResolution, hasDiscovery: !!status.toolDiscovery, probeCommands: executedSandboxCommands.filter((c) => c.includes("NEMOCLAW_MCP_PROBE")).length, @@ -691,7 +697,7 @@ describe("MCP status wire-level credential-resolution probe", { timeout: 15_000 errorLines.length = 0; logLines.length = 0; await bridge.dispatchMcpBridgeCommand("alpha", ["status", "github", "--tools"]); - process.stdout.write(JSON.stringify({ rejectedExitCode, rejection, rendered: logLines })); + writeHarnessResult(JSON.stringify({ rejectedExitCode, rejection, rendered: logLines })); `, ); const payload = JSON.parse(stdout) as { @@ -717,7 +723,7 @@ describe("MCP add post-add credential-resolution probe", () => { await bridge.dispatchMcpBridgeCommand("alpha", [ "add", "github", "--url", "https://api.githubcopilot.com/mcp/", "--env", "GITHUB_TOKEN", ]); - process.stdout.write(JSON.stringify({ + writeHarnessResult(JSON.stringify({ logLines, errorLines, probed: executedSandboxCommands.some((c) => c.includes("NEMOCLAW_MCP_PROBE")), @@ -773,7 +779,7 @@ describe("MCP add post-add credential-resolution probe", () => { exitCode: process.exitCode ?? 0, }); } - process.stdout.write(JSON.stringify(outcomes)); + writeHarnessResult(JSON.stringify(outcomes)); `, ); const outcomes = JSON.parse(stdout) as Array<{ @@ -801,7 +807,7 @@ describe("MCP add post-add credential-resolution probe", () => { await bridge.dispatchMcpBridgeCommand("alpha", [ "add", "github", "--url", "https://api.githubcopilot.com/mcp/", "--env", "GITHUB_TOKEN", ]); - process.stdout.write(JSON.stringify({ errorLines, exitCode: process.exitCode ?? 0 })); + writeHarnessResult(JSON.stringify({ errorLines, exitCode: process.exitCode ?? 0 })); `, { probeHttpStatus: 400 }, ); @@ -824,7 +830,7 @@ describe("MCP add post-add credential-resolution probe", () => { await bridge.dispatchMcpBridgeCommand("alpha", [ "add", "github", "--url", "https://api.githubcopilot.com/mcp/", "--env", "GITHUB_TOKEN", "--no-probe", ]); - process.stdout.write(JSON.stringify({ + writeHarnessResult(JSON.stringify({ errorLines, probed: executedSandboxCommands.some((c) => c.includes("NEMOCLAW_MCP_PROBE")), exitCode: process.exitCode ?? 0, diff --git a/src/lib/actions/sandbox/mcp-bridge-tool-discovery.test.ts b/src/lib/actions/sandbox/mcp-bridge-tool-discovery.test.ts index d09182ffdef..524dbb82625 100644 --- a/src/lib/actions/sandbox/mcp-bridge-tool-discovery.test.ts +++ b/src/lib/actions/sandbox/mcp-bridge-tool-discovery.test.ts @@ -228,12 +228,15 @@ describe("MCP tool discovery host boundary (#6901)", () => { { status: 1, stdout: `${marker}\nBearer should-not-leak`, - stderr: "authorization: should-not-leak", + stderr: + "authorization: should-not-leak\n[SECURITY] proxy startup refused Authorization=should-not-leak", }, entry, marker, ); expect(result.detail).toContain("rebuild the sandbox"); + expect(result.detail).toContain("exit 1"); + expect(result.detail).toContain("proxy startup refused"); expect(JSON.stringify(result)).not.toContain("should-not-leak"); }); diff --git a/src/lib/actions/sandbox/mcp-bridge-tool-discovery.ts b/src/lib/actions/sandbox/mcp-bridge-tool-discovery.ts index a50db124efb..d3369860200 100644 --- a/src/lib/actions/sandbox/mcp-bridge-tool-discovery.ts +++ b/src/lib/actions/sandbox/mcp-bridge-tool-discovery.ts @@ -136,8 +136,14 @@ export function classifyMcpToolDiscoveryResult( ): NonNullable { if (result === null) return failure("sandbox unreachable"); if (result.status !== 0) { + const safeFailure = `${result.stderr}\n${result.stdout}` + .split(/\r?\n/u) + .filter((line) => /^(?:\[SECURITY\] |Managed startup )/u.test(line)) + .map((line) => redactBridgeSecretsForDisplay(line, entry)) + .reverse() + .find((line) => safeString(line, MCP_TOOL_DISCOVERY_MAX_DETAIL_BYTES)); return failure( - "MCP tool discovery runtime failed to start; rebuild the sandbox if the image predates this diagnostic", + `MCP tool discovery runtime failed to start (exit ${String(result.status)})${safeFailure ? `: ${safeFailure}` : ""}; rebuild the sandbox if the image predates this diagnostic`, ); } const output = extractSandboxExecCommandStdoutFromStreams( diff --git a/src/lib/actions/sandbox/mcp-bridge.ts b/src/lib/actions/sandbox/mcp-bridge.ts index d501611c7c2..6b81455f5e7 100644 --- a/src/lib/actions/sandbox/mcp-bridge.ts +++ b/src/lib/actions/sandbox/mcp-bridge.ts @@ -326,7 +326,7 @@ export async function dispatchMcpBridgeCommand( const agent = getSandboxAgent(sandbox); const statuses = await statusMcpBridge(sandboxName); if (json) - console.log(JSON.stringify(buildJsonSummary(sandboxName, agent, statuses), null, 2)); + process.stdout.write(`${JSON.stringify(buildJsonSummary(sandboxName, agent, statuses), null, 2)}\n`); else renderMcpBridgeList(sandboxName, statuses, agent); return; } @@ -348,12 +348,12 @@ export async function dispatchMcpBridgeCommand( discoverTools: tools, }); if (json) { - console.log( - JSON.stringify( + process.stdout.write( + `${JSON.stringify( server ? statuses[0] : buildJsonSummary(sandboxName, agent, statuses), null, 2, - ), + )}\n`, ); } else renderMcpBridgeStatus(sandboxName, statuses, agent); return; diff --git a/src/lib/actions/sandbox/policy-channel-agent-gate.test.ts b/src/lib/actions/sandbox/policy-channel-agent-gate.test.ts index 984e78e45f1..d382cb81de7 100644 --- a/src/lib/actions/sandbox/policy-channel-agent-gate.test.ts +++ b/src/lib/actions/sandbox/policy-channel-agent-gate.test.ts @@ -97,9 +97,8 @@ describe("addSandboxChannel agent gate", () => { const errorText = (errSpy.mock.calls as unknown[][]) .map((call) => call.map(String).join(" ")) .join("\n"); - expect(errorText).toMatch(/Channel 'discord' does not support agent 'custom-agent'/); - expect(errorText).toMatch(/Channel-supported agents: openclaw, hermes/); - expect(errorText).toMatch(/Channels supported by agent 'custom-agent': \(none\)/); + expect(errorText).toContain("This channel does not support the configured agent."); + expect(errorText).not.toContain("custom-agent"); expect(loadPresetForSandboxMock).not.toHaveBeenCalled(); expect(applyPresetMock).not.toHaveBeenCalled(); @@ -178,9 +177,8 @@ describe("channel lifecycle agent gate", () => { const errorText = (errSpy.mock.calls as unknown[][]) .map((call) => call.map(String).join(" ")) .join("\n"); - expect(errorText).toMatch(/Channel 'googlechat' does not support agent 'custom-agent'/); - expect(errorText).toMatch(/Channel-supported agents: openclaw, hermes/); - expect(errorText).toMatch(/Channels supported by agent 'custom-agent': \(none\)/); + expect(errorText).toContain("This channel does not support the configured agent."); + expect(errorText).not.toContain("custom-agent"); expect(configuredChannelsMock).not.toHaveBeenCalled(); expect(disabledChannelsMock).not.toHaveBeenCalled(); diff --git a/src/lib/actions/sandbox/policy-channel-dependencies.ts b/src/lib/actions/sandbox/policy-channel-dependencies.ts index 19870d1ac79..53563986f87 100644 --- a/src/lib/actions/sandbox/policy-channel-dependencies.ts +++ b/src/lib/actions/sandbox/policy-channel-dependencies.ts @@ -66,12 +66,12 @@ function gatewayRunner(gatewayName: string): typeof runOpenshell { */ export const policyChannelDependencies = { /** Use stopped Docker cleanup only after both in-sandbox cleanup attempts fail. */ - clearStoppedDockerSandboxChannelState( + clearStoppedSandboxStateRoots( sandboxName: string, paths: readonly string[], - ): ReturnType { + ): ReturnType { const cleanup = require("../../sandbox/privileged-exec") as PrivilegedExecModule; - return cleanup.clearStoppedDockerSandboxChannelState(sandboxName, paths); + return cleanup.clearStoppedSandboxStateRoots(sandboxName, paths); }, deleteMessagingProviderWithRecovery( providerName: string, diff --git a/src/lib/actions/sandbox/policy-channel.ts b/src/lib/actions/sandbox/policy-channel.ts index ee585895303..c6a516c245a 100644 --- a/src/lib/actions/sandbox/policy-channel.ts +++ b/src/lib/actions/sandbox/policy-channel.ts @@ -26,7 +26,6 @@ import { createBuiltInMessagingHookRegistry, createBuiltInRenderTemplateResolver, createMessagingPreEnableHookInputs, - formatSupportedMessagingAgentIds, getMessagingManifestAvailabilityContext, isMessagingChannelSupportedByAgent, isMessagingHookConflictError, @@ -564,9 +563,8 @@ async function applyExternalPreset( refreshSandboxPolicyContextFile(sandboxName); } return result !== false; - } catch (err: unknown) { - const message = err instanceof Error ? err.message : String(err); - console.error(` Failed to apply preset '${loaded.presetName}': ${message}`); + } catch { + console.error(` Failed to apply preset '${loaded.presetName}': validation failed.`); return false; } } @@ -648,7 +646,7 @@ export function listSandboxChannels(sandboxName: string) { console.log(""); console.log(` Known messaging channels for sandbox '${sandboxName}':`); if (availableChannels.length === 0) { - console.log(` (none supported by agent '${agent.name}')`); + console.log(" (none supported by this agent)"); } for (const manifest of availableChannels) { console.log(` ${manifest.id} — ${manifest.description ?? manifest.displayName}`); @@ -656,14 +654,6 @@ export function listSandboxChannels(sandboxName: string) { console.log(""); } -function formatAvailableChannelsForAgent(agent: AgentDefinition): string { - return ( - availableManifestChannelsForAgent(agent) - .map((manifest) => manifest.id) - .join(", ") || "(none)" - ); -} - // Map a channel + token-env-key to the OpenShell provider name onboarding // uses for it. Mirrors the names in src/lib/onboard.ts:3201-3221 so a // channels-add upsert collides with (i.e. updates) the same provider that @@ -933,11 +923,7 @@ async function applyChannelAddToGatewayAndRegistry( return null; } } catch (err) { - console.error( - ` ✗ Failed to register '${channelName}' providers with the gateway: ${ - err instanceof Error ? err.message : String(err) - }`, - ); + console.error(" ✗ Failed to register channel providers with the gateway."); if ( policyChannelDependencies.isMessagingProviderBindingConflict(err) || policyChannelDependencies.isMessagingProviderMutationFailure(err) @@ -957,14 +943,10 @@ async function applyChannelAddToGatewayAndRegistry( (providerName) => !createdProviders.has(providerName), ); if (updatedProviderNames.length > 0) { - console.error( - ` ${YW}⚠${R} Updated provider state remains for ${updatedProviderNames.join(", ")}; resolve the conflicting provider, then rerun '${CLI_NAME} ${sandboxName} channels add ${channelName}'.`, - ); + console.error(` ${YW}⚠${R} Updated provider state remains; resolve it and retry.`); } if (cleanupFailures.length > 0) { - console.error( - ` ${YW}⚠${R} Could not remove newly created providers ${cleanupFailures.join(", ")}; rerun '${CLI_NAME} ${sandboxName} channels remove ${channelName}'.`, - ); + console.error(` ${YW}⚠${R} Could not remove newly created providers; retry cleanup.`); } cleanupCredentialFreePolicy?.(); process.exit(1); @@ -1241,9 +1223,9 @@ async function planSandboxChannelAdd( }); MessagingSetupApplier.writePlanToEnv(plan); return plan; - } catch (error) { + } catch { console.error(` Failed to plan messaging channel '${channelId}'.`); - console.error(` ${error instanceof Error ? error.message : String(error)}`); + console.error(" Inspect the redacted channel diagnostics for details."); process.exit(1); } } @@ -1342,11 +1324,7 @@ function assertAddChannelPlanActive( const missing = channelPlan?.inputs.filter((input) => input.required && !inputAvailable(input)) ?? []; if (missing.length > 0) { - console.error( - ` Missing required input(s) for channel '${manifest.id}': ${missing - .map(formatMissingInput) - .join(", ")}.`, - ); + console.error(" Missing required inputs for this channel."); if ( manifest.auth.mode === "host-qr" && getMessagingToken(manifest.credentials[0]?.providerEnvKey) @@ -1371,10 +1349,6 @@ function inputAvailable(input: SandboxMessagingChannelPlan["inputs"][number]): b return typeof input.value === "string" ? input.value.trim().length > 0 : true; } -function formatMissingInput(input: SandboxMessagingChannelPlan["inputs"][number]): string { - return input.sourceEnv ? `${input.inputId} (${input.sourceEnv})` : input.inputId; -} - function hydrateAddChannelEnvFromStoredState(sandboxName: string): void { const savedSession = safeLoadOnboardSession(); hydrateMessagingChannelConfig(getStoredMessagingChannelConfig(sandboxName, savedSession)); @@ -1459,15 +1433,7 @@ async function addSandboxChannelUnlocked( const agent = resolveAgentForSandbox(sandboxName); if (!channelSupportedByAgent(manifest, agent)) { - console.error( - ` Channel '${canonical}' does not support agent '${agent.name}' for sandbox '${sandboxName}'.`, - ); - console.error( - ` Channel-supported agents: ${formatSupportedMessagingAgentIds(manifest.supportedAgents)}.`, - ); - console.error( - ` Channels supported by agent '${agent.name}': ${formatAvailableChannelsForAgent(agent)}.`, - ); + console.error(" This channel does not support the configured agent."); process.exit(1); } @@ -1738,9 +1704,8 @@ export function applyChannelPresetIfAvailable( } refreshSandboxPolicyContextFile(sandboxName); return true; - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - console.error(` ${YW}⚠${R} Failed to apply '${channelName}' policy preset: ${msg}`); + } catch { + console.error(` ${YW}⚠${R} Failed to apply '${channelName}' policy preset.`); console.error( ` Restore the preset YAML and re-run: ${CLI_NAME} ${sandboxName} channels ${retryAction} ${channelName}`, ); @@ -1792,27 +1757,27 @@ function isSafeChannelStatePath(p: string): boolean { const CHANNEL_CLEAR_SENTINEL = "NEMOCLAW_CHANNEL_CLEAR_OK"; const STOPPED_WECHAT_CLEANUP_FAILURE_GUIDANCE = { "sandbox-registry-unavailable": "Restore the NemoClaw sandbox registry entry.", - "driver-not-docker": "Restore normal OpenShell lifecycle access for this non-Docker sandbox.", + "provider-cleanup-unavailable": "Restore the selected runtime provider's cleanup capability.", "state-paths-invalid": "Restore the channel's declared state-path contract.", - "docker-discovery-failed": "Start Docker or restore access to its daemon.", - "no-eligible-stopped-container": "Restore the registered stopped OpenShell container.", - "container-ownership-invalid": "Reconcile the sandbox registry and Docker container identity.", - "container-inspection-failed": "Restore Docker inspection access for the stopped container.", - "container-not-stopped": "Stop the registered sandbox container before retrying removal.", - "sandbox-volume-unavailable": "Restore a single writable Docker volume at /sandbox.", + "runtime-discovery-failed": "Restore access to the selected runtime provider.", + "no-eligible-stopped-runtime": "Restore the registered stopped OpenShell container.", + "runtime-ownership-invalid": "Reconcile the sandbox registry and runtime resource identity.", + "runtime-inspection-failed": "Restore inspection access for the stopped runtime resource.", + "runtime-not-stopped": "Stop the registered sandbox container before retrying removal.", + "state-resource-unavailable": "Restore the single writable runtime state resource at /sandbox.", "cleanup-helper-image-unavailable": "Restore the pinned NemoClaw cleanup image locally.", "cleanup-helper-ownership-invalid": "Remove the conflicting cleanup helper container.", "cleanup-helper-reconciliation-failed": "Reconcile the named cleanup helper container.", "cleanup-state-tree-unsafe": "Inspect the stopped sandbox volume; recreate the sandbox if its state tree is untrusted.", "cleanup-deletion-unconfirmed": "Restore writable access to the stopped sandbox volume.", - "cleanup-helper-failed": "Inspect the stopped sandbox and Docker daemon.", - "container-revalidation-failed": "Reconcile the stopped container identity and state.", + "cleanup-helper-failed": "Inspect the stopped sandbox and selected runtime provider.", + "runtime-revalidation-failed": "Reconcile the stopped container identity and state.", "lifecycle-authority-unavailable": "Finish the active lifecycle transition or repair its lock.", } as const; type StoppedWechatCleanupFailure = Exclude< - ReturnType<(typeof policyChannelDependencies)["clearStoppedDockerSandboxChannelState"]>, + ReturnType<(typeof policyChannelDependencies)["clearStoppedSandboxStateRoots"]>, { readonly cleared: true } >; @@ -1835,7 +1800,7 @@ function stoppedWechatCleanupFailureGuidance( /** * Wipe durable channel state before rebuild can preserve an obsolete auth blob. - * OpenShell exec runs first, followed by SSH and the stopped WeChat Docker fallback. + * OpenShell exec runs first, followed by SSH and the selected provider's stopped-state fallback. * Fixes #3998. */ function clearSandboxChannelDurableState( @@ -1861,7 +1826,7 @@ function clearSandboxChannelDurableState( result = executeSandboxCommand(sandboxName, cmd); } if (!sentinelSeen(result) && agent.name === "openclaw" && channelName === "wechat") { - const stoppedCleanup = policyChannelDependencies.clearStoppedDockerSandboxChannelState( + const stoppedCleanup = policyChannelDependencies.clearStoppedSandboxStateRoots( sandboxName, paths, ); @@ -1873,14 +1838,14 @@ function clearSandboxChannelDurableState( options.allowAbsentStoppedState && [ "sandbox-registry-unavailable", - "driver-not-docker", - "no-eligible-stopped-container", + "provider-cleanup-unavailable", + "no-eligible-stopped-runtime", ].includes(stoppedCleanup.failure) ) { return true; } console.error( - ` ${YW}⚠${R} Stopped-Docker cleanup failed (${stoppedCleanup.failure}). ` + + ` ${YW}⚠${R} Stopped-runtime cleanup failed (${stoppedCleanup.failure}). ` + `${stoppedWechatCleanupFailureGuidance(sandboxName, stoppedCleanup)} Then retry removal.`, ); } @@ -1918,9 +1883,8 @@ export function removeChannelPresetIfPresent(sandboxName: string, channelName: s } refreshSandboxPolicyContextFile(sandboxName); return true; - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - console.error(` ${YW}⚠${R} Failed to remove '${channelName}' policy preset: ${msg}`); + } catch { + console.error(` ${YW}⚠${R} Failed to remove '${channelName}' policy preset.`); console.error( ` Run manually after rebuild with: ${CLI_NAME} ${sandboxName} policy remove ${channelName}`, ); @@ -1994,12 +1958,12 @@ async function removeSandboxChannelUnlocked( // Channels with durable account or session state store auth blobs inside // the sandbox that survive a rebuild via the state_dirs backup. Tear those // down FIRST so a cleanup failure leaves the registry/policy untouched. - // OpenClaw WeChat can additionally recover through a stopped Docker volume + // OpenClaw WeChat can additionally recover through a provider-owned stopped-state // helper because the same missing account file may block its entrypoint. // Bailing here is the only way to keep #3998 from recurring on cleanup // error. OpenClaw WeChat also checks for physical residue after an earlier // interrupted removal erased its logical plan or policy record. A missing - // registry, non-Docker driver, or absent stopped container remains a quiet + // registry, unavailable provider cleanup, or absent stopped runtime remains a quiet // no-op only when no logical residue exists (#4001 review). if ( requiresStateCleanupBeforeTeardown && @@ -2125,15 +2089,7 @@ async function sandboxChannelsSetEnabled( const agent = resolveAgentForSandbox(sandboxName); const availableChannels = availableManifestChannelsForAgent(agent); if (!availableChannels.some((candidate) => candidate.id === canonical)) { - console.error( - ` Channel '${canonical}' does not support agent '${agent.name}' for sandbox '${sandboxName}'.`, - ); - console.error( - ` Channel-supported agents: ${formatSupportedMessagingAgentIds(manifest.supportedAgents)}.`, - ); - console.error( - ` Channels supported by agent '${agent.name}': ${formatAvailableChannelsForAgent(agent)}.`, - ); + console.error(" This channel does not support the configured agent."); process.exit(1); } diff --git a/src/lib/actions/sandbox/process-recovery.test.ts b/src/lib/actions/sandbox/process-recovery.test.ts index 2d2132a9bc9..aa42b171cc0 100644 --- a/src/lib/actions/sandbox/process-recovery.test.ts +++ b/src/lib/actions/sandbox/process-recovery.test.ts @@ -595,6 +595,18 @@ describe("confirmRecoveredSandboxGatewayManaged scope", () => { expect(requestGatewaySupervisorAction).toHaveBeenCalledWith("my-sandbox", "probe"); }); + it("accepts the same managed controller proof for a Podman sandbox", () => { + requestGatewaySupervisorAction.mockClear(); + expect( + confirmRecoveredSandboxGatewayManaged("my-sandbox", { + getSandboxImpl: () => ({ ...openClawEntry, openshellDriver: "podman" }), + getSessionAgentImpl: () => null, + requestGatewaySupervisorActionImpl: requestGatewaySupervisorAction, + }), + ).toBe(true); + expect(requestGatewaySupervisorAction).toHaveBeenCalledWith("my-sandbox", "probe"); + }); + it("does not control custom agents or non-direct OpenShell drivers", () => { requestGatewaySupervisorAction.mockClear(); expect( diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index 47549515674..25e12adb9f1 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -2,7 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 import { randomBytes } from "node:crypto"; -import { dockerSpawnSync } from "../../adapters/docker"; import { stripAnsi } from "../../adapters/openshell/client"; import { captureOpenshell, @@ -28,7 +27,8 @@ import { ROOT, shellQuote } from "../../runner"; import { isDirectSandboxFallbackUnavailableError, isPinnedSandboxContainerIdentityChangedError, - privilegedSandboxExecArgv, + executePrivilegedSandboxCommand as executeProviderPrivilegedSandboxCommand, + resolvePrivilegedSandboxTarget, withPrivilegedSandboxExecutionLease, } from "../../sandbox/privileged-exec"; import { withTimerBoundShieldsMutationLock } from "../../shields/timer-bound-lock"; @@ -82,7 +82,10 @@ import { } from "./sandbox-exec-output"; import { type ManagedSupervisorRelaunch, + recoverRegisteredRuntimeProviderSandbox, relaunchManagedSupervisorSession, + usesManagedGatewayController, + usesLegacyManagedGatewayRecovery, } from "./supervisor-relaunch"; export type { SandboxForwardHealth, SandboxForwardListEntry } from "./forward-health"; export { @@ -126,12 +129,11 @@ function commandTransportDependencies(): CommandTransportDependencies { buildSandboxExecMarkedCommand, buildSubprocessEnv, captureSandboxSshConfig, - dockerSpawnSync, + executePrivilegedSandboxCommand: executeProviderPrivilegedSandboxCommand, extractSandboxExecCommandStdout, getOpenshellBinary, isDirectSandboxFallbackUnavailableError, openshellProbeTimeoutMs: OPENSHELL_PROBE_TIMEOUT_MS, - privilegedSandboxExecArgv, root: ROOT, withPrivilegedSandboxExecutionLease, }; @@ -194,19 +196,15 @@ export function executePrivilegedSandboxCommand( sandboxName, "sandbox process recovery controller", () => { - const argv = privilegedSandboxExecArgv(sandboxName, [...command], false, true); - const result = dockerSpawnSync(argv, { - cwd: ROOT, - encoding: "utf-8", - env: buildSubprocessEnv(), - stdio: ["ignore", "pipe", "pipe"], + const result = executeProviderPrivilegedSandboxCommand(sandboxName, command, { + sanitizeEnvironment: true, timeout, }); if (result.error) return null; return { status: result.status ?? 1, - stdout: String(result.stdout || ""), - stderr: String(result.stderr || ""), + stdout: result.stdout.toString("utf8"), + stderr: result.stderr.toString("utf8"), }; }, ); @@ -236,26 +234,21 @@ function executeGatewaySupervisorActionPinned( const nonce = randomBytes(32).toString("hex"); try { return withPrivilegedSandboxExecutionLease(sandboxName, `gateway supervisor ${action}`, () => { - const argv = privilegedSandboxExecArgv( + const targetContainerId = + expectedContainerId ?? resolvePrivilegedSandboxTarget(sandboxName).resourceHandle; + const result = executeProviderPrivilegedSandboxCommand( sandboxName, [MANAGED_GATEWAY_CONTROL_PATH, action, nonce], - false, - true, - expectedContainerId, + { + sanitizeEnvironment: true, + expectedResourceHandle: targetContainerId, + timeout, + }, ); - const controlPathIndex = argv.lastIndexOf(MANAGED_GATEWAY_CONTROL_PATH); - const targetContainerId = controlPathIndex > 0 ? argv[controlPathIndex - 1] : null; - const result = dockerSpawnSync(argv, { - cwd: ROOT, - encoding: "utf-8", - env: process.env, - stdio: ["ignore", "pipe", "pipe"], - timeout, - }); - if (result.error) return null; const status = result.status ?? 1; - const stdout = String(result.stdout || "").trim(); - let stderr = String(result.stderr || "").trim(); + const stdout = result.stdout.toString("utf8").trim(); + let stderr = result.stderr.toString("utf8").trim(); + if (result.error) return null; const restartingContainerMatch = stderr.match(DOCKER_CONTAINER_RESTARTING_ERROR); const managedControlRestartingContainerId = status === 1 && @@ -654,8 +647,7 @@ export function confirmRecoveredSandboxGatewayManaged( const persistedAgent = entry.agent ?? "openclaw"; if (persistedAgent !== "openclaw" && persistedAgent !== "hermes") return null; - const driver = entry.openshellDriver?.trim().toLowerCase() ?? null; - if (driver !== null && driver !== "docker" && driver !== "vm") return null; + if (!usesManagedGatewayController(entry)) return null; const getSessionAgent = options.getSessionAgentImpl ?? agentRuntime.getSessionAgent; const agent = getSessionAgent(sandboxName); @@ -702,6 +694,7 @@ export async function isSandboxGatewayRunningForStatus( type SandboxProcessRecovery = | { kind: "managed"; managedControlCompletion?: ManagedGatewayControlCompletion } | { kind: "custom" } + | { kind: "provider" } | { kind: "relaunched"; relaunch: ManagedSupervisorRelaunch }; function recoverSandboxProcesses( @@ -733,6 +726,18 @@ function recoverSandboxProcesses( quiet || printGatewayRestartFailure(sandboxName, "unsupported agent", detail); return null; } + const persistedSandbox = registry.getSandbox(sandboxName); + // Providers that launch NemoClaw's managed in-sandbox controller recover the + // gateway through that controller. Restarting their runtime first replaces + // the still-healthy supervisor and changes gateway parentage. + if (persistedSandbox && !usesManagedGatewayController(persistedSandbox)) { + const result = recoverRegisteredRuntimeProviderSandbox(persistedSandbox); + if (result) { + if (result.exitCode === 0) return { kind: "provider" }; + if (!quiet && result.message) console.error(result.message); + return null; + } + } const recoveredSsh = (result: SandboxCommandResult | null): SandboxProcessRecovery | null => result && result.status === 0 && hasGatewayRecoveryMarker(result) ? { kind: "custom" } : null; const recoverManagedGateway = (): SandboxProcessRecovery | null => { @@ -1726,7 +1731,8 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( // Host-forward recovery requires an OpenShell-ready sandbox. Managed // recovery has already passed its authenticated control and health gates; // a replacement also rechecks its pinned identity before readiness. - const recoveryRequiresReadiness = recovery.kind === "managed" || relaunch; + const recoveryRequiresReadiness = + recovery.kind === "managed" || recovery.kind === "provider" || relaunch; const waitForRecoveryReadiness = () => { const readinessOptions: RecreatedSandboxOpenShellReadyOptions = { beforeProbe: relaunch diff --git a/src/lib/actions/sandbox/rebuild-agent-base-image-preflight.test.ts b/src/lib/actions/sandbox/rebuild-agent-base-image-preflight.test.ts index 95cb6519a4c..1588b425f25 100644 --- a/src/lib/actions/sandbox/rebuild-agent-base-image-preflight.test.ts +++ b/src/lib/actions/sandbox/rebuild-agent-base-image-preflight.test.ts @@ -180,7 +180,8 @@ describe("ensureRebuildAgentBaseImage", () => { const output = error.mock.calls.flat().join("\n"); expect(output).toContain("Rebuild preflight failed"); expect(output).toContain("agent base image could not be built"); - expect(output).toContain("Failed to build Hermes Agent base image (exit 23)"); + expect(output).toContain("Inspect the redacted rebuild diagnostics for details."); + expect(output).not.toContain("Failed to build Hermes Agent base image (exit 23)"); expect(output).toContain("Sandbox is untouched"); }); diff --git a/src/lib/actions/sandbox/rebuild-dcode-preflight.ts b/src/lib/actions/sandbox/rebuild-dcode-preflight.ts index a846b765488..cd71f7c049a 100644 --- a/src/lib/actions/sandbox/rebuild-dcode-preflight.ts +++ b/src/lib/actions/sandbox/rebuild-dcode-preflight.ts @@ -450,7 +450,7 @@ function resolvePinnedDcodeBaseImage( } if (!warned) { warned = true; - console.warn(` Warning: failed to remove temporary DCode base image '${imageRef}'.`); + console.warn(" Warning: failed to remove the temporary DCode base image."); } return false; }; diff --git a/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts b/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts index d1bad3d537d..3c00260e265 100644 --- a/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts +++ b/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts @@ -36,6 +36,7 @@ const mocks = vi.hoisted(() => ({ ), stopNimContainer: vi.fn(), stopNimContainerByName: vi.fn(), + settleAgentForwardPortsForRebuild: vi.fn(), })); vi.mock("../../adapters/openshell/runtime", () => ({ @@ -52,6 +53,10 @@ vi.mock("../../inference/nim", () => ({ stopNimContainerByName: mocks.stopNimContainerByName, })); +vi.mock("../../tunnel/agent-forward-stop", () => ({ + settleAgentForwardPortsForRebuild: mocks.settleAgentForwardPortsForRebuild, +})); + vi.mock("../../state/registry", async (importOriginal) => ({ ...(await importOriginal()), getSandbox: mocks.getSandbox, @@ -129,6 +134,7 @@ describe("rebuild destroy phase", () => { mocks.waitUntil.mockImplementation( (condition: () => boolean) => condition() || condition() || condition(), ); + mocks.settleAgentForwardPortsForRebuild.mockReturnValue(true); }); afterEach(() => { @@ -962,6 +968,10 @@ describe("rebuild destroy phase", () => { it("journals the delete boundary before the destructive command (#7734)", async () => { const order: string[] = []; const recreateJournal = stubRecreateJournal(); + mocks.settleAgentForwardPortsForRebuild.mockImplementation(() => { + order.push("forward:stop"); + return true; + }); vi.mocked(recreateJournal.markDeleting).mockImplementation(() => { order.push("journal:deleting"); }); @@ -987,7 +997,12 @@ describe("rebuild destroy phase", () => { onDeleted: vi.fn(), }); - expect(order).toEqual(["journal:deleting", "openshell:delete"]); + expect(order).toEqual([ + "forward:stop", + "journal:deleting", + "openshell:delete", + "forward:stop", + ]); }); it("reattaches MCP providers when the delete boundary cannot be journaled (#7734)", async () => { diff --git a/src/lib/actions/sandbox/rebuild-destroy-phase.ts b/src/lib/actions/sandbox/rebuild-destroy-phase.ts index 4755eb3dc24..b8cf4060ce5 100644 --- a/src/lib/actions/sandbox/rebuild-destroy-phase.ts +++ b/src/lib/actions/sandbox/rebuild-destroy-phase.ts @@ -13,6 +13,7 @@ import { redactFull } from "../../security/redact"; import { parseSandboxPhase } from "../../state/gateway"; import { registryEntryGatewayPort } from "../../state/gateway-registry"; import * as registry from "../../state/registry"; +import { settleAgentForwardPortsForRebuild } from "../../tunnel/agent-forward-stop"; import type { RebuildBackupManifest } from "./rebuild-backup-phase"; import type { RebuildBail, RebuildLog } from "./rebuild-credential-preflight"; import { type RebuildSandboxEntry, warnUnpreservedUserManagedFiles } from "./rebuild-flow-helpers"; @@ -352,23 +353,6 @@ export async function runRebuildDestroyPhase( } } - // MCP preparation can await external systems. Re-read the registry at the - // synchronous delete edge so those checks and deletion use one target. - if (!rebuildDeleteTargetMatchesRegistry(deleteTarget)) { - const mcpRecoveryFailure = await reattachMcpAfterDeleteFailure( - sandboxName, - rebuildDetachedMcpProviderEntries, - rebuildScrubbedMcpAdapterEntries, - ); - relockShieldsIfNeeded(true); - bail( - mcpRecoveryFailure - ? `Sandbox delete target changed during rebuild preparation; MCP provider recovery also failed: ${mcpRecoveryFailure}` - : "Sandbox delete target changed during rebuild preparation.", - ); - return null; - } - if (validateAtDeleteEdge) { let validation: RebuildDeleteValidationResult; try { @@ -398,6 +382,26 @@ export async function runRebuildDestroyPhase( } } + // Rebuild keeps the gateway/session alive, but the replacement must reclaim + // the old sandbox's host forwards. Stop only forwards proven to belong to + // this registered sandbox, then re-read the registry at the synchronous + // delete edge so cleanup and deletion still use one target. + settleAgentForwardPortsForRebuild(sandboxName, { info: log, warn: log }); + if (!rebuildDeleteTargetMatchesRegistry(deleteTarget)) { + const mcpRecoveryFailure = await reattachMcpAfterDeleteFailure( + sandboxName, + rebuildDetachedMcpProviderEntries, + rebuildScrubbedMcpAdapterEntries, + ); + relockShieldsIfNeeded(true); + bail( + mcpRecoveryFailure + ? `Sandbox delete target changed during rebuild preparation; MCP provider recovery also failed: ${mcpRecoveryFailure}` + : "Sandbox delete target changed during rebuild preparation.", + ); + return null; + } + // MCP adapter entries are already detached and scrubbed here. A journal write // that fails must reattach them before the rebuild gives up, or the still // running sandbox is left without its MCP wiring. @@ -511,6 +515,12 @@ export async function runRebuildDestroyPhase( bail(`Sandbox deletion could not be journaled: ${redactFull(detail)}`); return null; } + if (!settleAgentForwardPortsForRebuild(sandboxName, { info: log, warn: log })) { + bail( + `Sandbox '${sandboxName}' was deleted, but its host port forwards were not released; retry rebuild after the forward listener stops.`, + ); + return null; + } try { cleanupDockerOrphanAfterDelete?.(); } catch (error) { diff --git a/src/lib/actions/sandbox/rebuild-flow-helpers.ts b/src/lib/actions/sandbox/rebuild-flow-helpers.ts index a03de05f0ce..53a92be6b46 100644 --- a/src/lib/actions/sandbox/rebuild-flow-helpers.ts +++ b/src/lib/actions/sandbox/rebuild-flow-helpers.ts @@ -38,16 +38,18 @@ import * as shields from "../../shields"; import type { SandboxEntry } from "../../state/registry"; import { load as loadRegistry } from "../../state/registry/persistence"; import * as sandboxState from "../../state/sandbox"; +import { removeStaleRebuildDockerOrphan } from "../../onboard/openshell-docker-sandbox-containers"; import * as userManagedFilesProbe from "../../state/user-managed-files-probe"; import { getReconciledSandboxGatewayState, printSandboxGatewayStateHint, printWrongGatewayActiveGuidance, + usesLegacyRuntimeLifecycleCompatibility, } from "./gateway-state"; import { openRebuildShieldsWindow, type RebuildShieldsWindow } from "./rebuild-shields"; import * as snapshotBackup from "./snapshot/backup-authority"; -export { removeStaleRebuildDockerOrphan } from "../../onboard/openshell-docker-sandbox-containers"; +export { removeStaleRebuildDockerOrphan }; export type RebuildSandboxEntry = SandboxEntry & { agents?: unknown[] }; @@ -210,6 +212,16 @@ export async function resolveRebuildLiveState( if (reconciled.state === "missing") { if (options.authoritativeRecoveryPolicyAvailable === true) { + if (usesLegacyRuntimeLifecycleCompatibility(sb)) { + try { + removeStaleRebuildDockerOrphan(sandboxName, sb.openshellDriver, log); + } catch (error) { + bail( + `Stale-recovery Docker orphan cleanup failed: ${error instanceof Error ? error.message : String(error)}.`, + ); + return null; + } + } log( "Stale-sandbox recovery: the sandbox is absent, but its transaction-bound policy handoff is intact", ); @@ -411,7 +423,7 @@ export function ensureRebuildAgentBaseImage( const message = err instanceof Error ? err.message : String(err); console.error(""); console.error(` ${_RD}Rebuild preflight failed:${R} agent base image could not be built.`); - console.error(` ${message}`); + console.error(" Inspect the redacted rebuild diagnostics for details."); console.error(""); console.error(" Sandbox is untouched — no data was lost."); bail(message); diff --git a/src/lib/actions/sandbox/rebuild-flow-recovery.test.ts b/src/lib/actions/sandbox/rebuild-flow-recovery.test.ts index e6aff0e03f9..87523797a85 100644 --- a/src/lib/actions/sandbox/rebuild-flow-recovery.test.ts +++ b/src/lib/actions/sandbox/rebuild-flow-recovery.test.ts @@ -449,7 +449,7 @@ describe("rebuildSandbox flow: recovery", () => { ).rejects.toThrow("Prepared backup recovery"); expect(harness.errorSpy).toHaveBeenCalledWith( - expect.stringContaining("MCP bridge restore incomplete: MCP restore boom"), + expect.stringContaining("MCP bridge restore incomplete; inspect redacted diagnostics"), ); expect(harness.relockSpy).toHaveBeenCalled(); }); @@ -616,7 +616,7 @@ describe("rebuildSandbox flow: recovery", () => { expect(output).toContain("MCP bridge definitions were preserved but not fully refreshed"); expect(output).not.toContain("rebuilt successfully"); expect(harness.errorSpy).toHaveBeenCalledWith( - expect.stringContaining("MCP bridge restore incomplete: MCP restore boom"), + expect.stringContaining("MCP bridge restore incomplete; inspect redacted diagnostics"), ); }); }); diff --git a/src/lib/actions/sandbox/rebuild-hermes-cron-restore.test.ts b/src/lib/actions/sandbox/rebuild-hermes-cron-restore.test.ts index b8187d5b3e5..a9164704e02 100644 --- a/src/lib/actions/sandbox/rebuild-hermes-cron-restore.test.ts +++ b/src/lib/actions/sandbox/rebuild-hermes-cron-restore.test.ts @@ -8,23 +8,12 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { validateHermesCronRestoreBackup } from "../../state/rebuild/hermes-cron-restore-backup"; const processMocks = vi.hoisted(() => ({ - dockerSpawnSync: vi.fn(), - privilegedSandboxExecArgv: vi.fn((_sandboxName: string, command: string[]) => [ - "exec", - "container-id", - ...command, - ]), + executePrivilegedSandboxCommand: vi.fn(), })); -vi.mock("../../adapters/docker", async (importOriginal) => ({ - ...(await importOriginal()), - dockerSpawnSync: processMocks.dockerSpawnSync, -})); - -vi.mock("../../sandbox/privileged-exec", async (importOriginal) => ({ - ...(await importOriginal()), - isDirectSandboxFallbackUnavailableError: () => false, - privilegedSandboxExecArgv: processMocks.privilegedSandboxExecArgv, +vi.mock("./process-recovery", async (importOriginal) => ({ + ...(await importOriginal()), + executePrivilegedSandboxCommand: processMocks.executePrivilegedSandboxCommand, })); import { @@ -111,7 +100,7 @@ function receipt( } function completionFailure(stderr: string): unknown { - processMocks.dockerSpawnSync.mockReturnValue({ status: 1, stdout: "", stderr }); + processMocks.executePrivilegedSandboxCommand.mockReturnValue({ status: 1, stdout: "", stderr }); try { completeHermesCronRestoreAfterGatewayReplacement( "alpha", @@ -156,8 +145,7 @@ describe("Hermes cron rebuild restore contract", () => { let backupPath: string; beforeEach(() => { - processMocks.dockerSpawnSync.mockReset(); - processMocks.privilegedSandboxExecArgv.mockClear(); + processMocks.executePrivilegedSandboxCommand.mockReset(); backupPath = mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-cron-")); }); @@ -215,17 +203,19 @@ describe("Hermes cron rebuild restore contract", () => { }); it("binds validation to the begin receipt identity", () => { - processMocks.dockerSpawnSync.mockImplementation((argv: string[]) => { - const action = argv.includes("validate") ? "validate" : "begin"; - return { status: 0, stdout: receipt(action), stderr: "" }; - }); + processMocks.executePrivilegedSandboxCommand.mockImplementation( + (_sandboxName: string, argv: string[]) => { + const action = argv.includes("validate") ? "validate" : "begin"; + return { status: 0, stdout: receipt(action), stderr: "" }; + }, + ); const identity = beginHermesCronRestore("alpha"); validateHermesCronRestore("alpha", identity); expect(identity).toEqual({ pid: 41, start_time: 902, drain_token: "restore-token" }); - expect(processMocks.privilegedSandboxExecArgv).toHaveBeenCalledTimes(2); - expect(processMocks.privilegedSandboxExecArgv.mock.calls[1]?.[1]).toEqual([ + expect(processMocks.executePrivilegedSandboxCommand).toHaveBeenCalledTimes(2); + expect(processMocks.executePrivilegedSandboxCommand.mock.calls[1]?.[1]).toEqual([ "/opt/hermes/.venv/bin/python", "-I", "/usr/local/lib/nemoclaw/hermes-cron-restore-control.py", @@ -240,37 +230,46 @@ describe("Hermes cron rebuild restore contract", () => { it("passes an untrusted drain token as one argv value", () => { const untrustedToken = "restore-token'; touch /tmp/advisor-owned; #"; - processMocks.dockerSpawnSync.mockImplementation((argv: string[]) => ({ - status: 0, - stdout: receipt(argv.includes("validate") ? "validate" : "begin", 41, 902, untrustedToken), - stderr: "", - })); + processMocks.executePrivilegedSandboxCommand.mockImplementation( + (_sandboxName: string, argv: string[]) => ({ + status: 0, + stdout: receipt(argv.includes("validate") ? "validate" : "begin", 41, 902, untrustedToken), + stderr: "", + }), + ); const identity = beginHermesCronRestore("alpha"); validateHermesCronRestore("alpha", identity); - const validateArgv = processMocks.privilegedSandboxExecArgv.mock.calls[1]?.[1]; + const validateArgv = processMocks.executePrivilegedSandboxCommand.mock.calls[1]?.[1]; expect(validateArgv?.at(-1)).toBe(`--drain-token=${untrustedToken}`); }); it("keeps a leading-hyphen drain token attached to its option", () => { const leadingHyphenToken = "-restore-token"; - processMocks.dockerSpawnSync.mockImplementation((argv: string[]) => ({ - status: 0, - stdout: receipt(argv.includes("validate") ? "validate" : "begin", 41, 902, leadingHyphenToken), - stderr: "", - })); + processMocks.executePrivilegedSandboxCommand.mockImplementation( + (_sandboxName: string, argv: string[]) => ({ + status: 0, + stdout: receipt( + argv.includes("validate") ? "validate" : "begin", + 41, + 902, + leadingHyphenToken, + ), + stderr: "", + }), + ); const identity = beginHermesCronRestore("alpha"); validateHermesCronRestore("alpha", identity); - expect(processMocks.privilegedSandboxExecArgv.mock.calls[1]?.[1]).toContain( + expect(processMocks.executePrivilegedSandboxCommand.mock.calls[1]?.[1]).toContain( "--drain-token=-restore-token", ); }); it("keeps dispatch drained when state restore is incomplete", () => { - processMocks.dockerSpawnSync.mockReturnValue({ + processMocks.executePrivilegedSandboxCommand.mockReturnValue({ status: 0, stdout: receipt("begin"), stderr: "", @@ -279,17 +278,19 @@ describe("Hermes cron rebuild restore contract", () => { expect(() => runHermesCronRestoreTransaction("alpha", () => ({ restoreSucceeded: false })), ).toThrow("state restore was incomplete"); - expect(processMocks.dockerSpawnSync).toHaveBeenCalledOnce(); - expect(processMocks.privilegedSandboxExecArgv.mock.calls[0]?.[1]).toContain("begin"); + expect(processMocks.executePrivilegedSandboxCommand).toHaveBeenCalledOnce(); + expect(processMocks.executePrivilegedSandboxCommand.mock.calls[0]?.[1]).toContain("begin"); }); it("keeps dispatch held after restore validation until gateway replacement (#8472)", () => { const events: string[] = []; - processMocks.dockerSpawnSync.mockImplementation((argv: string[]) => { - const action = argv.includes("validate") ? "validate" : "begin"; - events.push(action); - return { status: 0, stdout: receipt(action), stderr: "" }; - }); + processMocks.executePrivilegedSandboxCommand.mockImplementation( + (_sandboxName: string, argv: string[]) => { + const action = argv.includes("validate") ? "validate" : "begin"; + events.push(action); + return { status: 0, stdout: receipt(action), stderr: "" }; + }, + ); const transaction = runHermesCronRestoreTransaction( "alpha", @@ -308,7 +309,7 @@ describe("Hermes cron rebuild restore contract", () => { }); it("completes the held gate against the replacement gateway identity (#8472)", () => { - processMocks.dockerSpawnSync.mockReturnValue({ + processMocks.executePrivilegedSandboxCommand.mockReturnValue({ status: 0, stdout: receipt("complete", 77, 903), stderr: "", @@ -325,7 +326,7 @@ describe("Hermes cron rebuild restore contract", () => { { pid: 77, start_time: 903, drain_token: "restore-token" }, ), ).toEqual({ pid: 77, start_time: 903, drain_token: "restore-token" }); - expect(processMocks.privilegedSandboxExecArgv).toHaveBeenCalledWith( + expect(processMocks.executePrivilegedSandboxCommand).toHaveBeenCalledWith( "alpha", [ "/opt/hermes/.venv/bin/python", @@ -342,13 +343,12 @@ describe("Hermes cron rebuild restore contract", () => { "--replacement-start-time", "903", ], - false, - true, + 130_000, ); }); it("rejects completion that did not bind to a replacement identity (#8472)", () => { - processMocks.dockerSpawnSync.mockReturnValue({ + processMocks.executePrivilegedSandboxCommand.mockReturnValue({ status: 0, stdout: receipt("complete"), stderr: "", @@ -378,7 +378,7 @@ describe("Hermes cron rebuild restore contract", () => { { pid: 77, start_time: 903, drain_token: "restore-token" }, ), ).toThrow("requires the held drain token"); - expect(processMocks.dockerSpawnSync).not.toHaveBeenCalled(); + expect(processMocks.executePrivilegedSandboxCommand).not.toHaveBeenCalled(); }); it("rejects completion when the replacement carries a different drain token (#8472)", () => { @@ -389,7 +389,7 @@ describe("Hermes cron rebuild restore contract", () => { { pid: 77, start_time: 903, drain_token: "different-token" }, ), ).toThrow("changed the held drain token"); - expect(processMocks.dockerSpawnSync).not.toHaveBeenCalled(); + expect(processMocks.executePrivilegedSandboxCommand).not.toHaveBeenCalled(); }); it("classifies the structured drain-marker rollback failure (#8472)", () => { @@ -419,7 +419,7 @@ describe("Hermes cron rebuild restore contract", () => { }); it("rejects completion while replacement agents are still active (#8472)", () => { - processMocks.dockerSpawnSync.mockReturnValue({ + processMocks.executePrivilegedSandboxCommand.mockReturnValue({ status: 0, stdout: receipt("complete", 77, 903, "restore-token", { active_agents: 1 }), stderr: "", @@ -439,7 +439,7 @@ describe("Hermes cron rebuild restore contract", () => { }); it("observes the replacement identity without releasing the held gate (#8472)", () => { - processMocks.dockerSpawnSync.mockReturnValue({ + processMocks.executePrivilegedSandboxCommand.mockReturnValue({ status: 0, stdout: receipt("observe", 77, 903), stderr: "", @@ -452,7 +452,7 @@ describe("Hermes cron rebuild restore contract", () => { drain_token: "restore-token", }), ).toEqual({ pid: 77, start_time: 903, drain_token: "restore-token" }); - expect(processMocks.privilegedSandboxExecArgv.mock.calls[0]?.[1]).toEqual([ + expect(processMocks.executePrivilegedSandboxCommand.mock.calls[0]?.[1]).toEqual([ "/opt/hermes/.venv/bin/python", "-I", "/usr/local/lib/nemoclaw/hermes-cron-restore-control.py", @@ -469,7 +469,7 @@ describe("Hermes cron rebuild restore contract", () => { ["dispatch-reactivated", false], ["operator-drain-preserved", true], ] as const)("returns the %s recovery disposition", (disposition, operatorDrainActive) => { - processMocks.dockerSpawnSync.mockReturnValue({ + processMocks.executePrivilegedSandboxCommand.mockReturnValue({ status: 0, stdout: receipt("recover", 41, 902, "restore-token", { disposition, @@ -480,7 +480,7 @@ describe("Hermes cron rebuild restore contract", () => { }); expect(recoverHermesCronRestore("alpha")).toBe(disposition); - expect(processMocks.privilegedSandboxExecArgv).toHaveBeenCalledWith( + expect(processMocks.executePrivilegedSandboxCommand).toHaveBeenCalledWith( "alpha", [ "/opt/hermes/.venv/bin/python", @@ -488,37 +488,35 @@ describe("Hermes cron rebuild restore contract", () => { "/usr/local/lib/nemoclaw/hermes-cron-restore-control.py", "recover", ], - false, - true, + 130_000, ); }); - it.each([ - "gate-prepared", - "not-required", - ] as const)("returns the %s pre-repair disposition", (disposition) => { - processMocks.dockerSpawnSync.mockReturnValue({ - status: 0, - stdout: preparationReceipt(disposition), - stderr: "", - }); - - expect(prepareHermesCronRestoreRecovery("alpha")).toBe(disposition); - expect(processMocks.privilegedSandboxExecArgv).toHaveBeenCalledWith( - "alpha", - [ - "/opt/hermes/.venv/bin/python", - "-I", - "/usr/local/lib/nemoclaw/hermes-cron-restore-control.py", - "prepare-recover", - ], - false, - true, - ); - }); + it.each(["gate-prepared", "not-required"] as const)( + "returns the %s pre-repair disposition", + (disposition) => { + processMocks.executePrivilegedSandboxCommand.mockReturnValue({ + status: 0, + stdout: preparationReceipt(disposition), + stderr: "", + }); + + expect(prepareHermesCronRestoreRecovery("alpha")).toBe(disposition); + expect(processMocks.executePrivilegedSandboxCommand).toHaveBeenCalledWith( + "alpha", + [ + "/opt/hermes/.venv/bin/python", + "-I", + "/usr/local/lib/nemoclaw/hermes-cron-restore-control.py", + "prepare-recover", + ], + 25_000, + ); + }, + ); it("rejects an inconsistent pre-repair receipt", () => { - processMocks.dockerSpawnSync.mockReturnValue({ + processMocks.executePrivilegedSandboxCommand.mockReturnValue({ status: 0, stdout: preparationReceipt("gate-prepared", { drain_acquired: false }), stderr: "", @@ -533,13 +531,13 @@ describe("Hermes cron rebuild restore contract", () => { `/opt/hermes/.venv/bin/python: can't open file '/usr/local/lib/nemoclaw/hermes-cron-restore-control.py': [Errno 2] No such file or directory`, "hermes-cron-restore-control.py: error: argument action: invalid choice: 'prepare-recover'", ])("keeps pre-repair compatible with a legacy Hermes sandbox: %s", (stderr) => { - processMocks.dockerSpawnSync.mockReturnValue({ status: 2, stdout: "", stderr }); + processMocks.executePrivilegedSandboxCommand.mockReturnValue({ status: 2, stdout: "", stderr }); expect(prepareHermesCronRestoreRecovery("alpha")).toBe("unsupported"); }); it("does not hide a current controller pre-repair failure", () => { - processMocks.dockerSpawnSync.mockReturnValue({ + processMocks.executePrivilegedSandboxCommand.mockReturnValue({ status: 1, stdout: "", stderr: "NemoClaw cron restore release recovery record metadata is unsafe", @@ -551,24 +549,26 @@ describe("Hermes cron rebuild restore contract", () => { }); it("composes the recovery transport budget from every controller phase (#7806)", () => { - processMocks.dockerSpawnSync.mockImplementation((argv: string[]) => { - const stdout = argv.includes("prepare-recover") - ? preparationReceipt("not-required") - : receipt(argv.includes("recover") ? "recover" : "begin"); - return { status: 0, stdout, stderr: "" }; - }); + processMocks.executePrivilegedSandboxCommand.mockImplementation( + (_sandboxName: string, argv: string[]) => { + const stdout = argv.includes("prepare-recover") + ? preparationReceipt("not-required") + : receipt(argv.includes("recover") ? "recover" : "begin"); + return { status: 0, stdout, stderr: "" }; + }, + ); beginHermesCronRestore("alpha"); prepareHermesCronRestoreRecovery("alpha"); recoverHermesCronRestore("alpha"); - expect(processMocks.dockerSpawnSync.mock.calls[0]?.[1]).toMatchObject({ timeout: 70_000 }); - expect(processMocks.dockerSpawnSync.mock.calls[1]?.[1]).toMatchObject({ timeout: 25_000 }); - expect(processMocks.dockerSpawnSync.mock.calls[2]?.[1]).toMatchObject({ timeout: 130_000 }); + expect(processMocks.executePrivilegedSandboxCommand.mock.calls[0]?.[2]).toBe(70_000); + expect(processMocks.executePrivilegedSandboxCommand.mock.calls[1]?.[2]).toBe(25_000); + expect(processMocks.executePrivilegedSandboxCommand.mock.calls[2]?.[2]).toBe(130_000); }); it("returns not-required when no NemoClaw recovery gate exists", () => { - processMocks.dockerSpawnSync.mockReturnValue({ + processMocks.executePrivilegedSandboxCommand.mockReturnValue({ status: 0, stdout: notRequiredRecoveryReceipt(), stderr: "", @@ -578,7 +578,7 @@ describe("Hermes cron rebuild restore contract", () => { }); it("accepts not-required while preserving an independent operator drain", () => { - processMocks.dockerSpawnSync.mockReturnValue({ + processMocks.executePrivilegedSandboxCommand.mockReturnValue({ status: 0, stdout: notRequiredRecoveryReceipt({ operator_drain_active: true, @@ -591,7 +591,7 @@ describe("Hermes cron rebuild restore contract", () => { }); it("rejects an inconsistent recovery receipt", () => { - processMocks.dockerSpawnSync.mockReturnValue({ + processMocks.executePrivilegedSandboxCommand.mockReturnValue({ status: 0, stdout: receipt("recover", 41, 902, "restore-token", { disposition: "operator-drain-preserved", @@ -608,13 +608,13 @@ describe("Hermes cron rebuild restore contract", () => { `/opt/hermes/.venv/bin/python: can't open file '/usr/local/lib/nemoclaw/hermes-cron-restore-control.py': [Errno 2] No such file or directory`, "hermes-cron-restore-control.py: error: argument action: invalid choice: 'recover'", ])("keeps recovery compatible with a legacy Hermes sandbox: %s", (stderr) => { - processMocks.dockerSpawnSync.mockReturnValue({ status: 2, stdout: "", stderr }); + processMocks.executePrivilegedSandboxCommand.mockReturnValue({ status: 2, stdout: "", stderr }); expect(recoverHermesCronRestore("alpha")).toBe("unsupported"); }); it("does not hide a current controller recovery failure", () => { - processMocks.dockerSpawnSync.mockReturnValue({ + processMocks.executePrivilegedSandboxCommand.mockReturnValue({ status: 1, stdout: "", stderr: "Hermes cron restore drain marker is invalid", diff --git a/src/lib/actions/sandbox/rebuild-mcp-phase.ts b/src/lib/actions/sandbox/rebuild-mcp-phase.ts index fb1a52180fb..1e0cafa4315 100644 --- a/src/lib/actions/sandbox/rebuild-mcp-phase.ts +++ b/src/lib/actions/sandbox/rebuild-mcp-phase.ts @@ -157,10 +157,8 @@ export async function restoreMcpAfterRebuild( await restoreMcpBridgesAfterRebuild(sandboxName, entries); console.log(` ${G}✓${R} MCP bridges restored`); return true; - } catch (error) { - console.error( - ` ${YW}⚠${R} MCP bridge restore incomplete: ${error instanceof Error ? error.message : String(error)}`, - ); + } catch { + console.error(` ${YW}⚠${R} MCP bridge restore incomplete; inspect redacted diagnostics.`); return false; } } diff --git a/src/lib/actions/sandbox/rebuild-target-runtime.test.ts b/src/lib/actions/sandbox/rebuild-target-runtime.test.ts index e03cc004781..0b549a78e15 100644 --- a/src/lib/actions/sandbox/rebuild-target-runtime.test.ts +++ b/src/lib/actions/sandbox/rebuild-target-runtime.test.ts @@ -144,6 +144,7 @@ describe("preflightRebuildTargetRuntime GPU route", () => { selectedRoute, gatewayPort: 8080, log, + reverifyBridgeReachability: expect.any(Function), }, ); expect(bail).not.toHaveBeenCalled(); diff --git a/src/lib/actions/sandbox/rebuild-target-runtime.ts b/src/lib/actions/sandbox/rebuild-target-runtime.ts index 4c3e5cdb6b1..9899c9a4c7f 100644 --- a/src/lib/actions/sandbox/rebuild-target-runtime.ts +++ b/src/lib/actions/sandbox/rebuild-target-runtime.ts @@ -13,6 +13,7 @@ import { import { shouldManageDashboardForAgent } from "../../onboard/dashboard-runtime"; import { isLinuxDockerDriverGatewayEnabled } from "../../onboard/docker-driver-platform"; import { enforceDockerGpuPatchPreserveNetwork } from "../../onboard/docker-gpu-local-inference"; +import { verifySandboxBridgeGatewayReachableOrExit } from "../../onboard/gateway-sandbox-reachability"; import { initialDockerGpuRoute, resolveDockerGpuRoutePlan } from "../../onboard/docker-gpu-route"; import { isDockerDesktopWslRuntime } from "../../onboard/docker-gpu-sandbox-create"; import { resolveSandboxGatewayName } from "../../onboard/gateway-binding"; @@ -191,6 +192,11 @@ export async function preflightRebuildTargetRuntime( selectedRoute, gatewayPort: recreateOptions.targetGatewayPort, log, + reverifyBridgeReachability: () => + verifySandboxBridgeGatewayReachableOrExit(true, { + skip: false, + port: recreateOptions.targetGatewayPort, + }), }); } catch (err) { printRebuildPreflightFailure( diff --git a/src/lib/actions/sandbox/runtime-env.test.ts b/src/lib/actions/sandbox/runtime-env.test.ts index 73a7a704f67..cb2e109f97b 100644 --- a/src/lib/actions/sandbox/runtime-env.test.ts +++ b/src/lib/actions/sandbox/runtime-env.test.ts @@ -12,6 +12,8 @@ import { wrapOpenClawAgentCommandWithRuntimeEnv, } from "./runtime-env"; +const TEST_PATH = `${path.dirname(process.execPath)}:/usr/bin:/bin`; + describe("wrapExecCommandWithRuntimeEnv", () => { it("sources the trusted runtime env and preserves each original argv element (#4504)", () => { const command = ["openclaw", "agent", "-m", "hello world", "quote'and\"double"]; @@ -47,7 +49,7 @@ describe("wrapExecCommandWithRuntimeEnv", () => { const result = spawnSync(wrapped[0], wrapped.slice(1), { encoding: "utf-8", - env: { ...process.env }, + env: { PATH: TEST_PATH }, }); expect(result.status, result.stderr).toBe(0); @@ -62,7 +64,7 @@ describe("wrapExecCommandWithRuntimeEnv", () => { ]); const result = spawnSync(wrapped[0], wrapped.slice(1), { encoding: "utf-8", - env: { ...process.env, OPENCLAW_GATEWAY_TOKEN: "super-secret-gateway-token" }, + env: { PATH: TEST_PATH, OPENCLAW_GATEWAY_TOKEN: "super-secret-gateway-token" }, }); expect(result.status, result.stderr).toBe(0); @@ -79,7 +81,7 @@ describe("wrapExecCommandWithRuntimeEnv", () => { const result = spawnSync(wrapped[0], wrapped.slice(1), { encoding: "utf-8", env: { - ...process.env, + PATH: TEST_PATH, HTTP_PROXY: "http://10.200.0.1:3128", NEMOCLAW_OPENCLAW_ALLOW_INSECURE_PRIVATE_WS: "1", NEMOCLAW_OPENCLAW_GATEWAY_URL: "ws://10.200.0.2:18789", @@ -101,7 +103,7 @@ describe("wrapExecCommandWithRuntimeEnv", () => { try { const result = spawnSync(wrapped[0], wrapped.slice(1), { encoding: "utf-8", - env: { ...process.env, BASH_ENV: bashEnv }, + env: { PATH: TEST_PATH, BASH_ENV: bashEnv }, }); expect(result.status, result.stderr).toBe(0); expect(result.stdout).toBe("COMMAND_RAN"); @@ -117,7 +119,10 @@ describe("wrapExecCommandWithRuntimeEnv", () => { "/usr/bin/printf", "SHOULD_NOT_RUN", ]); - const result = spawnSync(wrapped[0], wrapped.slice(1), { encoding: "utf-8" }); + const result = spawnSync(wrapped[0], wrapped.slice(1), { + encoding: "utf-8", + env: { PATH: TEST_PATH }, + }); expect(result.status).toBe(127); expect(result.stdout).not.toContain("SHOULD_NOT_RUN"); @@ -135,7 +140,7 @@ describe("wrapExecCommandWithRuntimeEnv", () => { ]); const result = spawnSync(wrapped[0], wrapped.slice(1), { encoding: "utf-8", - env: { ...process.env, OPENCLAW_GATEWAY_TOKEN: "test-gateway-token" }, + env: { PATH: TEST_PATH, OPENCLAW_GATEWAY_TOKEN: "test-gateway-token" }, }); expect(result.status, result.stderr).toBe(0); diff --git a/src/lib/actions/sandbox/sandbox-gateway-routing.ts b/src/lib/actions/sandbox/sandbox-gateway-routing.ts index 22bf2b3f683..b271c0cf4ae 100644 --- a/src/lib/actions/sandbox/sandbox-gateway-routing.ts +++ b/src/lib/actions/sandbox/sandbox-gateway-routing.ts @@ -18,6 +18,7 @@ import { } from "../../adapters/openshell/timeouts"; import { GATEWAY_PORT } from "../../core/ports"; import { resolveGatewayName, resolveSandboxGatewayName } from "../../onboard/gateway-binding"; +import { resolveRegisteredRuntimeProvider } from "../../onboard/runtime-provider/selection"; import { isGatewayHealthy } from "../../state/gateway"; import * as registry from "../../state/registry"; @@ -47,7 +48,10 @@ export function probeGatewayMetadataHealth(gatewayName: string): boolean { } export function usesGatewayMetadataProbe(driver: string | null | undefined): boolean { - return driver === "docker" || driver === "vm"; + if (driver === "vm") return true; + if (!driver) return false; + const provider = resolveRegisteredRuntimeProvider(driver); + return provider?.gateway.launcher === "nemoclaw"; } /** diff --git a/src/lib/actions/sandbox/snapshot/backup-authority.test.ts b/src/lib/actions/sandbox/snapshot/backup-authority.test.ts index b6ace737f39..efc3cb9d833 100644 --- a/src/lib/actions/sandbox/snapshot/backup-authority.test.ts +++ b/src/lib/actions/sandbox/snapshot/backup-authority.test.ts @@ -6,18 +6,14 @@ import { createHash } from "node:crypto"; import { beforeEach, describe, expect, it, vi } from "vitest"; const privilegedCaptureMocks = vi.hoisted(() => ({ - dockerSpawnSync: vi.fn(), - privilegedSandboxExecArgv: vi.fn(() => ["exec", "container", "python3"]), + executePrivilegedSandboxCommand: vi.fn(), withPrivilegedSandboxExecutionLease: vi.fn( (_sandboxName: string, _operation: string, run: () => unknown) => run(), ), })); -vi.mock("../../../adapters/docker/exec", () => ({ - dockerSpawnSync: privilegedCaptureMocks.dockerSpawnSync, -})); vi.mock("../../../sandbox/privileged-exec", () => ({ - privilegedSandboxExecArgv: privilegedCaptureMocks.privilegedSandboxExecArgv, + executePrivilegedSandboxCommand: privilegedCaptureMocks.executePrivilegedSandboxCommand, withPrivilegedSandboxExecutionLease: privilegedCaptureMocks.withPrivilegedSandboxExecutionLease, })); @@ -186,14 +182,13 @@ function explicitLlamaSandbox(agent: "openclaw" | "hermes" | "langchain-deepagen describe("managed snapshot backup authority", () => { beforeEach(() => { - privilegedCaptureMocks.dockerSpawnSync.mockReset(); - privilegedCaptureMocks.privilegedSandboxExecArgv.mockClear(); + privilegedCaptureMocks.executePrivilegedSandboxCommand.mockReset(); privilegedCaptureMocks.withPrivilegedSandboxExecutionLease.mockClear(); }); it("captures the exact OpenClaw configuration with bounded privileged execution", () => { const data = Buffer.from('{"models":{"default":"nvidia/test"}}\n'); - privilegedCaptureMocks.dockerSpawnSync.mockReturnValue({ + privilegedCaptureMocks.executePrivilegedSandboxCommand.mockReturnValue({ status: 0, signal: null, error: undefined, @@ -213,24 +208,19 @@ describe("managed snapshot backup authority", () => { "OpenClaw config snapshot capture", expect.any(Function), ); - expect(privilegedCaptureMocks.privilegedSandboxExecArgv).toHaveBeenCalledWith( + expect(privilegedCaptureMocks.executePrivilegedSandboxCommand).toHaveBeenCalledWith( "alpha", expect.arrayContaining(["/usr/bin/python3", "-I", "-S", "-c"]), - false, - true, - ); - expect(privilegedCaptureMocks.dockerSpawnSync).toHaveBeenCalledWith( - ["exec", "container", "python3"], expect.objectContaining({ - encoding: null, + sanitizeEnvironment: true, timeout: 30_000, - maxBuffer: 17 * 1024 * 1024, + maxOutputBytes: 17 * 1024 * 1024, }), ); }); it("recognizes only the fixed missing-file failure protocol", () => { - privilegedCaptureMocks.dockerSpawnSync.mockReturnValue({ + privilegedCaptureMocks.executePrivilegedSandboxCommand.mockReturnValue({ status: 2, signal: null, error: undefined, @@ -248,7 +238,7 @@ describe("managed snapshot backup authority", () => { }); it("returns a fixed failure reason when privileged capture rejects unsafe file metadata", () => { - privilegedCaptureMocks.dockerSpawnSync.mockReturnValue({ + privilegedCaptureMocks.executePrivilegedSandboxCommand.mockReturnValue({ status: 11, signal: null, error: undefined, @@ -269,7 +259,7 @@ describe("managed snapshot backup authority", () => { }); it("bounds and redacts untrusted privileged stderr", () => { - privilegedCaptureMocks.dockerSpawnSync.mockReturnValue({ + privilegedCaptureMocks.executePrivilegedSandboxCommand.mockReturnValue({ status: 10, signal: null, error: undefined, @@ -284,10 +274,7 @@ describe("managed snapshot backup authority", () => { }); expect(result).toMatchObject({ outcome: "failed" }); - const failedResult = result as Extract< - NonNullable, - { outcome: "failed" } - >; + const failedResult = result as Extract, { outcome: "failed" }>; const error = failedResult.error ?? ""; expect(error).toContain("permission denied apiKey="); expect(error).not.toContain("secret-value"); @@ -296,7 +283,7 @@ describe("managed snapshot backup authority", () => { }); it("does not confuse an unrecognized exit 2 with a missing config", () => { - privilegedCaptureMocks.dockerSpawnSync.mockReturnValue({ + privilegedCaptureMocks.executePrivilegedSandboxCommand.mockReturnValue({ status: 2, signal: null, error: undefined, @@ -344,39 +331,39 @@ describe("managed snapshot backup authority", () => { ] as const)("rejects $input before privileged capture", ({ request }) => { expect(captureOpenClawStateFile("alpha", request)).toBeNull(); expect(privilegedCaptureMocks.withPrivilegedSandboxExecutionLease).not.toHaveBeenCalled(); - expect(privilegedCaptureMocks.dockerSpawnSync).not.toHaveBeenCalled(); + expect(privilegedCaptureMocks.executePrivilegedSandboxCommand).not.toHaveBeenCalled(); }); it.each(["openclaw", "hermes", "langchain-deepagents-code"] as const)( "captures and republishes exact %s provider authority", (agent) => { - const entry = sandbox(agent); - const getSandbox = vi.fn(() => entry); - const requireProvider = vi.fn(() => provider()); - const captureRuntime = vi.fn(() => runtime()); + const entry = sandbox(agent); + const getSandbox = vi.fn(() => entry); + const requireProvider = vi.fn(() => provider()); + const captureRuntime = vi.fn(() => runtime()); const backup = vi.fn((_name: string, options: BackupOptions = {}) => successfulBackup(options), ); - const result = backupSandboxStateWithManagedAuthority( - "alpha", - { name: "stable" }, - { getSandbox, requireProvider, captureRuntime, backup }, - ); + const result = backupSandboxStateWithManagedAuthority( + "alpha", + { name: "stable" }, + { getSandbox, requireProvider, captureRuntime, backup }, + ); - expect(result.success).toBe(true); - expect(backup).toHaveBeenCalledWith( - "alpha", - expect.objectContaining({ - name: "stable", - workload: entry.workload, - runtimeSnapshot: runtime(), - validateBeforePublish: expect.any(Function), - }), - ); - expect(getSandbox).toHaveBeenCalledTimes(2); - expect(requireProvider).toHaveBeenCalledTimes(2); - expect(captureRuntime).toHaveBeenCalledTimes(2); + expect(result.success).toBe(true); + expect(backup).toHaveBeenCalledWith( + "alpha", + expect.objectContaining({ + name: "stable", + workload: entry.workload, + runtimeSnapshot: runtime(), + validateBeforePublish: expect.any(Function), + }), + ); + expect(getSandbox).toHaveBeenCalledTimes(2); + expect(requireProvider).toHaveBeenCalledTimes(2); + expect(captureRuntime).toHaveBeenCalledTimes(2); }, ); @@ -476,51 +463,48 @@ describe("managed snapshot backup authority", () => { expect(captureRuntime).not.toHaveBeenCalled(); }); - it.each([ - "openclaw", - "hermes", - "langchain-deepagents-code", - ] as const)("captures and confirms exact %s host-local inference authority", (agent) => { - const entry = hostLocalSandbox(agent); - const prepared = { - providerId: "mxc", - sandboxName: "alpha", - serializedReceipt: entry.hostLocalInferenceReceipt, - sandboxAuthority: { model: entry.model }, - }; - const prepareHostLocalInference = vi.fn(() => prepared); - const confirmHostLocalInference = vi.fn(); - const backup = vi.fn((_name: string, options: BackupOptions = {}) => successfulBackup(options)); + it.each(["openclaw", "hermes", "langchain-deepagents-code"] as const)( + "captures and confirms exact %s host-local inference authority", + (agent) => { + const entry = hostLocalSandbox(agent); + const prepared = { + providerId: "mxc", + sandboxName: "alpha", + serializedReceipt: entry.hostLocalInferenceReceipt, + sandboxAuthority: { model: entry.model }, + }; + const prepareHostLocalInference = vi.fn(() => prepared); + const confirmHostLocalInference = vi.fn(); + const backup = vi.fn((_name: string, options: BackupOptions = {}) => + successfulBackup(options), + ); - const result = backupSandboxStateWithManagedAuthority( - "alpha", - { name: "host-local" }, - { - getSandbox: () => entry, - requireProvider: () => provider(), - captureRuntime: vi.fn() as never, - prepareHostLocalInference: prepareHostLocalInference as never, - confirmHostLocalInference: confirmHostLocalInference as never, - backup, - }, - ); + const result = backupSandboxStateWithManagedAuthority( + "alpha", + { name: "host-local" }, + { + getSandbox: () => entry, + requireProvider: () => provider(), + captureRuntime: vi.fn() as never, + prepareHostLocalInference: prepareHostLocalInference as never, + confirmHostLocalInference: confirmHostLocalInference as never, + backup, + }, + ); - expect(result.success).toBe(true); - expect(backup).toHaveBeenCalledWith( - "alpha", - expect.objectContaining({ - name: "host-local", - hostLocalInferenceReceipt: entry.hostLocalInferenceReceipt, - validateBeforePublish: expect.any(Function), - }), - ); - expect(prepareHostLocalInference).toHaveBeenCalledWith(expect.anything(), entry); - expect(confirmHostLocalInference).toHaveBeenCalledWith( - expect.anything(), - entry, - prepared, - ); - }); + expect(result.success).toBe(true); + expect(backup).toHaveBeenCalledWith( + "alpha", + expect.objectContaining({ + name: "host-local", + hostLocalInferenceReceipt: entry.hostLocalInferenceReceipt, + validateBeforePublish: expect.any(Function), + }), + ); + expect(prepareHostLocalInference).toHaveBeenCalledWith(expect.anything(), entry); + expect(confirmHostLocalInference).toHaveBeenCalledWith(expect.anything(), entry, prepared); + }, + ); it.each([ ["agent", { agent: "hermes" }], @@ -550,7 +534,7 @@ describe("managed snapshot backup authority", () => { "lifecycleGeneration", ] as const; expect(fields.some((field) => candidate[field] !== initial[field])).toBe(true); - throw new Error("sandbox authority changed after lifecycle preparation"); + throw new Error("sandbox authority changed after lifecycle preparation"); }); const backup = vi.fn((_name: string, options: BackupOptions = {}) => successfulBackup(options)); @@ -608,39 +592,40 @@ describe("managed snapshot backup authority", () => { secondRuntime: runtime("session-2"), error: "runtime changed during backup", }, - ])("rejects $label drift before manifest publication", ({ - secondEntry, - secondRuntime, - error, - }) => { - const initialEntry = sandbox("openclaw"); - const getSandbox = vi - .fn<() => SandboxEntry | null>() - .mockReturnValueOnce(initialEntry) - .mockReturnValueOnce(secondEntry); - const captureRuntime = vi - .fn<() => ReturnType>() - .mockReturnValueOnce(runtime()) - .mockReturnValueOnce(secondRuntime); - const backup = vi.fn((_name: string, options: BackupOptions = {}) => successfulBackup(options)); + ])( + "rejects $label drift before manifest publication", + ({ secondEntry, secondRuntime, error }) => { + const initialEntry = sandbox("openclaw"); + const getSandbox = vi + .fn<() => SandboxEntry | null>() + .mockReturnValueOnce(initialEntry) + .mockReturnValueOnce(secondEntry); + const captureRuntime = vi + .fn<() => ReturnType>() + .mockReturnValueOnce(runtime()) + .mockReturnValueOnce(secondRuntime); + const backup = vi.fn((_name: string, options: BackupOptions = {}) => + successfulBackup(options), + ); - const result = backupSandboxStateWithManagedAuthority( - "alpha", - {}, - { - getSandbox, - requireProvider: () => provider(), - captureRuntime: captureRuntime as ( - bundle: RuntimeProviderBundle, - entry: SandboxEntry, - ) => ReturnType, - backup, - }, - ); + const result = backupSandboxStateWithManagedAuthority( + "alpha", + {}, + { + getSandbox, + requireProvider: () => provider(), + captureRuntime: captureRuntime as ( + bundle: RuntimeProviderBundle, + entry: SandboxEntry, + ) => ReturnType, + backup, + }, + ); - expect(result).toMatchObject({ - success: false, - error: expect.stringContaining(error), - }); - }); + expect(result).toMatchObject({ + success: false, + error: expect.stringContaining(error), + }); + }, + ); }); diff --git a/src/lib/actions/sandbox/snapshot/backup-authority.ts b/src/lib/actions/sandbox/snapshot/backup-authority.ts index 620e32a7a79..c6fa09738f4 100644 --- a/src/lib/actions/sandbox/snapshot/backup-authority.ts +++ b/src/lib/actions/sandbox/snapshot/backup-authority.ts @@ -3,7 +3,6 @@ import { isDeepStrictEqual } from "node:util"; -import { dockerSpawnSync } from "../../../adapters/docker/exec"; import type { RuntimeProviderBundle } from "../../../onboard/runtime-provider/contract"; import { CURRENT_RUNTIME_PROVIDER_BUNDLES } from "../../../onboard/runtime-provider/current"; import { @@ -14,7 +13,7 @@ import { requireRuntimeProviderBundleForSandbox } from "../../../onboard/runtime import type { SandboxEntry } from "../../../state/registry/types"; import * as sandboxState from "../../../state/sandbox"; import { - privilegedSandboxExecArgv, + executePrivilegedSandboxCommand, withPrivilegedSandboxExecutionLease, } from "../../../sandbox/privileged-exec"; import { sanitizeReadinessText } from "../../../readiness/sanitize"; @@ -170,7 +169,7 @@ export function captureOpenClawStateFile( sandboxName, "OpenClaw config snapshot capture", () => { - const argv = privilegedSandboxExecArgv( + const result = executePrivilegedSandboxCommand( sandboxName, [ "/usr/bin/python3", @@ -181,15 +180,12 @@ export function captureOpenClawStateFile( OPENCLAW_CONFIG_DIRECTORY, OPENCLAW_CONFIG_NAME, ], - false, - true, + { + sanitizeEnvironment: true, + timeout: OPENCLAW_CONFIG_CAPTURE_TIMEOUT_MS, + maxOutputBytes: OPENCLAW_CONFIG_CAPTURE_MAX_BUFFER, + }, ); - const result = dockerSpawnSync(argv, { - encoding: null, - stdio: ["ignore", "pipe", "pipe"], - timeout: OPENCLAW_CONFIG_CAPTURE_TIMEOUT_MS, - maxBuffer: OPENCLAW_CONFIG_CAPTURE_MAX_BUFFER, - }); const protocolFailure = captureFailureProtocol(result.stderr); if ( result.status === 2 && diff --git a/src/lib/actions/sandbox/status-preflight.ts b/src/lib/actions/sandbox/status-preflight.ts index ac34bb46a03..b4f772bcb39 100644 --- a/src/lib/actions/sandbox/status-preflight.ts +++ b/src/lib/actions/sandbox/status-preflight.ts @@ -3,11 +3,15 @@ import { isTerminalSandboxPhase } from "../../state/gateway"; import type * as registry from "../../state/registry"; +import { + registeredRuntimeProviderSupportsContainerEngineOperation, + resolveRegisteredRuntimeProvider, +} from "../../onboard/runtime-provider/selection"; import { classifyGatewayFailure, + classifyObservedSandboxContainerFailure, classifySandboxContainerFailure, getLayerHeader, - isDockerDaemonReachable, type SandboxContainerFailureResult, } from "./gateway-failure-classifier"; @@ -61,6 +65,53 @@ const defaultSandboxContainerFailureProbe: SandboxContainerFailureProbe = ( dashboardPort, ) => classifySandboxContainerFailure(sandboxName, { dashboardPort }); +export function hasLegacyStatusRuntimeObservation(sb: registry.SandboxEntry | null): boolean { + const driverName = sb?.openshellDriver?.trim().toLowerCase(); + if (!driverName) return false; + const provider = resolveRegisteredRuntimeProvider(driverName); + if (!provider || provider.identity.id !== driverName) return false; + if ( + !registeredRuntimeProviderSupportsContainerEngineOperation(driverName, "gateway-inspection") + ) { + return false; + } + try { + // Socket-backed native providers own observation through their provider + // implementation. This legacy classifier remains only for the default + // container-engine transport until the status contract grows an + // operation-bearing provider observation surface. + return ( + provider.gateway.prepareHostRuntime({ + environment: process.env, + platform: process.platform, + }).socketPath === null + ); + } catch { + return false; + } +} + +export function usesManagedProviderGateway(sb: registry.SandboxEntry | null): boolean { + const driverName = sb?.openshellDriver?.trim().toLowerCase(); + if (!driverName) return false; + const provider = resolveRegisteredRuntimeProvider(driverName); + return ( + provider?.identity.id === driverName && + provider.gateway.launcher === "nemoclaw" && + provider.bootstrap.supported === true + ); +} + +function isSelectedRuntimeReachable(sb: registry.SandboxEntry): boolean { + const provider = resolveRegisteredRuntimeProvider(sb.openshellDriver); + if (!provider) return false; + try { + return provider.preflightDoctor.inspectHost().status !== "fail"; + } catch { + return false; + } +} + export interface ClassifySandboxStatusPreflightFailureDeps { dockerProbe?: DockerInfoProbe; sandboxContainerProbe?: SandboxContainerFailureProbe; @@ -68,9 +119,9 @@ export interface ClassifySandboxStatusPreflightFailureDeps { export function isDockerDaemonUnreachableForStatus( sb: registry.SandboxEntry | null, - probe: DockerInfoProbe = isDockerDaemonReachable, + probe: DockerInfoProbe = () => (sb ? isSelectedRuntimeReachable(sb) : false), ): boolean { - if (!sb || sb.openshellDriver !== "docker") return false; + if (!sb || !hasLegacyStatusRuntimeObservation(sb)) return false; return !probe(); } @@ -78,8 +129,23 @@ export async function classifySandboxContainerFailureForStatus( sb: registry.SandboxEntry | null, probe: SandboxContainerFailureProbe = defaultSandboxContainerFailureProbe, ): Promise { - if (!sb || sb.openshellDriver !== "docker") return null; - return probe(sb.name, sb.dashboardPort ?? null); + if (!sb) return null; + if (hasLegacyStatusRuntimeObservation(sb)) { + return probe(sb.name, sb.dashboardPort ?? null); + } + if (sb.workload?.kind !== "managed-image" || !usesManagedProviderGateway(sb)) return null; + const provider = resolveRegisteredRuntimeProvider(sb.openshellDriver); + if (provider?.snapshot.supported !== true) return null; + try { + const observation = provider.snapshot.preflight("backup", sb); + return classifyObservedSandboxContainerFailure( + sb.name, + observation.lifecycleState, + sb.dashboardPort, + ); + } catch { + return null; + } } /** diff --git a/src/lib/actions/sandbox/status-snapshot.ts b/src/lib/actions/sandbox/status-snapshot.ts index 63db3cc1876..98b8438ecb5 100644 --- a/src/lib/actions/sandbox/status-snapshot.ts +++ b/src/lib/actions/sandbox/status-snapshot.ts @@ -45,8 +45,10 @@ import { } from "./inference-route-health"; import { getSandboxStatusPreflight, + hasLegacyStatusRuntimeObservation, type SandboxStatusFailureLayer, type SandboxStatusPreflightResult, + usesManagedProviderGateway, withoutTerminalPhasePreflight, } from "./status-preflight"; import { @@ -422,7 +424,8 @@ export async function collectSandboxStatusSnapshot( const dockerRecovered = lookup.recoveredSandbox === true; const managedOpenClawDeliveryMustBeProven = lookup.state === "present" && - sb?.openshellDriver === "docker" && + sb !== null && + usesManagedProviderGateway(sb) && (sb.agent ?? "openclaw") === "openclaw" && lookup.phase === "Ready" && !opts.preflight?.failure; @@ -710,7 +713,10 @@ async function buildSandboxStatusReport( inferenceHealth, terminalRuntimeHealth, } = snapshot; - const dockerRuntime = lookup.state === "present" ? getSandboxDockerRuntime(sandboxName) : null; + const dockerRuntime = + lookup.state === "present" && hasLegacyStatusRuntimeObservation(sb) + ? getSandboxDockerRuntime(sandboxName) + : null; const phase = lookup.state === "present" ? (lookup.phase ?? null) : null; const effectivePreflight = withoutTerminalPhasePreflight( snapshot.postRecoveryPreflight ?? preflight, diff --git a/src/lib/actions/sandbox/stopped-sandbox-backup.test.ts b/src/lib/actions/sandbox/stopped-sandbox-backup.test.ts index fcc69dfe283..329f102f651 100644 --- a/src/lib/actions/sandbox/stopped-sandbox-backup.test.ts +++ b/src/lib/actions/sandbox/stopped-sandbox-backup.test.ts @@ -1,17 +1,31 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it, vi } from "vitest"; +import { beforeEach, describe, expect, it, vi } from "vitest"; const adapterMocks = vi.hoisted(() => ({ - dockerRun: vi.fn(), - dockerCapture: vi.fn(), + providerCapture: vi.fn(), backupWithAuthority: vi.fn(), })); -vi.mock("../../adapters/docker/run", () => ({ - dockerRun: adapterMocks.dockerRun, - dockerCapture: adapterMocks.dockerCapture, +vi.mock("../../onboard/runtime-provider/selection", () => ({ + resolveRegisteredRuntimeProvider: (providerId: string | null | undefined) => { + const normalized = String(providerId).trim().toLowerCase(); + return { + identity: { id: normalized }, + lifecycle: { + supported: true, + containerMutationTimeoutMs: normalized === "podman" ? 75_000 : 30_000, + }, + containerEngine: { + supported: true, + identities: [ + { operation: "sandbox-lifecycle", engineId: normalized, displayName: normalized }, + ], + capture: adapterMocks.providerCapture, + }, + }; + }, })); vi.mock("../../state/registry", () => ({ getSandbox: vi.fn(), @@ -34,22 +48,45 @@ import { startStoppedSandboxContainerForBackup, } from "./stopped-sandbox-backup"; +function lifecycleEngine(runtimeProviderId = "docker") { + return { runtimeProviderId, mutationTimeoutMs: 30_000, capture: vi.fn() }; +} + describe("startStoppedSandboxContainerForBackup", () => { - const deps = (over: Record = {}) => ({ - getSandboxDriver: vi.fn().mockReturnValue("docker"), - listSandboxNames: vi.fn().mockReturnValue(["my-sb"]), - listLabeledContainerNames: vi.fn().mockReturnValue(["openshell-my-sb-abc123"]), - dockerInspectStatus: vi.fn().mockReturnValue("exited"), - dockerStart: vi.fn().mockReturnValue("openshell-my-sb-abc123"), - ...over, - }); + const deps = (over: Record = {}) => { + const engine = lifecycleEngine(); + return { + getSandboxDriver: vi.fn().mockReturnValue("docker"), + listSandboxNames: vi.fn().mockReturnValue(["my-sb"]), + resolveLifecycleEngine: vi.fn().mockReturnValue(engine), + listLabeledContainerNames: vi.fn().mockReturnValue(["openshell-my-sb-abc123"]), + inspectStatus: vi.fn().mockReturnValue("exited"), + start: vi.fn().mockReturnValue(true), + ...over, + }; + }; - it("starts an exited docker-driver container and reports its name", () => { + it("starts an exited provider-owned container and records its provider", () => { const d = deps(); expect(startStoppedSandboxContainerForBackup("my-sb", d)).toEqual({ containerName: "openshell-my-sb-abc123", + runtimeProviderId: "docker", + }); + expect(d.start).toHaveBeenCalledWith(expect.any(Object), "openshell-my-sb-abc123"); + }); + + it("uses the same lifecycle path for a registered Podman provider", () => { + const podmanEngine = lifecycleEngine("podman"); + const d = deps({ + getSandboxDriver: vi.fn().mockReturnValue("podman"), + resolveLifecycleEngine: vi.fn().mockReturnValue(podmanEngine), + }); + + expect(startStoppedSandboxContainerForBackup("my-sb", d)).toEqual({ + containerName: "openshell-my-sb-abc123", + runtimeProviderId: "podman", }); - expect(d.dockerStart).toHaveBeenCalledWith("openshell-my-sb-abc123"); + expect(d.start).toHaveBeenCalledWith(podmanEngine, "openshell-my-sb-abc123"); }); it("excludes created pending registrations from container ownership (#9733)", () => { @@ -66,21 +103,22 @@ describe("startStoppedSandboxContainerForBackup", () => { }); const { listSandboxNames: _listSandboxNames, ...d } = deps({ listLabeledContainerNames: vi.fn().mockReturnValue(["openshell-my-assistant-12ab"]), - dockerInspectStatus: vi.fn().mockReturnValue("exited"), + inspectStatus: vi.fn().mockReturnValue("exited"), }); expect(startStoppedSandboxContainerForBackup("my", d)).toEqual({ containerName: "openshell-my-assistant-12ab", + runtimeProviderId: "docker", }); }); it("starts a created container (onboarded but never run)", () => { - const d = deps({ dockerInspectStatus: vi.fn().mockReturnValue("created") }); + const d = deps({ inspectStatus: vi.fn().mockReturnValue("created") }); expect(startStoppedSandboxContainerForBackup("my-sb", d)).not.toBeNull(); }); - it("leaves non-docker-driver sandboxes alone", () => { - const d = deps({ getSandboxDriver: vi.fn().mockReturnValue("kubernetes") }); + it("leaves providers without a container lifecycle engine alone", () => { + const d = deps({ resolveLifecycleEngine: vi.fn().mockReturnValue(null) }); expect(startStoppedSandboxContainerForBackup("my-sb", d)).toBeNull(); expect(d.listLabeledContainerNames).not.toHaveBeenCalled(); }); @@ -88,7 +126,7 @@ describe("startStoppedSandboxContainerForBackup", () => { it("returns null when no labeled container owns the sandbox name", () => { const d = deps({ listLabeledContainerNames: vi.fn().mockReturnValue([]) }); expect(startStoppedSandboxContainerForBackup("my-sb", d)).toBeNull(); - expect(d.dockerStart).not.toHaveBeenCalled(); + expect(d.start).not.toHaveBeenCalled(); }); it("refuses ambiguous labeled containers", () => { @@ -98,14 +136,14 @@ describe("startStoppedSandboxContainerForBackup", () => { .mockReturnValue(["openshell-my-sb-old", "openshell-my-sb-new"]), }); expect(startStoppedSandboxContainerForBackup("my-sb", d)).toBeNull(); - expect(d.dockerInspectStatus).not.toHaveBeenCalled(); - expect(d.dockerStart).not.toHaveBeenCalled(); + expect(d.inspectStatus).not.toHaveBeenCalled(); + expect(d.start).not.toHaveBeenCalled(); }); it("refuses a labeled container whose name does not belong to the sandbox", () => { const d = deps({ listLabeledContainerNames: vi.fn().mockReturnValue(["openshell-other-x"]) }); expect(startStoppedSandboxContainerForBackup("my-sb", d)).toBeNull(); - expect(d.dockerStart).not.toHaveBeenCalled(); + expect(d.start).not.toHaveBeenCalled(); }); it("leaves GPU recovery backup siblings to the dedicated recovery flow", () => { @@ -115,34 +153,43 @@ describe("startStoppedSandboxContainerForBackup", () => { .mockReturnValue(["openshell-my-sb-nemoclaw-gpu-backup-123"]), }); expect(startStoppedSandboxContainerForBackup("my-sb", d)).toBeNull(); - expect(d.dockerStart).not.toHaveBeenCalled(); + expect(d.start).not.toHaveBeenCalled(); }); it("leaves a running-but-not-Ready container alone (crash loop, gateway drift)", () => { - const d = deps({ dockerInspectStatus: vi.fn().mockReturnValue("running") }); + const d = deps({ inspectStatus: vi.fn().mockReturnValue("running") }); expect(startStoppedSandboxContainerForBackup("my-sb", d)).toBeNull(); - expect(d.dockerStart).not.toHaveBeenCalled(); + expect(d.start).not.toHaveBeenCalled(); }); it("leaves a paused container alone (#4495)", () => { - const d = deps({ dockerInspectStatus: vi.fn().mockReturnValue("paused") }); + const d = deps({ inspectStatus: vi.fn().mockReturnValue("paused") }); expect(startStoppedSandboxContainerForBackup("my-sb", d)).toBeNull(); - expect(d.dockerStart).not.toHaveBeenCalled(); + expect(d.start).not.toHaveBeenCalled(); }); - it("returns null when docker start fails", () => { - const d = deps({ dockerStart: vi.fn().mockReturnValue("") }); + it("returns null when the provider start operation fails", () => { + const d = deps({ start: vi.fn().mockReturnValue(false) }); expect(startStoppedSandboxContainerForBackup("my-sb", d)).toBeNull(); }); }); describe("isSandboxContainerDefinitivelyAbsent (#6520)", () => { - const deps = (over: Record = {}) => ({ - getSandboxDriver: vi.fn().mockReturnValue("docker"), - listLabeledContainerNames: vi.fn().mockReturnValue([]), - ...over, + beforeEach(() => { + adapterMocks.providerCapture.mockReset(); + vi.mocked(registry.getSandbox).mockReset(); }); + const deps = (over: Record = {}) => { + const engine = lifecycleEngine(); + return { + getSandboxDriver: vi.fn().mockReturnValue("docker"), + resolveLifecycleEngine: vi.fn().mockReturnValue(engine), + listLabeledContainerNames: vi.fn().mockReturnValue([]), + ...over, + }; + }; + it("reports absent when a successful labeled listing shows zero containers", () => { expect(isSandboxContainerDefinitivelyAbsent("my-sb", deps())).toBe(true); }); @@ -152,8 +199,8 @@ describe("isSandboxContainerDefinitivelyAbsent (#6520)", () => { expect(isSandboxContainerDefinitivelyAbsent("my-sb", d)).toBe(false); }); - it("fails closed for non-docker-driver sandboxes", () => { - const d = deps({ getSandboxDriver: vi.fn().mockReturnValue("kubernetes") }); + it("fails closed for providers without a container lifecycle engine", () => { + const d = deps({ resolveLifecycleEngine: vi.fn().mockReturnValue(null) }); expect(isSandboxContainerDefinitivelyAbsent("my-sb", d)).toBe(false); expect(d.listLabeledContainerNames).not.toHaveBeenCalled(); }); @@ -168,22 +215,19 @@ describe("isSandboxContainerDefinitivelyAbsent (#6520)", () => { throw new Error("corrupt sandboxes.json"); }); expect(isSandboxContainerDefinitivelyAbsent("my-sb")).toBe(false); - expect(adapterMocks.dockerRun).not.toHaveBeenCalled(); + expect(adapterMocks.providerCapture).not.toHaveBeenCalled(); }); - it("status-checks the default listing with ignoreError so a dead daemon fails closed, not the process", () => { - // runner.run() calls process.exit on a non-zero status unless ignoreError - // is set, and a swallowed listing error must never read as "absent": a - // failed `docker ps` has to surface as false, not as an exit and not as - // an empty listing. + it("fails closed when the provider listing command fails", () => { vi.mocked(registry.getSandbox).mockReturnValue({ openshellDriver: "docker", } as unknown as ReturnType); - adapterMocks.dockerRun.mockReturnValue({ status: 1, stdout: "" }); + adapterMocks.providerCapture.mockReturnValue({ status: 1, stdout: "", stderr: "down" }); expect(isSandboxContainerDefinitivelyAbsent("my-sb")).toBe(false); - expect(adapterMocks.dockerRun).toHaveBeenCalledWith( + expect(adapterMocks.providerCapture).toHaveBeenCalledWith( + "sandbox-lifecycle", expect.arrayContaining(["ps", "-a", "--filter", "label=openshell.ai/sandbox-name=my-sb"]), - expect.objectContaining({ ignoreError: true }), + 5_000, ); }); @@ -191,7 +235,7 @@ describe("isSandboxContainerDefinitivelyAbsent (#6520)", () => { vi.mocked(registry.getSandbox).mockReturnValue({ openshellDriver: "docker", } as unknown as ReturnType); - adapterMocks.dockerRun.mockReturnValue({ status: 0, stdout: "\n" }); + adapterMocks.providerCapture.mockReturnValue({ status: 0, stdout: "\n", stderr: "" }); expect(isSandboxContainerDefinitivelyAbsent("my-sb")).toBe(true); }); @@ -199,44 +243,75 @@ describe("isSandboxContainerDefinitivelyAbsent (#6520)", () => { vi.mocked(registry.getSandbox).mockReturnValue({ openshellDriver: "docker", } as unknown as ReturnType); - adapterMocks.dockerRun.mockReturnValue({ status: 0, stdout: "openshell-my-sb-abc\n" }); + adapterMocks.providerCapture.mockReturnValue({ + status: 0, + stdout: "openshell-my-sb-abc\n", + stderr: "", + }); expect(isSandboxContainerDefinitivelyAbsent("my-sb")).toBe(false); }); }); describe("returnSandboxContainerToStopped", () => { - it("reports success when docker stop echoes the name and inspect confirms exited", () => { - const dockerStop = vi.fn().mockReturnValue("openshell-my-sb-abc123"); - const dockerInspectStatus = vi.fn().mockReturnValue("exited"); + beforeEach(() => adapterMocks.providerCapture.mockReset()); + + const started = { + containerName: "openshell-my-sb-abc123", + runtimeProviderId: "podman", + }; + + it("uses the recorded provider and confirms the container stopped", () => { + const engine = lifecycleEngine("podman"); + const resolveLifecycleEngine = vi.fn().mockReturnValue(engine); + const stop = vi.fn().mockReturnValue(true); + const inspectStatus = vi.fn().mockReturnValue("exited"); expect( - returnSandboxContainerToStopped("openshell-my-sb-abc123", { - dockerStop, - dockerInspectStatus, + returnSandboxContainerToStopped(started, { + resolveLifecycleEngine, + stop, + inspectStatus, }), ).toBe(true); - expect(dockerStop).toHaveBeenCalledWith("openshell-my-sb-abc123"); - expect(dockerInspectStatus).toHaveBeenCalledWith("openshell-my-sb-abc123"); + expect(resolveLifecycleEngine).toHaveBeenCalledWith("podman"); + expect(stop).toHaveBeenCalledWith(engine, "openshell-my-sb-abc123"); + expect(inspectStatus).toHaveBeenCalledWith(engine, "openshell-my-sb-abc123"); + }); + + it("uses the Podman lifecycle timeout when restoring a stopped container", () => { + adapterMocks.providerCapture + .mockReturnValueOnce({ status: 0, stdout: "", stderr: "" }) + .mockReturnValueOnce({ status: 0, stdout: "exited\n", stderr: "" }); + + expect(returnSandboxContainerToStopped(started)).toBe(true); + expect(adapterMocks.providerCapture).toHaveBeenNthCalledWith( + 1, + "sandbox-lifecycle", + ["stop", "openshell-my-sb-abc123"], + 75_000, + ); }); - it("reports failure when docker stop produces no output", () => { - const dockerStop = vi.fn().mockReturnValue(""); - const dockerInspectStatus = vi.fn(); + it("reports failure when the provider stop operation fails", () => { + const engine = lifecycleEngine("podman"); + const stop = vi.fn().mockReturnValue(false); + const inspectStatus = vi.fn(); expect( - returnSandboxContainerToStopped("openshell-my-sb-abc123", { - dockerStop, - dockerInspectStatus, + returnSandboxContainerToStopped(started, { + resolveLifecycleEngine: vi.fn().mockReturnValue(engine), + stop, + inspectStatus, }), ).toBe(false); - expect(dockerInspectStatus).not.toHaveBeenCalled(); + expect(inspectStatus).not.toHaveBeenCalled(); }); - it("reports failure when the container is still running after docker stop", () => { - const dockerStop = vi.fn().mockReturnValue("openshell-my-sb-abc123"); - const dockerInspectStatus = vi.fn().mockReturnValue("running"); + it("reports failure when the container is still running after stop", () => { + const engine = lifecycleEngine("podman"); expect( - returnSandboxContainerToStopped("openshell-my-sb-abc123", { - dockerStop, - dockerInspectStatus, + returnSandboxContainerToStopped(started, { + resolveLifecycleEngine: vi.fn().mockReturnValue(engine), + stop: vi.fn().mockReturnValue(true), + inspectStatus: vi.fn().mockReturnValue("running"), }), ).toBe(false); }); diff --git a/src/lib/actions/sandbox/stopped-sandbox-backup.ts b/src/lib/actions/sandbox/stopped-sandbox-backup.ts index 0547c65fc18..1c47637374a 100644 --- a/src/lib/actions/sandbox/stopped-sandbox-backup.ts +++ b/src/lib/actions/sandbox/stopped-sandbox-backup.ts @@ -1,16 +1,10 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { dockerContainerInspectFormat } from "../../adapters/docker/inspect"; -import { dockerCapture, dockerRun } from "../../adapters/docker/run"; import { retryUntilAsync } from "../../core/retry"; import { resolveSandboxContainerOwner } from "../../domain/sandbox/container-owner"; -import { - findLabeledSandboxContainers, - OPENSHELL_MANAGED_BY_LABEL, - OPENSHELL_MANAGED_BY_VALUE, - OPENSHELL_SANDBOX_NAME_LABEL, -} from "../../onboard/docker-driver-sandbox-recovery"; +import type { RuntimeProviderCommandCapture } from "../../onboard/runtime-provider/contract"; +import { resolveRegisteredRuntimeProvider } from "../../onboard/runtime-provider/selection"; import * as registry from "../../state/registry"; import * as sandboxState from "../../state/sandbox"; import * as snapshotBackup from "./snapshot/backup-authority"; @@ -25,18 +19,94 @@ function readSandboxDriver(name: string): string | null | undefined { } } -const DOCKER_ABSENCE_PROBE_TIMEOUT_MS = 5_000; +interface SandboxLifecycleEngine { + readonly runtimeProviderId: string; + readonly mutationTimeoutMs: number; + capture(args: readonly string[], timeoutMs?: number): RuntimeProviderCommandCapture; +} + +function resolveSandboxLifecycleEngine( + driverName: string | null | undefined, +): SandboxLifecycleEngine | null { + const normalized = driverName?.trim().toLowerCase(); + if (!normalized) return null; + const provider = resolveRegisteredRuntimeProvider(normalized); + if ( + !provider || + provider.identity.id !== normalized || + provider.containerEngine.supported !== true + ) { + return null; + } + const containerEngine = provider.containerEngine; + if (!containerEngine.identities.some((identity) => identity.operation === "sandbox-lifecycle")) { + return null; + } + return { + runtimeProviderId: provider.identity.id, + mutationTimeoutMs: + provider.lifecycle.supported === true && provider.lifecycle.containerMutationTimeoutMs + ? provider.lifecycle.containerMutationTimeoutMs + : CONTAINER_ENGINE_MUTATION_TIMEOUT_MS, + capture: (args, timeoutMs) => containerEngine.capture("sandbox-lifecycle", args, timeoutMs), + }; +} + +const CONTAINER_ENGINE_PROBE_TIMEOUT_MS = 5_000; +const CONTAINER_ENGINE_MUTATION_TIMEOUT_MS = 30_000; +const OPENSHELL_MANAGED_BY_LABEL = "openshell.ai/managed-by"; +const OPENSHELL_MANAGED_BY_VALUE = "openshell"; +const OPENSHELL_SANDBOX_NAME_LABEL = "openshell.ai/sandbox-name"; + +function captureSucceeded(result: RuntimeProviderCommandCapture): boolean { + return result.status === 0 && result.error === undefined; +} + +function listLabeledContainerNames( + engine: SandboxLifecycleEngine, + sandboxName: string, +): string[] | null { + const result = engine.capture( + [ + "ps", + "-a", + "--filter", + `label=${OPENSHELL_MANAGED_BY_LABEL}=${OPENSHELL_MANAGED_BY_VALUE}`, + "--filter", + `label=${OPENSHELL_SANDBOX_NAME_LABEL}=${sandboxName}`, + "--format", + "{{.Names}}", + ], + CONTAINER_ENGINE_PROBE_TIMEOUT_MS, + ); + if (!captureSucceeded(result)) return null; + return result.stdout + .split(/\r?\n/u) + .map((line) => line.trim()) + .filter(Boolean); +} + +function inspectContainerStatus( + engine: SandboxLifecycleEngine, + containerName: string, +): string | null { + const result = engine.capture( + ["inspect", "--format", "{{.State.Status}}", containerName], + CONTAINER_ENGINE_PROBE_TIMEOUT_MS, + ); + return captureSucceeded(result) ? result.stdout.trim().toLowerCase() : null; +} /** - * Backup support for registered docker-driver sandboxes whose container is + * Backup support for registered container-backed sandboxes whose container is * stopped. `backup-all` skips sandboxes the gateway does not report Ready, * which under installer-strict mode (#6114) fails the whole run — but a * stopped container's state is backupable: the backup transport is SSH+tar * through the container's PID 1 and does not need the agent gateway, so - * `docker start` alone is enough to capture it (#6500). These helpers start - * such a container for the duration of the backup and return it to its - * stopped state afterwards, so the strict gate can pass without weakening - * what it protects. + * starting the provider-owned container is enough to capture it (#6500). + * These helpers start such a container for the duration of the backup and + * return it to its stopped state afterwards, so the strict gate can pass + * without weakening what it protects. * * Only containers whose `.State.Status` is `exited` or `created` qualify. * A running-but-not-Ready container (crash loop, gateway drift, paused) is @@ -46,27 +116,33 @@ const DOCKER_ABSENCE_PROBE_TIMEOUT_MS = 5_000; export interface StartedForBackup { containerName: string; + runtimeProviderId: string; } interface StartDeps { getSandboxDriver: (name: string) => string | null | undefined; listSandboxNames: () => string[]; - listLabeledContainerNames: (sandboxName: string) => string[]; - dockerInspectStatus: (containerName: string) => string; - dockerStart: (containerName: string) => string; + resolveLifecycleEngine: (driverName: string | null | undefined) => SandboxLifecycleEngine | null; + listLabeledContainerNames: ( + engine: SandboxLifecycleEngine, + sandboxName: string, + ) => string[] | null; + inspectStatus: (engine: SandboxLifecycleEngine, containerName: string) => string | null; + start: (engine: SandboxLifecycleEngine, containerName: string) => boolean; } const defaultStartDeps: StartDeps = { getSandboxDriver: readSandboxDriver, listSandboxNames: () => - registry.listSandboxes().sandboxes.filter(registry.isPublishedSandboxRegistration).map((entry) => entry.name), - listLabeledContainerNames: (sandboxName) => - findLabeledSandboxContainers(sandboxName).map((container) => container.name), - dockerInspectStatus: (containerName) => - dockerContainerInspectFormat("{{.State.Status}}", containerName, { ignoreError: true }), - // `docker start` echoes the container name on success and prints nothing to - // stdout on failure, so a non-empty capture doubles as the success signal. - dockerStart: (containerName) => dockerCapture(["start", containerName], { ignoreError: true }), + registry + .listSandboxes() + .sandboxes.filter(registry.isPublishedSandboxRegistration) + .map((entry) => entry.name), + resolveLifecycleEngine: resolveSandboxLifecycleEngine, + listLabeledContainerNames, + inspectStatus: inspectContainerStatus, + start: (engine, containerName) => + captureSucceeded(engine.capture(["start", containerName], engine.mutationTimeoutMs)), }; export function startStoppedSandboxContainerForBackup( @@ -74,12 +150,13 @@ export function startStoppedSandboxContainerForBackup( depsOverride: Partial = {}, ): StartedForBackup | null { const deps: StartDeps = { ...defaultStartDeps, ...depsOverride }; - if (deps.getSandboxDriver(sandboxName) !== "docker") return null; - const labeledContainerNames = deps.listLabeledContainerNames(sandboxName); + const engine = deps.resolveLifecycleEngine(deps.getSandboxDriver(sandboxName)); + if (!engine) return null; + const labeledContainerNames = deps.listLabeledContainerNames(engine, sandboxName); // Lifecycle mutation must fail closed on missing or ambiguous ownership. // Name matching alone is insufficient because starting a container executes // its entrypoint; label discovery establishes the OpenShell owner first. - if (labeledContainerNames.length !== 1) return null; + if (labeledContainerNames === null || labeledContainerNames.length !== 1) return null; const containerName = resolveSandboxContainerOwner( labeledContainerNames[0] ?? "", sandboxName, @@ -89,92 +166,72 @@ export function startStoppedSandboxContainerForBackup( // GPU recovery siblings must be renamed through the dedicated recovery flow // before they are startable as the sandbox's active container. if (/-nemoclaw-gpu-backup-\d+$/.test(containerName)) return null; - const status = deps.dockerInspectStatus(containerName).trim().toLowerCase(); + const status = deps.inspectStatus(engine, containerName); if (status !== "exited" && status !== "created") return null; - if (deps.dockerStart(containerName).trim() === "") return null; - return { containerName }; + if (!deps.start(engine, containerName)) return null; + return { containerName, runtimeProviderId: engine.runtimeProviderId }; } interface ContainerAbsenceDeps { getSandboxDriver: (name: string) => string | null | undefined; + resolveLifecycleEngine: (driverName: string | null | undefined) => SandboxLifecycleEngine | null; /** Labeled container names for the sandbox, or null when the listing itself * failed (dead daemon, timeout) and absence must not be concluded. */ - listLabeledContainerNames: (name: string) => string[] | null; + listLabeledContainerNames: ( + engine: SandboxLifecycleEngine, + sandboxName: string, + ) => string[] | null; } const defaultContainerAbsenceDeps: ContainerAbsenceDeps = { getSandboxDriver: readSandboxDriver, - // findLabeledSandboxContainers swallows docker errors (a dead daemon reads - // as "no containers"), which suits its recovery callers but not an absence - // proof. Run the same labeled listing status-checked instead: any spawn - // error, timeout, or non-zero exit yields null, never "absent". ignoreError - // prevents runner.run() from exiting the process when the listing fails. - listLabeledContainerNames: (name) => { - const result = dockerRun( - [ - "ps", - "-a", - "--filter", - `label=${OPENSHELL_MANAGED_BY_LABEL}=${OPENSHELL_MANAGED_BY_VALUE}`, - "--filter", - `label=${OPENSHELL_SANDBOX_NAME_LABEL}=${name}`, - "--format", - "{{.Names}}", - ], - { - encoding: "utf-8", - stdio: ["ignore", "pipe", "pipe"], - ignoreError: true, - suppressOutput: true, - timeout: DOCKER_ABSENCE_PROBE_TIMEOUT_MS, - }, - ); - if (result.error || result.status !== 0) return null; - return String(result.stdout || "") - .split(/\r?\n/) - .map((line) => line.trim()) - .filter(Boolean); - }, + resolveLifecycleEngine: resolveSandboxLifecycleEngine, + listLabeledContainerNames, }; /** - * Returns true only when the registered sandbox uses Docker and a successful - * labeled `docker ps -a` returns no matching container. + * Returns true only when the registered sandbox has a container lifecycle + * engine and a successful labeled listing returns no matching container. * - * Returns false when the driver is not Docker, the registry read fails, or the - * Docker listing fails or times out. Callers must separately confirm gateway - * absence and same-gateway binding before classifying a sandbox as stranded. + * Returns false when the provider has no container lifecycle, the registry + * read fails, or the listing fails or times out. Callers must separately + * confirm gateway absence and same-gateway binding before classifying a + * sandbox as stranded. */ export function isSandboxContainerDefinitivelyAbsent( sandboxName: string, depsOverride: Partial = {}, ): boolean { const deps: ContainerAbsenceDeps = { ...defaultContainerAbsenceDeps, ...depsOverride }; - if (deps.getSandboxDriver(sandboxName) !== "docker") return false; - const labeledContainerNames = deps.listLabeledContainerNames(sandboxName); + const engine = deps.resolveLifecycleEngine(deps.getSandboxDriver(sandboxName)); + if (!engine) return false; + const labeledContainerNames = deps.listLabeledContainerNames(engine, sandboxName); return labeledContainerNames !== null && labeledContainerNames.length === 0; } interface StopDeps { - dockerStop: (containerName: string) => string; - dockerInspectStatus: (containerName: string) => string; + resolveLifecycleEngine: (driverName: string | null | undefined) => SandboxLifecycleEngine | null; + stop: (engine: SandboxLifecycleEngine, containerName: string) => boolean; + inspectStatus: (engine: SandboxLifecycleEngine, containerName: string) => string | null; } const defaultStopDeps: StopDeps = { - dockerStop: (containerName) => dockerCapture(["stop", containerName], { ignoreError: true }), - dockerInspectStatus: (containerName) => - dockerContainerInspectFormat("{{.State.Status}}", containerName, { ignoreError: true }), + resolveLifecycleEngine: resolveSandboxLifecycleEngine, + stop: (engine, containerName) => + captureSucceeded(engine.capture(["stop", containerName], engine.mutationTimeoutMs)), + inspectStatus: inspectContainerStatus, }; /** Return a container started by {@link startStoppedSandboxContainerForBackup} - * to its stopped state. Returns false when `docker stop` fails. */ + * to its stopped state. Returns false when the provider operation fails. */ export function returnSandboxContainerToStopped( - containerName: string, + started: StartedForBackup, depsOverride: Partial = {}, ): boolean { const deps: StopDeps = { ...defaultStopDeps, ...depsOverride }; - if (deps.dockerStop(containerName).trim() === "") return false; - return deps.dockerInspectStatus(containerName).trim().toLowerCase() === "exited"; + const engine = deps.resolveLifecycleEngine(started.runtimeProviderId); + if (!engine || !deps.stop(engine, started.containerName)) return false; + return deps.inspectStatus(engine, started.containerName) === "exited"; } interface BackupRetryDeps { diff --git a/src/lib/actions/sandbox/supervisor-relaunch.ts b/src/lib/actions/sandbox/supervisor-relaunch.ts index 6ed1f7e186d..7a31c6fa623 100644 --- a/src/lib/actions/sandbox/supervisor-relaunch.ts +++ b/src/lib/actions/sandbox/supervisor-relaunch.ts @@ -15,6 +15,7 @@ import { } from "../../onboard/docker-gpu-patch-finalize"; import { getDockerGpuSupervisorReconnectTimeoutSecs } from "../../onboard/docker-gpu-supervisor-reconnect"; import { recreateOpenShellDockerSandboxWithStartupCommand } from "../../onboard/docker-startup-command-patch"; +import { resolveRegisteredRuntimeProvider } from "../../onboard/runtime-provider/selection"; import { buildSandboxRuntimeEnvArgs } from "../../onboard/sandbox-create-launch"; import { readManagedWorkloadAuthority } from "../../onboard/workload/authority"; import { resolveDirectSandboxContainer } from "../../sandbox/privileged-exec"; @@ -66,6 +67,51 @@ export type ManagedSupervisorRelaunchDeps = { >["runCaptureOpenshell"]; }; +export type RegisteredRuntimeRecoveryResult = { + readonly exitCode: number; + readonly message?: string; +}; + +/** Whether the registered provider exposes the managed in-sandbox controller. */ +export function usesManagedGatewayController(entry: registry.SandboxEntry): boolean { + const provider = resolveRegisteredRuntimeProvider(entry.openshellDriver); + return ( + provider?.gateway.supported === true && + provider.gateway.launcher === "nemoclaw" && + provider.lifecycle.supported === true + ); +} + +/** Whether retained default-engine gateway compatibility logic applies. */ +export function usesLegacyManagedGatewayRecovery(entry: registry.SandboxEntry): boolean { + const provider = resolveRegisteredRuntimeProvider(entry.openshellDriver); + if ( + !provider || + provider.lifecycle.supported !== true || + provider.gateway.launcher !== "nemoclaw" + ) { + return false; + } + try { + return ( + provider.gateway.prepareHostRuntime({ + environment: process.env, + platform: process.platform, + }).socketPath === null + ); + } catch { + return false; + } +} + +/** Execute provider-owned recovery when the persisted provider registers it. */ +export function recoverRegisteredRuntimeProviderSandbox( + entry: registry.SandboxEntry, +): RegisteredRuntimeRecoveryResult | null { + const provider = resolveRegisteredRuntimeProvider(entry.openshellDriver); + return provider?.recovery.supported === true ? provider.recovery.recover(entry) : null; +} + function inspectContainer(containerId: string): DockerContainerInspect { return parseDockerInspectJson( dockerCapture(["inspect", "--type", "container", containerId], { @@ -157,7 +203,7 @@ export function relaunchManagedSupervisorSession( const entry = getSandbox(sandboxName); if (!entry) return null; const driver = entry.openshellDriver?.trim().toLowerCase() ?? null; - if (driver !== null && driver !== "docker" && driver !== "vm") return null; + if (!usesLegacyManagedGatewayRecovery(entry)) return null; const startupCommand = reconstructSupervisorLaunchCommand(sandboxName, entry, quiet, deps); if (startupCommand === null) return null; diff --git a/src/lib/actions/sandbox/terminal-runtime-health.ts b/src/lib/actions/sandbox/terminal-runtime-health.ts index d4a23cebf89..1b6a0c74a16 100644 --- a/src/lib/actions/sandbox/terminal-runtime-health.ts +++ b/src/lib/actions/sandbox/terminal-runtime-health.ts @@ -4,6 +4,10 @@ import { dockerSpawnSync } from "../../adapters/docker/exec"; import { resolveSandboxContainerOwner } from "../../domain/sandbox/container-owner"; import { findLabeledSandboxContainers } from "../../onboard/docker-driver-sandbox-recovery"; +import { + registeredRuntimeProviderSupportsContainerEngineOperation, + resolveRegisteredRuntimeProvider, +} from "../../onboard/runtime-provider/selection"; import { load as loadRegistry } from "../../state/registry/persistence"; /** @@ -113,12 +117,35 @@ const defaultDeps: TerminalRuntimeOomProbeDeps = { }), }; +function hasLegacyContainerHealthProbe(driverName: string | null | undefined): boolean { + const normalized = driverName?.trim().toLowerCase(); + if (!normalized) return false; + const provider = resolveRegisteredRuntimeProvider(normalized); + if ( + !provider || + provider.identity.id !== normalized || + !registeredRuntimeProviderSupportsContainerEngineOperation(normalized, "gateway-inspection") + ) { + return false; + } + try { + return ( + provider.gateway.prepareHostRuntime({ + environment: process.env, + platform: process.platform, + }).socketPath === null + ); + } catch { + return false; + } +} + export function probeTerminalRuntimeCgroupOom( sandboxName: string, depsOverride: Partial = {}, ): TerminalRuntimeOomProbeResult { const deps: TerminalRuntimeOomProbeDeps = { ...defaultDeps, ...depsOverride }; - if (deps.getSandboxDriver(sandboxName) !== "docker") { + if (!hasLegacyContainerHealthProbe(deps.getSandboxDriver(sandboxName))) { return { kind: "unavailable", detail: "cgroup OOM probe requires the docker driver" }; } // Reading the wrong container's counter would report a degraded sandbox that diff --git a/src/lib/actions/uninstall/hermes-uninstall-cleanup.ts b/src/lib/actions/uninstall/hermes-uninstall-cleanup.ts index f810f97cf24..5e9d8c88897 100644 --- a/src/lib/actions/uninstall/hermes-uninstall-cleanup.ts +++ b/src/lib/actions/uninstall/hermes-uninstall-cleanup.ts @@ -3,16 +3,19 @@ import type { ManagedHermesStateVolumeContext } from "../../onboard/managed-workload/hermes-state-volume"; import { normalizeRuntimeProviderIdentity } from "../../onboard/runtime-provider/access"; +import { CURRENT_RUNTIME_PROVIDER_BUNDLES } from "../../onboard/runtime-provider/current"; +import type { RuntimeProviderBundleRegistry } from "../../onboard/runtime-provider/contract"; import { removeManagedHermesStateVolume } from "../../onboard/sandbox-provider-cleanup"; export { stopHermesForwardWatchers } from "./hermes-forward-watcher-cleanup"; export { requiresManagedHermesStateVolume } from "../../onboard/managed-workload/hermes-state-volume"; export type { ManagedHermesStateVolumeContext }; -interface ManagedHermesStateVolumeRuntime { +export interface ManagedHermesStateVolumeRuntime { env: NodeJS.ProcessEnv; error(message: string): void; log(message: string): void; + runtimeProviders?: RuntimeProviderBundleRegistry; runDocker( args: string[], options?: { @@ -54,6 +57,7 @@ export function removeManagedHermesStateVolumes( ): boolean { for (const context of contexts) { const result = removeManagedHermesStateVolume(context, { + runtimeProviders: runtime.runtimeProviders ?? CURRENT_RUNTIME_PROVIDER_BUNDLES, runDocker: (args, options) => runtime.runDocker(["volume", ...args], { env: runtime.env, @@ -67,7 +71,9 @@ export function removeManagedHermesStateVolumes( return false; } if (result.status === "not-owned") { - runtime.warn(`Left Docker volume '${result.volumeName}' untouched because ${result.detail}.`); + runtime.warn( + `Left managed state volume '${result.volumeName}' untouched because ${result.detail}.`, + ); } else if (result.status === "removed") { runtime.log(`Removed managed Hermes state volume for '${context.sandboxName}'.`); } diff --git a/src/lib/actions/uninstall/run-plan-hermes-state-volume.test.ts b/src/lib/actions/uninstall/run-plan-hermes-state-volume.test.ts index a0965b1acb9..2129604a181 100644 --- a/src/lib/actions/uninstall/run-plan-hermes-state-volume.test.ts +++ b/src/lib/actions/uninstall/run-plan-hermes-state-volume.test.ts @@ -6,6 +6,12 @@ import os from "node:os"; import path from "node:path"; import { describe, expect, it, vi } from "vitest"; +import type { PodmanBoundContainerEngine } from "../../adapters/podman"; +import { createHermesStateVolumeDockerHarness } from "../../onboard/__test-helpers__/hermes-state-volume"; +import { createDockerRuntimeProviderBundle } from "../../onboard/runtime-provider/docker"; +import { createPodmanRuntimeProviderBundle } from "../../onboard/runtime-provider/podman"; +import { createRuntimeProviderBundleRegistry } from "../../onboard/runtime-provider/registry"; +import { removeManagedHermesStateVolumes } from "./hermes-uninstall-cleanup"; import { withSuccessfulPreUninstallBackup } from "../../../../test/support/uninstall-managed-gateway-test-support"; import { @@ -145,6 +151,21 @@ async function runManagedHermesVolumeUninstall( rmSync: fs.rmSync, run, runDocker, + runtimeProviders: createRuntimeProviderBundleRegistry([ + [ + "docker", + createDockerRuntimeProviderBundle({ + captureHostCommand: (_command, args) => { + const result = runDocker(args); + return { + status: result.status ?? 1, + stdout: result.stdout, + stderr: result.stderr, + }; + }, + }), + ], + ]), }, ); @@ -164,6 +185,80 @@ async function runManagedHermesVolumeUninstall( } describe("managed Hermes state volume uninstall", () => { + it("dispatches a Podman-owned volume through provider cleanup authority", () => { + const volume = createHermesStateVolumeDockerHarness(); + volume.runDocker([ + "create", + "--label", + "io.nvidia.nemoclaw.hermes-state.managed=true", + "--label", + "io.nvidia.nemoclaw.hermes-state.sandbox=hermes", + "--label", + "io.nvidia.nemoclaw.hermes-state.schema=1", + "--label", + "io.nvidia.nemoclaw.hermes-state.target=/sandbox/.hermes", + "nemoclaw-hermes-state-v1-hermes", + ]); + const capture = vi.fn((args: readonly string[]) => { + const result = volume.runDocker(args.slice(1)); + return { + status: result.status ?? 1, + stdout: String(result.stdout ?? ""), + stderr: String(result.stderr ?? ""), + }; + }); + const engine = ( + operation: PodmanBoundContainerEngine["operation"], + operationCapture = vi.fn(), + ) => + ({ + operation, + engineId: "podman", + displayName: "Podman", + authorityId: `podman:${operation}`, + endpointAuthorityId: "podman:test-endpoint", + capture: operationCapture, + captureHost: operationCapture, + assertAuthority: vi.fn(), + }) satisfies PodmanBoundContainerEngine; + const provider = createPodmanRuntimeProviderBundle({ + engines: { + hostDoctor: engine("host-doctor"), + sandboxLifecycle: engine("sandbox-lifecycle"), + workloadCleanup: engine("workload-cleanup", capture), + }, + }); + const runDocker = vi.fn(() => { + throw new Error("Podman uninstall reached Docker"); + }); + + expect( + removeManagedHermesStateVolumes( + [ + { + agentName: "hermes", + runtimeProviderId: "podman", + sandboxName: "hermes", + workloadKind: "managed-image", + }, + ], + { + env: {}, + error: vi.fn(), + log: vi.fn(), + runDocker, + runtimeProviders: createRuntimeProviderBundleRegistry([["podman", provider]]), + warn: vi.fn(), + }, + ), + ).toBe(true); + expect(capture).toHaveBeenCalledWith( + ["volume", "rm", "nemoclaw-hermes-state-v1-hermes"], + 30_000, + ); + expect(runDocker).not.toHaveBeenCalled(); + }); + it.each([ ["local uninstall", false], ["destructive uninstall", true], @@ -189,7 +284,7 @@ describe("managed Hermes state volume uninstall", () => { expect(harness.volumePresent()).toBe(true); expect(harness.dockerCalls).not.toContainEqual(["volume", "rm", harness.volumeName]); expect(harness.errors.join("\n")).toContain( - `Left Docker volume '${harness.volumeName}' untouched because the exact NemoClaw ownership labels are absent or changed.`, + `Left managed state volume '${harness.volumeName}' untouched because the exact NemoClaw ownership labels are absent or changed.`, ); } finally { harness.cleanup(); diff --git a/src/lib/actions/uninstall/run-plan.ts b/src/lib/actions/uninstall/run-plan.ts index bf7eac4518a..a252caad627 100644 --- a/src/lib/actions/uninstall/run-plan.ts +++ b/src/lib/actions/uninstall/run-plan.ts @@ -87,6 +87,7 @@ import { } from "../../state/gateway-registry"; import { managedHermesStateVolumeContext, + type ManagedHermesStateVolumeRuntime, type ManagedHermesStateVolumeContext, removeManagedHermesStateVolumes, requiresManagedHermesStateVolume, @@ -162,6 +163,7 @@ export interface UninstallRunDeps { runHuggingFaceCacheDataCleanup?: (options?: SpawnSyncOptions) => RunResult; runLocalModelRuntimeCleanup?: (options?: SpawnSyncOptions) => RunResult; runManagedLlamaCppRuntimeCleanup?: (sandboxName: string, gatewayPort: number) => RunResult; + runtimeProviders?: ManagedHermesStateVolumeRuntime["runtimeProviders"]; sleep?: (milliseconds: number) => void; hasPortableRuntimeCleanup?: (stateDir: string) => boolean; runPortableRuntimeCleanupTransaction?: ( @@ -536,6 +538,7 @@ interface UninstallRuntime { runHuggingFaceCacheDataCleanup: (options?: SpawnSyncOptions) => RunResult; runLocalModelRuntimeCleanup: (options?: SpawnSyncOptions) => RunResult; runManagedLlamaCppRuntimeCleanup: (sandboxName: string, gatewayPort: number) => RunResult; + runtimeProviders: ManagedHermesStateVolumeRuntime["runtimeProviders"]; sleep: (milliseconds: number) => void; hasPortableRuntimeCleanup: (stateDir: string) => boolean; runPortableRuntimeCleanupTransaction: ( @@ -660,6 +663,7 @@ function buildRuntime(deps: UninstallRunDeps): UninstallRuntime { stderr: result.reason, }; }), + runtimeProviders: deps.runtimeProviders, sleep: deps.sleep ?? sleepMs, hasPortableRuntimeCleanup: deps.hasPortableRuntimeCleanup ?? hasPortableRuntimeCleanup, runPortableRuntimeCleanupTransaction: @@ -2975,7 +2979,7 @@ function executeOpenShellResourceCleanup( !portableRuntimeCleanup && !externallySupervised && !scopedToSelectedGateway && - managedHermesStateVolumes.some(requiresManagedHermesStateVolume) && + managedHermesStateVolumes.some((context) => requiresManagedHermesStateVolume(context)) && dockerIsAvailable(runtime) ) { // An unreachable gateway can leave a stopped sandbox container attached to the state volume. diff --git a/src/lib/adapters/container-engine.ts b/src/lib/adapters/container-engine.ts index 7ff6b699d85..0b8204cb0af 100644 --- a/src/lib/adapters/container-engine.ts +++ b/src/lib/adapters/container-engine.ts @@ -183,6 +183,7 @@ function operationCommandEnvironment( function replacementCommandEnvironment( explicit: Readonly>, + allowedNames: ReadonlySet, ): Readonly> { if (typeof explicit !== "object" || explicit === null || Array.isArray(explicit)) { throw new Error("Container engine command environment is invalid."); @@ -198,7 +199,8 @@ function replacementCommandEnvironment( !ENVIRONMENT_NAME_PATTERN.test(name) || ENGINE_ENV_NAMES.has(name) || (!COMMAND_ENV_NAMES.has(name) && - !COMMAND_ENV_PREFIXES.some((prefix) => name.startsWith(prefix))) + !COMMAND_ENV_PREFIXES.some((prefix) => name.startsWith(prefix)) && + !allowedNames.has(name)) ) { throw new Error("Container engine command environment name is invalid."); } @@ -362,7 +364,7 @@ export function createContainerEngineCommand( }), ); const commandEnvironment = options.commandEnvironment - ? replacementCommandEnvironment(options.commandEnvironment) + ? replacementCommandEnvironment(options.commandEnvironment, allowedEnvironmentNames) : undefined; const capture = options.capture ?? defaultCapture; const run = ( diff --git a/src/lib/adapters/openshell/sandbox-observer-cli.test.ts b/src/lib/adapters/openshell/sandbox-observer-cli.test.ts index 8dc2cab2c27..79a27f3e751 100644 --- a/src/lib/adapters/openshell/sandbox-observer-cli.test.ts +++ b/src/lib/adapters/openshell/sandbox-observer-cli.test.ts @@ -258,6 +258,8 @@ describe("CLI OpenShell sandbox observer", () => { }); expect(run).toHaveBeenCalledWith(["sandbox", "list", "-g", "nemoclaw"], { ignoreError: true, + killProcessTreeOnTimeout: true, + killSignal: "SIGKILL", suppressOutput: true, timeout: 9_000, }); diff --git a/src/lib/adapters/openshell/sandbox-observer-cli.ts b/src/lib/adapters/openshell/sandbox-observer-cli.ts index 88eb146cbb9..bb9689c089e 100644 --- a/src/lib/adapters/openshell/sandbox-observer-cli.ts +++ b/src/lib/adapters/openshell/sandbox-observer-cli.ts @@ -71,7 +71,13 @@ export type CliOpenShellSandboxObserverDeps = Readonly<{ export type RunSandboxCommand = ( args: string[], - options: { ignoreError: true; suppressOutput: true; timeout: number }, + options: { + ignoreError: true; + killProcessTreeOnTimeout: true; + killSignal: "SIGKILL"; + suppressOutput: true; + timeout: number; + }, ) => Readonly<{ status?: number | null; stdout?: string | Buffer | null; @@ -233,6 +239,8 @@ export function createCliOpenShellSandboxObserverFromRunner( capture: (args, options) => { const result = run(args, { ignoreError: true, + killProcessTreeOnTimeout: true, + killSignal: "SIGKILL", suppressOutput: true, timeout: options.timeout, }); @@ -255,8 +263,7 @@ export function createCliOpenShellLegacyPodReadinessProbe( deps: CliOpenShellSandboxObserverDeps, ): OpenShellSandboxReadinessProbe { return async (request) => { - const gatewayArgs = - request.target.kind === "named" ? ["-g", request.target.gatewayName] : []; + const gatewayArgs = request.target.kind === "named" ? ["-g", request.target.gatewayName] : []; const result = await deps.capture( [ "doctor", diff --git a/src/lib/adapters/podman/index.test.ts b/src/lib/adapters/podman/index.test.ts index ba839c8bd98..8022a8b90e7 100644 --- a/src/lib/adapters/podman/index.test.ts +++ b/src/lib/adapters/podman/index.test.ts @@ -206,6 +206,99 @@ describe("Podman container engine command adapter", () => { ); }); + it("prepares one exact managed-bootstrap workspace root through Podman's user namespace", ({ + onTestFinished, + }) => { + const directory = fs.realpathSync( + fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-podman-workspace-")), + ); + onTestFinished(() => fs.rmSync(directory, { recursive: true, force: true })); + const capture = vi.fn((_executable, args) => { + const payload = JSON.parse(args.at(-1) ?? "{}") as Record; + return { status: 0, stdout: JSON.stringify(payload), stderr: "" }; + }); + const engine = createPodmanContainerEngine({ + operation: "managed-bootstrap", + socketAuthority: AUTHORITY, + executable: "/usr/bin/podman", + executableAuthorityDeps: executableAuthorityDeps(), + assertAuthority: vi.fn(), + commandEnvironment: { + HOME: "/home/podman", + XDG_RUNTIME_DIR: "/run/user/1000", + CONTAINERS_CONF: "/tmp/native-podman-containers.conf", + CONTAINERS_STORAGE_CONF: "/tmp/native-podman-storage.conf", + }, + capture, + }); + + expect( + engine.prepareManagedWorkspaceRoot?.({ path: directory, uid: 0, gid: 999, mode: 0o1775 }), + ).toMatchObject({ + path: directory, + uid: 0, + gid: 999, + mode: 0o1775, + }); + expect(capture).toHaveBeenCalledOnce(); + expect(capture.mock.calls[0]?.[1].slice(0, 2)).toEqual([ + "unshare", + fs.realpathSync(process.execPath), + ]); + expect(JSON.parse(capture.mock.calls[0]?.[1].at(-1) ?? "{}")).toMatchObject({ + path: directory, + uid: 0, + gid: 999, + mode: 0o1775, + }); + expect(capture.mock.calls[0]?.[4]).toEqual({ + HOME: "/home/podman", + XDG_RUNTIME_DIR: "/run/user/1000", + CONTAINERS_CONF: "/tmp/native-podman-containers.conf", + CONTAINERS_STORAGE_CONF: "/tmp/native-podman-storage.conf", + }); + expect(() => engine.captureHost(["unshare", "id"])).toThrow( + "forbids ambient host command capture", + ); + }); + + it("prepares one exact managed-bootstrap volume root without recursive ownership changes", ({ + onTestFinished, + }) => { + const directory = fs.realpathSync( + fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-podman-volume-")), + ); + onTestFinished(() => fs.rmSync(directory, { recursive: true, force: true })); + const capture = vi.fn((_executable, args) => { + const payload = JSON.parse(args.at(-1) ?? "{}") as Record; + return { status: 0, stdout: JSON.stringify(payload), stderr: "" }; + }); + const engine = createPodmanContainerEngine({ + operation: "managed-bootstrap", + socketAuthority: AUTHORITY, + executable: "/usr/bin/podman", + executableAuthorityDeps: executableAuthorityDeps(), + assertAuthority: vi.fn(), + capture, + }); + + expect( + engine.prepareManagedVolumeRoot?.({ + path: directory, + uid: 1000, + gid: 1000, + mode: 0o2770, + }), + ).toMatchObject({ path: directory, uid: 1000, gid: 1000, mode: 0o2770 }); + expect(capture).toHaveBeenCalledOnce(); + expect(JSON.parse(capture.mock.calls[0]?.[1].at(-1) ?? "{}")).toMatchObject({ + path: directory, + uid: 1000, + gid: 1000, + mode: 0o2770, + }); + }); + it("shares only socket authority across real operation-scoped engines", () => { const common = { socketAuthority: AUTHORITY, diff --git a/src/lib/adapters/podman/index.ts b/src/lib/adapters/podman/index.ts index d85f656ec69..8467d2c936e 100644 --- a/src/lib/adapters/podman/index.ts +++ b/src/lib/adapters/podman/index.ts @@ -26,13 +26,38 @@ import { // Immutable metadata is checked before and after every dispatch. Rehash the // full executable before every 64th command within this operation. const EXECUTABLE_CONTENT_REVALIDATION_COMMAND_INTERVAL = 64; +const MANAGED_ROOT_HELPER_EXECUTABLE = fs.realpathSync(process.execPath); +const MANAGED_ROOT_DESCRIPTOR_SCRIPT = ` +const fs = require("node:fs"); +const payload = JSON.parse(process.argv[1]); +const flags = fs.constants.O_RDONLY | fs.constants.O_DIRECTORY | fs.constants.O_NOFOLLOW; +const descriptor = fs.openSync(payload.path, flags); +try { + const before = fs.fstatSync(descriptor, { bigint: true }); + if (!before.isDirectory() || before.nlink < 1n || before.dev.toString() !== payload.device || before.ino.toString() !== payload.inode) { + throw new Error("managed root identity changed before descriptor-bound mutation"); + } + fs.fchownSync(descriptor, payload.uid, payload.gid); + fs.fchmodSync(descriptor, payload.mode); + const after = fs.fstatSync(descriptor, { bigint: true }); + if (!after.isDirectory() || after.nlink < 1n || after.dev !== before.dev || after.ino !== before.ino) { + throw new Error("managed root identity changed during descriptor-bound mutation"); + } + process.stdout.write(JSON.stringify({ device: after.dev.toString(), inode: after.ino.toString(), uid: Number(after.uid), gid: Number(after.gid), mode: Number(after.mode & 0o7777n) })); +} finally { + fs.closeSync(descriptor); +} +`; export interface PodmanContainerEngineOptions { readonly operation: | "host-doctor" + | "gateway-inspection" | "host-local-inference" + | "managed-bootstrap" | "sandbox-lifecycle" - | "state-mutation"; + | "state-mutation" + | "workload-cleanup"; readonly socketAuthority: PodmanSocketAuthority; readonly executable?: string; readonly executableAuthority?: PodmanExecutableAuthority; @@ -51,9 +76,41 @@ export interface PodmanContainerEngine extends ContainerEngine { readonly endpointAuthorityId: string; } +export interface PodmanManagedWorkspaceRootReceipt { + readonly path: string; + readonly device: string; + readonly inode: string; + readonly uid: number; + readonly gid: number; + readonly mode: 0o755 | 0o1775; +} + +export interface PodmanManagedVolumeRootReceipt { + readonly path: string; + readonly device: string; + readonly inode: string; + readonly uid: number; + readonly gid: number; + readonly mode: number; +} + /** Podman engine whose exact socket and executable authority can be revalidated on demand. */ export interface PodmanBoundContainerEngine extends PodmanContainerEngine { readonly assertAuthority: () => void; + /** Exact local user-namespace mutation available only to managed bootstrap. */ + readonly prepareManagedWorkspaceRoot?: (input: { + readonly path: string; + readonly uid: number; + readonly gid: number; + readonly mode: 0o755 | 0o1775; + }) => PodmanManagedWorkspaceRootReceipt; + /** Exact, non-recursive local user-namespace mutation for one managed volume root. */ + readonly prepareManagedVolumeRoot?: (input: { + readonly path: string; + readonly uid: number; + readonly gid: number; + readonly mode: number; + }) => PodmanManagedVolumeRootReceipt; } export function resolvePodmanExecutablePath(env: NodeJS.ProcessEnv = process.env): string { @@ -133,7 +190,10 @@ export function createPodmanContainerEngine( ): PodmanBoundContainerEngine { const assertAuthority = options.assertAuthority ?? assertPodmanSocketAuthority; const protectsRuntimeMutation = - options.operation === "host-local-inference" || options.operation === "state-mutation"; + options.operation === "host-local-inference" || + options.operation === "managed-bootstrap" || + options.operation === "state-mutation" || + options.operation === "workload-cleanup"; const executable = options.executable ?? options.executableAuthority?.executablePath ?? @@ -172,7 +232,9 @@ export function createPodmanContainerEngine( allowedEnvironmentNames: options.operation === "host-local-inference" ? ["NGC_API_KEY", "NIM_NGC_API_KEY", "OLLAMA_CONTEXT_LENGTH"] - : [], + : options.operation === "managed-bootstrap" + ? ["CONTAINERS_CONF", "CONTAINERS_STORAGE_CONF"] + : [], commandEnvironment: options.commandEnvironment, capture: options.capture, guard: (phase) => { @@ -219,8 +281,108 @@ export function createPodmanContainerEngine( assertAuthority: () => assertBoundAuthority(true), }; if (!protectsRuntimeMutation) return Object.freeze(boundEngine); + const prepareManagedVolumeRoot = (input: { + readonly path: string; + readonly uid: number; + readonly gid: number; + readonly mode: number; + }): PodmanManagedVolumeRootReceipt => { + if (options.operation !== "managed-bootstrap") { + throw new Error("Podman volume-root preparation requires managed-bootstrap authority."); + } + if ( + !path.isAbsolute(input.path) || + path.normalize(input.path) !== input.path || + input.path === path.parse(input.path).root || + fs.realpathSync(input.path) !== input.path + ) { + throw new Error("Podman managed volume mountpoint is invalid."); + } + if ( + !Number.isSafeInteger(input.uid) || + input.uid < 0 || + input.uid > 2_147_483_647 || + !Number.isSafeInteger(input.gid) || + input.gid < 1 || + input.gid > 2_147_483_647 || + !Number.isSafeInteger(input.mode) || + input.mode < 0 || + input.mode > 0o7777 || + (input.mode & 0o002) !== 0 + ) { + throw new Error("Podman managed volume root metadata is invalid."); + } + const before = fs.lstatSync(input.path, { bigint: true }); + if (!before.isDirectory() || before.isSymbolicLink() || before.nlink < 1n) { + throw new Error("Podman managed volume mountpoint is not one stable directory."); + } + const payload = JSON.stringify({ + path: input.path, + device: before.dev.toString(), + inode: before.ino.toString(), + uid: input.uid, + gid: input.gid, + mode: input.mode, + }); + const result = boundEngine.captureHost( + ["unshare", MANAGED_ROOT_HELPER_EXECUTABLE, "-e", MANAGED_ROOT_DESCRIPTOR_SCRIPT, payload], + 15_000, + ); + if (result.status !== 0 || result.error) { + const detail = (result.stderr || result.stdout || result.error?.message || "unknown failure") + .replace(/\s+/gu, " ") + .trim() + .slice(-600); + throw new Error( + `Podman managed volume descriptor preparation failed (exit ${String(result.status)}): ${detail}`, + ); + } + let observed: unknown; + try { + observed = JSON.parse(result.stdout); + } catch { + throw new Error("Podman managed volume descriptor preparation returned invalid JSON."); + } + if ( + !observed || + typeof observed !== "object" || + Array.isArray(observed) || + (observed as Record).device !== before.dev.toString() || + (observed as Record).inode !== before.ino.toString() || + (observed as Record).uid !== input.uid || + (observed as Record).gid !== input.gid || + (observed as Record).mode !== input.mode + ) { + throw new Error("Podman managed volume authority is invalid after descriptor preparation."); + } + return Object.freeze({ + path: input.path, + device: before.dev.toString(), + inode: before.ino.toString(), + uid: input.uid, + gid: input.gid, + mode: input.mode, + }); + }; + const prepareManagedWorkspaceRoot = (input: { + readonly path: string; + readonly uid: number; + readonly gid: number; + readonly mode: 0o755 | 0o1775; + }): PodmanManagedWorkspaceRootReceipt => { + const receipt = prepareManagedVolumeRoot({ + path: input.path, + uid: input.uid, + gid: input.gid, + mode: input.mode, + }); + return Object.freeze({ ...receipt, mode: input.mode }); + }; return Object.freeze({ ...boundEngine, + ...(options.operation === "managed-bootstrap" + ? { prepareManagedVolumeRoot, prepareManagedWorkspaceRoot } + : {}), captureHost: () => { throw new Error(`Podman ${options.operation} forbids ambient host command capture.`); }, diff --git a/src/lib/adapters/sandbox/command-transport.test.ts b/src/lib/adapters/sandbox/command-transport.test.ts index 7c4fca84dcd..66f3f2dff91 100644 --- a/src/lib/adapters/sandbox/command-transport.test.ts +++ b/src/lib/adapters/sandbox/command-transport.test.ts @@ -55,12 +55,15 @@ function createDependencies( output: "Host openshell-alpha.default\n HostName 127.0.0.1\n", status: 0, })), - dockerSpawnSync: vi.fn(() => spawnResult("fallback-output")), + executePrivilegedSandboxCommand: vi.fn(() => ({ + status: 0, + stdout: "fallback-output", + stderr: "", + })), extractSandboxExecCommandStdout: vi.fn((output: string) => output), getOpenshellBinary: vi.fn(() => "/usr/bin/openshell"), isDirectSandboxFallbackUnavailableError: vi.fn(() => false), openshellProbeTimeoutMs: 5000, - privilegedSandboxExecArgv: vi.fn(() => ["exec", "container-id", "sh", "-c", "marked:id"]), root: "/repo", withPrivilegedSandboxExecutionLease: ( _sandboxName: string, @@ -222,8 +225,7 @@ describe("sandbox command transport privileged execution lease", () => { allowLocalDockerFallback: false, }), ).toBeNull(); - expect(deps.privilegedSandboxExecArgv).not.toHaveBeenCalled(); - expect(deps.dockerSpawnSync).not.toHaveBeenCalled(); + expect(deps.executePrivilegedSandboxCommand).not.toHaveBeenCalled(); }); it("holds one lease across OpenShell failure and the complete local fallback", () => { @@ -259,9 +261,9 @@ describe("sandbox command transport privileged execution lease", () => { assertLeaseHeld("environment"); return { PATH: "/usr/bin" }; }), - dockerSpawnSync: vi.fn(() => { - assertLeaseHeld("fallback-spawn"); - return spawnResult("fallback-output"); + executePrivilegedSandboxCommand: vi.fn(() => { + assertLeaseHeld("fallback-execution"); + return { status: 0, stdout: "fallback-output", stderr: "" }; }), extractSandboxExecCommandStdout: vi.fn((output: string) => { assertLeaseHeld(`parse:${output}`); @@ -271,10 +273,6 @@ describe("sandbox command transport privileged execution lease", () => { assertLeaseHeld("openshell-resolution"); return "/usr/bin/openshell"; }), - privilegedSandboxExecArgv: vi.fn(() => { - assertLeaseHeld("fallback-resolution"); - return ["exec", "container-id", "sh", "-c", "marked:id"]; - }), withPrivilegedSandboxExecutionLease: withLease, }); mocks.spawnSync.mockImplementation(() => { @@ -295,9 +293,7 @@ describe("sandbox command transport privileged execution lease", () => { "environment", "openshell-spawn", "parse:unmarked-output", - "fallback-resolution", - "environment", - "fallback-spawn", + "fallback-execution", "parse:fallback-output", "lease:released", ]); @@ -322,8 +318,7 @@ describe("sandbox command transport privileged execution lease", () => { ); expect(deps.buildSandboxExecMarkedCommand).not.toHaveBeenCalled(); expect(deps.getOpenshellBinary).not.toHaveBeenCalled(); - expect(deps.privilegedSandboxExecArgv).not.toHaveBeenCalled(); - expect(deps.dockerSpawnSync).not.toHaveBeenCalled(); + expect(deps.executePrivilegedSandboxCommand).not.toHaveBeenCalled(); expect(mocks.spawnSync).not.toHaveBeenCalled(); }); }); diff --git a/src/lib/adapters/sandbox/command-transport.ts b/src/lib/adapters/sandbox/command-transport.ts index 12f6a8997dd..f5cae0e43f5 100644 --- a/src/lib/adapters/sandbox/command-transport.ts +++ b/src/lib/adapters/sandbox/command-transport.ts @@ -23,15 +23,20 @@ export type CommandTransportDependencies = { sandboxName: string, options: { ignoreError: boolean; timeout: number }, ) => { output: string; status: number | null }; - dockerSpawnSync: ( - args: readonly string[], - options: Parameters[2], - ) => ReturnType; + executePrivilegedSandboxCommand: ( + sandboxName: string, + command: readonly string[], + options: { readonly sanitizeEnvironment: boolean; readonly timeout: number }, + ) => { + readonly status: number | null; + readonly stdout: string | Buffer; + readonly stderr: string | Buffer; + readonly error?: unknown; + }; extractSandboxExecCommandStdout: (output: string) => string | null; getOpenshellBinary: () => string; isDirectSandboxFallbackUnavailableError: (error: unknown) => boolean; openshellProbeTimeoutMs: number; - privilegedSandboxExecArgv: (sandboxName: string, command: string[]) => string[]; root: string; withPrivilegedSandboxExecutionLease: ( sandboxName: string, @@ -107,7 +112,12 @@ export function executeSandboxCommandTransport( function parseSandboxCommandResult( deps: CommandTransportDependencies, - result: ReturnType, + result: { + readonly status: number | null; + readonly stdout: string | Buffer; + readonly stderr: string | Buffer; + readonly error?: unknown; + }, ): SandboxCommandResult | null { if (result.error) return null; const stdout = typeof result.stdout === "string" ? result.stdout : String(result.stdout || ""); @@ -121,17 +131,20 @@ function parseSandboxCommandResult( }; } -function executeLocalDockerSandboxCommand( +function executeLocalSandboxCommand( deps: CommandTransportDependencies, sandboxName: string, markedCommand: string, timeout: number, ): SandboxCommandResult | null { - let argv: string[]; try { - argv = deps.privilegedSandboxExecArgv(sandboxName, ["sh", "-c", markedCommand]); + const result = deps.executePrivilegedSandboxCommand(sandboxName, ["sh", "-c", markedCommand], { + sanitizeEnvironment: true, + timeout, + }); + return parseSandboxCommandResult(deps, result); } catch (error) { - // Docker discovery failure or a stopped/nonexistent direct container means + // Provider discovery failure or a stopped/nonexistent runtime resource means // there is no local fallback. Identity refusals, unsupported drivers, // registry corruption, and ambiguous matches are security-boundary // diagnostics: let callers surface them instead of collapsing them into an @@ -139,18 +152,6 @@ function executeLocalDockerSandboxCommand( if (deps.isDirectSandboxFallbackUnavailableError(error)) return null; throw error; } - - try { - const result = deps.dockerSpawnSync(argv, { - encoding: "utf-8", - env: deps.buildSubprocessEnv(), - stdio: ["ignore", "pipe", "pipe"], - timeout, - }); - return parseSandboxCommandResult(deps, result); - } catch { - return null; - } } export function executeSandboxExecCommandTransport( @@ -199,7 +200,7 @@ export function executeSandboxExecCommandTransport( // refusal cannot be caught and retried against changing container state. // The outer execution lease covers argv resolution and the complete fallback // subprocess lifetime without an unleased gap after the OpenShell attempt. - return executeLocalDockerSandboxCommand(deps, sandboxName, markedCommand, effectiveTimeout); + return executeLocalSandboxCommand(deps, sandboxName, markedCommand, effectiveTimeout); }, ); } diff --git a/src/lib/agent/base-image.ts b/src/lib/agent/base-image.ts index 580ff068846..23eb7fe089d 100644 --- a/src/lib/agent/base-image.ts +++ b/src/lib/agent/base-image.ts @@ -624,7 +624,7 @@ export function ensureAgentBaseImage( `Hermes final image does not accept base image ref '${pinnedBaseImageTag}'; use the tracked official digest or a repository-built local base`, ); } - console.log(` \u2713 Base image built: ${pinnedBaseImageTag}`); + console.log(" \u2713 Base image built."); const resolutionMetadata = createLocalResolutionMetadata( resolutionOptions, pinnedBaseImageTag, @@ -711,7 +711,7 @@ export function ensureAgentBaseImage( : ` (exit ${buildResult.status ?? "unknown"})`; throw new Error(`Failed to build ${agent.displayName} base image${detail}`); } - console.log(` \u2713 Base image built: ${baseImageTag}`); + console.log(" \u2713 Base image built."); const resolutionMetadata = createLocalResolutionMetadata(resolutionOptions, baseImageTag); return { imageTag: baseImageTag, @@ -720,7 +720,7 @@ export function ensureAgentBaseImage( }; } - console.log(` Base image exists: ${baseImageTag}`); + console.log(" Base image exists."); const resolutionMetadata = createLocalResolutionMetadata(resolutionOptions, baseImageTag); return { imageTag: baseImageTag, diff --git a/src/lib/agent/list-command.ts b/src/lib/agent/list-command.ts index cb603c7888e..b2beb0fe225 100644 --- a/src/lib/agent/list-command.ts +++ b/src/lib/agent/list-command.ts @@ -23,6 +23,6 @@ export function renderAgentRuntimeList( .join("\n"); } -export function printAgentRuntimeList(log: (message: string) => void = console.log): void { +export function printAgentRuntimeList(log: (message: string) => void): void { log(renderAgentRuntimeList()); } diff --git a/src/lib/agent/onboard.ts b/src/lib/agent/onboard.ts index 0caaf86616d..d020850466f 100644 --- a/src/lib/agent/onboard.ts +++ b/src/lib/agent/onboard.ts @@ -170,13 +170,6 @@ function sleep(seconds: number): void { sleepSeconds(seconds); } -/** - * Resolve the CLI command name used for agent-specific recovery guidance. - */ -function agentCliName(agent: AgentDefinition): string { - return getAgentBranding(agent.name).cli; -} - const HERMES_TIRITH_MARKER_ABSENT = "tirith marker: absent"; const HERMES_STARTUP_DIAGNOSTICS_SCRIPT = ` set +e @@ -256,11 +249,8 @@ async function failAgentSetup( "agent_setup", details.length > 0 ? `${message}\n${details.join("\n")}` : message, ); - console.error(` \u2717 ${message}`); - for (const line of details) { - console.error(` ${line}`); - } - console.error(` Check: ${agentCliName(agent)} ${sandboxName} logs --follow`); + console.error(" \u2717 Agent setup failed."); + console.error(" Check the sandbox logs for redacted diagnostics."); process.exit(1); } diff --git a/src/lib/core/gateway-address.ts b/src/lib/core/gateway-address.ts index 147d12970c6..b71fa33ccaa 100644 --- a/src/lib/core/gateway-address.ts +++ b/src/lib/core/gateway-address.ts @@ -13,8 +13,9 @@ export type GatewayBindAddress = export function parseGatewayBindAddress( envVar = "NEMOCLAW_GATEWAY_BIND_ADDRESS", fallback: GatewayBindAddress = DEFAULT_GATEWAY_BIND_ADDRESS, + environment: NodeJS.ProcessEnv = process.env, ): GatewayBindAddress { - const raw = process.env[envVar]; + const raw = environment[envVar]; if (raw === undefined || raw === "") return fallback; const trimmed = String(raw).trim(); if (trimmed === DEFAULT_GATEWAY_BIND_ADDRESS) return DEFAULT_GATEWAY_BIND_ADDRESS; diff --git a/src/lib/core/version.test.ts b/src/lib/core/version.test.ts index a693cc387c8..d3e083c5d56 100644 --- a/src/lib/core/version.test.ts +++ b/src/lib/core/version.test.ts @@ -6,7 +6,12 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; -import { getBuildIdentity, getVersion, validateBuildIdentity } from "./version"; +import { + getBuildIdentity, + getVersion, + resolveSourceBuildIdentity, + validateBuildIdentity, +} from "./version"; const repoRoot = join(import.meta.dirname, "..", "..", ".."); @@ -114,6 +119,18 @@ describe("lib/version", () => { expect(getBuildIdentity({ rootDir: testDir })).toEqual(identity); }); + it("preserves an exact compiled identity when rebuilding the same source revision", () => { + const identity = { + nemoclawVersion: "0.0.113-195-ga52f16721", + sourceRevision: `a52f16721${"5".repeat(31)}`, + }; + mkdirSync(join(testDir, "dist")); + writeFileSync(join(testDir, "dist", "build-identity.json"), JSON.stringify(identity)); + writeFileSync(join(testDir, ".source-revision"), identity.sourceRevision); + + expect(resolveSourceBuildIdentity({ rootDir: testDir })).toEqual(identity); + }); + it("rejects a described version whose revision does not match (#7777)", () => { expect(() => validateBuildIdentity({ diff --git a/src/lib/core/version.ts b/src/lib/core/version.ts index 9f5ee44159c..376b02815e0 100644 --- a/src/lib/core/version.ts +++ b/src/lib/core/version.ts @@ -124,9 +124,14 @@ export function validateBuildIdentity(identity: Readonly): BuildI export function resolveSourceBuildIdentity(opts: VersionOptions = {}): BuildIdentity { const root = rootDirectory(opts); + const sourceRevision = resolveSourceRevision(root); + const existingIdentity = readBuildIdentity(root); + if (existingIdentity?.sourceRevision === sourceRevision) { + return existingIdentity; + } return validateBuildIdentity({ nemoclawVersion: resolveSourceVersion(root), - sourceRevision: resolveSourceRevision(root), + sourceRevision, }); } diff --git a/src/lib/inference/gateway-route-mutation-lock.test.ts b/src/lib/inference/gateway-route-mutation-lock.test.ts index f93280c0676..13813cf1fc3 100644 --- a/src/lib/inference/gateway-route-mutation-lock.test.ts +++ b/src/lib/inference/gateway-route-mutation-lock.test.ts @@ -1,11 +1,15 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { AsyncResource } from "node:async_hooks"; import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { describe, expect, it, vi } from "vitest"; -import { withGatewayRouteMutationLock } from "./gateway-route-mutation-lock"; +import { + withGatewayRouteMutationLock, + withGatewayRouteMutationLockSync, +} from "./gateway-route-mutation-lock"; describe("gateway route mutation lock", () => { it("serializes separate operations for the same gateway", async () => { @@ -82,6 +86,36 @@ describe("gateway route mutation lock", () => { } }); + it("makes sync and async operations contend in the same explicit nondefault state directory", async () => { + const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "nemoclaw-gateway-cross-mode-")); + const outsideLockContext = new AsyncResource("gateway-route-cross-mode-test"); + const options = { stateDir, pollIntervalMs: 1, timeoutMs: 5_000 }; + const contentionOptions = { ...options, timeoutMs: 25 }; + try { + await withGatewayRouteMutationLock( + "nemoclaw-18080", + () => { + expect(() => + outsideLockContext.runInAsyncScope(() => + withGatewayRouteMutationLockSync( + "nemoclaw-18080", + () => "must-not-enter", + contentionOptions, + ), + ), + ).toThrow(); + }, + options, + ); + expect( + withGatewayRouteMutationLockSync("nemoclaw-18080", () => "entered-after-release", options), + ).toBe("entered-after-release"); + } finally { + outsideLockContext.emitDestroy(); + await fs.rm(stateDir, { recursive: true, force: true }); + } + }); + it("keeps cross-gateway router onboarding publication ahead of teardown", async () => { const homeDir = await fs.mkdtemp(path.join(os.tmpdir(), "nemoclaw-router-port-lock-")); const homedirSpy = vi.spyOn(os, "homedir").mockReturnValue(homeDir); diff --git a/src/lib/inference/gateway-route-mutation-lock.ts b/src/lib/inference/gateway-route-mutation-lock.ts index 697fc5dd734..97251ac60b8 100644 --- a/src/lib/inference/gateway-route-mutation-lock.ts +++ b/src/lib/inference/gateway-route-mutation-lock.ts @@ -4,7 +4,11 @@ import os from "node:os"; import path from "node:path"; -import { type McpLifecycleLockOptions, withMcpLifecycleLock } from "../state/mcp-lifecycle-lock"; +import { + type McpLifecycleLockOptions, + withMcpLifecycleLock, + withMcpLifecycleLockSync, +} from "../state/mcp-lifecycle-lock"; import { resolveSharedLocalAdapterStateRoot } from "./local-adapter-lifecycle"; const GATEWAY_ROUTE_LOCK_PREFIX = "gateway-route:"; @@ -33,6 +37,20 @@ export function withGatewayRouteMutationLock( ); } +export function withGatewayRouteMutationLockSync( + gatewayName: string, + operation: () => T, + options: McpLifecycleLockOptions = {}, +): T { + const normalizedGatewayName = gatewayName.trim(); + if (!normalizedGatewayName) throw new Error("OpenShell gateway name is required."); + return withMcpLifecycleLockSync( + `${GATEWAY_ROUTE_LOCK_PREFIX}${normalizedGatewayName}`, + operation, + options, + ); +} + /** Serialize current-user lifecycle changes for one Model Router port across gateways. */ export function withModelRouterPortLifecycleLock( port: number, diff --git a/src/lib/inference/https-pin-runtime-adapter.test.ts b/src/lib/inference/https-pin-runtime-adapter.test.ts index c6f6d77e090..0af99fd8221 100644 --- a/src/lib/inference/https-pin-runtime-adapter.test.ts +++ b/src/lib/inference/https-pin-runtime-adapter.test.ts @@ -1015,37 +1015,6 @@ describe("createHttpsPinRuntimeAdapterServer OpenShell bridge source restriction }); }); -describe("discoverOpenShellBridgeSourceCidrs (#6141)", () => { - it("accepts only validated subnets from the inspected OpenShell Docker network", () => { - const capture = vi.fn(() => - JSON.stringify([ - { Subnet: "172.17.0.0/16", Gateway: "172.17.0.1" }, - { Subnet: "fd00:1234::/64", Gateway: "fd00:1234::1" }, - { Subnet: "not-a-cidr" }, - ]), - ) as unknown as NonNullable[0]>; - - expect(__test.discoverOpenShellBridgeSourceCidrs(capture)).toEqual([ - "172.17.0.0/16", - "fd00:1234::/64", - ]); - expect(capture).toHaveBeenCalledWith( - ["docker", "network", "inspect", "openshell-docker", "--format", "{{json .IPAM.Config}}"], - { ignoreError: true }, - ); - }); - - it("fails closed when the OpenShell bridge has no valid source subnet", () => { - const capture = vi.fn(() => "[]") as unknown as NonNullable< - Parameters[0] - >; - - expect(() => __test.discoverOpenShellBridgeSourceCidrs(capture)).toThrow( - /refusing to expose the credential-bearing HTTPS Pin Runtime adapter/, - ); - }); -}); - describe("adapter recovery lock (#6141)", () => { // The statically-imported `__test.LOCK_PATH` above is derived from this // machine's real os.homedir() at module-evaluation time, same as a real, @@ -1344,6 +1313,7 @@ describe("HTTPS Pin Runtime adapter child environment (#6141)", () => { describe("ensureHttpsPinRuntimeAdapter preflight-before-credential ordering (#6141)", () => { const privateLookup: EndpointDnsLookupFn = async () => [{ address: "10.48.203.205", family: 4 }]; const publicLookup: EndpointDnsLookupFn = async () => [{ address: "93.184.216.34", family: 4 }]; + const discoverAllowedSourceCidrs = () => ["172.17.0.0/16"]; it("rejects a cleartext HTTP endpoint at the exported lifecycle boundary", async () => { await expect( @@ -1354,6 +1324,7 @@ describe("ensureHttpsPinRuntimeAdapter preflight-before-credential ordering (#61 providerType: "openai", credentialValue: "sk-secret", lookup: publicLookup, + discoverAllowedSourceCidrs, }), ).rejects.toThrow("requires an HTTPS endpoint URL"); }); @@ -1369,6 +1340,7 @@ describe("ensureHttpsPinRuntimeAdapter preflight-before-credential ordering (#61 // message would mention "credential" instead of the SSRF reason. credentialValue: "", lookup: privateLookup, + discoverAllowedSourceCidrs, }), ).rejects.toThrow(/resolves to private\/internal address/); }); @@ -1382,6 +1354,7 @@ describe("ensureHttpsPinRuntimeAdapter preflight-before-credential ordering (#61 providerType: "openai", credentialValue: " ", lookup: publicLookup, + discoverAllowedSourceCidrs, }), ).rejects.toThrow(/requires a non-empty credential value/); }); @@ -1395,6 +1368,7 @@ describe("ensureHttpsPinRuntimeAdapter preflight-before-credential ordering (#61 providerType: "openai", credentialValue: "", lookup: publicLookup, + discoverAllowedSourceCidrs, }), ).rejects.toThrow(/requires a DNS-resolved public address/); }); @@ -1411,6 +1385,7 @@ describe("ensureHttpsPinRuntimeAdapter preflight-before-credential ordering (#61 providerType: "openai", credentialValue: "sk-secret", lookup: failingLookup, + discoverAllowedSourceCidrs, }), ).rejects.toThrow(/cannot resolve endpoint host/); }); diff --git a/src/lib/inference/https-pin-runtime-adapter.ts b/src/lib/inference/https-pin-runtime-adapter.ts index 766a84d3329..7c379d4082d 100644 --- a/src/lib/inference/https-pin-runtime-adapter.ts +++ b/src/lib/inference/https-pin-runtime-adapter.ts @@ -106,7 +106,6 @@ const STALE_LOCK_MS = 30_000; const PROCESS_EXIT_WAIT_ATTEMPTS = 30; const PROCESS_EXIT_WAIT_MS = 100; const ADAPTER_PROTOCOL_VERSION = "3"; -const OPEN_SHELL_DOCKER_NETWORK = "openshell-docker"; interface AdapterIdentity { protocolVersion: string; @@ -186,7 +185,7 @@ function isLoopbackRemoteAddress(remoteAddress: string | undefined): boolean { return normalized === "127.0.0.1" || normalized === "::1"; } -/** Parse one exact Docker IPAM subnet before it becomes a route-source capability. */ +/** Parse one exact runtime-provider subnet before it becomes a route-source capability. */ function normalizeAllowedSourceCidr(value: string): string | null { const candidate = value.trim(); const slash = candidate.lastIndexOf("/"); @@ -242,43 +241,6 @@ function routeSourcePolicyDigest(cidrs: readonly string[]): string { .digest("hex"); } -function discoverOpenShellBridgeSourceCidrs(capture: typeof runCapture = runCapture): string[] { - let raw = ""; - try { - raw = capture( - [ - "docker", - "network", - "inspect", - OPEN_SHELL_DOCKER_NETWORK, - "--format", - "{{json .IPAM.Config}}", - ], - { ignoreError: true }, - ); - } catch { - raw = ""; - } - try { - const parsed = JSON.parse(raw.trim()) as unknown; - if (!Array.isArray(parsed)) throw new Error("expected Docker IPAM array"); - const cidrs = parsed - .map((entry) => - entry && typeof entry === "object" && typeof (entry as JsonObject).Subnet === "string" - ? String((entry as JsonObject).Subnet) - : "", - ) - .map(normalizeAllowedSourceCidr) - .filter((entry): entry is string => Boolean(entry)); - if (cidrs.length > 0) return [...new Set(cidrs)]; - } catch { - // Fall through to the fail-closed error below. - } - throw new Error( - `Cannot determine the ${OPEN_SHELL_DOCKER_NETWORK} bridge source CIDR; refusing to expose the credential-bearing HTTPS Pin Runtime adapter.`, - ); -} - function controlChallengeProof( controlToken: string, nonce: string, @@ -453,7 +415,7 @@ function parseRoutePutBody(raw: JsonObject): RouteRuntime { export function createHttpsPinRuntimeAdapterServer(options: { controlToken: string; /** - * Exact OpenShell Docker IPAM subnets. Direct unit callers may omit this + * Exact runtime-provider sandbox source subnets. Direct unit callers may omit this * and get loopback-only behavior; the spawned production adapter always * receives inspected bridge CIDRs in its authenticated bootstrap. */ @@ -1214,7 +1176,7 @@ export async function ensureHttpsPinRuntimeAdapter(options: { providerType: HttpsPinCredentialProviderType; credentialValue: string; lookup?: EndpointDnsLookupFn; - discoverAllowedSourceCidrs?: () => string[]; + discoverAllowedSourceCidrs: () => readonly string[]; }): Promise<{ baseUrl: string; localBaseUrl: string; @@ -1266,7 +1228,7 @@ export async function ensureHttpsPinRuntimeAdapter(options: { options.endpointUrl, ); const allowedSourceCidrs = buildAllowedRouteSourceMatcher( - options.discoverAllowedSourceCidrs?.() ?? discoverOpenShellBridgeSourceCidrs(), + options.discoverAllowedSourceCidrs(), ).cidrs; // Keep the lifecycle lock through the whole adapter-registration // transaction. In particular, persistRouteState is a read/modify/write of @@ -1576,7 +1538,6 @@ export const __test = { withAdapterLock, computeRespawnState, buildAllowedRouteSourceMatcher, - discoverOpenShellBridgeSourceCidrs, extractPersistedAllowedSourceCidrs, findReusableAdapterControlToken, revokeRouteLocked, diff --git a/src/lib/inference/serving/profile-list.test.ts b/src/lib/inference/serving/profile-list.test.ts index 7e82022e3ee..6fff15217fc 100644 --- a/src/lib/inference/serving/profile-list.test.ts +++ b/src/lib/inference/serving/profile-list.test.ts @@ -11,37 +11,41 @@ import { } from "./profile-list"; describe("serving profile discovery", () => { - it("lists every compiled preset with stable selection metadata (#8384)", () => { - const catalog = loadServingCatalog(); - const entries = listServingProfiles(catalog, { - evaluateCompatibility: (_catalog, preset) => - preset.metadata.id === catalog.presets[0]?.metadata.id - ? { compatible: true, incompatibilityReason: null } - : { compatible: false, incompatibilityReason: "Test host requirement is not met." }, - }); + it( + "lists every compiled preset with stable selection metadata (#8384)", + { timeout: 30_000 }, + () => { + const catalog = loadServingCatalog(); + const entries = listServingProfiles(catalog, { + evaluateCompatibility: (_catalog, preset) => + preset.metadata.id === catalog.presets[0]?.metadata.id + ? { compatible: true, incompatibilityReason: null } + : { compatible: false, incompatibilityReason: "Test host requirement is not met." }, + }); - expect(entries.map(({ id }) => id)).toEqual( - [...catalog.presets].map(({ metadata }) => metadata.id).sort(), - ); - entries.forEach((entry) => { - expect(entry).toMatchObject({ - id: expect.any(String), - displayName: expect.any(String), - backend: expect.any(String), - model: expect.any(String), - topology: expect.any(String), - selectionMode: expect.stringMatching(/^(automatic|explicit-only|disabled)$/u), - supportState: expect.stringMatching(/^(supported|experimental|disabled)$/u), - validationLevel: expect.stringMatching(/^(schema|software|hardware)$/u), - compatible: expect.any(Boolean), + expect(entries.map(({ id }) => id)).toEqual( + [...catalog.presets].map(({ metadata }) => metadata.id).sort(), + ); + entries.forEach((entry) => { + expect(entry).toMatchObject({ + id: expect.any(String), + displayName: expect.any(String), + backend: expect.any(String), + model: expect.any(String), + topology: expect.any(String), + selectionMode: expect.stringMatching(/^(automatic|explicit-only|disabled)$/u), + supportState: expect.stringMatching(/^(supported|experimental|disabled)$/u), + validationLevel: expect.stringMatching(/^(schema|software|hardware)$/u), + compatible: expect.any(Boolean), + }); + expect( + entry.compatible ? entry.incompatibilityReason : typeof entry.incompatibilityReason, + ).toBe(entry.compatible ? null : "string"); }); - expect( - entry.compatible ? entry.incompatibilityReason : typeof entry.incompatibilityReason, - ).toBe(entry.compatible ? null : "string"); - }); - expect(entries.some(({ compatible }) => compatible)).toBe(true); - expect(entries.some(({ compatible }) => !compatible)).toBe(true); - }); + expect(entries.some(({ compatible }) => compatible)).toBe(true); + expect(entries.some(({ compatible }) => !compatible)).toBe(true); + }, + ); it("renders IDs, selection state, support state, and compatibility (#8384)", () => { const output = renderServingProfiles([ diff --git a/src/lib/messaging/applier/agent-config.ts b/src/lib/messaging/applier/agent-config.ts index ee2a408f322..f509a9a57ca 100644 --- a/src/lib/messaging/applier/agent-config.ts +++ b/src/lib/messaging/applier/agent-config.ts @@ -232,7 +232,6 @@ function readHermesRuntimeAliasRender( const expectedPattern = `^openshell:resolve:env:v[0-9]+_${sourceKey}$`; const expectedValue = `openshell:resolve:env:${sourceKey}`; if (alias.match !== expectedPattern || alias.value !== expectedValue) return []; - const result = runOpenshell( [ "sandbox", @@ -529,6 +528,7 @@ function applyEnvLines( plan: SandboxMessagingPlan, existing: string | undefined, render: readonly SandboxMessagingEnvLinesRenderPlan[], + additionalLines: readonly string[] = [], ): string { const desired = new Map(); const rawDesiredLines: string[] = []; @@ -542,6 +542,11 @@ function applyEnvLines( } } } + for (const line of additionalLines) { + const key = readEnvLineKey(line); + if (!key) throw new Error("Messaging runtime credential alias line is invalid."); + desired.set(key, line); + } const stale = staleCredentialEnvKeys(plan, new Set(desired.keys())); const written = new Set(); diff --git a/src/lib/messaging/applier/setup-applier-credential-env.test.ts b/src/lib/messaging/applier/setup-applier-credential-env.test.ts index 241c76ff432..aeff2612aa1 100644 --- a/src/lib/messaging/applier/setup-applier-credential-env.test.ts +++ b/src/lib/messaging/applier/setup-applier-credential-env.test.ts @@ -1,9 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; -import { restoreEnvBulk } from "../../../../test/helpers/env-test-helpers"; import { createBuiltInChannelManifestRegistry, createBuiltInRenderTemplateResolver, @@ -58,28 +57,24 @@ function buildHermesTelegramPlan( } async function buildHermesWechatPlan(): Promise { - const original = { - WECHAT_ACCOUNT_ID: process.env.WECHAT_ACCOUNT_ID, - WECHAT_ALLOWED_IDS: process.env.WECHAT_ALLOWED_IDS, - }; - process.env.WECHAT_ACCOUNT_ID = "wechat-account"; - process.env.WECHAT_ALLOWED_IDS = "wechat-user"; - try { - return await planner().buildPlan({ - sandboxName: "demo", - agent: "hermes", - workflow: "rebuild", - isInteractive: false, - configuredChannels: ["wechat"], - credentialAvailability: { WECHAT_BOT_TOKEN: true }, - }); - } finally { - restoreEnvBulk(original); - } + vi.stubEnv("WECHAT_ACCOUNT_ID", "wechat-account"); + vi.stubEnv("WECHAT_BASE_URL", "https://ilinkai.wechat.com"); + vi.stubEnv("WECHAT_ALLOWED_IDS", "wechat-user"); + return planner().buildPlan({ + sandboxName: "demo", + agent: "hermes", + workflow: "rebuild", + isInteractive: false, + configuredChannels: ["wechat"], + credentialAvailability: { WECHAT_BOT_TOKEN: true }, + }); } /** An in-memory sandbox filesystem behind the `cat`/write calls the applier makes. */ -function sandboxFiles(seed: Readonly>): { +function sandboxFiles( + seed: Readonly>, + runtimeEnv: Readonly> = {}, +): { readonly files: Record; readonly writes: string[]; readonly runOpenshell: MessagingOpenShellRunner; @@ -87,6 +82,9 @@ function sandboxFiles(seed: Readonly>): { const files: Record = { ...seed }; const writes: string[] = []; const runOpenshell: MessagingOpenShellRunner = (args, options) => { + const script = args.includes("sh") ? String(args.at(-1)) : ""; + const envProbe = /printf '%s' "[$][{]([A-Za-z_][A-Za-z0-9_]*)-[}]"/u.exec(script)?.[1]; + const envProbeResult = envProbe ? { status: 0, stdout: runtimeEnv[envProbe] ?? "" } : null; const target = String(args.at(-1)); const reading = args.includes("cat") && options?.input === undefined; const written = options?.input; @@ -95,19 +93,24 @@ function sandboxFiles(seed: Readonly>): { writes.push(target); return { status: 0 }; }; - return written !== undefined - ? write(written) - : reading - ? { - status: files[target] === undefined ? 1 : 0, - stdout: files[target] ?? "", - } - : { status: 1 }; + return ( + envProbeResult ?? + (written !== undefined + ? write(written) + : reading + ? { + status: files[target] === undefined ? 1 : 0, + stdout: files[target] ?? "", + } + : { status: 1 }) + ); }; return { files, writes, runOpenshell }; } describe("MessagingSetupApplier credential env cleanup", () => { + afterEach(() => vi.unstubAllEnvs()); + it("drops a stale credential env line written in the export form", async () => { const plan = await buildHermesTelegramPlan(); const { files, runOpenshell } = sandboxFiles({ diff --git a/src/lib/messaging/channels/discord/policy/hermes.yaml b/src/lib/messaging/channels/discord/policy/hermes.yaml index 6678670b55b..4d7fd53ce02 100644 --- a/src/lib/messaging/channels/discord/policy/hermes.yaml +++ b/src/lib/messaging/channels/discord/policy/hermes.yaml @@ -60,4 +60,6 @@ network_policies: binaries: - { path: /usr/local/bin/node } - { path: /usr/bin/python3* } + - { path: /usr/bin/python3.13 } + - { path: /opt/hermes/.venv/bin/python3 } - { path: /opt/hermes/.venv/bin/python } diff --git a/src/lib/messaging/channels/slack/policy/hermes.yaml b/src/lib/messaging/channels/slack/policy/hermes.yaml index 8c19367f434..5e025cc4665 100644 --- a/src/lib/messaging/channels/slack/policy/hermes.yaml +++ b/src/lib/messaging/channels/slack/policy/hermes.yaml @@ -80,4 +80,6 @@ network_policies: binaries: - { path: /usr/local/bin/hermes } - { path: /usr/bin/python3* } + - { path: /usr/bin/python3.13 } + - { path: /opt/hermes/.venv/bin/python3 } - { path: /opt/hermes/.venv/bin/python } diff --git a/src/lib/messaging/channels/teams/policy/hermes.yaml b/src/lib/messaging/channels/teams/policy/hermes.yaml index c5a80131ab7..94723f54d84 100644 --- a/src/lib/messaging/channels/teams/policy/hermes.yaml +++ b/src/lib/messaging/channels/teams/policy/hermes.yaml @@ -87,4 +87,6 @@ network_policies: binaries: - { path: /usr/local/bin/hermes } - { path: /usr/bin/python3* } + - { path: /usr/bin/python3.13 } + - { path: /opt/hermes/.venv/bin/python3 } - { path: /opt/hermes/.venv/bin/python } diff --git a/src/lib/messaging/channels/telegram/policy/hermes.yaml b/src/lib/messaging/channels/telegram/policy/hermes.yaml index 0ddae3e1655..7382f91805a 100644 --- a/src/lib/messaging/channels/telegram/policy/hermes.yaml +++ b/src/lib/messaging/channels/telegram/policy/hermes.yaml @@ -22,4 +22,6 @@ network_policies: binaries: - { path: /usr/local/bin/node } - { path: /usr/bin/python3* } + - { path: /usr/bin/python3.13 } + - { path: /opt/hermes/.venv/bin/python3 } - { path: /opt/hermes/.venv/bin/python } diff --git a/src/lib/messaging/channels/wechat/hooks/implementations.test.ts b/src/lib/messaging/channels/wechat/hooks/implementations.test.ts index 8c4e674f34a..3ac80f68ba7 100644 --- a/src/lib/messaging/channels/wechat/hooks/implementations.test.ts +++ b/src/lib/messaging/channels/wechat/hooks/implementations.test.ts @@ -313,6 +313,7 @@ describe("WeChat hook implementations", () => { }, channels: { "openclaw-weixin": { + enabled: true, channelConfigUpdatedAt: "2026-05-25T00:00:00.000Z", accounts: { "wechat-account": { diff --git a/src/lib/messaging/channels/wechat/hooks/seed-openclaw-account.ts b/src/lib/messaging/channels/wechat/hooks/seed-openclaw-account.ts index dd7e1b3658a..0b1312ce62c 100644 --- a/src/lib/messaging/channels/wechat/hooks/seed-openclaw-account.ts +++ b/src/lib/messaging/channels/wechat/hooks/seed-openclaw-account.ts @@ -108,6 +108,7 @@ export function buildWechatSeedOpenClawAccountOutputs( }, channels: { [WECHAT_PLUGIN_ID]: { + enabled: true, channelConfigUpdatedAt: savedAt, accounts: { [accountId]: { diff --git a/src/lib/messaging/channels/wechat/policy/hermes.yaml b/src/lib/messaging/channels/wechat/policy/hermes.yaml index 73696ef4456..82623a61164 100644 --- a/src/lib/messaging/channels/wechat/policy/hermes.yaml +++ b/src/lib/messaging/channels/wechat/policy/hermes.yaml @@ -30,4 +30,6 @@ network_policies: binaries: - { path: /usr/local/bin/hermes } - { path: /usr/bin/python3* } + - { path: /usr/bin/python3.13 } + - { path: /opt/hermes/.venv/bin/python3 } - { path: /opt/hermes/.venv/bin/python } diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index dad6121b2ae..8279155474d 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -509,9 +509,9 @@ const preflightUtils: typeof import("./onboard/preflight") = require("./onboard/ const clusterImagePatch: typeof import("./cluster-image-patch") = require("./cluster-image-patch"); const overlayfsAutoFix: typeof import("./onboard/overlayfs-auto-fix") = require("./onboard/overlayfs-auto-fix"); const { assessHost, checkPortAvailable, ensureSwap, getMemoryInfo } = preflightUtils; -const { - assertDockerBridgeAndContainerDnsHealthy, -}: typeof import("./onboard/bridge-dns-preflight") = require("./onboard/bridge-dns-preflight"); +const runtimeEffectfulPreflight: typeof import("./onboard/machine/runtime-effectful-preflight") = require("./onboard/machine/runtime-effectful-preflight"); +const assertRuntimeProviderHealthy = + runtimeEffectfulPreflight.bindConfiguredRuntimeProviderHealth(isNonInteractive); const agentOnboard = require("./agent/onboard"); const agentDefs = require("./agent/defs"); @@ -1521,8 +1521,8 @@ const { const { getSandboxRuntimeRegistryFields, hasSandboxGpuDrift, updateReusedSandboxMetadata } = sandboxRegistryMetadata.createSandboxRegistryMetadataHelpers({ - getOpenShellComputeDriverName: () => - dockerDriverPlatform.resolveCurrentOpenShellComputePlan().driverName, + getCurrentRuntimeProviderId: () => + setupNimFlow.resolveCurrentRuntimeProviderBundle().identity.id, getInstalledOpenshellVersion, runCaptureOpenshell, }); @@ -2976,7 +2976,7 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { ...options, allowStorageRemediation: !isGatewayExternallySupervised(), }), - assertDockerBridgeAndContainerDnsHealthy, + assertRuntimeProviderHealthy, resolveSandboxGpuConfig, validateSandboxGpuPreflight, skippedStepMessage, @@ -3342,8 +3342,7 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { ensureAgentDashboardForward: ensureFinalizationAgentDashboardForward, setDefaultSandbox: registry.setDefault, verifyWebSearchInsideSandbox, - toSessionUpdates: (updates) => - toSessionUpdates(updates as Parameters[0]), + toSessionUpdates, removeLegacyCredentialsFile, cleanupStaleHostFiles, getChatUiUrl: () => process.env.CHAT_UI_URL || `http://127.0.0.1:${DASHBOARD_PORT}`, @@ -3522,6 +3521,7 @@ module.exports = { startDockerDriverGateway, findAvailableDashboardPort, startGatewayForRecovery, + managedWorkloadOnboard, ...{ openshellArgv, runOpenshell, runCaptureOpenshell, sleepSeconds }, agentSupportsWebSearch, agentSupportsWebSearchProvider, diff --git a/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts b/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts index 9d13143e74c..b0ccde59c1c 100644 --- a/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/__test-helpers__/sandbox-gpu-create-flow.ts @@ -104,6 +104,7 @@ export function createGpuFlowDeps( capture: (args, options) => { const stdout = runCaptureOpenshell(args, { ignoreError: true, + killProcessTreeOnTimeout: true, timeout: options.timeout, }); return { status: 0, output: stdout, stdout, stderr: "" }; @@ -175,7 +176,7 @@ export function resetGpuFlowMocks(): void { } export function createGpuFlowTestHarness(mocks: Record>) { - const readyCheckOptions = { ignoreError: true, timeout: 5_000 }; + const readyCheckOptions = { ignoreError: true, killProcessTreeOnTimeout: true, timeout: 5_000 }; const failedProof: SandboxGpuProofResult = { status: "failed", cudaVerified: false, diff --git a/src/lib/onboard/compute/plan.test.ts b/src/lib/onboard/compute/plan.test.ts index 2a669346800..9f2a7f8b07e 100644 --- a/src/lib/onboard/compute/plan.test.ts +++ b/src/lib/onboard/compute/plan.test.ts @@ -3,9 +3,32 @@ import { describe, expect, it } from "vitest"; import { isLinuxDockerDriverGatewayEnabled } from "../docker-driver-platform"; -import { resolveCurrentOpenShellComputePlan, usesManagedDockerGateway } from "./plan"; +import { resolveCurrentOpenShellComputePlan, usesManagedLocalGateway } from "./plan"; describe("current OpenShell compute plan", () => { + it("selects the registered native Podman provider once at the configuration boundary (#9145)", () => { + expect( + resolveCurrentOpenShellComputePlan("linux", "x64", { + NEMOCLAW_GATEWAY_RUNTIME: "podman", + }), + ).toEqual({ + driverName: "podman", + gatewayLauncher: "nemoclaw", + }); + }); + + it("keeps the portable profile independent from native Podman selection (#9145)", () => { + expect( + resolveCurrentOpenShellComputePlan("linux", "x64", { + NEMOCLAW_EXPERIMENTAL_PROFILE: "portable", + NEMOCLAW_GATEWAY_RUNTIME: "podman", + }), + ).toEqual({ + driverName: "docker", + gatewayLauncher: "nemoclaw", + }); + }); + it.each([ { label: "Linux x64", @@ -42,29 +65,26 @@ describe("current OpenShell compute plan", () => { driverName: "kubernetes", gatewayLauncher: "openshell", }, - ])("preserves the existing driver and gateway-launch behavior on $label (#7744)", ({ - platform, - arch, - driverName, - gatewayLauncher, - }) => { - expect(resolveCurrentOpenShellComputePlan(platform, arch)).toEqual({ - driverName, - gatewayLauncher, - }); - expect(isLinuxDockerDriverGatewayEnabled(platform, arch)).toBe(driverName === "docker"); - }); + ])( + "preserves the existing driver and gateway-launch behavior on $label (#7744)", + ({ platform, arch, driverName, gatewayLauncher }) => { + expect(resolveCurrentOpenShellComputePlan(platform, arch)).toEqual({ + driverName, + gatewayLauncher, + }); + expect(isLinuxDockerDriverGatewayEnabled(platform, arch)).toBe(driverName === "docker"); + }, + ); it.each([ { driverName: "docker", gatewayLauncher: "nemoclaw", expected: true }, { driverName: "docker", gatewayLauncher: "openshell", expected: false }, - { driverName: "podman", gatewayLauncher: "nemoclaw", expected: false }, + { driverName: "podman", gatewayLauncher: "nemoclaw", expected: true }, { driverName: "mxc", gatewayLauncher: "nemoclaw", expected: false }, - ] as const)("reports Docker lifecycle ownership as $expected for $driverName with the $gatewayLauncher launcher (#7744)", ({ - driverName, - gatewayLauncher, - expected, - }) => { - expect(usesManagedDockerGateway({ driverName, gatewayLauncher })).toBe(expected); - }); + ] as const)( + "reports managed gateway ownership as $expected for $driverName with the $gatewayLauncher launcher (#7744)", + ({ driverName, gatewayLauncher, expected }) => { + expect(usesManagedLocalGateway({ driverName, gatewayLauncher })).toBe(expected); + }, + ); }); diff --git a/src/lib/onboard/compute/plan.ts b/src/lib/onboard/compute/plan.ts index 03ff7672180..71b83519e64 100644 --- a/src/lib/onboard/compute/plan.ts +++ b/src/lib/onboard/compute/plan.ts @@ -23,6 +23,11 @@ export interface OpenShellComputePlan { readonly gatewayLauncher: OpenShellGatewayLauncher; } +export interface CurrentOpenShellRuntimeSelection { + readonly plan: OpenShellComputePlan; + readonly provider: RuntimeProviderBundle; +} + export function projectRuntimeProviderComputePlan( bundle: RuntimeProviderBundle, ): OpenShellComputePlan { @@ -39,11 +44,27 @@ export function projectRuntimeProviderComputePlan( export function resolveCurrentOpenShellComputePlan( platform: NodeJS.Platform = process.platform, arch: NodeJS.Architecture = process.arch, + env: NodeJS.ProcessEnv = process.env, ): OpenShellComputePlan { - return projectRuntimeProviderComputePlan(resolveCurrentRuntimeProviderBundle(platform, arch)); + return resolveCurrentOpenShellRuntimeSelection(platform, arch, env).plan; +} + +/** Resolve the configured provider once and project every central selection from that bundle. */ +export function resolveCurrentOpenShellRuntimeSelection( + platform: NodeJS.Platform = process.platform, + arch: NodeJS.Architecture = process.arch, + env: NodeJS.ProcessEnv = process.env, +): CurrentOpenShellRuntimeSelection { + const provider = resolveCurrentRuntimeProviderBundle( + platform, + arch, + CURRENT_RUNTIME_PROVIDER_BUNDLES, + env, + ); + return { plan: projectRuntimeProviderComputePlan(provider), provider }; } -export function usesManagedDockerGateway( +export function usesManagedLocalGateway( plan: Pick, providers: RuntimeProviderBundleRegistry = CURRENT_RUNTIME_PROVIDER_BUNDLES, ): boolean { @@ -54,6 +75,6 @@ export function usesManagedDockerGateway( return ( bundle?.gateway.launcher === plan.gatewayLauncher && plan.gatewayLauncher === "nemoclaw" && - engine?.engineId === "docker" + engine !== null ); } diff --git a/src/lib/onboard/created-sandbox-finalization.test.ts b/src/lib/onboard/created-sandbox-finalization.test.ts index 28ec63eb85c..54ef954b702 100644 --- a/src/lib/onboard/created-sandbox-finalization.test.ts +++ b/src/lib/onboard/created-sandbox-finalization.test.ts @@ -19,6 +19,7 @@ import { finalizeCreatedSandbox, } from "./created-sandbox-finalization"; import { getDcodeSelectionDrift } from "./dcode-selection-drift"; +import { dashboardForwardControlRuntime } from "./dashboard-forward-control"; import type { HermesPortableConfiguredReceipt } from "./experimental/hermes-portable-receipt"; import { pendingSandboxCreateIdentityForBoundary } from "./sandbox-create/identity-boundary"; import type { SandboxGpuCreateFlowResult } from "./sandbox-gpu-create-flow"; @@ -27,6 +28,41 @@ import type { CreatedSandboxRegistrationInput } from "./sandbox-registration"; const fixtures: string[] = []; +describe("ordinary managed sandbox completion", () => { + it.each(["docker", "podman"] as const)( + "does not mutate attached provider generations after %s sandbox startup", + (openshellDriver) => { + const providerExistsInGateway = vi.fn(() => true); + + expect( + completeOrdinaryOnboardSandboxCreation( + { + sandboxName: "alpha", + sandboxWasLiveDefault: false, + gatewayPort: 8080, + runtimeFields: { openshellDriver } as SandboxEntry, + messagingProviders: ["alpha-slack", "alpha-slack"], + liveExists: true, + }, + { + setDefault: vi.fn(), + runFile: vi.fn(), + scriptsDir: "/tmp/scripts", + gatewayName: "nemoclaw", + providerExistsInGateway, + armCancelRollback: vi.fn(), + markCancellationRecovery: vi.fn(), + dockerInfoFormat: vi.fn(() => "true"), + runCapture: vi.fn(() => ""), + revalidateSandboxIdentity: vi.fn(), + }, + ), + ).toBe("alpha"); + expect(providerExistsInGateway).toHaveBeenCalledTimes(2); + }, + ); +}); + afterEach(() => { delete process.env.NEMOCLAW_OPENSHELL_BIN; for (const fixture of fixtures.splice(0)) fs.rmSync(fixture, { recursive: true, force: true }); @@ -392,16 +428,39 @@ describe("created DCode sandbox finalization", () => { reservation: {} as never, checkpoint: pendingSandboxCreateIdentityForBoundary(verifiedCreateBoundary), } as NonNullable; - const runCaptureOpenshell = vi.fn(() => - [ - "Sandbox: dcode", - "Route: inference", - "Provider: compatible-endpoint", - `Model: openai:${model}`, - "Endpoint: https://inference.local/v1", - "Runtime: Deep Agents Code (terminal)", - ].join("\n"), - ); + const runCaptureOpenshell = vi + .fn() + .mockReturnValueOnce( + ["SANDBOX BIND PORT PID STATUS", "alpha 127.0.0.1 18789 101 running"].join("\n"), + ) + .mockReturnValue( + [ + "Sandbox: dcode", + "Route: inference", + "Provider: compatible-endpoint", + `Model: openai:${model}`, + "Endpoint: https://inference.local/v1", + "Runtime: Deep Agents Code (terminal)", + ].join("\n"), + ); + const ensureDashboardForward = vi.fn(() => 8643); + const preservedSibling = { + bind: "127.0.0.1", + gatewayName: "nemoclaw", + lifecycleGeneration: "generation-alpha", + lifecycleLiveIdentityFingerprint: "b".repeat(64), + openshellDriver: "podman", + pid: 101, + port: "18789", + sandboxName: "alpha", + }; + vi.spyOn(dashboardForwardControlRuntime, "getSandbox").mockReturnValue({ + name: "alpha", + gatewayName: preservedSibling.gatewayName, + lifecycleGeneration: preservedSibling.lifecycleGeneration, + lifecycleLiveIdentityFingerprint: preservedSibling.lifecycleLiveIdentityFingerprint, + openshellDriver: preservedSibling.openshellDriver, + }); vi.spyOn(process, "exit").mockImplementation((code): never => { throw new Error(`exit ${code}`); }); @@ -474,7 +533,7 @@ describe("created DCode sandbox finalization", () => { "http://127.0.0.1:8643", { config: null, enabled: false }, vi.fn(), - vi.fn(), + ensureDashboardForward, vi.fn(), vi.fn(), vi.fn(), @@ -505,6 +564,10 @@ describe("created DCode sandbox finalization", () => { })), ] as unknown as Parameters; const completion = createOnboardCreatedSandboxCompletion(...completionArgs); + expect(runCaptureOpenshell).toHaveBeenCalledOnce(); + expect(runCaptureOpenshell).toHaveBeenCalledWith(["forward", "list"], { + ignoreError: true, + }); const created = { createResult: { status: 0, output: "", sawProgress: true }, route: "native", @@ -534,12 +597,25 @@ describe("created DCode sandbox finalization", () => { created, null, "disabled", - false, + true, () => ({ lifecycleGeneration: "generation-1" }), lifecycle, ), ).rejects.toThrow("exit 1"); - expect(runCaptureOpenshell).toHaveBeenCalledOnce(); + expect(runCaptureOpenshell).toHaveBeenCalledTimes(2); + expect(ensureDashboardForward).toHaveBeenCalledWith("dcode", "http://127.0.0.1:8643", { + rollbackSandboxOnFailure: true, + preservedSiblingForwards: [preservedSibling], + revalidateSandboxIdentity: expect.any(Function), + }); + + runCaptureOpenshell.mockClear(); + const portableCompletionArgs = [...completionArgs] as Parameters< + typeof createOnboardCreatedSandboxCompletion + >; + portableCompletionArgs[9] = true; + createOnboardCreatedSandboxCompletion(...portableCompletionArgs); + expect(runCaptureOpenshell).not.toHaveBeenCalled(); }); it("does not publish registry metadata when live validation fails (#6311)", () => { @@ -1100,10 +1176,34 @@ describe("created sandbox completion actions", () => { dashboard: { chatUiUrl: "http://127.0.0.1:8643", initialHermesState: { config: null, enabled: false }, + preservedSiblingForwards: [ + { + bind: "127.0.0.1", + gatewayName: "nemoclaw", + lifecycleGeneration: "generation-alpha", + lifecycleLiveIdentityFingerprint: "b".repeat(64), + openshellDriver: "podman", + pid: 101, + port: "18789", + sandboxName: "alpha", + }, + ], releasePort: async () => { order.push("dashboard-release"); }, - ensureForward: () => { + ensureForward: (_sandboxName, _chatUiUrl, options) => { + expect(options.preservedSiblingForwards).toEqual([ + { + bind: "127.0.0.1", + gatewayName: "nemoclaw", + lifecycleGeneration: "generation-alpha", + lifecycleLiveIdentityFingerprint: "b".repeat(64), + openshellDriver: "podman", + pid: 101, + port: "18789", + sandboxName: "alpha", + }, + ]); order.push("dashboard-forward"); return 8644; }, diff --git a/src/lib/onboard/created-sandbox-finalization.ts b/src/lib/onboard/created-sandbox-finalization.ts index bc023627988..6f689ba60a2 100644 --- a/src/lib/onboard/created-sandbox-finalization.ts +++ b/src/lib/onboard/created-sandbox-finalization.ts @@ -23,7 +23,13 @@ import * as buildContext from "../build-context"; import { resolveSandboxImageTagFromCreateOutput } from "../domain/sandbox/image-tag"; import { restoreDefaultAfterRecreate } from "./default-preservation"; import { createDcodeSelectionDriftReader } from "./dcode-selection-drift"; +import { + captureLiveSiblingDashboardForwards, + type DashboardForwardOptions, + type PreservedDashboardForward, +} from "./dashboard-forward-control"; import * as dockerGpuLocalInference from "./docker-gpu-local-inference"; +import { shouldManageDashboardForAgent } from "./dashboard-runtime"; import type { HermesDashboardOnboardState } from "./hermes-dashboard"; import type { HermesPortableConfiguredReceipt } from "./experimental/hermes-portable-receipt"; import { warnIfLandlockUnsupported } from "./landlock-warning"; @@ -128,14 +134,15 @@ export interface CreatedSandboxCompletionOptions { readonly dashboard: { readonly chatUiUrl: string; readonly initialHermesState: HermesDashboardOnboardState; + readonly preservedSiblingForwards: readonly PreservedDashboardForward[]; readonly releasePort: () => Promise; readonly ensureForward: ( sandboxName: string, chatUiUrl: string, - options: { - rollbackSandboxOnFailure: true; - revalidateSandboxIdentity?: (operation: string) => void; - }, + options: Pick< + DashboardForwardOptions, + "rollbackSandboxOnFailure" | "preservedSiblingForwards" | "revalidateSandboxIdentity" + > & { rollbackSandboxOnFailure: true }, ) => number; readonly getForwardPort: (chatUiUrl: string) => string; readonly resolveHermesState: (port: number) => HermesDashboardOnboardState; @@ -385,6 +392,7 @@ export function createCreatedSandboxCompletionActions( ); dashboardPort = options.dashboard.ensureForward(options.finalization.sandboxName, chatUiUrl, { rollbackSandboxOnFailure: true, + preservedSiblingForwards: options.dashboard.preservedSiblingForwards, revalidateSandboxIdentity: deps.revalidateSandboxIdentity, }); deps.revalidateSandboxIdentity?.( @@ -633,6 +641,17 @@ export function createOnboardCreatedSandboxCompletion( ): CreatedSandboxCompletionActions { const { provider, model, preferredInferenceApi, endpointUrl } = inference; const { createIntent, resolvedCreateIntent } = createContext; + // This constructor runs before the potentially long create/build operation. + // Preserve only siblings proven live now; finalization may safely restore + // those exact forwards if their SSH processes die while the target builds. + // Portable lifecycle keeps its separate forwarding behavior unchanged. + const preservedSiblingForwards = + portableLifecycle || !shouldManageDashboardForAgent(agent ?? null) + ? [] + : captureLiveSiblingDashboardForwards( + runCaptureOpenshell(["forward", "list"], { ignoreError: true }), + sandboxName, + ); return createCreatedSandboxCompletionActions( { finalization: { @@ -694,6 +713,7 @@ export function createOnboardCreatedSandboxCompletion( dashboard: { chatUiUrl, initialHermesState: initialHermesDashboardState, + preservedSiblingForwards, releasePort: releaseDashboardPort, ensureForward: ensureDashboardForward, getForwardPort: getDashboardForwardPort, diff --git a/src/lib/onboard/credential-provider-registration.test.ts b/src/lib/onboard/credential-provider-registration.test.ts index 2095e40c984..1ac19957754 100644 --- a/src/lib/onboard/credential-provider-registration.test.ts +++ b/src/lib/onboard/credential-provider-registration.test.ts @@ -45,8 +45,10 @@ function providerMetadata( return { status: 0, stdout: [ + `Id: provider-${name}`, `Name: ${name}`, `Type: ${type}`, + "Resource version: 1", `Credential keys: ${credentialKey}`, "Config keys: ", ].join("\n"), @@ -141,6 +143,38 @@ describe("credential provider registration", () => { } }); + it("leaves unrelated provider batches on the production path while the live override is installed", () => { + vi.stubEnv("NEMOCLAW_RUN_LIVE_E2E", "1"); + const tokenDefs: MessagingTokenDef[] = [ + { + name: "e2e-oc-ch-cycle-discord-bridge", + envKey: "DISCORD_BOT_TOKEN", + token: null, + providerType: "generic", + }, + ]; + const runOpenshell = vi.fn(); + const override = vi.fn(() => ["e2e-oc-ch-cycle-googlechat-bridge"]); + const restore = installLiveE2eCredentialProviderRegistrationOverride({ + expectedName: "e2e-oc-ch-cycle-googlechat-bridge", + expectedType: "google-chat-bridge", + upsert: override, + }); + try { + expect( + credentialProviderRegistrationDependencies.upsertMessagingProviders( + tokenDefs, + runOpenshell, + {}, + ), + ).toEqual([]); + expect(override).not.toHaveBeenCalled(); + } finally { + restore(); + vi.unstubAllEnvs(); + } + }); + it("resolves the provider upsert dependency when registration executes", () => { const session = { stagedCredentialProviders: [] } as unknown as Session; const runOpenshell = vi.fn(); diff --git a/src/lib/onboard/credential-provider-registration.ts b/src/lib/onboard/credential-provider-registration.ts index 363d36ea112..165b75bccd6 100644 --- a/src/lib/onboard/credential-provider-registration.ts +++ b/src/lib/onboard/credential-provider-registration.ts @@ -74,13 +74,15 @@ export const credentialProviderRegistrationDependencies = { ): string[] { const override = liveE2eCredentialProviderOverride(); if (override) { - const expected = tokenDefs.filter( - ({ envKey, name, providerType }) => - envKey === "GOOGLE_CHAT_ACCESS_TOKEN" && - name === override.expectedName && - providerType === override.expectedType, - ); - if (expected.length !== 1) { + const selected = tokenDefs.filter(({ name }) => name === override.expectedName); + if (selected.length === 0) { + return providers.upsertMessagingProviders(tokenDefs, runOpenshell, options) as string[]; + } + if ( + selected.length !== 1 || + selected[0]?.envKey !== "GOOGLE_CHAT_ACCESS_TOKEN" || + selected[0]?.providerType !== override.expectedType + ) { throw new Error("Google Chat live E2E provider override received an unexpected plan."); } return override.upsert(tokenDefs, runOpenshell, options); diff --git a/src/lib/onboard/dashboard-forward-control.test.ts b/src/lib/onboard/dashboard-forward-control.test.ts index 7f7f5f7e0b1..27225b53744 100644 --- a/src/lib/onboard/dashboard-forward-control.test.ts +++ b/src/lib/onboard/dashboard-forward-control.test.ts @@ -3,7 +3,13 @@ import { describe, expect, it, vi } from "vitest"; -import { createSandboxForwardStopper } from "./dashboard-forward-control"; +import { + captureLiveSiblingDashboardForwards, + createSandboxForwardStopper, + mergePreservedDashboardForwards, + reconcileSiblingDashboardForwards, + revalidatePreservedDashboardForward, +} from "./dashboard-forward-control"; describe("createSandboxForwardStopper", () => { it("skips the stop when the forward-list capture fails (#8522)", () => { @@ -42,3 +48,190 @@ describe("createSandboxForwardStopper", () => { expect(runOpenshell).not.toHaveBeenCalled(); }); }); + +describe("sibling dashboard forward preservation", () => { + const header = "SANDBOX BIND PORT PID STATUS"; + const alphaOwner = { + name: "alpha", + gatewayName: "nemoclaw", + lifecycleGeneration: "generation-alpha", + lifecycleLiveIdentityFingerprint: "a".repeat(64), + openshellDriver: "podman", + }; + const getSandbox = (name: string) => (name === "alpha" ? alphaOwner : null); + const alpha = { + sandboxName: "alpha", + bind: "127.0.0.1", + port: "18789", + pid: 101, + gatewayName: "nemoclaw", + lifecycleGeneration: "generation-alpha", + lifecycleLiveIdentityFingerprint: "a".repeat(64), + openshellDriver: "podman", + }; + const beta = { sandboxName: "beta", bind: "127.0.0.1", port: "18790" }; + + it("restores a sibling that becomes dead while starting the target forward", () => { + const beforeCreate = `${header}\nalpha 127.0.0.1 18789 101 running`; + const beforeTargetStart = `${header}\nalpha 127.0.0.1 18789 101 dead`; + let restored = false; + const fetch = vi.fn(() => + restored + ? `${header}\nalpha 127.0.0.1 18789 303 running\nbeta 127.0.0.1 18790 202 running` + : `${header}\nalpha 127.0.0.1 18789 101 dead\nbeta 127.0.0.1 18790 202 running`, + ); + const restore = vi.fn(() => { + restored = true; + return { ok: true }; + }); + + expect( + reconcileSiblingDashboardForwards({ + preserved: mergePreservedDashboardForwards( + captureLiveSiblingDashboardForwards(beforeCreate, "beta", getSandbox), + captureLiveSiblingDashboardForwards(beforeTargetStart, "beta", getSandbox), + ), + target: beta, + fetch, + revalidateLive: (forward, snapshot) => + revalidatePreservedDashboardForward(forward, snapshot, getSandbox), + restore, + }), + ).toEqual({ ok: true }); + expect(restore).toHaveBeenCalledExactlyOnceWith(alpha); + }); + + it("deduplicates a sibling that remains live across create and finalization", () => { + const live = `${header}\nalpha 127.0.0.1 18789 101 running`; + + expect( + mergePreservedDashboardForwards( + captureLiveSiblingDashboardForwards(live, "beta", getSandbox), + captureLiveSiblingDashboardForwards(live, "beta", getSandbox), + ), + ).toEqual([alpha]); + }); + + it("rejects an unbounded pre-create sibling snapshot", () => { + const forwards = Array.from({ length: 129 }, (_, index) => ({ + bind: "127.0.0.1", + gatewayName: "nemoclaw", + lifecycleGeneration: `generation-${index}`, + lifecycleLiveIdentityFingerprint: index.toString(16).padStart(64, "0"), + openshellDriver: "podman", + pid: 1_000 + index, + port: String(20_000 + index), + sandboxName: `sibling-${index}`, + })); + + expect(() => mergePreservedDashboardForwards(forwards)).toThrow( + "Merged sibling dashboard-forward snapshot exceeds its bound.", + ); + }); + + it("does not revive a sibling that was already dead before target startup", () => { + const before = `${header}\nalpha 127.0.0.1 18789 101 dead`; + const snapshot = `${header}\nbeta 127.0.0.1 18790 202 running`; + const restore = vi.fn(() => ({ ok: true })); + + expect( + reconcileSiblingDashboardForwards({ + preserved: captureLiveSiblingDashboardForwards(before, "beta", getSandbox), + target: beta, + fetch: () => snapshot, + revalidateLive: (forward, live) => + revalidatePreservedDashboardForward(forward, live, getSandbox), + restore, + }), + ).toEqual({ ok: true }); + expect(restore).not.toHaveBeenCalled(); + }); + + it("fails when restoring a sibling kills the newly started target", () => { + const before = `${header}\nalpha 127.0.0.1 18789 101 running`; + let restored = false; + const fetch = () => + restored + ? `${header}\nalpha 127.0.0.1 18789 303 running\nbeta 127.0.0.1 18790 202 dead` + : `${header}\nalpha 127.0.0.1 18789 101 dead\nbeta 127.0.0.1 18790 202 running`; + + expect( + reconcileSiblingDashboardForwards({ + preserved: captureLiveSiblingDashboardForwards(before, "beta", getSandbox), + target: beta, + fetch, + revalidateLive: (forward, snapshot) => + revalidatePreservedDashboardForward(forward, snapshot, getSandbox), + restore: () => { + restored = true; + return { ok: true }; + }, + }), + ).toEqual({ + ok: false, + diagnostic: "Forward beta:18790 did not remain live after sibling reconciliation.", + }); + }); + + it("rejects a same-name replacement before restoring its dead forward", () => { + const before = `${header}\nalpha 127.0.0.1 18789 101 running`; + const preserved = captureLiveSiblingDashboardForwards(before, "beta", getSandbox)[0]!; + + expect( + revalidatePreservedDashboardForward( + preserved, + `${header}\nalpha 127.0.0.1 18789 101 dead`, + () => ({ ...alphaOwner, lifecycleGeneration: "generation-replacement" }), + ), + ).toBe(false); + }); + + it("rejects an already-live tuple owned by a replacement forward process", () => { + const preserved = captureLiveSiblingDashboardForwards( + `${header}\nalpha 127.0.0.1 18789 101 running`, + "beta", + getSandbox, + ); + const replacement = + `${header}\nalpha 127.0.0.1 18789 303 running\n` + "beta 127.0.0.1 18790 202 running"; + const restore = vi.fn(() => ({ ok: true })); + + expect( + reconcileSiblingDashboardForwards({ + preserved, + target: beta, + fetch: () => replacement, + revalidateLive: (forward, snapshot) => + revalidatePreservedDashboardForward(forward, snapshot, getSandbox), + restore, + }), + ).toEqual({ + ok: false, + diagnostic: "Forward alpha:18789 changed live ownership.", + }); + expect(restore).not.toHaveBeenCalled(); + }); + + it("rejects an already-live tuple after its registry generation changes", () => { + const live = `${header}\nalpha 127.0.0.1 18789 101 running`; + const preserved = captureLiveSiblingDashboardForwards(live, "beta", getSandbox); + const replacementOwner = () => ({ + ...alphaOwner, + lifecycleGeneration: "generation-replacement", + }); + + expect( + reconcileSiblingDashboardForwards({ + preserved, + target: beta, + fetch: () => `${live}\nbeta 127.0.0.1 18790 202 running`, + revalidateLive: (forward, snapshot) => + revalidatePreservedDashboardForward(forward, snapshot, replacementOwner), + restore: vi.fn(() => ({ ok: true })), + }), + ).toEqual({ + ok: false, + diagnostic: "Forward alpha:18789 changed live ownership.", + }); + }); +}); diff --git a/src/lib/onboard/dashboard-forward-control.ts b/src/lib/onboard/dashboard-forward-control.ts index 9b819fe9605..6c70903f7be 100644 --- a/src/lib/onboard/dashboard-forward-control.ts +++ b/src/lib/onboard/dashboard-forward-control.ts @@ -2,24 +2,247 @@ // SPDX-License-Identifier: Apache-2.0 import { bestEffortForwardStopForSandbox } from "./forward-cleanup"; +import { load as loadRegistry } from "../state/registry/persistence"; +import { isPublishedSandboxRegistration } from "../state/registry/route-reservation"; +import type { SandboxEntry } from "../state/registry/types"; +import { parseForwardList, type ForwardEntry } from "../state/sandbox-session"; + +const MAX_PRESERVED_SIBLING_FORWARDS = 128; + +type DashboardForwardTarget = Pick; + +export type PreservedDashboardForward = DashboardForwardTarget & { + readonly gatewayName: string; + readonly lifecycleGeneration: string; + readonly lifecycleLiveIdentityFingerprint: string; + readonly openshellDriver: string; + readonly pid: number; +}; + +export const dashboardForwardControlRuntime = { + getSandbox: (name: string): SandboxEntry | null => loadRegistry().sandboxes[name] ?? null, +}; export interface DashboardForwardOptions { rollbackSandboxOnFailure?: boolean; gatewayName?: string; preserveSandboxPorts?: Array; + /** Exact siblings observed live before a long sandbox create/build begins. */ + preservedSiblingForwards?: readonly PreservedDashboardForward[]; allowPortReallocation?: boolean; revalidateSandboxIdentity?: (operation: string) => void; onForwardStarted?: (port: number) => void; } +function isExactLiveForward( + entries: readonly ForwardEntry[], + expected: DashboardForwardTarget, +): boolean { + return entries.some( + (entry) => + entry.sandboxName === expected.sandboxName && + entry.bind === expected.bind && + entry.port === expected.port && + entry.status === "running", + ); +} + +function canonicalForwardEntry(entry: ForwardEntry): entry is ForwardEntry & { pid: number } { + const port = Number(entry.port); + return ( + (entry.bind === "127.0.0.1" || entry.bind === "0.0.0.0") && + Number.isInteger(port) && + port >= 1 && + port <= 65_535 && + String(port) === entry.port && + Number.isSafeInteger(entry.pid) && + (entry.pid ?? 0) > 0 && + entry.status === "running" + ); +} + +function canonicalPreservedForward(forward: PreservedDashboardForward): boolean { + const port = Number(forward.port); + return ( + forward.sandboxName.length > 0 && + forward.sandboxName === forward.sandboxName.trim() && + (forward.bind === "127.0.0.1" || forward.bind === "0.0.0.0") && + Number.isInteger(port) && + port >= 1 && + port <= 65_535 && + String(port) === forward.port && + Number.isSafeInteger(forward.pid) && + forward.pid > 0 && + /^[A-Za-z0-9._:-]{1,128}$/u.test(forward.gatewayName) && + /^[A-Za-z0-9._:/=+-]{1,512}$/u.test(forward.lifecycleGeneration) && + /^[a-f0-9]{64}$/u.test(forward.lifecycleLiveIdentityFingerprint) && + /^[a-z][a-z0-9-]{0,62}$/u.test(forward.openshellDriver) + ); +} + +function preservedForward( + entry: ForwardEntry & { pid: number }, + owner: SandboxEntry | null | undefined, +): PreservedDashboardForward | null { + if ( + !owner || + !isPublishedSandboxRegistration(owner) || + owner.name !== entry.sandboxName || + typeof owner.gatewayName !== "string" || + typeof owner.lifecycleGeneration !== "string" || + typeof owner.lifecycleLiveIdentityFingerprint !== "string" || + !/^[a-f0-9]{64}$/u.test(owner.lifecycleLiveIdentityFingerprint) || + typeof owner.openshellDriver !== "string" + ) { + return null; + } + return { + bind: entry.bind, + gatewayName: owner.gatewayName, + lifecycleGeneration: owner.lifecycleGeneration, + lifecycleLiveIdentityFingerprint: owner.lifecycleLiveIdentityFingerprint, + openshellDriver: owner.openshellDriver, + pid: entry.pid, + port: entry.port, + sandboxName: entry.sandboxName, + }; +} + +/** Snapshot only live sibling forwards; pre-existing dead rows are not recovery authority. */ +export function captureLiveSiblingDashboardForwards( + output: string | null | undefined, + sandboxName: string, + getSandbox: ( + name: string, + ) => SandboxEntry | null | undefined = dashboardForwardControlRuntime.getSandbox, +): PreservedDashboardForward[] { + const captured = parseForwardList(output) + .filter(canonicalForwardEntry) + .filter((entry) => entry.sandboxName !== sandboxName) + .flatMap((entry) => preservedForward(entry, getSandbox(entry.sandboxName)) ?? []); + if (captured.length > MAX_PRESERVED_SIBLING_FORWARDS) { + throw new Error("Live sibling dashboard-forward snapshot exceeds its bound."); + } + return captured; +} + +/** Revalidate the exact published sandbox owner and original forward process before mutation. */ +export function revalidatePreservedDashboardForward( + forward: PreservedDashboardForward, + output: string | null, + getSandbox: ( + name: string, + ) => SandboxEntry | null | undefined = dashboardForwardControlRuntime.getSandbox, +): boolean { + if (!revalidatePreservedDashboardOwner(forward, getSandbox)) return false; + return parseForwardList(output).some( + (entry) => + entry.sandboxName === forward.sandboxName && + entry.bind === forward.bind && + entry.port === forward.port && + entry.pid === forward.pid, + ); +} + +export function revalidatePreservedDashboardOwner( + forward: PreservedDashboardForward, + getSandbox: ( + name: string, + ) => SandboxEntry | null | undefined = dashboardForwardControlRuntime.getSandbox, +): boolean { + if (!canonicalPreservedForward(forward)) return false; + const owner = getSandbox(forward.sandboxName); + if (!owner) return false; + return ( + isPublishedSandboxRegistration(owner) && + owner.gatewayName === forward.gatewayName && + owner.lifecycleGeneration === forward.lifecycleGeneration && + owner.lifecycleLiveIdentityFingerprint === forward.lifecycleLiveIdentityFingerprint && + owner.openshellDriver === forward.openshellDriver + ); +} + +/** Combine pre-create and finalization observations without duplicating one exact forward. */ +export function mergePreservedDashboardForwards( + ...groups: readonly (readonly PreservedDashboardForward[])[] +): PreservedDashboardForward[] { + const merged = new Map(); + for (const group of groups) { + for (const forward of group) { + if (!canonicalPreservedForward(forward)) { + throw new Error("Preserved sibling dashboard-forward authority is invalid."); + } + const key = `${forward.sandboxName}\0${forward.lifecycleGeneration}\0${forward.lifecycleLiveIdentityFingerprint}\0${forward.bind}\0${forward.port}\0${String(forward.pid)}`; + if (!merged.has(key)) { + if (merged.size >= MAX_PRESERVED_SIBLING_FORWARDS) { + throw new Error("Merged sibling dashboard-forward snapshot exceeds its bound."); + } + merged.set(key, forward); + } + } + } + return [...merged.values()]; +} + +/** Restore siblings lost during one forward start and prove the new owner remains live. */ +export function reconcileSiblingDashboardForwards(input: { + readonly preserved: readonly PreservedDashboardForward[]; + readonly target: DashboardForwardTarget; + readonly fetch: () => string | null; + readonly revalidateLive: (forward: PreservedDashboardForward, snapshot: string) => boolean; + readonly restore: (forward: PreservedDashboardForward) => { + readonly ok: boolean; + readonly diagnostic?: string; + }; +}): { readonly ok: true } | { readonly ok: false; readonly diagnostic: string } { + for (const forward of input.preserved) { + const snapshot = input.fetch(); + if (snapshot === null) { + return { ok: false, diagnostic: "OpenShell forward ownership became unavailable." }; + } + if (isExactLiveForward(parseForwardList(snapshot), forward)) { + if (!input.revalidateLive(forward, snapshot)) { + return { + ok: false, + diagnostic: `Forward ${forward.sandboxName}:${forward.port} changed live ownership.`, + }; + } + continue; + } + const restored = input.restore(forward); + if (!restored.ok) { + return { + ok: false, + diagnostic: `Could not restore ${forward.sandboxName}:${forward.port}: ${restored.diagnostic ?? "forward start failed"}`, + }; + } + } + const finalSnapshot = input.fetch(); + if (finalSnapshot === null) { + return { ok: false, diagnostic: "OpenShell forward ownership became unavailable." }; + } + const finalEntries = parseForwardList(finalSnapshot); + const missing = [...input.preserved, input.target].find( + (forward) => !isExactLiveForward(finalEntries, forward), + ); + return missing + ? { + ok: false, + diagnostic: `Forward ${missing.sandboxName}:${missing.port} did not remain live after sibling reconciliation.`, + } + : { ok: true }; +} + export function normalizeDashboardForwardOptions(options: DashboardForwardOptions = {}): { rollbackSandboxOnFailure: boolean; preservedPorts: Set; + preservedSiblingForwards: readonly PreservedDashboardForward[]; allowPortReallocation: boolean; } { return { rollbackSandboxOnFailure: options.rollbackSandboxOnFailure === true, preservedPorts: new Set((options.preserveSandboxPorts ?? []).map((port) => String(port))), + preservedSiblingForwards: options.preservedSiblingForwards ?? [], allowPortReallocation: options.allowPortReallocation !== false, }; } diff --git a/src/lib/onboard/dashboard.ts b/src/lib/onboard/dashboard.ts index 66e2326cd33..d94dcb09fce 100644 --- a/src/lib/onboard/dashboard.ts +++ b/src/lib/onboard/dashboard.ts @@ -10,6 +10,7 @@ import { getInteractiveAgentCommand } from "../agent/gateway-restart-scripts"; import { DASHBOARD_PORT } from "../core/ports"; import { buildChain, buildControlUiUrls, buildFallbackControlUiUrls } from "../dashboard/contract"; import * as nim from "../inference/nim"; +import { withGatewayRouteMutationLockSync } from "../inference/gateway-route-mutation-lock"; import { runCapture as defaultRunCapture } from "../runner"; import { ensureAgentDashboardForward as ensureAgentDashboardForwardForAgent, @@ -20,9 +21,15 @@ import { ensureAgentFixedForward as ensureFixedAgentForward } from "./agent-fixe import { fetchAgentWebAuthTokenFromSandbox as fetchAgentWebAuthToken } from "./agent-web-auth-token"; import * as dashboardAccess from "./dashboard-access"; import { + captureLiveSiblingDashboardForwards, createSandboxForwardStopper, type DashboardForwardOptions, + mergePreservedDashboardForwards, normalizeDashboardForwardOptions, + type PreservedDashboardForward, + reconcileSiblingDashboardForwards, + revalidatePreservedDashboardForward, + revalidatePreservedDashboardOwner, } from "./dashboard-forward-control"; import { findAvailableDashboardPort, @@ -332,8 +339,12 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa options: DashboardForwardOptions = {}, ): number { chatUiUrl ||= `http://127.0.0.1:${CONTROL_UI_PORT}`; - const { rollbackSandboxOnFailure, preservedPorts, allowPortReallocation } = - normalizeDashboardForwardOptions(options); + const { + rollbackSandboxOnFailure, + preservedPorts, + preservedSiblingForwards: preCreateSiblingForwards, + allowPortReallocation, + } = normalizeDashboardForwardOptions(options); const { revalidateSandboxIdentity } = options; const messagingForward = resolveMessagingHostForwardForSandbox(sandboxName); if (messagingForward) preservedPorts.add(String(messagingForward.port)); @@ -347,6 +358,10 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa }); const stopForwardForSandbox = makeStopForwardForSandbox(); let existingForwards = deps.runCaptureOpenshell(["forward", "list"], { ignoreError: true }); + const preservedSiblingForwards = mergePreservedDashboardForwards( + preCreateSiblingForwards, + captureLiveSiblingDashboardForwards(existingForwards, sandboxName), + ); const preferredEntry = findForwardEntry(existingForwards, String(preferredPort)); if ( preferredEntry && @@ -460,6 +475,85 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa ); } } + if (fwdOk && preservedSiblingForwards.length > 0) { + const targetForward = { + sandboxName, + bind: actualTarget.startsWith("0.0.0.0:") ? "0.0.0.0" : "127.0.0.1", + port: String(actualPort), + }; + const siblingResult = reconcileSiblingDashboardForwards({ + preserved: preservedSiblingForwards, + target: targetForward, + fetch: () => + deps.runCaptureOpenshell(["forward", "list"], { + timeout: OPENSHELL_PROBE_TIMEOUT_MS, + }), + revalidateLive: (forward, snapshot) => + revalidatePreservedDashboardForward(forward, snapshot, deps.getSandbox as never), + restore: (forward) => { + return withGatewayRouteMutationLockSync(forward.gatewayName, () => { + const ownership = deps.runCaptureOpenshell(["forward", "list"], { + timeout: OPENSHELL_PROBE_TIMEOUT_MS, + }); + if ( + !revalidatePreservedDashboardForward(forward, ownership, deps.getSandbox as never) + ) { + return { ok: false, diagnostic: "recorded forward ownership changed" }; + } + const port = Number(forward.port); + const makeSiblingStopper = () => + createSandboxForwardStopper({ + runOpenshell: deps.runOpenshell, + runCaptureOpenshell: deps.runCaptureOpenshell, + sandboxName: forward.sandboxName, + }); + const stopResult = makeSiblingStopper()(port); + if (stopResult === "list-failed" || stopResult === "owned-other") { + return { ok: false, diagnostic: `forward stop returned ${stopResult}` }; + } + waitForStoppedForwardPortRelease(port, deps.isPortBoundOnHost ?? isPortBoundOnHost, { + sleep: (milliseconds) => deps.sleep(milliseconds / 1_000), + }); + if (!revalidatePreservedDashboardOwner(forward, deps.getSandbox as never)) { + return { ok: false, diagnostic: "sandbox owner changed before forward restart" }; + } + const forwardTarget = + forward.bind === "0.0.0.0" ? `0.0.0.0:${forward.port}` : forward.port; + const restarted = runDetachedForwardStartWithRetries( + buildDetachedForwardStartSpawn( + deps.openshellArgv([ + "forward", + "start", + "--background", + forwardTarget, + forward.sandboxName, + ]), + ), + () => + deps.runCaptureOpenshell(["forward", "list"], { + timeout: OPENSHELL_PROBE_TIMEOUT_MS, + }), + { port, sandboxName: forward.sandboxName }, + () => { + deps.sleep(1); + makeSiblingStopper()(port); + }, + { onProgress: buildForwardStartProgressLogger(port) }, + ); + return revalidatePreservedDashboardOwner(forward, deps.getSandbox as never) + ? restarted + : { ok: false, diagnostic: "sandbox owner changed after forward restart" }; + }); + }, + }); + if (!siblingResult.ok) { + const error = new Error( + `Starting the dashboard forward for '${sandboxName}' disrupted a live sibling: ${siblingResult.diagnostic}`, + ); + if (rollbackSandboxOnFailure) rollbackSandboxAndExit(sandboxName, error); + throw error; + } + } if (fwdOk) options.onForwardStarted?.(actualPort); if (fwdOk && rollbackSandboxOnFailure) { ensureMessagingHostForwardForSandbox({ @@ -658,7 +752,8 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa console.log(""); console.log(`${indent} Or open a sandbox shell first:`); console.log(`${indent} ${deps.cliName()} ${sandboxName} connect`); - console.log(`${indent} then run: ${getInteractiveAgentCommand(agent, agent?.name)}`); + void getInteractiveAgentCommand(agent, agent?.name); + console.log(`${indent} then run the configured interactive agent command`); } function printDashboard( diff --git a/src/lib/onboard/docker-driver-gateway-config-toml.test.ts b/src/lib/onboard/docker-driver-gateway-config-toml.test.ts index 02c951c00a4..af045df8abd 100644 --- a/src/lib/onboard/docker-driver-gateway-config-toml.test.ts +++ b/src/lib/onboard/docker-driver-gateway-config-toml.test.ts @@ -31,6 +31,7 @@ import { buildDockerDriverGatewayRuntimeMarker, writeDockerDriverGatewayRuntimeMarker, } from "./docker-driver-gateway-runtime-marker"; +import { prepareNativePodmanGatewayHostRuntime } from "./runtime-provider/podman-runtime-surfaces"; const SCOPED_NAMESPACE_PROOF_DRIVER = path.join( process.cwd(), @@ -51,6 +52,14 @@ function legacyGatewayIdForStateDir(stateDir: string): string { return leaf ? `nemoclaw-${leaf}` : "nemoclaw"; } +function podmanGatewayRuntime(env: Record) { + return prepareNativePodmanGatewayHostRuntime({ + environment: { ...process.env, ...env }, + platform: "linux", + socketPath: env.OPENSHELL_PODMAN_SOCKET, + }); +} + function writePreScopedGatewayConfig( stateDir: string, includeDefaultNamespace = false, @@ -68,11 +77,20 @@ function writePreScopedGatewayConfig( ); const gatewayId = legacyGatewayIdForStateDir(stateDir); const jwtBundle = ensureDockerDriverGatewayJwtBundle(stateDir); + const gatewayRuntime = + driver === "podman" + ? prepareNativePodmanGatewayHostRuntime({ + environment: { ...process.env, ...env }, + platform: "linux", + socketPath: env.OPENSHELL_PODMAN_SOCKET, + }) + : undefined; let toml = buildDockerDriverGatewayConfigToml( env, "/usr/bin/openshell-sandbox", jwtBundle, gatewayId, + gatewayRuntime, ); toml = toml.replace( /^sandbox_namespace = .*\n/m, @@ -338,7 +356,9 @@ describe("docker-driver-gateway config TOML", () => { }); expect(() => - prepareDockerDriverGatewayConfigEnv(podmanEnv, stateDir, "/usr/bin/openshell-sandbox"), + prepareDockerDriverGatewayConfigEnv(podmanEnv, stateDir, "/usr/bin/openshell-sandbox", { + gatewayRuntime: podmanGatewayRuntime(podmanEnv), + }), ).toThrow( /already configures a 'docker'-driver OpenShell gateway.*this run selected the 'podman' driver.*NemoClaw-managed state.*nemoclaw uninstall.*preserves externally managed or supervised state.*lifecycle authority.*NEMOCLAW_GATEWAY_PORT.*separate state directory.*NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR/s, ); @@ -385,10 +405,14 @@ describe("docker-driver-gateway config TOML", () => { }); expect(() => - prepareDockerDriverGatewayConfigEnv(podmanEnv, stateDir, "/usr/bin/openshell-sandbox"), + prepareDockerDriverGatewayConfigEnv(podmanEnv, stateDir, "/usr/bin/openshell-sandbox", { + gatewayRuntime: podmanGatewayRuntime(podmanEnv), + }), ).toThrow(/driver config is incomplete/); expect(() => - prepareDockerDriverGatewayConfigEnv(podmanEnv, stateDir, "/usr/bin/openshell-sandbox"), + prepareDockerDriverGatewayConfigEnv(podmanEnv, stateDir, "/usr/bin/openshell-sandbox", { + gatewayRuntime: podmanGatewayRuntime(podmanEnv), + }), ).not.toThrow(/already configures a 'docker'-driver/); expect(fs.readFileSync(configPath, "utf-8")).toBe(malformedToml); } finally { @@ -436,7 +460,9 @@ describe("docker-driver-gateway config TOML", () => { }); expect(() => - prepareDockerDriverGatewayConfigEnv(podmanEnv, stateDir, "/usr/bin/openshell-sandbox"), + prepareDockerDriverGatewayConfigEnv(podmanEnv, stateDir, "/usr/bin/openshell-sandbox", { + gatewayRuntime: podmanGatewayRuntime(podmanEnv), + }), ).toThrow(/already configures a 'docker'-driver/); printOnboardResumeHint(true, console.error); const joined = errSpy.mock.calls.map((call) => String(call[0])).join("\n"); @@ -758,7 +784,13 @@ describe("docker-driver-gateway config TOML", () => { }); env.OPENSHELL_PODMAN_SOCKET = path.join(stateDir, "new-podman.sock"); - prepareDockerDriverGatewayConfigEnv(env, stateDir, "/usr/bin/openshell-sandbox"); + prepareDockerDriverGatewayConfigEnv(env, stateDir, "/usr/bin/openshell-sandbox", { + gatewayRuntime: prepareNativePodmanGatewayHostRuntime({ + environment: { ...process.env, ...env }, + platform: "linux", + socketPath: env.OPENSHELL_PODMAN_SOCKET, + }), + }); const rewritten = fs.readFileSync(configPath, "utf-8"); expect(rewritten).toContain('compute_drivers = ["podman"]'); diff --git a/src/lib/onboard/docker-driver-gateway-config.ts b/src/lib/onboard/docker-driver-gateway-config.ts index 69cd30b2b78..e5977c5d515 100644 --- a/src/lib/onboard/docker-driver-gateway-config.ts +++ b/src/lib/onboard/docker-driver-gateway-config.ts @@ -17,8 +17,12 @@ import { type DockerDriverGatewayJwtBundle, ensureDockerDriverGatewayJwtBundle, } from "./docker-driver-gateway-jwt-bundle"; -import { PORTABLE_HOST_GATEWAY_IP } from "./docker-driver-platform"; import { parseDockerDriverGatewayRuntimeMarker } from "./docker-driver-gateway-runtime-marker"; +import type { RuntimeProviderGatewayHostRuntime } from "./runtime-provider/contract"; +import { + resolveConfiguredRuntimeProvider, + resolveRegisteredRuntimeProvider, +} from "./runtime-provider/selection"; import { noteOnboardResumeHintShown } from "./resume-hint"; export type { DockerDriverGatewayJwtBundle } from "./docker-driver-gateway-jwt-bundle"; @@ -31,8 +35,6 @@ const LEGACY_DOCKER_DRIVER_GATEWAY_JWT_TTL_SECS = 3600; const PRE_AUTH_DOCKER_DRIVER_GATEWAY_VERSION = "0.0.44"; export const NEMOCLAW_OPENSHELL_SANDBOX_NAMESPACE_ENV = "NEMOCLAW_OPENSHELL_SANDBOX_NAMESPACE"; -type DockerDriverGatewayDriver = "docker" | "podman"; - interface FileIdentity { dev: number; ino: number; @@ -59,6 +61,46 @@ interface LegacyJwtBundleProof { jwtDir: string; } +function resolveGatewayRuntimeProjection( + gatewayEnv: Record, + runtime?: RuntimeProviderGatewayHostRuntime, +): RuntimeProviderGatewayHostRuntime { + if (runtime) return runtime; + const configuredProviderId = gatewayEnv.OPENSHELL_DRIVERS?.trim(); + const provider = configuredProviderId + ? resolveRegisteredRuntimeProvider(configuredProviderId) + : resolveConfiguredRuntimeProvider(); + if (!provider?.gateway.supported) { + throw new Error("The configured runtime provider does not support gateway configuration."); + } + return provider.gateway.prepareHostRuntime({ + environment: { ...process.env, ...gatewayEnv }, + platform: process.platform, + socketPath: gatewayEnv.OPENSHELL_PODMAN_SOCKET, + }); +} + +function alternateGatewayRuntimeProjection( + runtime: RuntimeProviderGatewayHostRuntime, + driver: string, + driverConfig: Record, +): RuntimeProviderGatewayHostRuntime { + const namespace = driverConfig.sandbox_namespace; + const hostGatewayIp = driverConfig.host_gateway_ip; + const socketPath = driverConfig.socket_path; + return { + ...runtime, + openShellDriver: driver, + socketPath: isNonEmptyString(socketPath) ? socketPath : null, + gatewayConfig: { + ...runtime.gatewayConfig, + sandboxNamespace: isNonEmptyString(namespace) ? "scoped" : "omitted", + hostGatewayIp: isNonEmptyString(hostGatewayIp) ? hostGatewayIp : null, + includeSupervisorBin: isNonEmptyString(driverConfig.supervisor_bin), + }, + }; +} + interface LegacyGatewayIdentity { configProof: ExistingConfigProof; gatewayId: string; @@ -292,8 +334,8 @@ class CrossDriverGatewayConflictError extends Error {} function crossDriverGatewayConflict( configPath: string, stateDir: string, - requestedDriver: DockerDriverGatewayDriver, - configuredDriver: DockerDriverGatewayDriver, + requestedDriver: string, + configuredDriver: string, ): Error { return new CrossDriverGatewayConflictError( `Refusing to rewrite ${configPath}: it already configures a '${configuredDriver}'-driver ` + @@ -387,10 +429,11 @@ function isNonEmptyString(value: unknown): value is string { function existingGatewayIdentityFromConfig( stateDir: string, - driver: DockerDriverGatewayDriver, + runtime: RuntimeProviderGatewayHostRuntime, allowOpenShell0044PreAuthDatabase = false, ): DockerDriverGatewayIdentity | null { const configPath = path.join(stateDir, DOCKER_DRIVER_GATEWAY_CONFIG_NAME); + const driver = runtime.openShellDriver; let configFile: OpenRegularFile | null = null; let configProof: ExistingConfigProof | null = null; let state: fs.Stats; @@ -493,13 +536,23 @@ function existingGatewayIdentityFromConfig( const drivers = asTomlTable(openshell?.drivers); const driverConfig = asTomlTable(drivers?.[driver]); if (!driverConfig && parsed && openshell && gateway && gatewayJwt && drivers) { - const otherDriver: DockerDriverGatewayDriver = driver === "docker" ? "podman" : "docker"; - if (asTomlTable(drivers[otherDriver])) { + const configuredDriver = Object.entries(drivers).find( + ([candidate, config]) => candidate !== driver && asTomlTable(config) !== null, + ); + if (configuredDriver) { + const [otherDriver, otherDriverConfigValue] = configuredDriver; + const otherDriverConfig = asTomlTable(otherDriverConfigValue); + if (!otherDriverConfig) throw new Error("unreachable gateway driver classification"); let otherIdentity: DockerDriverGatewayIdentity | null = null; try { + const otherRuntime = alternateGatewayRuntimeProjection( + runtime, + otherDriver, + otherDriverConfig, + ); otherIdentity = existingGatewayIdentityFromConfig( stateDir, - otherDriver, + otherRuntime, allowOpenShell0044PreAuthDatabase, ); if (otherIdentity) { @@ -514,26 +567,16 @@ function existingGatewayIdentityFromConfig( if (!parsed || !openshell || !gateway || !gatewayJwt || !drivers || !driverConfig) { throw ambiguousGatewayConfig(configPath, "the config does not match NemoClaw's schema"); } - const requiredDriverFields = - driver === "docker" - ? [ - "grpc_endpoint", - "network_name", - "supervisor_image", - "guest_tls_ca", - "guest_tls_cert", - "guest_tls_key", - ] - : [ - "grpc_endpoint", - "host_gateway_ip", - "socket_path", - "network_name", - "supervisor_image", - "guest_tls_ca", - "guest_tls_cert", - "guest_tls_key", - ]; + const requiredDriverFields = [ + "grpc_endpoint", + ...(runtime.gatewayConfig.hostGatewayIp === null ? [] : ["host_gateway_ip"]), + ...(runtime.socketPath === null ? [] : ["socket_path"]), + "network_name", + "supervisor_image", + "guest_tls_ca", + "guest_tls_cert", + "guest_tls_key", + ]; if (!requiredDriverFields.every((field) => isNonEmptyString(driverConfig[field]))) { throw ambiguousGatewayConfig(configPath, "the driver config is incomplete"); } @@ -544,11 +587,13 @@ function existingGatewayIdentityFromConfig( const legacyGatewayId = legacyGatewayIdForStateDir(stateDir); const scopedGatewayId = gatewayIdForStateDir(stateDir); const hasLegacyNamespace = - driver === "docker" + runtime.gatewayConfig.sandboxNamespace === "scoped" ? namespace === undefined || namespace === "default" : namespace === undefined; const hasScopedNamespace = - driver === "docker" ? namespace === scopedGatewayId : namespace === undefined; + runtime.gatewayConfig.sandboxNamespace === "scoped" + ? namespace === scopedGatewayId + : namespace === undefined; const isLegacy = configuredGatewayId === legacyGatewayId && hasLegacyNamespace; const isScoped = configuredGatewayId === scopedGatewayId && hasScopedNamespace; if (!isLegacy && !isScoped) { @@ -564,8 +609,8 @@ function existingGatewayIdentityFromConfig( const parsedEnv: Record = { OPENSHELL_LOCAL_TLS_DIR: path.join(stateDir, "tls"), + OPENSHELL_DRIVERS: runtime.openShellDriver, }; - if (driver === "podman") parsedEnv.OPENSHELL_DRIVERS = "podman"; if (driverConfig.enable_bind_mounts === true) { parsedEnv.NEMOCLAW_DOCKER_ENABLE_BIND_MOUNTS = "1"; } @@ -575,6 +620,10 @@ function existingGatewayIdentityFromConfig( assignStringEnv(parsedEnv, "OPENSHELL_DOCKER_SUPERVISOR_IMAGE", driverConfig.supervisor_image); const configuredSandboxBin = typeof driverConfig.supervisor_bin === "string" ? driverConfig.supervisor_bin : undefined; + const configuredRuntime = { + ...runtime, + socketPath: typeof driverConfig.socket_path === "string" ? driverConfig.socket_path : null, + } satisfies RuntimeProviderGatewayHostRuntime; const canonicalToml = buildDockerDriverGatewayConfigTomlForIdentity( parsedEnv, configuredSandboxBin, @@ -582,6 +631,7 @@ function existingGatewayIdentityFromConfig( String(configuredGatewayId), typeof namespace === "string" ? namespace : null, canonicalGatewayJwtTtl, + configuredRuntime, ); if (originalToml !== canonicalToml) { throw ambiguousGatewayConfig( @@ -623,13 +673,12 @@ function existingGatewayIdentityFromConfig( function resolveDockerDriverGatewayIdentity( stateDir: string, gatewayEnv: Record, + runtime: RuntimeProviderGatewayHostRuntime, allowOpenShell0044PreAuthDatabase = false, ): DockerDriverGatewayIdentity { - const driver: DockerDriverGatewayDriver = - gatewayEnv.OPENSHELL_DRIVERS === "podman" ? "podman" : "docker"; const existing = existingGatewayIdentityFromConfig( stateDir, - driver, + runtime, allowOpenShell0044PreAuthDatabase, ); if (existing) return existing; @@ -641,8 +690,9 @@ function resolveDockerDriverGatewayIdentity( export function hasStateScopedSandboxNamespace(stateDir: string): boolean { let identity: DockerDriverGatewayIdentity | null = null; try { - identity = existingGatewayIdentityFromConfig(stateDir, "docker"); - return identity?.kind === "scoped"; + const runtime = resolveGatewayRuntimeProjection({}); + identity = existingGatewayIdentityFromConfig(stateDir, runtime); + return runtime.gatewayConfig.sandboxNamespace === "scoped" && identity?.kind === "scoped"; } catch { return false; } finally { @@ -666,21 +716,31 @@ function buildDockerDriverGatewayConfigTomlForIdentity( gatewayId = "nemoclaw", sandboxNamespace: string | null = gatewayId, gatewayJwtTtlSecs = DOCKER_DRIVER_GATEWAY_JWT_TTL_SECS, + projectedRuntime?: RuntimeProviderGatewayHostRuntime, ): string { - const driver = gatewayEnv.OPENSHELL_DRIVERS === "podman" ? "podman" : "docker"; + const runtime = resolveGatewayRuntimeProjection(gatewayEnv, projectedRuntime); + const driver = runtime.openShellDriver; const localTlsDir = jwtBundle ? gatewayLocalTlsDir(gatewayEnv) : undefined; const dockerEntries: [string, string | boolean | undefined][] = [ ["enable_bind_mounts", gatewayEnv.NEMOCLAW_DOCKER_ENABLE_BIND_MOUNTS === "1" || undefined], - ["sandbox_namespace", driver === "docker" ? (sandboxNamespace ?? undefined) : undefined], + [ + "sandbox_namespace", + runtime.gatewayConfig.sandboxNamespace === "scoped" + ? (sandboxNamespace ?? undefined) + : undefined, + ], ["grpc_endpoint", gatewayEnv.OPENSHELL_GRPC_ENDPOINT], - ["host_gateway_ip", driver === "podman" ? PORTABLE_HOST_GATEWAY_IP : undefined], - ["socket_path", driver === "podman" ? gatewayEnv.OPENSHELL_PODMAN_SOCKET : undefined], + ["host_gateway_ip", runtime.gatewayConfig.hostGatewayIp ?? undefined], + ["socket_path", runtime.socketPath ?? undefined], ["network_name", gatewayEnv.OPENSHELL_DOCKER_NETWORK_NAME], ["supervisor_image", gatewayEnv.OPENSHELL_DOCKER_SUPERVISOR_IMAGE], // OpenShell 0.0.99 accepts supervisor_bin only for the Docker driver. // The Podman schema rejects the entire driver table when this Docker-only // field is present, so portable onboarding must rely on supervisor_image. - ["supervisor_bin", driver === "docker" ? (sandboxBin ?? undefined) : undefined], + [ + "supervisor_bin", + runtime.gatewayConfig.includeSupervisorBin ? (sandboxBin ?? undefined) : undefined, + ], ["guest_tls_ca", localTlsDir ? path.join(localTlsDir, "ca.crt") : undefined], ["guest_tls_cert", localTlsDir ? path.join(localTlsDir, "client", "tls.crt") : undefined], ["guest_tls_key", localTlsDir ? path.join(localTlsDir, "client", "tls.key") : undefined], @@ -743,6 +803,7 @@ export function buildDockerDriverGatewayConfigToml( sandboxBin?: string | null, jwtBundle?: DockerDriverGatewayJwtBundle | null, gatewayId = "nemoclaw", + runtime?: RuntimeProviderGatewayHostRuntime, ): string { return buildDockerDriverGatewayConfigTomlForIdentity( gatewayEnv, @@ -750,6 +811,8 @@ export function buildDockerDriverGatewayConfigToml( jwtBundle, gatewayId, gatewayId, + DOCKER_DRIVER_GATEWAY_JWT_TTL_SECS, + runtime, ); } @@ -758,6 +821,7 @@ function writeDockerDriverGatewayConfigWithIdentity( gatewayEnv: Record, sandboxBin: string | null | undefined, identity: DockerDriverGatewayIdentity, + runtime: RuntimeProviderGatewayHostRuntime, ): string { const configPath = path.join(stateDir, DOCKER_DRIVER_GATEWAY_CONFIG_NAME); if (identity.kind === "legacy") { @@ -772,6 +836,8 @@ function writeDockerDriverGatewayConfigWithIdentity( identity.jwtProof.bundle, identity.gatewayId, identity.sandboxNamespace, + DOCKER_DRIVER_GATEWAY_JWT_TTL_SECS, + runtime, ), 0o600, () => { @@ -792,7 +858,13 @@ function writeDockerDriverGatewayConfigWithIdentity( const jwtBundle = ensureDockerDriverGatewayJwtBundle(stateDir); writeRestrictedFileAtomic( configPath, - buildDockerDriverGatewayConfigToml(gatewayEnv, sandboxBin, jwtBundle, identity.gatewayId), + buildDockerDriverGatewayConfigToml( + gatewayEnv, + sandboxBin, + jwtBundle, + identity.gatewayId, + runtime, + ), 0o600, identity.configProof ? () => assertExistingConfigProof(identity.configProof!) : undefined, ); @@ -806,12 +878,15 @@ export function writeDockerDriverGatewayConfig( stateDir: string, gatewayEnv: Record, sandboxBin?: string | null, + projectedRuntime?: RuntimeProviderGatewayHostRuntime, ): string { + const runtime = resolveGatewayRuntimeProjection(gatewayEnv, projectedRuntime); return writeDockerDriverGatewayConfigWithIdentity( stateDir, gatewayEnv, sandboxBin, - resolveDockerDriverGatewayIdentity(stateDir, gatewayEnv), + resolveDockerDriverGatewayIdentity(stateDir, gatewayEnv, runtime), + runtime, ); } @@ -819,13 +894,18 @@ export function prepareDockerDriverGatewayConfigEnv( gatewayEnv: Record, stateDir: string, sandboxBin?: string | null, - options: { allowOpenShell0044PreAuthDatabase?: boolean } = {}, + options: { + allowOpenShell0044PreAuthDatabase?: boolean; + gatewayRuntime?: RuntimeProviderGatewayHostRuntime; + } = {}, ): Record { + const runtime = resolveGatewayRuntimeProjection(gatewayEnv, options.gatewayRuntime); let identity: DockerDriverGatewayIdentity; try { identity = resolveDockerDriverGatewayIdentity( stateDir, gatewayEnv, + runtime, options.allowOpenShell0044PreAuthDatabase === true, ); } catch (error) { @@ -843,8 +923,9 @@ export function prepareDockerDriverGatewayConfigEnv( gatewayEnv, sandboxBin, identity, + runtime, ); - if (gatewayEnv.OPENSHELL_DRIVERS === "podman") { + if (runtime.gatewayConfig.sandboxNamespace === "omitted") { delete gatewayEnv[NEMOCLAW_OPENSHELL_SANDBOX_NAMESPACE_ENV]; } else { gatewayEnv[NEMOCLAW_OPENSHELL_SANDBOX_NAMESPACE_ENV] = identity.sandboxNamespace; diff --git a/src/lib/onboard/docker-driver-gateway-env.test.ts b/src/lib/onboard/docker-driver-gateway-env.test.ts index 929033eeef5..a7f87001cb9 100644 --- a/src/lib/onboard/docker-driver-gateway-env.test.ts +++ b/src/lib/onboard/docker-driver-gateway-env.test.ts @@ -79,6 +79,7 @@ describe("buildDockerDriverGatewayEnv", () => { it("uses the Docker driver on macOS without VM helper state", () => { const env = buildDockerDriverGatewayEnv({ platform: "darwin", + architecture: "arm64", stateDir: "/tmp/nemoclaw-gateway", getDockerSupervisorImage: () => "ghcr.io/nvidia/openshell/supervisor:0.0.37", resolveSandboxBin: () => "/usr/local/bin/openshell-sandbox", @@ -143,6 +144,7 @@ describe("buildDockerDriverGatewayEnv", () => { it("builds the exact rootless gateway network contract for the portable profile", () => { vi.stubEnv("NEMOCLAW_EXPERIMENTAL_PROFILE", "portable"); + vi.stubEnv("NEMOCLAW_GATEWAY_RUNTIME", "docker"); vi.stubEnv("CONTAINERS_CONF", "/tmp/nemoclaw-portable/containers.conf"); const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-portable-gateway-")); try { @@ -173,6 +175,36 @@ describe("buildDockerDriverGatewayEnv", () => { } }); + it("selects the native Podman gateway without changing portable-only environment", () => { + vi.stubEnv("NEMOCLAW_GATEWAY_RUNTIME", "podman"); + vi.stubEnv("CONTAINERS_CONF", "/tmp/nemoclaw-portable/containers.conf"); + const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-native-podman-gateway-")); + try { + const env = buildDockerDriverGatewayEnv({ + platform: "linux", + stateDir, + getDockerSupervisorImage: () => "supervisor:test", + resolveSandboxBin: () => "/usr/bin/openshell-sandbox", + }); + expect(env).toMatchObject({ + OPENSHELL_DRIVERS: "podman", + OPENSHELL_BIND_ADDRESS: "0.0.0.0", + OPENSHELL_GRPC_ENDPOINT: `https://${PORTABLE_HOST_GATEWAY_IP}:8080`, + }); + expect(path.isAbsolute(env.OPENSHELL_PODMAN_SOCKET)).toBe(true); + expect(env.OPENSHELL_PODMAN_SOCKET).toMatch(/\/podman\/podman\.sock$/u); + expect(env.CONTAINERS_CONF).toBeUndefined(); + expect(env.NETAVARK_FW).toBeUndefined(); + const toml = fs.readFileSync(env.OPENSHELL_GATEWAY_CONFIG, "utf-8"); + expect(toml).toContain('compute_drivers = ["podman"]'); + expect(toml).toContain(`socket_path = "${env.OPENSHELL_PODMAN_SOCKET}"`); + expect(toml).not.toContain("supervisor_bin"); + } finally { + vi.unstubAllEnvs(); + fs.rmSync(stateDir, { recursive: true, force: true }); + } + }); + it.each([ ["a relative path", "run/user/1001/podman/podman.sock"], ["trailing whitespace", "/run/user/1001/podman/podman.sock "], @@ -199,9 +231,7 @@ describe("buildDockerDriverGatewayEnv", () => { }); }); - describe("writeDockerGatewayDebEnvOverride", () => { - it("rejects an env file swapped to a symlink after opening without writing its target", () => { const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-gateway-env-")); const envDir = path.join(tempHome, ".config", "openshell"); @@ -244,7 +274,6 @@ describe("writeDockerGatewayDebEnvOverride", () => { } }); - it("uses the provided HOME as the config root fallback when XDG_CONFIG_HOME is unset", () => { const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-gateway-env-home-")); const envFile = path.join(tempHome, ".config", "openshell", "gateway.env"); @@ -275,6 +304,7 @@ describe("writeDockerGatewayDebEnvOverride", () => { const envFile = path.join(tempHome, ".config", "openshell", "gateway.env"); const gatewayEnv = buildDockerDriverGatewayEnv({ platform: "darwin", + architecture: "arm64", stateDir: path.join(tempHome, "state"), getDockerSupervisorImage: () => "ghcr.io/nvidia/openshell/supervisor:0.0.72", resolveSandboxBin: () => null, diff --git a/src/lib/onboard/docker-driver-gateway-env.ts b/src/lib/onboard/docker-driver-gateway-env.ts index aef1882076a..a540e13f65e 100644 --- a/src/lib/onboard/docker-driver-gateway-env.ts +++ b/src/lib/onboard/docker-driver-gateway-env.ts @@ -5,7 +5,12 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import { dockerCapture, dockerRun } from "../adapters/docker/run"; import { openRegularFileNoFollow } from "../adapters/fs/regular-file"; +import { + DOCKER_NETWORK_IPAM_INSPECT_FORMAT, + parseDockerNetworkIpamEntries, +} from "./experimental/docker-network-authority"; import { GATEWAY_BIND_ADDRESS, getGatewayConnectHost, @@ -34,6 +39,8 @@ import { PORTABLE_HOST_GATEWAY_IP, resolveDockerDriverNetworkName, } from "./docker-driver-platform"; +import type { RuntimeProviderGatewayHostRuntime } from "./runtime-provider/contract"; +import { resolveConfiguredRuntimeProvider } from "./runtime-provider/selection"; export { getGatewayHttpsEndpoint, startPackageManagedDockerDriverGateway }; @@ -69,15 +76,132 @@ export const DOCKER_DRIVER_GATEWAY_RUNTIME_ENV_KEYS = [ export interface BuildDockerDriverGatewayEnvOptions { platform?: NodeJS.Platform; + architecture?: NodeJS.Architecture; gatewayPort?: number; stateDir: string; dockerNetworkName?: string; podmanSocketPath?: string; + gatewayHostRuntime?: RuntimeProviderGatewayHostRuntime; getDockerSupervisorImage: () => string; resolveSandboxBin: () => string | null; enableBindMounts?: boolean; } +function preparePortableGatewayHostRuntime( + socketPath?: string, + platform: NodeJS.Platform = process.platform, +): RuntimeProviderGatewayHostRuntime { + const run = (args: readonly string[], timeoutMs: number) => { + const result = dockerRun([...args], { + timeout: timeoutMs, + ignoreError: true, + suppressOutput: true, + }); + const error = result.error as NodeJS.ErrnoException | undefined; + return { + status: result.status ?? null, + signal: result.signal, + error: error?.message, + errorCode: error?.code ?? null, + timedOut: error?.code === "ETIMEDOUT", + stderr: result.stderr, + stdout: result.stdout, + }; + }; + const inspectNetwork = (networkName: string) => { + const raw = dockerCapture( + ["network", "inspect", "--format", DOCKER_NETWORK_IPAM_INSPECT_FORMAT, networkName], + { ignoreError: true }, + ); + return (parseDockerNetworkIpamEntries(raw) ?? []).find( + ({ gatewayIp }) => gatewayIp && !gatewayIp.includes(":"), + ); + }; + return { + providerId: "docker", + openShellDriver: "podman", + bindAddress: WILDCARD_GATEWAY_BIND_ADDRESS, + grpcHost: PORTABLE_HOST_GATEWAY_IP, + sshGatewayHost: getGatewayConnectHost(), + portCheckHost: WILDCARD_GATEWAY_BIND_ADDRESS, + socketPath: socketPath ?? null, + requiredServerIpSans: [PORTABLE_HOST_GATEWAY_IP], + sandboxHostAddress: PORTABLE_HOST_GATEWAY_IP, + usesHostGatewayRoute: true, + // The portable compatibility profile historically discovers OpenShell's + // Docker-compatible labels. Keep that independent from native Podman. + resourceOwnership: { + label: "openshell.ai/managed-by", + value: "openshell", + }, + gatewayConfig: { + sandboxNamespace: "omitted", + hostGatewayIp: PORTABLE_HOST_GATEWAY_IP, + includeSupervisorBin: false, + processOwnership: "scoped-namespace", + }, + network: { + sandboxSourceCidrs: () => { + const network = inspectNetwork(resolveDockerDriverNetworkName()); + return network?.subnet ? [network.subnet] : []; + }, + inspect: inspectNetwork, + usesHostGatewayRoute: () => { + if (platform !== "linux") return true; + const info = dockerCapture( + ["info", "--format", "{{.OperatingSystem}}\n{{range .Labels}}{{.}}\n{{end}}"], + { ignoreError: true }, + ); + return /Docker Desktop|com\.docker\.desktop\./iu.test(info); + }, + run, + ensureProbeImageCached: (image) => { + const inspect = run(["image", "inspect", image], 10_000); + if (inspect.status === 0) return { ok: true, alreadyCached: true }; + if (inspect.status === null || inspect.errorCode) { + return { + ok: false, + reason: "inspect_unavailable", + details: inspect.error ?? String(inspect.stderr ?? "").trim(), + }; + } + const pull = run(["pull", image], 120_000); + if (pull.status === 0) return { ok: true, alreadyCached: false }; + return { + ok: false, + reason: pull.timedOut ? "pull_timeout" : "pull_failed", + details: pull.error ?? String(pull.stderr ?? "").trim(), + }; + }, + }, + }; +} + +export function prepareConfiguredGatewayHostRuntime( + options: { + architecture?: NodeJS.Architecture; + environment?: NodeJS.ProcessEnv; + platform?: NodeJS.Platform; + socketPath?: string; + } = {}, +): RuntimeProviderGatewayHostRuntime { + const environment = options.environment ?? process.env; + const platform = options.platform ?? process.platform; + const architecture = options.architecture ?? process.arch; + if (isPortableExperimentalProfile(environment)) { + return preparePortableGatewayHostRuntime(options.socketPath, platform); + } + const provider = resolveConfiguredRuntimeProvider(platform, architecture, environment); + if (!provider.gateway.supported) { + throw new Error("The selected runtime provider does not support a host-managed gateway."); + } + return provider.gateway.prepareHostRuntime({ + environment, + platform, + socketPath: options.socketPath, + }); +} + export type PackageManagedDockerDriverGatewayWithEnvOverrideOptions = Omit< PackageManagedDockerDriverGatewayOptions, "prepareOpenShellGatewayUserServiceEnv" @@ -87,30 +211,38 @@ export type PackageManagedDockerDriverGatewayWithEnvOverrideOptions = Omit< home?: string; }; -export function getGatewayPortCheckOptions(): { host: string } { +export function getGatewayPortCheckOptions(env: NodeJS.ProcessEnv = process.env): { host: string } { return { - host: isPortableExperimentalProfile() ? WILDCARD_GATEWAY_BIND_ADDRESS : GATEWAY_BIND_ADDRESS, + host: prepareConfiguredGatewayHostRuntime({ environment: env }).portCheckHost, }; } export function getGatewayStartNetworkEnv( gatewayPort: number = GATEWAY_PORT, + env: NodeJS.ProcessEnv = process.env, + platform: NodeJS.Platform = process.platform, ): Record { - const portable = isPortableExperimentalProfile(); + const runtime = prepareConfiguredGatewayHostRuntime({ environment: env, platform }); return { - OPENSHELL_BIND_ADDRESS: portable ? WILDCARD_GATEWAY_BIND_ADDRESS : GATEWAY_BIND_ADDRESS, + OPENSHELL_BIND_ADDRESS: runtime.bindAddress, OPENSHELL_SERVER_PORT: String(gatewayPort), - OPENSHELL_SSH_GATEWAY_HOST: getGatewayConnectHost(), + OPENSHELL_SSH_GATEWAY_HOST: runtime.sshGatewayHost, OPENSHELL_SSH_GATEWAY_PORT: String(gatewayPort), }; } -export function assertDockerDriverGatewayBindAddressSafe(gatewayEnv: Record): void { +export function assertDockerDriverGatewayBindAddressSafe( + gatewayEnv: Record, + environment: NodeJS.ProcessEnv = process.env, + platform: NodeJS.Platform = process.platform, +): void { if (gatewayEnv.OPENSHELL_BIND_ADDRESS !== WILDCARD_GATEWAY_BIND_ADDRESS) return; + const selectedRuntime = prepareConfiguredGatewayHostRuntime({ environment, platform }); if ( - gatewayEnv.OPENSHELL_DRIVERS === "podman" && + selectedRuntime.sandboxHostAddress !== null && gatewayEnv.OPENSHELL_GRPC_ENDPOINT === - `https://${PORTABLE_HOST_GATEWAY_IP}:${gatewayEnv.OPENSHELL_SERVER_PORT}` + `https://${selectedRuntime.grpcHost}:${gatewayEnv.OPENSHELL_SERVER_PORT}` && + gatewayEnv.OPENSHELL_SSH_GATEWAY_HOST === selectedRuntime.sshGatewayHost ) { return; } @@ -197,8 +329,11 @@ function assertGatewayJwtFile(key: string, filePath: string): void { ); } -export function assertDockerDriverGatewayAuthConfigSafe(gatewayEnv: Record): void { - assertDockerDriverGatewayBindAddressSafe(gatewayEnv); +export function assertDockerDriverGatewayAuthConfigSafe( + gatewayEnv: Record, + environment: NodeJS.ProcessEnv = process.env, +): void { + assertDockerDriverGatewayBindAddressSafe(gatewayEnv, environment); const configPath = gatewayEnv.OPENSHELL_GATEWAY_CONFIG?.trim(); if (!configPath) { throw new Error("OpenShell Docker-driver gateway requires OPENSHELL_GATEWAY_CONFIG"); @@ -234,44 +369,55 @@ export function warnIfGatewayWildcardBindAddress(): void { export function buildDockerDriverGatewayEnv({ platform = process.platform, + architecture = process.arch, gatewayPort = GATEWAY_PORT, stateDir, dockerNetworkName, podmanSocketPath, + gatewayHostRuntime, getDockerSupervisorImage, resolveSandboxBin, enableBindMounts = false, }: BuildDockerDriverGatewayEnvOptions): Record { const portable = isPortableExperimentalProfile(); + const runtime = + gatewayHostRuntime ?? + prepareConfiguredGatewayHostRuntime({ + architecture, + platform, + socketPath: podmanSocketPath, + }); const resolvedDockerNetworkName = dockerNetworkName ?? resolveDockerDriverNetworkName(); const env: Record = { - OPENSHELL_DRIVERS: portable ? "podman" : "docker", - ...getGatewayStartNetworkEnv(gatewayPort), + NEMOCLAW_RUNTIME_PROVIDER_ID: runtime.providerId, + OPENSHELL_DRIVERS: runtime.openShellDriver, + OPENSHELL_BIND_ADDRESS: runtime.bindAddress, + OPENSHELL_SERVER_PORT: String(gatewayPort), + OPENSHELL_SSH_GATEWAY_HOST: runtime.sshGatewayHost, + OPENSHELL_SSH_GATEWAY_PORT: String(gatewayPort), ...buildDockerDriverGatewayLocalTlsEnv(stateDir), OPENSHELL_DB_URL: `sqlite:${path.join(stateDir, "openshell.db")}`, - OPENSHELL_GRPC_ENDPOINT: portable - ? `https://${PORTABLE_HOST_GATEWAY_IP}:${gatewayPort}` - : getDockerDriverGatewayEndpoint(gatewayPort), + OPENSHELL_GRPC_ENDPOINT: `https://${runtime.grpcHost}:${gatewayPort}`, OPENSHELL_DOCKER_NETWORK_NAME: resolvedDockerNetworkName, OPENSHELL_DOCKER_SUPERVISOR_IMAGE: getDockerSupervisorImage(), }; if (enableBindMounts) env.NEMOCLAW_DOCKER_ENABLE_BIND_MOUNTS = "1"; - if (portable) { - env.NETAVARK_FW = "iptables"; - if (podmanSocketPath !== undefined) { - const rawSocketPath = String(podmanSocketPath); - const normalizedSocketPath = rawSocketPath.trim(); - if ( - normalizedSocketPath === "" || - rawSocketPath !== normalizedSocketPath || - /[\0\r\n]/u.test(rawSocketPath) || - !path.isAbsolute(normalizedSocketPath) || - path.normalize(normalizedSocketPath) !== normalizedSocketPath - ) { - throw new Error("OpenShell Podman gateway socket must be a safe normalized absolute path."); - } - env.OPENSHELL_PODMAN_SOCKET = normalizedSocketPath; + if (portable) env.NETAVARK_FW = "iptables"; + if (runtime.socketPath !== null) { + const rawSocketPath = String(runtime.socketPath); + const normalizedSocketPath = rawSocketPath.trim(); + if ( + normalizedSocketPath === "" || + rawSocketPath !== normalizedSocketPath || + /[\0\r\n]/u.test(rawSocketPath) || + !path.isAbsolute(normalizedSocketPath) || + path.normalize(normalizedSocketPath) !== normalizedSocketPath + ) { + throw new Error("OpenShell Podman gateway socket must be a safe normalized absolute path."); } + env.OPENSHELL_PODMAN_SOCKET = normalizedSocketPath; + } + if (portable) { const containersConf = process.env.CONTAINERS_CONF?.trim(); if (containersConf) env.CONTAINERS_CONF = containersConf; } @@ -287,6 +433,7 @@ export function buildDockerDriverGatewayEnv({ // prepared recovery may safely attach the first scoped identity to it. allowOpenShell0044PreAuthDatabase: process.env.NEMOCLAW_RESTORE_LATEST_BACKUP_ON_RECREATE === "1", + gatewayRuntime: runtime, }); return env; } @@ -404,7 +551,7 @@ export function startPackageManagedDockerDriverGatewayWithEnvOverride( const env = optionsWithEnv.env ?? process.env; const gatewayPort = Number(gatewayEnv.OPENSHELL_SERVER_PORT ?? GATEWAY_PORT); if (gatewayPort !== DEFAULT_GATEWAY_PORT) return Promise.resolve(false); - assertDockerDriverGatewayAuthConfigSafe(gatewayEnv); + assertDockerDriverGatewayAuthConfigSafe(gatewayEnv, env); const effectiveHome = home ?? optionsWithEnv.env?.HOME ?? os.homedir(); return startPackageManagedDockerDriverGateway({ ...options, diff --git a/src/lib/onboard/docker-driver-gateway-local-tls.test.ts b/src/lib/onboard/docker-driver-gateway-local-tls.test.ts index af5ccde28ac..67ff6811e87 100644 --- a/src/lib/onboard/docker-driver-gateway-local-tls.test.ts +++ b/src/lib/onboard/docker-driver-gateway-local-tls.test.ts @@ -246,6 +246,28 @@ describe("docker-driver-gateway-local-tls", () => { } }); + it("adds the rootless host gateway SAN for the native Podman runtime", () => { + const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-podman-tls-")); + const calls: string[][] = []; + try { + expect(() => + ensureDockerDriverGatewayLocalTlsBundle({ + env: { NEMOCLAW_GATEWAY_RUNTIME: "podman" }, + gatewayBin: "/opt/openshell/openshell-gateway", + platform: "linux", + stateDir, + spawnSyncImpl: ((_command: string, args: string[]) => { + calls.push(args); + return { status: 0, stdout: "", stderr: "" }; + }) as never, + }), + ).toThrow("did not create a complete"); + expect(calls[0]).toEqual(expect.arrayContaining(["--server-san", PORTABLE_HOST_GATEWAY_IP])); + } finally { + fs.rmSync(stateDir, { recursive: true, force: true }); + } + }); + it("preserves an existing complete mTLS bundle without regenerating certs", () => { const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-gateway-tls-")); const contents = writeBundle(stateDir, TEST_CERT_PEM, TEST_KEY_PEM); @@ -267,8 +289,11 @@ describe("docker-driver-gateway-local-tls", () => { expect(bundle.localTlsDir).toBe(path.join(stateDir, "tls")); expect(certgenCalls).toBe(0); - expect(Object.entries(contents).every(([filePath, content]) => - Object.is(fs.readFileSync(filePath, "utf-8"), content))).toBe(true); + expect( + Object.entries(contents).every(([filePath, content]) => + Object.is(fs.readFileSync(filePath, "utf-8"), content), + ), + ).toBe(true); expect(fs.statSync(paths.serverKeyPath).mode & 0o777).toBe(0o600); expect(fs.statSync(paths.clientKeyPath).mode & 0o777).toBe(0o600); } finally { diff --git a/src/lib/onboard/docker-driver-gateway-local-tls.ts b/src/lib/onboard/docker-driver-gateway-local-tls.ts index 2989daa57d6..ac383017482 100644 --- a/src/lib/onboard/docker-driver-gateway-local-tls.ts +++ b/src/lib/onboard/docker-driver-gateway-local-tls.ts @@ -7,6 +7,7 @@ import fs from "node:fs"; import path from "node:path"; import { isPortableExperimentalProfile, PORTABLE_HOST_GATEWAY_IP } from "./docker-driver-platform"; +import { resolveConfiguredRuntimeProvider } from "./runtime-provider/selection"; // See docs/security/gateway-authentication-controls.mdx for the public compatibility boundary. export const DOCKER_DRIVER_GATEWAY_LOCAL_TLS_DIR_NAME = "tls"; @@ -27,6 +28,7 @@ export type DockerDriverGatewayLocalTlsBundle = { export interface EnsureDockerDriverGatewayLocalTlsBundleOptions { env?: NodeJS.ProcessEnv; gatewayBin: string; + platform?: NodeJS.Platform; spawnSyncImpl?: typeof spawnSync; stateDir: string; } @@ -190,13 +192,25 @@ function normalizeDockerDriverGatewayLocalTlsBundlePermissions( export function ensureDockerDriverGatewayLocalTlsBundle({ env = process.env, gatewayBin, + platform = process.platform, spawnSyncImpl = spawnSync, stateDir, }: EnsureDockerDriverGatewayLocalTlsBundleOptions): DockerDriverGatewayLocalTlsBundle { const bundle = getDockerDriverGatewayLocalTlsBundle(stateDir); - const requiredServerIpSans = isPortableExperimentalProfile(env) - ? [...REQUIRED_SERVER_IP_SANS, PORTABLE_HOST_GATEWAY_IP] - : REQUIRED_SERVER_IP_SANS; + const portable = isPortableExperimentalProfile(env); + const requiredProviderIpSans = portable + ? [PORTABLE_HOST_GATEWAY_IP] + : (() => { + const provider = resolveConfiguredRuntimeProvider(platform, process.arch, env); + if (!provider.gateway.supported) { + throw new Error("The selected runtime provider does not support a host-managed gateway."); + } + return provider.gateway.prepareHostRuntime({ + environment: env, + platform, + }).requiredServerIpSans; + })(); + const requiredServerIpSans = [...REQUIRED_SERVER_IP_SANS, ...requiredProviderIpSans]; fs.mkdirSync(stateDir, { recursive: true, mode: 0o700 }); fs.chmodSync(stateDir, 0o700); if (dockerDriverGatewayLocalTlsBundleIsComplete(stateDir, requiredServerIpSans)) { @@ -216,7 +230,7 @@ export function ensureDockerDriverGatewayLocalTlsBundle({ "localhost", "--server-san", "127.0.0.1", - ...(isPortableExperimentalProfile(env) ? ["--server-san", PORTABLE_HOST_GATEWAY_IP] : []), + ...requiredProviderIpSans.flatMap((ipAddress) => ["--server-san", ipAddress]), ], { encoding: "utf-8", diff --git a/src/lib/onboard/docker-driver-gateway-process-identity.ts b/src/lib/onboard/docker-driver-gateway-process-identity.ts index b8b262d8814..3e1128581a9 100644 --- a/src/lib/onboard/docker-driver-gateway-process-identity.ts +++ b/src/lib/onboard/docker-driver-gateway-process-identity.ts @@ -52,8 +52,7 @@ export function hasDockerDriverGatewayEnvironment( ): boolean { if (!env) return false; return ( - env.OPENSHELL_DRIVERS === "docker" || - env.OPENSHELL_DRIVERS === "podman" || + Boolean(env.OPENSHELL_DRIVERS?.trim()) || Boolean(env.OPENSHELL_DOCKER_SUPERVISOR_IMAGE) || env.OPENSHELL_GRPC_ENDPOINT === expectedEndpoint ); diff --git a/src/lib/onboard/docker-driver-gateway-runtime-marker.ts b/src/lib/onboard/docker-driver-gateway-runtime-marker.ts index d462ff4a029..21f52402520 100644 --- a/src/lib/onboard/docker-driver-gateway-runtime-marker.ts +++ b/src/lib/onboard/docker-driver-gateway-runtime-marker.ts @@ -3,14 +3,36 @@ import crypto from "node:crypto"; import fs from "node:fs"; +import os from "node:os"; import path from "node:path"; +import { DEFAULT_GATEWAY_PORT, resolveGatewayStateDirForPort } from "./gateway/state-dir"; + export const DOCKER_DRIVER_GATEWAY_RUNTIME_MARKER_VERSION = 1; +export function resolveDockerDriverGatewayStateDir( + env: NodeJS.ProcessEnv = process.env, + homeDir: string = env.HOME || os.homedir(), + gatewayPort: number = DEFAULT_GATEWAY_PORT, +): string { + return resolveGatewayStateDirForPort({ + configured: env.NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR, + home: homeDir, + port: gatewayPort, + }); +} + +export function resolveDockerDriverGatewayPidFile( + env: NodeJS.ProcessEnv = process.env, + homeDir: string = env.HOME || os.homedir(), +): string { + return path.join(resolveDockerDriverGatewayStateDir(env, homeDir), "openshell-gateway.pid"); +} + export type DockerDriverGatewayRuntimeMarker = { version: typeof DOCKER_DRIVER_GATEWAY_RUNTIME_MARKER_VERSION; pid: number; - driver: "docker"; + driver: string; platform: NodeJS.Platform; arch: NodeJS.Architecture; endpoint: string; @@ -31,6 +53,7 @@ export type DockerDriverGatewayRuntimeMarkerInput = { platform?: NodeJS.Platform; arch?: NodeJS.Architecture; createdAt?: string; + runtimeProviderId?: string; }; export type DockerDriverGatewayRuntimeMarkerDrift = { reason: string }; @@ -67,11 +90,16 @@ export function buildDockerDriverGatewayRuntimeMarker({ platform = process.platform, arch = process.arch, createdAt = new Date().toISOString(), + runtimeProviderId = desiredEnv.NEMOCLAW_RUNTIME_PROVIDER_ID ?? "docker", }: DockerDriverGatewayRuntimeMarkerInput): DockerDriverGatewayRuntimeMarker { + const driver = runtimeProviderId.trim().toLowerCase(); + if (!/^[a-z][a-z0-9-]{0,63}$/u.test(driver)) { + throw new Error("Gateway runtime provider identity is invalid."); + } return { version: DOCKER_DRIVER_GATEWAY_RUNTIME_MARKER_VERSION, pid, - driver: "docker", + driver, platform, arch, endpoint, @@ -88,7 +116,8 @@ function isRuntimeMarker(value: unknown): value is DockerDriverGatewayRuntimeMar const marker = value as Partial; return ( marker.version === DOCKER_DRIVER_GATEWAY_RUNTIME_MARKER_VERSION && - marker.driver === "docker" && + typeof marker.driver === "string" && + /^[a-z][a-z0-9-]{0,63}$/u.test(marker.driver) && Number.isInteger(marker.pid) && typeof marker.platform === "string" && typeof marker.arch === "string" && @@ -173,7 +202,11 @@ export function getDockerDriverGatewayRuntimeMarkerDrift( const desired = buildDockerDriverGatewayRuntimeMarker(expected); if (marker.pid !== desired.pid) return { reason: `runtime marker pid=${marker.pid} (expected ${desired.pid})` }; - if (marker.driver !== "docker") return { reason: `runtime marker driver=${marker.driver}` }; + if (marker.driver !== desired.driver) { + return { + reason: `runtime marker provider=${marker.driver} (expected ${desired.driver})`, + }; + } if (marker.platform !== desired.platform) { return { reason: `runtime marker platform=${marker.platform} (expected ${desired.platform})` }; } diff --git a/src/lib/onboard/docker-driver-gateway-runtime.test.ts b/src/lib/onboard/docker-driver-gateway-runtime.test.ts index d6d74247947..53f766efa42 100644 --- a/src/lib/onboard/docker-driver-gateway-runtime.test.ts +++ b/src/lib/onboard/docker-driver-gateway-runtime.test.ts @@ -356,6 +356,14 @@ describe("docker-driver gateway runtime helpers", () => { ], ]); const { helpers, runCapture } = makeHelpers({ + loadDockerDriverGatewayEnv: () => ({ + ...dockerDriverGatewayEnv, + buildDockerDriverGatewayEnv: (options) => + dockerDriverGatewayEnv.buildDockerDriverGatewayEnv({ + ...options, + architecture: "arm64", + }), + }), runCapture: vi.fn((args) => processOutput.get(args.join(" ")) ?? ""), }); const desiredEnv = helpers.getDockerDriverGatewayEnv(null, "darwin"); diff --git a/src/lib/onboard/docker-driver-gateway-service.ts b/src/lib/onboard/docker-driver-gateway-service.ts index b9ad827b9fb..c485a08e922 100644 --- a/src/lib/onboard/docker-driver-gateway-service.ts +++ b/src/lib/onboard/docker-driver-gateway-service.ts @@ -31,6 +31,9 @@ export const NEMOCLAW_OPENSHELL_GATEWAY_USER_SERVICE_MARKER = "NEMOCLAW_MANAGED_OPENSHELL_GATEWAY=1"; export const NEMOCLAW_OPENSHELL_GATEWAY_USER_SERVICE_MARKER_LINE = `# ${NEMOCLAW_OPENSHELL_GATEWAY_USER_SERVICE_MARKER}`; +/** Shared blocking wait used while observing native gateway readiness. */ +export const waitForOpenShellGatewayRetry = sleepSeconds; + export interface OpenShellGatewayUserServiceOptions { commandExists?: (command: string) => boolean; env?: NodeJS.ProcessEnv; diff --git a/src/lib/onboard/docker-driver-platform.ts b/src/lib/onboard/docker-driver-platform.ts index 77a6d9a9792..aa337b68a56 100644 --- a/src/lib/onboard/docker-driver-platform.ts +++ b/src/lib/onboard/docker-driver-platform.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { resolveCurrentOpenShellComputePlan, usesManagedDockerGateway } from "./compute/plan"; +import { resolveCurrentOpenShellComputePlan, usesManagedLocalGateway } from "./compute/plan"; export { resolveCurrentOpenShellComputePlan } from "./compute/plan"; @@ -30,5 +30,5 @@ export function isLinuxDockerDriverGatewayEnabled( platform: NodeJS.Platform = process.platform, arch: NodeJS.Architecture = process.arch, ): boolean { - return usesManagedDockerGateway(resolveCurrentOpenShellComputePlan(platform, arch)); + return usesManagedLocalGateway(resolveCurrentOpenShellComputePlan(platform, arch)); } diff --git a/src/lib/onboard/docker-gpu-local-inference.ts b/src/lib/onboard/docker-gpu-local-inference.ts index 8758b0c8ba5..7d24fa308d8 100644 --- a/src/lib/onboard/docker-gpu-local-inference.ts +++ b/src/lib/onboard/docker-gpu-local-inference.ts @@ -125,7 +125,7 @@ export async function enforceDockerGpuPatchPreserveNetwork( provider: string | null | undefined, config: DockerGpuLocalInferenceConfig, options: DockerGpuLocalInferenceOptions & { - reverifyBridgeReachability?: () => void | Promise; + reverifyBridgeReachability: () => void | Promise; }, ): Promise { if (!isLocalInferenceProvider(provider)) return false; @@ -137,23 +137,10 @@ export async function enforceDockerGpuPatchPreserveNetwork( "loopback is not reachable from the sandbox network namespace, so OpenClaw routes through " + "the OpenShell-managed inference path (host networking is not needed for GPU device access).", ); - await ( - options.reverifyBridgeReachability ?? - (() => defaultReverifyBridgeReachability(options.gatewayPort)) - )(); + await options.reverifyBridgeReachability(); return true; } -/** Re-run the sandbox→gateway bridge reachability probe (with UFW auto-fix). */ -function defaultReverifyBridgeReachability(gatewayPort?: number): Promise { - const { verifySandboxBridgeGatewayReachableOrExit } = - require("./gateway-sandbox-reachability") as typeof import("./gateway-sandbox-reachability"); - return verifySandboxBridgeGatewayReachableOrExit(true, { - skip: false, - ...(gatewayPort === undefined ? {} : { port: gatewayPort }), - }); -} - export type SandboxExecResult = { status: number; stdout: string; diff --git a/src/lib/onboard/docker-gpu-patch-clone.test.ts b/src/lib/onboard/docker-gpu-patch-clone.test.ts index ee360e1f276..63071e4b980 100644 --- a/src/lib/onboard/docker-gpu-patch-clone.test.ts +++ b/src/lib/onboard/docker-gpu-patch-clone.test.ts @@ -164,7 +164,15 @@ describe("Docker GPU clone envelope", () => { }); it("builds clone args that preserve OpenShell labels, mounts, and runtime settings", () => { - const args = buildDockerGpuCloneRunArgs(inspectFixture(), buildDockerGpuMode("gpus")); + const inspect = inspectFixture(); + inspect.HostConfig!.Annotations = { "io.container.manager": "libpod" }; + inspect.HostConfig!.Mounts!.push({ + Type: "image", + Source: "ghcr.io/nvidia/openshell/sandbox:v0.0.106", + Target: "/opt/openshell/bin", + ReadOnly: true, + }); + const args = buildDockerGpuCloneRunArgs(inspect, buildDockerGpuMode("gpus")); expect(args).toEqual( expect.arrayContaining([ @@ -178,6 +186,8 @@ describe("Docker GPU clone envelope", () => { "OPENSHELL_ENDPOINT=http://host.openshell.internal:8080/", "--env", "OPENSHELL_TEST=1", + "--annotation", + "io.container.manager=libpod", "--label", "openshell.ai/managed-by=openshell", "--label", @@ -186,6 +196,8 @@ describe("Docker GPU clone envelope", () => { "/host:/container:rw", "--mount", "type=tmpfs,dst=/tmp/nemoclaw-exact-main-driver-config,tmpfs-size=16777216,tmpfs-mode=1777", + "--mount", + "type=image,src=ghcr.io/nvidia/openshell/sandbox:v0.0.106,dst=/opt/openshell/bin", "--network", "openshell-docker", "--network-alias", @@ -251,7 +263,7 @@ describe("Docker GPU clone envelope", () => { const inspect = inspectFixture(); inspect.HostConfig!.Ulimits = [ { Name: "core", Soft: 0, Hard: -1 }, - { Name: "nofile", Soft: 1024, Hard: 1024 }, + { Name: "RLIMIT_NOFILE", Soft: 1024, Hard: 1024 }, ]; const args = buildDockerGpuCloneRunArgs(inspect, buildDockerGpuMode("startup-command"), { @@ -271,6 +283,7 @@ describe("Docker GPU clone envelope", () => { "nproc=512:512", ]), ); + expect(args).not.toContain("RLIMIT_NOFILE=1024:1024"); expect(args).not.toContain("nofile=1024:1024"); }); @@ -297,13 +310,34 @@ describe("Docker GPU clone envelope", () => { }); it("uses exact managed-bootstrap container, entrypoint, and command overrides", () => { + const inspect = inspectFixture(); + Object.assign(inspect.Config!, { + ExposedPorts: { "2222/tcp": {} }, + Healthcheck: { + Test: ["CMD-SHELL", "test -S /run/openshell/ssh.sock"], + Interval: 10_000_000_000, + Timeout: 2_000_000_000, + StartPeriod: 5_000_000_000, + Retries: 10, + }, + StopTimeout: 45, + }); + Object.assign(inspect.HostConfig!, { + Annotations: { "io.container.manager": "libpod" }, + NetworkMode: "bridge", + OomScoreAdj: 0, + PortBindings: { + "2222/tcp": [{ HostIp: "0.0.0.0", HostPort: "33513" }], + }, + }); const args = buildDockerGpuCloneRunArgs( - inspectFixture(), + inspect, buildDockerGpuMode("startup-command"), { containerName: "openshell-alpha-bootstrap-stage", containerEntrypoint: "/usr/local/bin/nemoclaw-managed-bootstrap", containerCommand: ["--request", "/run/nemoclaw/bootstrap-request.json"], + preserveManagedLaunchSpec: true, }, ); @@ -311,6 +345,30 @@ describe("Docker GPU clone envelope", () => { expect(args).toEqual( expect.arrayContaining(["--entrypoint", "/usr/local/bin/nemoclaw-managed-bootstrap"]), ); + expect(args).toEqual( + expect.arrayContaining([ + "--expose", + "2222/tcp", + "--publish", + "0.0.0.0:33513:2222/tcp", + "--health-cmd", + "test -S /run/openshell/ssh.sock", + "--health-interval", + "10000000000ns", + "--health-timeout", + "2000000000ns", + "--health-start-period", + "5000000000ns", + "--health-retries", + "10", + "--stop-timeout", + "45", + "--oom-score-adj", + "500", + "--network", + "openshell-docker", + ]), + ); expect(args.slice(args.indexOf("openshell/sandbox:abc"))).toEqual([ "openshell/sandbox:abc", "--request", diff --git a/src/lib/onboard/docker-gpu-patch-clone.ts b/src/lib/onboard/docker-gpu-patch-clone.ts index 616bfccab23..b69bc97d9f1 100644 --- a/src/lib/onboard/docker-gpu-patch-clone.ts +++ b/src/lib/onboard/docker-gpu-patch-clone.ts @@ -75,6 +75,100 @@ function pushStringFlag(args: string[], flag: string, value: unknown): void { if (normalized) args.push(flag, normalized); } +function managedPortKey(value: string): string { + const match = /^(\d{1,5})\/(tcp|udp|sctp)$/u.exec(value); + const port = Number(match?.[1]); + if (!match || !Number.isSafeInteger(port) || port < 1 || port > 65_535) { + throw new Error(`Managed bootstrap Docker port '${value}' is invalid.`); + } + return value; +} + +function managedPublishedPort(hostIp: unknown, hostPort: unknown, containerPort: string): string { + const ip = String(hostIp ?? "").trim(); + const published = String(hostPort ?? "").trim(); + if (!/^\d{1,5}$/u.test(published) || Number(published) < 1 || Number(published) > 65_535) { + throw new Error(`Managed bootstrap Docker binding for '${containerPort}' is invalid.`); + } + if (ip.includes("\0") || /\s/u.test(ip)) { + throw new Error(`Managed bootstrap Docker binding for '${containerPort}' is invalid.`); + } + const address = ip.includes(":") && !ip.startsWith("[") ? `[${ip}]` : ip; + return address ? `${address}:${published}:${containerPort}` : `${published}:${containerPort}`; +} + +function pushManagedPortArgs(args: string[], inspect: DockerContainerInspect): void { + const exposed = inspect.Config?.ExposedPorts ?? {}; + const bindings = inspect.HostConfig?.PortBindings ?? {}; + for (const port of new Set([...Object.keys(exposed), ...Object.keys(bindings)])) { + const normalizedPort = managedPortKey(port); + args.push("--expose", normalizedPort); + const entries = bindings[port]; + if (entries === null || entries === undefined) continue; + if (!Array.isArray(entries)) { + throw new Error(`Managed bootstrap Docker bindings for '${port}' are invalid.`); + } + for (const entry of entries) { + args.push( + "--publish", + managedPublishedPort(entry?.HostIp, entry?.HostPort, normalizedPort), + ); + } + } +} + +function managedDuration(value: unknown, label: string): string | null { + if (value === undefined || value === null) return null; + if (!Number.isSafeInteger(value) || (value as number) < 0) { + throw new Error(`Managed bootstrap Docker ${label} is invalid.`); + } + return `${String(value)}ns`; +} + +function pushManagedHealthcheckArgs(args: string[], inspect: DockerContainerInspect): void { + const healthcheck = inspect.Config?.Healthcheck; + if (healthcheck === undefined || healthcheck === null) return; + const test = healthcheck.Test ?? []; + if (!Array.isArray(test) || test.some((entry) => typeof entry !== "string")) { + throw new Error("Managed bootstrap Docker healthcheck command is invalid."); + } + if (test.length === 1 && test[0] === "NONE") { + args.push("--no-healthcheck"); + } else if (test.length === 2 && test[0] === "CMD-SHELL" && test[1]) { + args.push("--health-cmd", test[1]); + } else { + throw new Error("Managed bootstrap Docker healthcheck command cannot be reproduced exactly."); + } + for (const [flag, value, label] of [ + ["--health-interval", healthcheck.Interval, "healthcheck interval"], + ["--health-timeout", healthcheck.Timeout, "healthcheck timeout"], + ["--health-start-period", healthcheck.StartPeriod, "healthcheck start period"], + ["--health-start-interval", healthcheck.StartInterval, "healthcheck start interval"], + ] as const) { + const duration = managedDuration(value, label); + if (duration !== null) args.push(flag, duration); + } + if (healthcheck.Retries !== undefined && healthcheck.Retries !== null) { + if (!Number.isSafeInteger(healthcheck.Retries) || healthcheck.Retries < 0) { + throw new Error("Managed bootstrap Docker healthcheck retries are invalid."); + } + args.push("--health-retries", String(healthcheck.Retries)); + } +} + +function managedNetworkMode(inspect: DockerContainerInspect, configured: unknown): string { + const mode = String(configured ?? "").trim(); + const networks = Object.keys(inspect.NetworkSettings?.Networks ?? {}); + if ( + networks.length === 1 && + ["", "bridge", "default", "podman"].includes(mode) && + !["bridge", "default", "podman"].includes(networks[0]!) + ) { + return networks[0]!; + } + return mode; +} + function normalizeRequiredUlimit(ulimit: DockerUlimit): DockerUlimit { const name = String(ulimit.name).trim(); if (!/^[a-z][a-z0-9_]*$/u.test(name)) { @@ -91,6 +185,13 @@ function normalizeRequiredUlimit(ulimit: DockerUlimit): DockerUlimit { return { name, soft: ulimit.soft, hard: ulimit.hard }; } +export function normalizeDockerUlimitName(name: unknown): string { + const normalized = String(name ?? "").trim(); + return /^RLIMIT_[A-Z][A-Z0-9_]*$/u.test(normalized) + ? normalized.slice("RLIMIT_".length).toLowerCase() + : normalized; +} + export function validateRequiredDockerUlimits( required: readonly DockerUlimit[] | null | undefined, ): void { @@ -103,7 +204,7 @@ function dockerUlimits( ): DockerUlimit[] { const merged = new Map(); for (const ulimit of inspect.HostConfig?.Ulimits ?? []) { - const name = String(ulimit.Name ?? "").trim(); + const name = normalizeDockerUlimitName(ulimit.Name); const soft = ulimit.Soft; const hard = ulimit.Hard; if ( @@ -222,6 +323,32 @@ function dockerVolumeMountValue(mount: DockerStructuredMount): string { return values.join(","); } +function dockerImageMountValue(mount: DockerStructuredMount): string { + if (String(mount.Consistency ?? "") !== "") { + throw new Error("Docker image mount consistency is not supported during recreation."); + } + assertUnusedMountOption(mount.BindOptions, "BindOptions for an image mount"); + assertUnusedMountOption(mount.VolumeOptions, "VolumeOptions for an image mount"); + assertUnusedMountOption(mount.TmpfsOptions, "TmpfsOptions for an image mount"); + + const source = String(mount.Source ?? "").trim(); + if (!source || /[\0,]/u.test(source)) { + throw new Error("Docker image mount source is invalid."); + } + const target = mountValue(mount.Target, "target"); + if (!target.startsWith("/")) { + throw new Error("Docker structured mount target must be an absolute container path."); + } + if (!optionalMountBoolean(mount.ReadOnly, "ReadOnly")) { + throw new Error("Docker image mounts must remain read-only during recreation."); + } + const values = [`type=image`, `src=${source}`, `dst=${target}`]; + // Podman's Docker-compatible API translates Docker's `readonly` flag to + // `ro=true`, which its image-mount parser rejects. Image mounts default to + // read-only when the option is omitted. + return values.join(","); +} + function dockerStructuredMountArgs(inspect: DockerContainerInspect): string[] { const args: string[] = []; for (const mount of inspect.HostConfig?.Mounts ?? []) { @@ -235,6 +362,9 @@ function dockerStructuredMountArgs(inspect: DockerContainerInspect): string[] { case "volume": args.push("--mount", dockerVolumeMountValue(mount)); break; + case "image": + args.push("--mount", dockerImageMountValue(mount)); + break; default: throw new Error(`Unsupported Docker structured mount type '${String(mount.Type)}'.`); } @@ -482,6 +612,34 @@ export function buildDockerGpuCloneRunArgs( pushStringFlag(args, "--workdir", config.WorkingDir); if (config.Tty) args.push("--tty"); if (config.OpenStdin) args.push("--interactive"); + if (options.preserveManagedLaunchSpec) { + pushManagedPortArgs(args, inspect); + pushManagedHealthcheckArgs(args, inspect); + if (config.StopTimeout !== undefined && config.StopTimeout !== null) { + if (!Number.isSafeInteger(config.StopTimeout) || config.StopTimeout < 0) { + throw new Error("Managed bootstrap Docker stop timeout is invalid."); + } + args.push("--stop-timeout", String(config.StopTimeout)); + } + if (host.OomScoreAdj !== undefined && host.OomScoreAdj !== null) { + if ( + !Number.isSafeInteger(host.OomScoreAdj) || + host.OomScoreAdj < -1_000 || + host.OomScoreAdj > 1_000 + ) { + throw new Error("Managed bootstrap Docker OOM score adjustment is invalid."); + } + // Native Podman is rootless and clamps an inspected request of zero to + // the API-service user's effective floor when the container starts. + // Request that stable effective value up front so stopped and running + // inspection report one launch contract. + const oomScoreAdj = + host.Annotations?.["io.container.manager"] === "libpod" && host.OomScoreAdj === 0 + ? 500 + : host.OomScoreAdj; + args.push("--oom-score-adj", String(oomScoreAdj)); + } + } const sandboxCommand = openshellSandboxCommandEnvValue(options.openshellSandboxCommand); const omitOciImageUser = shouldOmitOpenShellOciImageUser( @@ -506,6 +664,11 @@ export function buildDockerGpuCloneRunArgs( args.push("--env", `${OPENSHELL_SANDBOX_COMMAND_ENV}=${sandboxCommand}`); } + const annotations = host.Annotations || {}; + for (const key of Object.keys(annotations).sort()) { + const value = annotations[key]; + if (value !== undefined && value !== null) args.push("--annotation", `${key}=${value}`); + } const labels = config.Labels || {}; for (const key of Object.keys(labels).sort()) { const value = labels[key]; @@ -513,7 +676,10 @@ export function buildDockerGpuCloneRunArgs( } for (const bind of stringArray(host.Binds)) args.push("--volume", bind); args.push(...dockerStructuredMountArgs(inspect)); - const networkMode = options.networkMode ?? host.NetworkMode; + const configuredNetworkMode = options.networkMode ?? host.NetworkMode; + const networkMode = options.preserveManagedLaunchSpec + ? managedNetworkMode(inspect, configuredNetworkMode) + : configuredNetworkMode; pushStringFlag(args, "--network", networkMode); for (const alias of dockerNetworkAliases(inspect, networkMode)) args.push("--network-alias", alias); diff --git a/src/lib/onboard/docker-gpu-patch-types.ts b/src/lib/onboard/docker-gpu-patch-types.ts index 972cd858f54..37262e4d77f 100644 --- a/src/lib/onboard/docker-gpu-patch-types.ts +++ b/src/lib/onboard/docker-gpu-patch-types.ts @@ -125,6 +125,8 @@ export type DockerGpuCloneRunOptions = { containerCommand?: readonly string[] | null; /** Stopped staging name used before exact-name cutover. */ containerName?: string | null; + /** Preserve managed-bootstrap-only launch fields during stopped replacement. */ + preserveManagedLaunchSpec?: boolean; /** * Extra supplementary group IDs to add to the recreated container via * `--group-add`. On Jetson these are the host group(s) owning the Tegra GPU @@ -209,6 +211,15 @@ export type DockerContainerInspect = { AttachStderr?: boolean; Env?: string[] | null; Labels?: Record | null; + ExposedPorts?: Record> | null; + Healthcheck?: { + Test?: string[] | null; + Interval?: number; + Timeout?: number; + StartPeriod?: number; + StartInterval?: number; + Retries?: number; + } | null; Entrypoint?: string[] | string | null; Cmd?: string[] | string | null; User?: string; @@ -225,7 +236,14 @@ export type DockerContainerInspect = { Restarting?: boolean; Dead?: boolean; } | null; + Mounts?: Array<{ + Type?: string; + Source?: string; + Destination?: string; + RW?: boolean; + }> | null; HostConfig?: { + Annotations?: Record | null; Binds?: string[] | null; Mounts?: Array<{ Type?: string; @@ -248,6 +266,7 @@ export type DockerContainerInspect = { }> | null; NetworkMode?: string; PortBindings?: Record | null> | null; + Tmpfs?: Record | null; RestartPolicy?: { Name?: string; MaximumRetryCount?: number } | null; CapAdd?: string[] | null; CapDrop?: string[] | null; @@ -263,6 +282,7 @@ export type DockerContainerInspect = { CpusetCpus?: string; CpusetMems?: string; PidsLimit?: number | null; + OomScoreAdj?: number | null; ConsoleSize?: number[] | null; Privileged?: boolean; Init?: boolean; diff --git a/src/lib/onboard/fatal-runtime-preflight.test.ts b/src/lib/onboard/fatal-runtime-preflight.test.ts index 59a095206e3..7449276d799 100644 --- a/src/lib/onboard/fatal-runtime-preflight.test.ts +++ b/src/lib/onboard/fatal-runtime-preflight.test.ts @@ -23,6 +23,7 @@ import { runReadinessGatedRuntimePreflight, } from "./fatal-runtime-preflight"; import type { HostAssessment } from "./preflight"; +import type { SandboxGpuConfig } from "./sandbox-gpu-mode"; function hostWithRuntime(runtime: HostAssessment["runtime"]): HostAssessment { return { @@ -357,7 +358,7 @@ describe("report-backed runtime readiness (#7411)", () => { }), detectGpu: () => null, warnIfHostProxyMissesLoopback: vi.fn(), - assertDockerBridgeAndContainerDnsHealthy: bridge, + assertRuntimeProviderHealthy: bridge, validateSandboxGpuPreflight: validateGpu, exitProcess, }, @@ -422,6 +423,19 @@ describe("report-backed runtime readiness (#7411)", () => { expect(exit).not.toHaveBeenCalled(); }, ); + + it.skipIf(!isLinuxDockerDriverGatewayEnabled())( + "allows Podman when the native gateway runtime is explicit", + () => { + vi.stubEnv("NEMOCLAW_GATEWAY_RUNTIME", "podman"); + const exit = vi.fn(); + assertOnboardHostReadiness(hostWithRuntime("podman"), null, { + explicitlyOptedOutGpuPassthrough: false, + exitProcess: exit as never, + }); + expect(exit).not.toHaveBeenCalled(); + }, + ); }); describe("runFatalOnboardRuntimePreflight", () => { @@ -441,7 +455,7 @@ describe("runFatalOnboardRuntimePreflight", () => { nimCapable: true, }), warnIfHostProxyMissesLoopback: vi.fn(), - assertDockerBridgeAndContainerDnsHealthy: vi.fn(), + assertRuntimeProviderHealthy: vi.fn(), validateSandboxGpuPreflight: vi.fn(), exitProcess: vi.fn(() => { throw new Error("unexpected exit"); @@ -462,7 +476,7 @@ describe("runFatalOnboardRuntimePreflight", () => { assessHost: assess, detectGpu: () => null, warnIfHostProxyMissesLoopback: vi.fn(), - assertDockerBridgeAndContainerDnsHealthy: vi.fn(), + assertRuntimeProviderHealthy: vi.fn(), validateSandboxGpuPreflight: vi.fn(), }, ); @@ -479,7 +493,10 @@ describe("runFatalOnboardRuntimePreflight", () => { assessHost: () => hostWithRuntime("docker"), detectGpu: () => null, warnIfHostProxyMissesLoopback: vi.fn(), - assertDockerBridgeAndContainerDnsHealthy: bridge, + assertRuntimeProviderHealthy: (_host: HostAssessment, config: SandboxGpuConfig) => { + gpu(config); + bridge(); + }, validateSandboxGpuPreflight: gpu, }; @@ -521,7 +538,7 @@ describe("runFatalOnboardRuntimePreflight", () => { assessHost: () => hostWithRuntime("docker"), detectGpu: () => null, warnIfHostProxyMissesLoopback: vi.fn(), - assertDockerBridgeAndContainerDnsHealthy: bridge, + assertRuntimeProviderHealthy: bridge, validateSandboxGpuPreflight: validateGpu, exitProcess, }; @@ -557,7 +574,10 @@ describe("readiness-gated runtime preflight", () => { }, assessHost, detectGpu: () => null, - assertDockerBridgeAndContainerDnsHealthy: bridge, + assertRuntimeProviderHealthy: (_host, config) => { + validateGpu(config); + bridge(); + }, validateSandboxGpuPreflight: validateGpu, }, ); @@ -596,7 +616,10 @@ describe("readiness-gated runtime preflight", () => { collectGatewayReadiness, assessHost, detectGpu: () => null, - assertDockerBridgeAndContainerDnsHealthy: bridge, + assertRuntimeProviderHealthy: (_host, config) => { + validateGpu(config); + bridge(); + }, validateSandboxGpuPreflight: validateGpu, }, ); @@ -680,7 +703,10 @@ describe("readiness-gated runtime preflight", () => { }, detectGpu, warnIfHostProxyMissesLoopback: vi.fn(), - assertDockerBridgeAndContainerDnsHealthy: () => calls.push("bridge-dns"), + assertRuntimeProviderHealthy: () => { + calls.push("gpu-validation"); + calls.push("bridge-dns"); + }, validateSandboxGpuPreflight: () => calls.push("gpu-validation"), }, ); @@ -713,7 +739,7 @@ describe("readiness-gated runtime preflight", () => { assessHost: wslDockerDesktopHost, detectGpu, warnIfHostProxyMissesLoopback: vi.fn(), - assertDockerBridgeAndContainerDnsHealthy: vi.fn(), + assertRuntimeProviderHealthy: vi.fn(), validateSandboxGpuPreflight: vi.fn(), }, ); @@ -743,7 +769,7 @@ describe("readiness-gated runtime preflight", () => { assessHost: wslDockerDesktopHost, detectGpu: () => null, warnIfHostProxyMissesLoopback: vi.fn(), - assertDockerBridgeAndContainerDnsHealthy: bridge, + assertRuntimeProviderHealthy: bridge, validateSandboxGpuPreflight: validateGpu, exitProcess, }, @@ -773,7 +799,10 @@ describe("readiness-gated runtime preflight", () => { }, detectGpu: () => null, warnIfHostProxyMissesLoopback: vi.fn(), - assertDockerBridgeAndContainerDnsHealthy: () => calls.push("bridge"), + assertRuntimeProviderHealthy: () => { + calls.push("gpu"); + calls.push("bridge"); + }, validateSandboxGpuPreflight: () => calls.push("gpu"), }, ); @@ -808,7 +837,10 @@ describe("readiness-gated runtime preflight", () => { }, detectGpu: () => null, warnIfHostProxyMissesLoopback: vi.fn(), - assertDockerBridgeAndContainerDnsHealthy: () => calls.push("bridge"), + assertRuntimeProviderHealthy: () => { + calls.push("gpu"); + calls.push("bridge"); + }, validateSandboxGpuPreflight: () => calls.push("gpu"), }, ); @@ -862,7 +894,7 @@ describe("readiness-gated runtime preflight", () => { assessHost: () => hostWithRuntime("docker"), detectGpu: () => null, warnIfHostProxyMissesLoopback: vi.fn(), - assertDockerBridgeAndContainerDnsHealthy: bridge, + assertRuntimeProviderHealthy: bridge, validateSandboxGpuPreflight: gpu, exitProcess: exit as never, }, @@ -884,7 +916,7 @@ describe("GPU trust-gate rejection reason propagation (#9000)", () => { assessHost: () => host, detectGpu, warnIfHostProxyMissesLoopback: vi.fn(), - assertDockerBridgeAndContainerDnsHealthy: vi.fn(), + assertRuntimeProviderHealthy: vi.fn(), validateSandboxGpuPreflight: vi.fn(), }); diff --git a/src/lib/onboard/fatal-runtime-preflight.ts b/src/lib/onboard/fatal-runtime-preflight.ts index 204a0b83e43..1230730b78a 100644 --- a/src/lib/onboard/fatal-runtime-preflight.ts +++ b/src/lib/onboard/fatal-runtime-preflight.ts @@ -24,12 +24,12 @@ import { } from "../readiness/onboard-admission"; import { composeSystemReadinessReport } from "../readiness/system"; import type { SystemReadinessReport } from "../readiness/types"; -import { assertDockerBridgeAndContainerDnsHealthy } from "./bridge-dns-preflight"; import { isLinuxDockerDriverGatewayEnabled, isPortableExperimentalProfile, } from "./docker-driver-platform"; import { warnIfHostProxyMissesLoopback } from "./http-proxy-preflight"; +import { assertConfiguredRuntimeProviderHealthy } from "./machine/runtime-effectful-preflight"; import { assessHost, type HostAssessment, planHostAdvisories } from "./preflight"; import { printCdiSpecUnavailableError, @@ -38,6 +38,7 @@ import { } from "./preflight-messages"; import { printRemediationActions } from "./remediation"; import { resolveSandboxGpuConfig, type SandboxGpuConfig } from "./sandbox-gpu-mode"; +import { resolveConfiguredRuntimeProvider } from "./runtime-provider/selection"; import { exitOnSandboxGpuConfigErrors, printJetsonNvidiaRuntimeUnavailableError, @@ -70,7 +71,7 @@ export interface FatalRuntimePreflightContext { */ detectGpu?: typeof detectGpu; warnIfHostProxyMissesLoopback?: typeof warnIfHostProxyMissesLoopback; - assertDockerBridgeAndContainerDnsHealthy?: typeof assertDockerBridgeAndContainerDnsHealthy; + assertRuntimeProviderHealthy?: typeof assertConfiguredRuntimeProviderHealthy; validateSandboxGpuPreflight?: typeof validateSandboxGpuPreflight; now?: () => Date; } @@ -162,10 +163,28 @@ export function assertOnboardSystemReadiness( options: OnboardHostReadinessOptions, ): SystemReadinessReport { const exitProcess = options.exitProcess ?? exitProcessByDefault; + const portable = isPortableExperimentalProfile(); + const managedLocalGatewayEnabled = + host.platform === "linux" && isLinuxDockerDriverGatewayEnabled("linux"); + const selectedRuntimeUsesProviderHostRoute = + !portable && managedLocalGatewayEnabled + ? (() => { + const provider = resolveConfiguredRuntimeProvider("linux"); + return ( + provider.gateway.supported && + provider.gateway.prepareHostRuntime({ + environment: process.env, + platform: "linux", + }).sandboxHostAddress !== null + ); + })() + : false; const admission = evaluateOnboardReadinessAdmission(readinessReport, { explicitlyOptedOutGpuPassthrough: options.explicitlyOptedOutGpuPassthrough, allowUnsupportedRuntime: - isPortableExperimentalProfile() || !isLinuxDockerDriverGatewayEnabled(), + portable || + selectedRuntimeUsesProviderHostRoute || + !managedLocalGatewayEnabled, allowStorageRemediation: options.allowStorageRemediation === true, allowPortableHostPreparation: options.allowPortableHostPreparation, allowDeferredN1xManagedVllm: @@ -292,7 +311,10 @@ function collectOnboardHostReadiness( wslDockerDesktopGpuProofPassed: runtimeGpu?.wslDockerDesktopGpuProofPassed, now, }); - const readinessReport = projectHostReadiness(snapshot, { ...getBuildIdentity(), now }); + const readinessReport = projectHostReadiness(snapshot, { + ...getBuildIdentity(), + now, + }); assertOnboardSystemReadiness(readinessReport, host, { explicitlyOptedOutGpuPassthrough: sandboxGpuConfig.mode === "0" || options.optedOutGpuPassthrough === true, @@ -311,7 +333,9 @@ function collectOnboardHostReadiness( readinessReport, sandboxGpuConfig, ...(runtimeGpu?.gpuTrustGateRejection || gpuTrustGateRejection - ? { gpuTrustGateRejection: runtimeGpu?.gpuTrustGateRejection ?? gpuTrustGateRejection } + ? { + gpuTrustGateRejection: runtimeGpu?.gpuTrustGateRejection ?? gpuTrustGateRejection, + } : {}), }, snapshot, @@ -363,7 +387,9 @@ async function collectAdmittedReadinessPair( assertOnboardGatewayReadiness(collectedGateway.projection, exitProcess); let evaluatedAt = now(); - let gateway = projectGatewayReadiness(collectedGateway.snapshot, { now: () => evaluatedAt }); + let gateway = projectGatewayReadiness(collectedGateway.snapshot, { + now: () => evaluatedAt, + }); assertOnboardGatewayReadiness(gateway, exitProcess); let host = projectCollectedHostReadiness(collectedHost, evaluatedAt); @@ -377,7 +403,9 @@ async function collectAdmittedReadinessPair( collectedGateway = await context.collectGatewayReadiness(); assertOnboardGatewayReadiness(collectedGateway.projection, exitProcess); evaluatedAt = now(); - gateway = projectGatewayReadiness(collectedGateway.snapshot, { now: () => evaluatedAt }); + gateway = projectGatewayReadiness(collectedGateway.snapshot, { + now: () => evaluatedAt, + }); assertOnboardGatewayReadiness(gateway, exitProcess); host = projectCollectedHostReadiness(host, evaluatedAt); } @@ -435,7 +463,10 @@ export function assertOnboardHostReadiness( wslDockerDesktopGpuProofPassed: options.wslDockerDesktopGpuProofPassed, now: observedAt ? () => new Date(observedAt) : now, }); - const readinessReport = projectHostReadiness(snapshot, { ...getBuildIdentity(), now }); + const readinessReport = projectHostReadiness(snapshot, { + ...getBuildIdentity(), + now, + }); return assertOnboardSystemReadiness(readinessReport, host, options); } @@ -446,18 +477,27 @@ export function runOnboardRuntimeEffectfulPreflightChecks( ): void { const exitProcess = context.exitProcess ?? exitProcessByDefault; exitOnSandboxGpuConfigErrors(result.sandboxGpuConfig, exitProcess); - console.log(" ✓ Docker is running"); (context.warnIfHostProxyMissesLoopback ?? warnIfHostProxyMissesLoopback)(); - (context.validateSandboxGpuPreflight ?? validateSandboxGpuPreflight)( - result.sandboxGpuConfig, - {}, - exitProcess, - ); - (context.assertDockerBridgeAndContainerDnsHealthy ?? assertDockerBridgeAndContainerDnsHealthy)( - result.host, - context.nonInteractive, - exitProcess, - ); + const assertRuntimeProviderHealthy = context.assertRuntimeProviderHealthy; + if (assertRuntimeProviderHealthy) { + assertRuntimeProviderHealthy( + result.host, + result.sandboxGpuConfig, + context.nonInteractive, + exitProcess, + ); + } else { + assertConfiguredRuntimeProviderHealthy( + result.host, + result.sandboxGpuConfig, + context.nonInteractive, + exitProcess, + { + validatePortableSandboxGpuPreflight: + context.validateSandboxGpuPreflight ?? validateSandboxGpuPreflight, + }, + ); + } if (result.host.runtime !== "unknown") { console.log(` ✓ Container runtime: ${result.host.runtime}`); } diff --git a/src/lib/onboard/gateway-binding.ts b/src/lib/onboard/gateway-binding.ts index b884f8a2877..7348527aefa 100644 --- a/src/lib/onboard/gateway-binding.ts +++ b/src/lib/onboard/gateway-binding.ts @@ -20,8 +20,23 @@ */ import type { GatewayReuseState } from "../state/gateway"; +import { + BASE_GATEWAY_COMPAT_CONTAINER_NAME, + BASE_GATEWAY_NAME, + isDefaultGatewayPort, + resolveGatewayCompatContainerName, + resolveGatewayName, +} from "./gateway-binding/identity"; import { DEFAULT_GATEWAY_PORT } from "./gateway/state-dir"; +export { + BASE_GATEWAY_COMPAT_CONTAINER_NAME, + BASE_GATEWAY_NAME, + isDefaultGatewayPort, + resolveGatewayCompatContainerName, + resolveGatewayName, +}; + export { assertManagedGatewayStateDirectoryParentTrusted, BASE_GATEWAY_STATE_DIR_NAME, @@ -34,25 +49,6 @@ export { UnsafeGatewayStateDirectoryError, } from "./gateway/state-dir"; -/** Gateway registration name used for the default gateway port. */ -export const BASE_GATEWAY_NAME = "nemoclaw"; -/** Docker-driver gateway compatibility container name for the default port. */ -export const BASE_GATEWAY_COMPAT_CONTAINER_NAME = "nemoclaw-openshell-gateway"; - -export function isDefaultGatewayPort(port: number): boolean { - return port === DEFAULT_GATEWAY_PORT; -} - -/** - * Resolve the OpenShell gateway registration name for a gateway port. The - * default port keeps the bare `nemoclaw` name for backward compatibility; any - * other port gets a `nemoclaw-` name so its lifecycle commands - * (add/select/remove/start/destroy) never target another sandbox's gateway. - */ -export function resolveGatewayName(port: number): string { - return isDefaultGatewayPort(port) ? BASE_GATEWAY_NAME : `${BASE_GATEWAY_NAME}-${port}`; -} - /** Resolve the gateway port encoded by a canonical NemoClaw gateway name. */ export function resolveGatewayPortFromName(gatewayName: string): number | null { if (gatewayName === BASE_GATEWAY_NAME) { @@ -172,12 +168,6 @@ export function resolveCoreOnboardGatewayBinding(options: { * `docker run --name ...` (and the pre-launch `docker rm`) from tearing down * the first sandbox's compat gateway container. */ -export function resolveGatewayCompatContainerName(port: number): string { - return isDefaultGatewayPort(port) - ? BASE_GATEWAY_COMPAT_CONTAINER_NAME - : `${BASE_GATEWAY_COMPAT_CONTAINER_NAME}-${port}`; -} - /** Gateway state classifiers from `state/gateway`, each bound to a gateway name. */ export interface GatewayNameBoundClassifiers { hasStaleGateway(gwInfoOutput?: string): boolean; diff --git a/src/lib/onboard/gateway-binding/identity.ts b/src/lib/onboard/gateway-binding/identity.ts new file mode 100644 index 00000000000..b22e3e460df --- /dev/null +++ b/src/lib/onboard/gateway-binding/identity.ts @@ -0,0 +1,33 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { DEFAULT_GATEWAY_PORT, GATEWAY_PORT } from "../../core/ports"; + +export { DEFAULT_GATEWAY_PORT, GATEWAY_PORT }; + +/** Gateway registration name used for the default gateway port. */ +export const BASE_GATEWAY_NAME = "nemoclaw"; +/** Docker-driver gateway state directory leaf name for the default port. */ +export const BASE_GATEWAY_STATE_DIR_NAME = "openshell-docker-gateway"; +/** Docker-driver gateway compatibility container name for the default port. */ +export const BASE_GATEWAY_COMPAT_CONTAINER_NAME = "nemoclaw-openshell-gateway"; + +export function isDefaultGatewayPort(port: number): boolean { + return port === DEFAULT_GATEWAY_PORT; +} + +export function resolveGatewayName(port: number): string { + return isDefaultGatewayPort(port) ? BASE_GATEWAY_NAME : `${BASE_GATEWAY_NAME}-${port}`; +} + +export function resolveGatewayStateDirName(port: number): string { + return isDefaultGatewayPort(port) + ? BASE_GATEWAY_STATE_DIR_NAME + : `${BASE_GATEWAY_STATE_DIR_NAME}-${port}`; +} + +export function resolveGatewayCompatContainerName(port: number): string { + return isDefaultGatewayPort(port) + ? BASE_GATEWAY_COMPAT_CONTAINER_NAME + : `${BASE_GATEWAY_COMPAT_CONTAINER_NAME}-${port}`; +} diff --git a/src/lib/onboard/gateway-runtime-selection.test.ts b/src/lib/onboard/gateway-runtime-selection.test.ts new file mode 100644 index 00000000000..d01d678b8dd --- /dev/null +++ b/src/lib/onboard/gateway-runtime-selection.test.ts @@ -0,0 +1,29 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + isPodmanGatewayRuntimeEnabled, + resolveNemoClawGatewayRuntime, +} from "./runtime-provider/configured-runtime"; + +describe("gateway runtime selection", () => { + it.each([{}, { NEMOCLAW_GATEWAY_RUNTIME: "" }, { NEMOCLAW_GATEWAY_RUNTIME: "docker" }])( + "keeps Docker as the default", + (env) => { + expect(resolveNemoClawGatewayRuntime(env)).toBe("docker"); + expect(isPodmanGatewayRuntimeEnabled(env)).toBe(false); + }, + ); + + it("restores the explicit native Podman selector", () => { + expect(resolveNemoClawGatewayRuntime({ NEMOCLAW_GATEWAY_RUNTIME: "podman" })).toBe("podman"); + }); + + it("rejects unknown values", () => { + expect(() => resolveNemoClawGatewayRuntime({ NEMOCLAW_GATEWAY_RUNTIME: "portable" })).toThrow( + 'NEMOCLAW_GATEWAY_RUNTIME must be either "docker" or "podman"', + ); + }); +}); diff --git a/src/lib/onboard/gateway-sandbox-reachability.test.ts b/src/lib/onboard/gateway-sandbox-reachability.test.ts index 38e9cc82fe4..72d8b7d04c4 100644 --- a/src/lib/onboard/gateway-sandbox-reachability.test.ts +++ b/src/lib/onboard/gateway-sandbox-reachability.test.ts @@ -15,6 +15,7 @@ import { PORTABLE_DOCKER_NETWORK_SUBNET, PORTABLE_HOST_GATEWAY_IP, } from "./experimental/portable-profile"; +import { prepareNativePodmanGatewayHostRuntime } from "./runtime-provider/podman-runtime-surfaces"; describe("gateway sandbox reachability route modeling", () => { it("parses Docker network IPAM config for subnet and gateway", () => { @@ -150,6 +151,46 @@ describe("isSandboxBridgeGatewayReachable", () => { expect(seen.args).not.toContain("host.openshell.internal:10.87.0.1"); }); + it("reaches the native Podman host gateway without selecting the portable profile", async () => { + const seen: { args: readonly string[] } = { args: [] }; + const inspect = vi.fn(() => ({ subnet: "10.89.0.0/24", gatewayIp: "10.89.0.1" })); + const run = vi.fn((args: readonly string[]) => { + seen.args = args; + return { status: 0 }; + }); + const ensureProbeImageCached = vi.fn(() => ({ ok: true, alreadyCached: true })); + const gatewayRuntime = { + ...prepareNativePodmanGatewayHostRuntime({ + environment: {}, + platform: "linux", + socketPath: "/run/user/1000/podman/podman.sock", + }), + network: { + sandboxSourceCidrs: () => ["10.89.0.0/24"], + inspect, + usesHostGatewayRoute: vi.fn(() => false), + run, + ensureProbeImageCached, + }, + }; + + const result = await isSandboxBridgeGatewayReachable({ + gatewayRuntime, + platform: "linux", + }); + + expect(result).toMatchObject({ + ok: true, + gatewayIp: PORTABLE_HOST_GATEWAY_IP, + routeKind: "provider_host_gateway", + }); + expect(inspect).toHaveBeenCalledWith("openshell-docker"); + expect(ensureProbeImageCached).toHaveBeenCalledOnce(); + expect(run).toHaveBeenCalledOnce(); + expect(seen.args).toContain(`host.openshell.internal:${PORTABLE_HOST_GATEWAY_IP}`); + expect(seen.args).not.toContain("host.openshell.internal:10.89.0.1"); + }); + it("does not call a missing Docker network a firewall failure", async () => { const result = await isSandboxBridgeGatewayReachable({ inspectNetworkImpl: () => undefined, diff --git a/src/lib/onboard/gateway-sandbox-reachability.ts b/src/lib/onboard/gateway-sandbox-reachability.ts index b2e2ba67eef..7d21e016cca 100644 --- a/src/lib/onboard/gateway-sandbox-reachability.ts +++ b/src/lib/onboard/gateway-sandbox-reachability.ts @@ -12,14 +12,12 @@ import os from "node:os"; -import { dockerCapture, dockerRun } from "../adapters/docker/run"; import { failLine, warnLine } from "../cli/terminal-style"; import { GATEWAY_PORT } from "../core/ports"; import { parseDockerDaemonObservation } from "../domain/docker-host"; import { cliDisplayName, cliName } from "./branding"; import { DEFAULT_DOCKER_DRIVER_NETWORK_NAME, - DOCKER_NETWORK_IPAM_INSPECT_FORMAT, parseDockerNetworkIpamEntries, resolveDockerDriverNetworkName, } from "./experimental/docker-network-authority"; @@ -27,13 +25,11 @@ import { isPortableExperimentalProfile, PORTABLE_HOST_GATEWAY_IP, } from "./experimental/portable-profile"; -import { - DOCKER_DESKTOP_WSL_INTEGRATION_HINT, - ensureProbeImageCached, - isDockerDaemonUnreachable, -} from "./preflight"; +import { DOCKER_DESKTOP_WSL_INTEGRATION_HINT, isDockerDaemonUnreachable } from "./preflight"; import type { UfwAutoApplyResult } from "./ufw-auto-apply"; import { isUfwAutoApplyOptedIn, tryAutoApplyUfwRule } from "./ufw-auto-apply"; +import type { RuntimeProviderGatewayHostRuntime } from "./runtime-provider/contract"; +import { prepareConfiguredGatewayHostRuntime } from "./docker-driver-gateway-env"; export type { UfwAutoApplyOptions, UfwAutoApplyResult } from "./ufw-auto-apply"; export { tryAutoApplyUfwRule } from "./ufw-auto-apply"; @@ -55,7 +51,11 @@ export type SandboxBridgeReachabilityReason = | "probe_timeout" | "veth_unsupported" | "docker_daemon_unreachable"; -export type SandboxBridgeRouteKind = "bridge_gateway" | "host_gateway" | "portable_host_gateway"; +export type SandboxBridgeRouteKind = + | "bridge_gateway" + | "host_gateway" + | "portable_host_gateway" + | "provider_host_gateway"; export interface DockerBridgeNetworkInfo { subnet?: string; @@ -100,8 +100,10 @@ export interface SandboxBridgeReachabilityOptions { runImpl?: (args: readonly string[], timeoutMs: number) => SandboxBridgeProbeRunResult; inspectNetworkImpl?: (networkName: string) => DockerBridgeNetworkInfo | undefined; usesHostGatewayRouteImpl?: () => boolean; + platform?: NodeJS.Platform; runtimeProbeImpl?: (args: readonly string[], timeoutMs: number) => SandboxBridgeProbeRunResult; + gatewayRuntime?: RuntimeProviderGatewayHostRuntime; /** Inject a precomputed image-cache result; bypasses real pre-pull. */ ensureImageCachedOverride?: import("./preflight").EnsureProbeImageCachedResult; } @@ -124,55 +126,23 @@ function parseDockerNetworkIpamConfig(raw: string): DockerBridgeNetworkInfo | un ); } -function defaultInspectNetwork(networkName: string): DockerBridgeNetworkInfo | undefined { - const raw = dockerCapture( - ["network", "inspect", "--format", DOCKER_NETWORK_IPAM_INSPECT_FORMAT, networkName], - { ignoreError: true }, - ); - return parseDockerNetworkIpamConfig(raw); -} - -function defaultUsesHostGatewayRoute(): boolean { - if (process.platform !== "linux") return true; - const info = dockerCapture( - ["info", "--format", "{{.OperatingSystem}}\n{{range .Labels}}{{.}}\n{{end}}"], - { ignoreError: true }, - ); - return /Docker Desktop|com\.docker\.desktop\./i.test(info); -} - -function defaultRunImpl(args: readonly string[], timeoutMs: number): SandboxBridgeProbeRunResult { - const result = dockerRun(args, { - timeout: timeoutMs, - ignoreError: true, - suppressOutput: true, - }); - const error = result.error as NodeJS.ErrnoException | undefined; - return { - status: result.status ?? null, - signal: result.signal, - error: error?.message, - timedOut: error?.code === "ETIMEDOUT", - errorCode: error?.code ?? null, - stderr: result.stderr, - stdout: result.stdout, - }; -} - function buildOpenShellDockerRoute( networkName: string, network: DockerBridgeNetworkInfo | undefined, usesHostGatewayRoute: boolean, - portableHostGatewayIp?: string, + providerHostGateway?: { + address: string; + routeKind: "portable_host_gateway" | "provider_host_gateway"; + }, ): OpenShellDockerRoute | undefined { if (!network) return undefined; - if (portableHostGatewayIp) { + if (providerHostGateway) { return { networkName, subnet: network.subnet, - gatewayIp: portableHostGatewayIp, - routeKind: "portable_host_gateway", - addHosts: [`${HOST_INTERNAL_NAME}:${portableHostGatewayIp}`], + gatewayIp: providerHostGateway.address, + routeKind: providerHostGateway.routeKind, + addHosts: [`${HOST_INTERNAL_NAME}:${providerHostGateway.address}`], }; } if (usesHostGatewayRoute) { @@ -304,19 +274,32 @@ export async function isSandboxBridgeGatewayReachable( const port = opts.port ?? GATEWAY_PORT; const timeoutSec = opts.timeoutSec ?? DEFAULT_PROBE_TIMEOUT_SEC; const probeImage = opts.probeImage ?? DEFAULT_PROBE_IMAGE; - const inspectNetwork = opts.inspectNetworkImpl ?? defaultInspectNetwork; - const usesHostGatewayRoute = opts.usesHostGatewayRouteImpl ?? defaultUsesHostGatewayRoute; - const runImpl = opts.runImpl ?? defaultRunImpl; const portableProfile = isPortableExperimentalProfile(); - const runtimeProbe = opts.runtimeProbeImpl ?? defaultRunImpl; + const platform = opts.platform ?? process.platform; + const managedGatewayRuntime = + opts.gatewayRuntime ?? + prepareConfiguredGatewayHostRuntime({ environment: process.env, platform }); + const providerHostGateway = portableProfile + ? { address: PORTABLE_HOST_GATEWAY_IP, routeKind: "portable_host_gateway" as const } + : managedGatewayRuntime?.sandboxHostAddress + ? { + address: managedGatewayRuntime.sandboxHostAddress, + routeKind: "provider_host_gateway" as const, + } + : undefined; + const inspectNetwork = opts.inspectNetworkImpl ?? managedGatewayRuntime.network.inspect; + const usesHostGatewayRoute = + opts.usesHostGatewayRouteImpl ?? managedGatewayRuntime.network.usesHostGatewayRoute; + const runImpl = opts.runImpl ?? managedGatewayRuntime.network.run; + const runtimeProbe = opts.runtimeProbeImpl ?? managedGatewayRuntime.network.run; const network = inspectNetwork(networkName); const route = buildOpenShellDockerRoute( networkName, network, - usesHostGatewayRoute(), - portableProfile ? PORTABLE_HOST_GATEWAY_IP : undefined, + managedGatewayRuntime.usesHostGatewayRoute === true || usesHostGatewayRoute(), + providerHostGateway, ); if (!route) { if (portableProfile) { @@ -354,7 +337,9 @@ export async function isSandboxBridgeGatewayReachable( // skip the pre-pull there unless the test supplies an explicit // ensureImageCachedOverride. if (opts.ensureImageCachedOverride !== undefined || opts.runImpl === undefined) { - const cached = opts.ensureImageCachedOverride ?? ensureProbeImageCached(probeImage); + const cached = + opts.ensureImageCachedOverride ?? + managedGatewayRuntime.network.ensureProbeImageCached(probeImage); if (!cached.ok) { // A wedged docker daemon (inspect_unavailable) is a fatal Docker // outage, not a probe/pull uncertainty — keep onboarding from @@ -535,6 +520,14 @@ export function formatSandboxBridgeUnreachableMessage( ].join("\n"); } + if (result.routeKind === "provider_host_gateway") { + return [ + failLine(`Sandbox containers cannot reach the gateway at ${HOST_INTERNAL_NAME}:${port}.`), + ` The probe mapped ${HOST_INTERNAL_NAME} to the selected runtime provider's host gateway.`, + ` Restart the selected container runtime and OpenShell gateway, then re-run \`${cliName()} onboard\`.`, + ].join("\n"); + } + if (result.routeKind === "host_gateway") { return [ failLine(`Sandbox containers cannot reach the gateway at ${HOST_INTERNAL_NAME}:${port}.`), diff --git a/src/lib/onboard/gateway/state-dir.ts b/src/lib/onboard/gateway/state-dir.ts index 310bade6c4e..d46fe2ca866 100644 --- a/src/lib/onboard/gateway/state-dir.ts +++ b/src/lib/onboard/gateway/state-dir.ts @@ -5,7 +5,7 @@ import fs from "node:fs"; import path from "node:path"; import { type OpenRegularFile, openRegularFileNoFollow } from "../../adapters/fs/regular-file"; -import { DEFAULT_GATEWAY_PORT, GATEWAY_PORT } from "../../core/ports"; +import { DEFAULT_GATEWAY_PORT, GATEWAY_PORT } from "../gateway-binding/identity"; export { DEFAULT_GATEWAY_PORT, GATEWAY_PORT }; diff --git a/src/lib/onboard/host-gateway-process-target.test.ts b/src/lib/onboard/host-gateway-process-target.test.ts index 4f72a505296..3ae574366a6 100644 --- a/src/lib/onboard/host-gateway-process-target.test.ts +++ b/src/lib/onboard/host-gateway-process-target.test.ts @@ -14,6 +14,7 @@ import { NEMOCLAW_OPENSHELL_SANDBOX_NAMESPACE_ENV, } from "./docker-driver-gateway-config"; import { writeDockerDriverGatewayRuntimeMarkerForStateDir } from "./docker-driver-gateway-runtime-marker"; +import { prepareNativePodmanGatewayHostRuntime } from "./runtime-provider/podman-runtime-surfaces"; import { HOST_GATEWAY_PGREP_PATTERN, type HostGatewayProcessDeps, @@ -64,6 +65,7 @@ function psResponses( cmdline: string | (() => string); exited: Set; processStatus?: RunResult; + provider?: "docker" | "podman"; }, ): [string, RunResponse][] { return [ @@ -90,12 +92,14 @@ function stopScopedTarget( pidFilePid?: number; port?: number; processStatus?: RunResult; + provider?: "docker" | "podman"; signalDenied?: boolean; } = {}, ) { const selectedPid = 9_999_601; const pid = overrides.pidFilePid ?? selectedPid; const stateDir = makeTempRoot("nemoclaw-scoped-target-"); + const provider = overrides.provider ?? "docker"; const pidFile = path.join(stateDir, "openshell-gateway.pid"); fs.writeFileSync(pidFile, `${String(pid)}\n`); const jwtBundle = ensureDockerDriverGatewayJwtBundle(stateDir); @@ -103,21 +107,33 @@ function stopScopedTarget( path.join(stateDir, "openshell-gateway.toml"), buildDockerDriverGatewayConfigToml( { - OPENSHELL_GRPC_ENDPOINT: "https://127.0.0.1:18080", + OPENSHELL_GRPC_ENDPOINT: + provider === "podman" ? "https://169.254.2.2:18080" : "https://127.0.0.1:18080", OPENSHELL_LOCAL_TLS_DIR: path.join(stateDir, "tls"), OPENSHELL_DOCKER_NETWORK_NAME: "openshell-docker", OPENSHELL_DOCKER_SUPERVISOR_IMAGE: "supervisor:test", + ...(provider === "podman" + ? { OPENSHELL_PODMAN_SOCKET: path.join(stateDir, "podman.sock") } + : {}), }, "/usr/bin/openshell-sandbox", jwtBundle, gatewayIdForStateDir(stateDir), + provider === "podman" + ? prepareNativePodmanGatewayHostRuntime({ + environment: {}, + platform: "linux", + socketPath: path.join(stateDir, "podman.sock"), + }) + : undefined, ), { mode: 0o600 }, ); writeDockerDriverGatewayRuntimeMarkerForStateDir(stateDir, { - desiredEnv: {}, + desiredEnv: { NEMOCLAW_RUNTIME_PROVIDER_ID: provider }, endpoint: `https://127.0.0.1:${String(overrides.markerPort ?? 18080)}`, pid: selectedPid, + platform: provider === "podman" ? "linux" : process.platform, }); const exited = new Set(); const markExited = (pid: number): true => { @@ -220,6 +236,21 @@ describe("stopHostGatewayProcesses target filtering", () => { expect(fs.existsSync(pidFile)).toBe(false); }); + it("uses the native provider runtime marker when its gateway schema omits namespaces", () => { + const platform = vi.spyOn(process, "platform", "get").mockReturnValue("linux"); + try { + const { kill, pidFile, result, run } = stopScopedTarget({ provider: "podman" }); + + expect(result.stopped).toEqual([9_999_601]); + expect(result.ownershipFailures).toEqual([]); + expect(kill).toHaveBeenCalledWith(9_999_601, "SIGTERM"); + expect(run.mock.calls.some(([command]) => command === "pgrep")).toBe(false); + expect(fs.existsSync(pidFile)).toBe(false); + } finally { + platform.mockRestore(); + } + }); + it.each([ ["PID file", { pidFilePid: 9_999_602 }], ["command line", { cmdline: "openshell-gateway[nemoclaw=nemoclaw;port=8080]" }], diff --git a/src/lib/onboard/host-gateway-process.ts b/src/lib/onboard/host-gateway-process.ts index 08b41707c71..27d90661386 100644 --- a/src/lib/onboard/host-gateway-process.ts +++ b/src/lib/onboard/host-gateway-process.ts @@ -3,11 +3,9 @@ import { type SpawnSyncOptions, spawnSync } from "node:child_process"; import fs from "node:fs"; -import os from "node:os"; import path from "node:path"; import { waitUntil } from "../core/wait"; -import { DEFAULT_GATEWAY_PORT, resolveGatewayStateDirForPort } from "./gateway/state-dir"; import { gatewayIdForStateDir, hasStateScopedSandboxNamespace, @@ -17,7 +15,10 @@ import { clearDockerDriverGatewayRuntimeMarker, getDockerDriverGatewayRuntimeMarkerPath, parseDockerDriverGatewayRuntimeMarker, + resolveDockerDriverGatewayPidFile, + resolveDockerDriverGatewayStateDir, } from "./docker-driver-gateway-runtime-marker"; +import { resolveRegisteredRuntimeProvider } from "./runtime-provider/selection"; import { canonicalGatewayTargetMatches, type OpenShellGatewayProcessTarget, @@ -25,6 +26,10 @@ import { } from "./gateway-process-identity"; export { hasStateScopedSandboxNamespace } from "./docker-driver-gateway-config"; +export { + resolveDockerDriverGatewayPidFile, + resolveDockerDriverGatewayStateDir, +} from "./docker-driver-gateway-runtime-marker"; export interface RunResult { status: number | null; @@ -122,25 +127,6 @@ function defaultCommandExists(command: string, env: NodeJS.ProcessEnv): boolean ); } -export function resolveDockerDriverGatewayStateDir( - env: NodeJS.ProcessEnv = process.env, - homeDir: string = env.HOME || os.homedir(), - gatewayPort: number = DEFAULT_GATEWAY_PORT, -): string { - return resolveGatewayStateDirForPort({ - configured: env.NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR, - home: homeDir, - port: gatewayPort, - }); -} - -export function resolveDockerDriverGatewayPidFile( - env: NodeJS.ProcessEnv = process.env, - homeDir: string = env.HOME || os.homedir(), -): string { - return path.join(resolveDockerDriverGatewayStateDir(env, homeDir), "openshell-gateway.pid"); -} - function defaultDeps(overrides: Partial = {}): HostGatewayProcessDeps { const env = overrides.env ?? process.env; return { @@ -358,9 +344,6 @@ function scopedGatewayOwnershipFailure( pidFile: string, target: { name: string; port: number }, ): string | null { - if (!hasStateScopedSandboxNamespace(stateDir)) { - return "gateway config does not prove an isolated sandbox namespace"; - } const uid = typeof process.getuid === "function" ? process.getuid() : -1; const pidText = readOwnedRuntimeFile(pidFile, uid); const markerText = readOwnedRuntimeFile(getDockerDriverGatewayRuntimeMarkerPath(stateDir), uid); @@ -368,6 +351,22 @@ function scopedGatewayOwnershipFailure( if (Number(pidText?.trim()) !== pid || marker?.pid !== pid) { return "PID file and runtime marker do not identify the same process"; } + const provider = resolveRegisteredRuntimeProvider(marker.driver); + if (!provider?.gateway.supported) { + return "runtime marker does not identify a registered gateway provider"; + } + let processOwnership: "scoped-namespace" | "runtime-marker"; + try { + processOwnership = provider.gateway.prepareHostRuntime({ + environment: deps.env, + platform: marker.platform, + }).gatewayConfig.processOwnership; + } catch { + return "runtime marker provider ownership could not be prepared"; + } + if (processOwnership === "scoped-namespace" && !hasStateScopedSandboxNamespace(stateDir)) { + return "gateway config does not prove an isolated sandbox namespace"; + } let markerPort = 0; try { markerPort = Number(new URL(marker.endpoint).port); @@ -381,7 +380,10 @@ function scopedGatewayOwnershipFailure( ) { return "runtime marker does not identify the selected gateway"; } - if (!processUsesStateScopedSandboxNamespace(pid, stateDir, deps)) { + if ( + processOwnership === "scoped-namespace" && + !processUsesStateScopedSandboxNamespace(pid, stateDir, deps) + ) { return "gateway process owner and loaded sandbox namespace cannot be proven"; } if ( diff --git a/src/lib/onboard/host-service-reachability.test.ts b/src/lib/onboard/host-service-reachability.test.ts index ac3b9e7d0c5..ec98b8dc0c3 100644 --- a/src/lib/onboard/host-service-reachability.test.ts +++ b/src/lib/onboard/host-service-reachability.test.ts @@ -9,6 +9,7 @@ import { describe, expect, it, vi } from "vitest"; import { PORTABLE_HOST_GATEWAY_IP } from "./experimental/portable-profile"; +import { prepareNativePodmanGatewayHostRuntime } from "./runtime-provider/podman-runtime-surfaces"; // Mock the docker adapter so the test never loads runner.ts (which requires // the compiled ./platform artifact unavailable in the test environment). @@ -131,6 +132,41 @@ describe("probeHostServiceSandboxReachability", () => { expect(capturedArgs).not.toContain("host.openshell.internal:10.89.0.1"); }); + it("routes native Podman probes through the sandbox host gateway", async () => { + let capturedArgs: readonly string[] = []; + const inspect = vi.fn(() => ({ subnet: "10.89.0.0/24", gatewayIp: "10.89.0.1" })); + const run = vi.fn((args: readonly string[]) => { + capturedArgs = args; + return { status: 0 }; + }); + const gatewayRuntime = { + ...prepareNativePodmanGatewayHostRuntime({ + environment: {}, + platform: "linux", + socketPath: "/run/user/1000/podman/podman.sock", + }), + network: { + sandboxSourceCidrs: () => ["10.89.0.0/24"], + inspect, + usesHostGatewayRoute: vi.fn(() => false), + run, + ensureProbeImageCached: vi.fn(() => ({ ok: true, alreadyCached: true })), + }, + }; + + const result = await probeHostServiceSandboxReachability({ + gatewayRuntime, + platform: "linux", + port: 11435, + }); + + expect(result).toMatchObject({ ok: true, reason: "ok" }); + expect(inspect).toHaveBeenCalledWith("openshell-docker"); + expect(run).toHaveBeenCalledOnce(); + expect(capturedArgs).toContain(`host.openshell.internal:${PORTABLE_HOST_GATEWAY_IP}`); + expect(capturedArgs).not.toContain("host.openshell.internal:10.89.0.1"); + }); + it("keeps portable host-gateway failures credential-free and inconclusive", async () => { vi.stubEnv("NEMOCLAW_EXPERIMENTAL_PROFILE", "portable"); vi.spyOn(process, "platform", "get").mockReturnValue("linux"); @@ -176,26 +212,26 @@ describe("formatHostServiceUnreachableMessage", () => { expect(msg).toContain("nemoclaw onboard"); }); - it.each([ - "nemohermes", - "nemo-deepagents", - ])("uses the invoked %s CLI in the recovery command (#8712)", (invokedAs) => { - vi.stubEnv("NEMOCLAW_INVOKED_AS", invokedAs); + it.each(["nemohermes", "nemo-deepagents"])( + "uses the invoked %s CLI in the recovery command (#8712)", + (invokedAs) => { + vi.stubEnv("NEMOCLAW_INVOKED_AS", invokedAs); - const msg = formatHostServiceUnreachableMessage( - { - ok: false, - reason: "tcp_failed", - port: 8081, - networkName: "openshell-docker", - subnet: "172.18.0.0/16", - gatewayIp: "172.18.0.1", - }, - { serviceLabel: "managed llama.cpp server" }, - ); + const msg = formatHostServiceUnreachableMessage( + { + ok: false, + reason: "tcp_failed", + port: 8081, + networkName: "openshell-docker", + subnet: "172.18.0.0/16", + gatewayIp: "172.18.0.1", + }, + { serviceLabel: "managed llama.cpp server" }, + ); - expect(msg).toContain(`Then rerun \`${invokedAs} onboard\`.`); - }); + expect(msg).toContain(`Then rerun \`${invokedAs} onboard\`.`); + }, + ); it("falls back to result.port when no explicit port option is given", () => { const msg = formatHostServiceUnreachableMessage( diff --git a/src/lib/onboard/host-service-reachability.ts b/src/lib/onboard/host-service-reachability.ts index 231d55d40f0..66173d8c3b1 100644 --- a/src/lib/onboard/host-service-reachability.ts +++ b/src/lib/onboard/host-service-reachability.ts @@ -17,11 +17,8 @@ * onboard successful. */ -import { dockerCapture, dockerRun } from "../adapters/docker/run"; -import { cliName } from "./branding"; import { DEFAULT_DOCKER_DRIVER_NETWORK_NAME, - DOCKER_NETWORK_IPAM_INSPECT_FORMAT, parseDockerNetworkIpamEntries, resolveDockerDriverNetworkName, } from "./experimental/docker-network-authority"; @@ -29,6 +26,9 @@ import { isPortableExperimentalProfile, PORTABLE_HOST_GATEWAY_IP, } from "./experimental/portable-profile"; +import type { RuntimeProviderGatewayHostRuntime } from "./runtime-provider/contract"; +import { prepareConfiguredGatewayHostRuntime } from "./docker-driver-gateway-env"; +export { formatHostServiceUnreachableMessage } from "./reachability/host-service-message"; export const DEFAULT_PROBE_NETWORK = DEFAULT_DOCKER_DRIVER_NETWORK_NAME; const HOST_INTERNAL_NAME = "host.openshell.internal"; @@ -69,54 +69,18 @@ export interface HostServiceReachabilityOptions { runImpl?: (args: readonly string[], timeoutMs: number) => ProbeRunResult; inspectNetworkImpl?: (networkName: string) => { subnet?: string; gatewayIp?: string } | undefined; usesHostGatewayRouteImpl?: () => boolean; + platform?: NodeJS.Platform; + gatewayRuntime?: RuntimeProviderGatewayHostRuntime; } function parseNetworkIpamConfig(raw: string): { subnet?: string; gatewayIp?: string } | undefined { for (const entry of parseDockerNetworkIpamEntries(raw) ?? []) { const { subnet, gatewayIp } = entry; - // Skip IPv6-only entries (contain colons) if (gatewayIp && !gatewayIp.includes(":")) return { subnet, gatewayIp }; } return undefined; } -function defaultInspectNetwork( - networkName: string, -): { subnet?: string; gatewayIp?: string } | undefined { - const raw = dockerCapture( - ["network", "inspect", "--format", DOCKER_NETWORK_IPAM_INSPECT_FORMAT, networkName], - { ignoreError: true }, - ); - return parseNetworkIpamConfig(raw); -} - -// Docker Desktop and VM-backed Docker use the runtime's host-gateway alias -// instead of the inspected bridge IP. These routes do not support native -// Docker bridge UFW remediation. -function defaultUsesHostGatewayRoute(): boolean { - if (process.platform !== "linux") return true; - const info = dockerCapture( - ["info", "--format", "{{.OperatingSystem}}\n{{range .Labels}}{{.}}\n{{end}}"], - { ignoreError: true }, - ); - return /Docker Desktop|com\.docker\.desktop\./i.test(info); -} - -function defaultRunImpl(args: readonly string[], timeoutMs: number): ProbeRunResult { - const result = dockerRun(args, { - timeout: timeoutMs, - ignoreError: true, - suppressOutput: true, - stdio: ["ignore", "pipe", "pipe"], - }); - return { - status: result.status ?? null, - signal: result.signal, - error: result.error?.message, - stderr: result.stderr, - }; -} - function outputTail(value: unknown): string | undefined { const raw = Buffer.isBuffer(value) ? value.toString("utf8") : value == null ? "" : String(value); const text = raw.trim(); @@ -136,10 +100,16 @@ export async function probeHostServiceSandboxReachability( const port = opts.port; const timeoutSec = opts.timeoutSec ?? PROBE_TIMEOUT_SEC; const probeImage = opts.probeImage ?? PROBE_IMAGE; - const inspectNetwork = opts.inspectNetworkImpl ?? defaultInspectNetwork; - const usesHostGatewayRoute = opts.usesHostGatewayRouteImpl ?? defaultUsesHostGatewayRoute; - const runImpl = opts.runImpl ?? defaultRunImpl; + const portableProfile = isPortableExperimentalProfile(); + const platform = opts.platform ?? process.platform; + const managedGatewayRuntime = + opts.gatewayRuntime ?? + prepareConfiguredGatewayHostRuntime({ environment: process.env, platform }); + const inspectNetwork = opts.inspectNetworkImpl ?? managedGatewayRuntime.network.inspect; + const usesHostGatewayRoute = + opts.usesHostGatewayRouteImpl ?? managedGatewayRuntime.network.usesHostGatewayRoute; + const runImpl = opts.runImpl ?? managedGatewayRuntime.network.run; const network = inspectNetwork(networkName); if (!network) { return { @@ -147,13 +117,16 @@ export async function probeHostServiceSandboxReachability( reason: "probe_unavailable", port, networkName, - detail: `Docker network "${networkName}" not found`, + detail: `Runtime network "${networkName}" not found`, }; } - - const portableProfile = isPortableExperimentalProfile(); - const isHostGateway = portableProfile ? false : usesHostGatewayRoute(); - const usesNonBridgeRoute = portableProfile || isHostGateway; + const providerHostAddress = portableProfile + ? PORTABLE_HOST_GATEWAY_IP + : managedGatewayRuntime.sandboxHostAddress; + const isHostGateway = + providerHostAddress === null && + (managedGatewayRuntime.usesHostGatewayRoute === true || usesHostGatewayRoute()); + const usesNonBridgeRoute = providerHostAddress !== null || isHostGateway; if (!usesNonBridgeRoute && !network.gatewayIp) { return { @@ -166,8 +139,8 @@ export async function probeHostServiceSandboxReachability( }; } - const hostInternalTarget = portableProfile - ? PORTABLE_HOST_GATEWAY_IP + const hostInternalTarget = providerHostAddress + ? providerHostAddress : isHostGateway ? "host-gateway" : (network.gatewayIp as string); @@ -236,32 +209,6 @@ export async function probeHostServiceSandboxReachability( }; } -export function formatHostServiceUnreachableMessage( - result: HostServiceReachabilityResult, - options: { serviceLabel: string; port?: number }, -): string { - if (result.ok || result.reason !== "tcp_failed") return ""; - - const port = options.port ?? result.port; - const allowCmd = - result.subnet && result.gatewayIp - ? ` sudo ufw allow from ${result.subnet} to ${result.gatewayIp} port ${port} proto tcp` - : result.subnet - ? ` sudo ufw allow from ${result.subnet} to any port ${port} proto tcp` - : [ - ` SUBNET=$(docker network inspect ${result.networkName ?? DEFAULT_PROBE_NETWORK} --format '{{(index .IPAM.Config 0).Subnet}}')`, - ` sudo ufw allow from "$SUBNET" to any port ${port} proto tcp`, - ].join("\n"); - - return [ - ` ✗ Sandbox containers cannot reach the ${options.serviceLabel} at ${HOST_INTERNAL_NAME}:${port}.`, - " A host firewall may be blocking traffic from the OpenShell Docker bridge.", - " To allow it:", - allowCmd, - ` Then rerun \`${cliName()} onboard\`.`, - ].join("\n"); -} - export const __test = { parseNetworkIpamConfig, }; diff --git a/src/lib/onboard/initial-policy-real-policy.test.ts b/src/lib/onboard/initial-policy-real-policy.test.ts index 24eba28afa5..d7b2b1d31a7 100644 --- a/src/lib/onboard/initial-policy-real-policy.test.ts +++ b/src/lib/onboard/initial-policy-real-policy.test.ts @@ -303,6 +303,8 @@ describe("initial sandbox policy real preset merge", () => { expect(slackBinaries).toEqual([ "/usr/local/bin/hermes", "/usr/bin/python3*", + "/usr/bin/python3.13", + "/opt/hermes/.venv/bin/python3", "/opt/hermes/.venv/bin/python", ]); @@ -449,19 +451,38 @@ describe("initial sandbox policy real preset merge", () => { ].flatMap((policyCase) => ["slack.com", "api.slack.com", "hooks.slack.com"].map((host) => ({ policyCase, host })), ), - )("keeps Slack credential rewrite for $policyCase.agent on $host", ({ policyCase, host }) => { - const effective = readPreparedPolicy( - prepareInitialSandboxCreatePolicy(policyCase.path, ["slack"], { - agentName: policyCase.agent, - }), - ); - const slackEndpoints = effective.network_policies?.slack?.endpoints ?? []; - const endpoint = slackEndpoints.find((candidate) => candidate.host === host); - expect(endpoint, `${policyCase.agent}:${host}`).toMatchObject({ - protocol: "rest", - request_body_credential_rewrite: true, - }); - }); + )( + "replaces permissive Slack access with credential-bound channel policy for $policyCase.agent on $host", + ({ policyCase, host }) => { + const sandboxName = policyCase.agent === "openclaw" ? "oc-slack" : "hm-slack"; + const effective = readPreparedPolicy( + prepareInitialSandboxCreatePolicy(policyCase.path, ["slack"], { + agentName: policyCase.agent, + sandboxName, + }), + ); + const slackEndpoints = effective.network_policies?.slack?.endpoints ?? []; + const endpoints = slackEndpoints.filter((candidate) => candidate.host === host); + expect(endpoints.length, `${policyCase.agent}:${host}`).toBeGreaterThan(0); + expect(endpoints).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + protocol: "rest", + request_body_credential_rewrite: true, + credential_binding: { + provider: + host === "slack.com" + ? expect.stringMatching(new RegExp(`^${sandboxName}-slack-(?:app|bridge)$`, "u")) + : `${sandboxName}-slack-bridge`, + }, + }), + ]), + ); + expect(endpoints).not.toEqual( + expect.arrayContaining([expect.objectContaining({ access: "full" })]), + ); + }, + ); it("materializes Hermes Discord credential bindings from the target sandbox name", () => { const sandboxName = "hermes-discord-e2e"; diff --git a/src/lib/onboard/initial-policy.test.ts b/src/lib/onboard/initial-policy.test.ts index 54a0ab4e048..a8395415065 100644 --- a/src/lib/onboard/initial-policy.test.ts +++ b/src/lib/onboard/initial-policy.test.ts @@ -10,13 +10,17 @@ import YAML from "yaml"; const logPresetScopeMock = vi.hoisted(() => vi.fn()); vi.mock("../policy", () => ({ - mergePresetNamesIntoPolicy: (policy: string, presetNames: string[]) => ({ - policy: `${policy.trimEnd()}\n${presetNames - .map((preset) => ` ${preset === "wechat" ? "wechat_bridge" : preset}: {}`) - .join("\n")}\n`, - appliedPresets: presetNames, - missingPresets: [], - }), + mergePresetNamesIntoPolicy: (policy: string, presetNames: string[]) => { + const additions = presetNames.flatMap((preset) => { + const policyKey = preset === "wechat" ? "wechat_bridge" : preset; + return new RegExp(`^ ${policyKey}:`, "mu").test(policy) ? [] : [` ${policyKey}: {}`]; + }); + return { + policy: additions.length === 0 ? policy : `${policy.trimEnd()}\n${additions.join("\n")}\n`, + appliedPresets: presetNames, + missingPresets: [], + }; + }, logPresetScope: logPresetScopeMock, })); @@ -654,22 +658,26 @@ network_policies: {} expect(fs.existsSync(prepared.policyPath)).toBe(false); }); - it("records an existing create-time preset without writing a temp policy", () => { + it("replaces an existing messaging key with its active channel preset", () => { const basePolicyPath = tmpPolicy("version: 1\nnetwork_policies:\n slack: {}\n"); - expect(prepareInitialSandboxCreatePolicy(basePolicyPath, ["slack"])).toEqual({ - policyPath: basePolicyPath, - appliedPresets: ["slack"], + const prepared = prepareInitialSandboxCreatePolicy(basePolicyPath, ["slack"], { + sandboxName: "active-slack", }); + + expect(prepared.policyPath).not.toBe(basePolicyPath); + expect(prepared.appliedPresets).toEqual(["slack"]); + expect(prepared.cleanup?.()).toBe(true); }); - it("records active channel policies already provided by an agent base policy", () => { + it("materializes active channel policy authority over an agent base entry", () => { const basePolicyPath = tmpPolicy("version: 1\nnetwork_policies:\n discord: {}\n"); - expect(prepareInitialSandboxCreatePolicy(basePolicyPath, ["discord"])).toEqual({ - policyPath: basePolicyPath, - appliedPresets: ["discord"], - }); + const prepared = prepareInitialSandboxCreatePolicy(basePolicyPath, ["discord"]); + + expect(prepared.policyPath).not.toBe(basePolicyPath); + expect(prepared.appliedPresets).toEqual(["discord"]); + expect(prepared.cleanup?.()).toBe(true); }); it("filters inactive Hermes messaging policies from the create-time policy", () => { diff --git a/src/lib/onboard/initial-policy.ts b/src/lib/onboard/initial-policy.ts index 81db004d9b3..0226d39b19e 100644 --- a/src/lib/onboard/initial-policy.ts +++ b/src/lib/onboard/initial-policy.ts @@ -539,11 +539,12 @@ function resolveInitialSandboxCreatePolicy( return result(dedupe(existingChannelPresets)); } - const existingCreateTimePresets = requestedCreateTimePresets.filter((preset) => - basePolicyNames.has(preset), + const messagingPresets = new Set(messagingCreateTimePresets); + const existingCreateTimePresets = requestedCreateTimePresets.filter( + (preset) => !messagingPresets.has(preset) && basePolicyNames.has(preset), ); const createTimePresets = requestedCreateTimePresets.filter( - (preset) => !basePolicyNames.has(preset), + (preset) => messagingPresets.has(preset) || !basePolicyNames.has(preset), ); if (createTimePresets.length === 0) { return result(dedupe([...existingChannelPresets, ...existingCreateTimePresets])); diff --git a/src/lib/onboard/lifecycle-contracts.md b/src/lib/onboard/lifecycle-contracts.md index af15f737410..b357e057cb7 100644 --- a/src/lib/onboard/lifecycle-contracts.md +++ b/src/lib/onboard/lifecycle-contracts.md @@ -134,10 +134,10 @@ runtime mutation | **New interactive or non-interactive onboard** — `onboard()` and `resolveOnboardEntryOptions` | Current flags, environment, and prompts. `MessagingWorkflowPlanner.buildPlan`, `prepareSandboxMessagingPreflight`, resource-profile selection, `resolveSandboxCreateIntent`, and `materializeSandboxCreatePlan` assemble policy, provider, package, resource, host-forward, and runtime-setup contributions. Non-interactive mode replaces prompts with defaults or hard aborts. | Consent/session/lock setup and preflight can persist local state, install OpenShell, or clean stale gateway artifacts before the gateway handler. Gateway reuse/recovery/start is the first provider-routing effect; inference-provider upserts follow. For OpenClaw, messaging selection and plan reconciliation complete before web-search or messaging provider registration. Each validated provider group is then created or updated and checkpointed before resource selection. A name with no live sandbox has no sandbox-destructive boundary; an existing target enters the recreate contract below. | Whole-step session plus machine snapshot. OpenClaw adds narrow checkpoints after each completed secret-free sandbox prompt group; sandbox registry registration is deferred until readiness and live validation. The session stores credential environment names, redacted endpoint metadata, legacy-value digests, and non-secret names of web-search and messaging providers registered for resume; real values remain process- or gateway-bound. | Readiness, post-create policy verification, dashboard forwarding, and cancellation failures preserve the live sandbox and an independent identity-bound recovery record. A later `destroy` refuses mutable-name deletion while that sandbox is live. After administrator identity-bound removal, destroy uses the record to qualify immutable runtime identity and, for Docker-backed sandboxes, exact container identities before residual cleanup and record retirement. A different explicit sandbox name starts a fresh session without changing the retained record. Exact provider-owned GPU cleanup can proceed through its owner receipt. NemoClaw attempts to remove temporary policy and build-context sources and reports cleanup failures with the onboarding error; post-create failures retain recovery state. Cancellation before sandbox creation can leave the session resumable. Shared inference providers remain gateway configuration and are not sandbox cleanup targets. Coverage: `transition-traces.test.ts`, `sandbox-create-intent-boundary.test.ts`, `sandbox-create-plan.test.ts`, and the focused cancellation, readiness, GPU cleanup, dashboard, policy-authority, destroy, and retained-recovery tests. Gap: gateway upserts can outlive a failed or interrupted create. | | **`--fresh` onboard** — `resolveOnboardEntryOptions`, `prepareFreshSession`, `createBaseImageResolutionContext` | Current flags/environment/prompts replace resumable intent. `--fresh` disables auto-resume and forces base-image resolution; it does not prove that the selected sandbox name is unused. | The first destructive effect is local: the prior onboard session is cleared before a new session is saved. A matching live sandbox can later reuse or recreate through the normal sandbox decision; `--fresh` does not itself delete it. | The new session and machine snapshot replace the old resume checkpoint. Credential and effect boundaries then match new onboard or live recreate. | The discarded resume checkpoint is not restored on later failure. Covered by `entry-options.test.ts`, `session-bootstrap.test.ts`, and base-image resolution tests. | | **Resume, re-onboard, or recreate** — `onboard()`, `prepareOnboardSession`, `decideSandboxResume`, live-sandbox handling in `createSandbox` | For `--resume`, the recorded session is authoritative and conflicting current name/provider/model/image/tool-disclosure hints are rejected. A new re-onboard run takes current flags, environment, and prompts as intent while registry/gateway state provides drift evidence. The machine resolves a complete secret-free create intent, including policy, messaging/provider, GPU, resource, disabled-channel, and agent inputs, before repair/removal or live recreation. | Ordinary live recreation conditionally backs up before provider cleanup, **delete**, and image removal. The recreate journal preserves the source registry row after deletion. Replacement registration commits the new row after readiness and validation. A selected pre-upgrade backup suppresses a new one; an explicit override permits recreation without backup. Resume registry removal and `repair-and-recreate` occur only after complete intent validation. Temporary policy/build artifacts remain materialization effects after the delete boundary. | Resume continues the recorded session/machine snapshot; non-resume re-onboard writes a new session first. OpenClaw records completed sandbox name, web search, messaging, and resource choices with explicit progress markers, including explicit `null` choices, while the complete create intent stays process-local and is not persisted or emitted. Raw credential values remain outside the session. A missing process value can be rebound only when the same OpenClaw session recorded successfully registering that provider and its live provider name, provider type, and credential key still match; otherwise interactive resume requests it again and non-interactive resume exits with environment-variable guidance. Credentials are checked before mutation and again immediately before materialization. | A failed replacement keeps the source registry row. Restore failures warn and can still publish the replacement; managed-DCode live-selection failure leaves a running, unregistered sandbox with manual-delete guidance. Checkpoint replay reuses an exact live sandbox after an interrupted create and backfills missing create/register receipts. Cancel rollback is not armed and there is no rebuild-style receipt rollback. Coverage: transition traces, create-intent characterization, checkpoint replay and resume guards, and sandbox-handler crash recovery. Gaps: early backup asymmetry and no rebuild-style cross-effect rollback. | -| **Rebuild or installer-driven upgrade** — `rebuildSandbox` in `rebuild-pipeline.ts`; `upgradeSandboxes` | Registry state is authoritative. A matching session may fill guarded legacy gaps only when its selection agrees; an unrelated/global session is never used. Ambient provider/model selection is quarantined by `isolateAmbientRecreateEnv`, apart from narrowly scoped legacy recovery. Legacy and custom-image rebuilds retain and fingerprint a prepared build context. Managed-image rebuilds instead stage an immutable image and startup-profile handoff, skip Dockerfile image preflight, and revalidate provider-bound workload authority before each deletion boundary. | Consent persistence, target-gateway selection/recovery, and target-preflight registry updates can precede disposable image build/probes. Backup is the first durable recovery checkpoint when available. Shields unlock, MCP detach/scrub, and NIM stop are destructive in-place effects before the **sandbox delete** boundary. Legacy and custom-image paths recheck prepared context and mutation-edge conditions before delete. Managed-image paths revalidate the exact provider-bound handoff before delete. | Durable checkpoints are the backup/recovery manifest when one exists and the rewritten recreate session; stale recovery can reach deletion without a manifest, making that session its first new durable checkpoint. Rollback receipts/snapshots are process-local. Credential metadata comes from the target or guarded fallback; raw credentials/providers are checked against current process/gateway state, while prepared installer recovery may reconstruct a missing gateway provider from a validated host credential. | In-process rollback best-effort restores registry/MCP retry metadata, but process death after non-MCP delete can still lose it. The inner onboarding consumes the exact managed-workload handoff or selects the legacy resource profile after deletion. Covered by rebuild, managed-workload authority, image-preflight, DCode, and messaging tests. Gaps: health-before-delete and atomic swap. Closed issue #5801 records the original gap; #6835 fixed only the printed recovery path. | -| **Stock Docker-driver managed-image onboarding** — managed-workload selection in `onboard-orchestration.ts` | Ordinary onboarding through the OpenShell Docker driver validates one complete all-agent catalog, immutable release and platform contracts, and selected-provider capabilities before selecting OpenClaw, Hermes, or LangChain Deep Agents Code. Portable onboarding, non-managed agents, and explicit `--from` custom images retain their legacy or custom workload paths. | The managed path skips Dockerfile build materialization, creates provider-bound bootstrap authority for the immutable image and startup profile, launches that workload, and registers the managed-workload receipt only after readiness. | Catalog contracts, bootstrap authority, and workload receipts are secret-free and identity-bound. Raw provider credentials retain their existing process and gateway boundaries. | Preparation and provider failures stop before registration; provider-owned bootstrap rollback and durable recovery own partial activation. Catalog, bootstrap, managed-image activation, and protected-runtime tests cover the shipped Docker-driver path. Native Podman remains outside the production provider registry and supported surface. | +| **Rebuild or installer-driven upgrade** — `rebuildSandbox` in `rebuild-pipeline.ts`; `upgradeSandboxes` | Registry state is authoritative for the target configuration, while the current OpenShell sandbox is authoritative for policy and live workspace state. A matching session may fill guarded legacy gaps only when its selection agrees; an unrelated/global session is never used. Ambient provider/model selection is quarantined by `isolateAmbientRecreateEnv`. Legacy and custom-image rebuilds retain and fingerprint a prepared build context. Managed-image rebuilds instead stage an immutable image and startup-profile handoff, skip Dockerfile image preflight, and revalidate provider-bound workload authority before each deletion boundary. At replacement create, the live OpenShell policy is the base for one ephemeral handoff that adds missing current-image baseline fields. Existing values, network keys, and same-name live network entries win, so the handoff does not overwrite host choices. | Consent persistence, target-gateway selection/recovery, and target-preflight registry updates can precede disposable image build/probes. Backup plus live-policy capture is the first durable recovery checkpoint. A missing live sandbox stops with clean replacement guidance before Shields, MCP, NIM, registry, or sandbox mutation. Shields unlock, MCP detach/scrub, and NIM stop are destructive in-place effects before the **sandbox delete** boundary. Legacy and custom-image paths recheck prepared context and mutation-edge conditions before delete. Managed-image paths revalidate the exact provider-bound handoff before delete. | Durable checkpoints are the backup/recovery manifest and the rewritten recreate session. The ephemeral replacement policy input is removed after create and is never stored as desired policy. A transaction-bound marker inside the backup lets an accepted replacement resume restore and post-restore before the journal clears. Rollback receipts/snapshots are process-local. Credential metadata comes from the target or guarded fallback; raw credentials/providers are checked against current process/gateway state, while prepared installer recovery may reconstruct a missing gateway provider from a validated host credential. | In-process rollback best-effort restores registry/MCP retry metadata. The inner onboarding consumes the exact managed-workload handoff or selects the legacy resource profile after deletion. No post-create policy receipt or containment gate exists; OpenShell owns the resulting policy lifecycle. Covered by rebuild, managed-workload authority, image-preflight, DCode, messaging, and accepted-replacement recovery tests. Gaps: health-before-delete and atomic swap. | +| **Stock managed-image onboarding through registered runtime providers** — managed-workload selection in `onboard-orchestration.ts` | Ordinary onboarding through the default Docker provider or explicitly selected native Podman provider validates one complete all-agent catalog, immutable release and platform contracts, and selected-provider capabilities before selecting OpenClaw, Hermes, or LangChain Deep Agents Code. Portable onboarding, non-managed agents, and explicit `--from` custom images retain their separate legacy or custom workload paths. | The managed path skips Dockerfile build materialization, creates provider-bound bootstrap authority for the immutable image and startup profile, launches that workload, and registers the managed-workload receipt only after readiness. Native Podman rejects legacy and custom Dockerfile workloads. | Catalog contracts, bootstrap authority, and workload receipts are secret-free and identity-bound. Raw provider credentials retain their existing process and gateway boundaries. | Preparation and provider failures stop before registration; provider-owned bootstrap rollback and durable recovery own partial activation. Catalog, bootstrap, managed-image activation, native Podman, and protected-runtime tests cover the registered Docker and Podman paths. | | **Managed snapshot clone handoff and provider transaction (internal and dormant)** — `prepareManagedWorkloadCloneHandoff`; `prepareManagedCloneProviderTransaction` | The current source registry row owns mutable operator intent; the selected snapshot owns immutable managed-workload and provider-runtime history. Handoff preparation proves the selected runtime provider and its `clone` capability, exact current registry generation and live-identity fingerprint, snapshot/source workload equivalence, snapshot runtime generation, and the state layer's selected-manifest/payload digest. It rebinds the secret-free startup profile, messaging intent, dashboard identity, and provider-owned contributions for OpenClaw, Hermes, or DCode without a central Podman-specific switch. Provider preparation then resolves active application bindings plus provider-contributed bindings, treating a live exact provider as reusable only when the destination registry independently proves that same logical binding. | The handoff and provider plan are inert. The internal materializer can create only bindings proven absent at preflight; it never updates or deletes an existing destination-owned provider. Immediately before each create it revalidates the source and optional destination registry rows plus the exact `SnapshotRestoreAuthority`. Production snapshot restore does not invoke this transaction and continues to reject cross-sandbox managed-image restore through `rejectManagedSnapshotCloneUntilRebind`; no user-visible clone support is advertised. | Both plans are deeply frozen and secret-free. A successful create produces an exact process-local ownership receipt; a non-zero create reconciled to an exact provider remains ambiguous and unowned. The receipt ledger remembers completed cleanup so a repeated cleanup cannot delete a later same-name provider. Raw credentials exist only in the explicit apply environment and one OpenShell child environment. | Failure rolls back only providers confirmed created by the exact in-process receipt, preserves collisions and ambiguous creates, reports incomplete cleanup for retry, and never rewrites a reused provider. `src/lib/onboard/managed-workload-clone-handoff.test.ts`, `src/lib/onboard/managed-startup-clone-rebinder.test.ts`, and `src/lib/actions/sandbox/snapshot-managed-clone-handoff-dormancy.test.ts` cover the all-agent, Docker/MXC-style provider, canonical-name, and fail-closed boundaries; provider transaction tests cover race, force-replace, disappearing-credential, rollback, and idempotent cleanup. This PR intentionally covers only the dormant contract. Epic [#7744](https://github.com/NVIDIA/NemoClaw/issues/7744) tracks destination creation/bootstrap, filesystem mutation-edge invocation, Hermes broker activation, durable recovery, protected E2E, and user-visible activation. | -| **Channel add/remove/start/stop** — `addSandboxChannel`, `removeSandboxChannel`, `sandboxChannelsSetEnabled` in `policy-channel.ts` | Add compiles and merges a manifest-derived channel delta with `MessagingWorkflowPlanner`. Start, stop, and remove transform the registry plan and rehydrate executable render/build/runtime/forward details from current manifests. | Token-backed add can mutate gateway credentials before policy and plan persistence; QR/in-sandbox-auth add skips that credential upsert. Start persists the enabled plan before policy; stop persists the disabled plan before the rebuild prompt. Remove clears QR-backed durable state when applicable, detaches gateway/bridge state, removes policy, then persists the plan. During `channels remove`, OpenClaw WeChat can clear its manifest-declared legacy state and current plugin account state from an identity-pinned stopped Docker volume after normal cleanup fails. If that cleanup fails, the command stops before policy and plan teardown. A queued rebuild has a separate delete boundary. | The compact registry messaging plan is authoritative; render/build/runtime/state/health entries and nested host-forward details are rehydrated rather than persisted. Channel mutations persist the registry plan but do not rewrite `Session.messagingPlan` or matching-session `policyPresets`. Raw tokens stay in process/gateway bindings. | `rollbackChannelAdd`, re-disable after failed start, and fail-closed QR-state cleanup provide partial compensation. Covered by `policy-channel*.test.ts`, `workflow-planner.test.ts`, and channel integration tests. Gaps: channel add has a separate `--force` conflict policy; add/remove effects can precede plan persistence, and persistence failures are not fully rolled back. | +| **Channel add/remove/start/stop** — `addSandboxChannel`, `removeSandboxChannel`, `sandboxChannelsSetEnabled` in `policy-channel.ts` | Add compiles and merges a manifest-derived channel delta with `MessagingWorkflowPlanner`. Start, stop, and remove transform the registry plan and rehydrate executable render/build/runtime/forward details from current manifests. | Token-backed add can mutate gateway credentials before policy and plan persistence; QR/in-sandbox-auth add skips that credential upsert. Start persists the enabled plan before policy; stop persists the disabled plan before the rebuild prompt. Remove clears QR-backed durable state when applicable, detaches gateway/bridge state, removes policy, then persists the plan. During `channels remove`, OpenClaw WeChat can clear its manifest-declared legacy state and current plugin account state from an identity-pinned provider-owned stopped-state resource after normal cleanup fails. If that cleanup fails, the command stops before policy and plan teardown. A queued rebuild has a separate delete boundary. | The compact registry messaging plan is authoritative; render/build/runtime/state/health entries and nested host-forward details are rehydrated rather than persisted. Channel mutations persist the registry plan but do not rewrite `Session.messagingPlan` or matching-session `policyPresets`. Raw tokens stay in process/gateway bindings. | `rollbackChannelAdd`, re-disable after failed start, and fail-closed QR-state cleanup provide partial compensation. Covered by `policy-channel*.test.ts`, `workflow-planner.test.ts`, and channel integration tests. Gaps: channel add has a separate `--force` conflict policy; add/remove effects can precede plan persistence, and persistence failures are not fully rolled back. | | **Provider, model, or credential-binding change** — `runInferenceSet` | CLI intent plus registry/session metadata. Target resolution and OpenShell preparation occur before locking. The target is re-resolved in the mutating phase under the sandbox lifecycle and timer-bound shields locks; that phase validates provider/model syntax, selected agent, shields state, and local reachability before the first write. | First mutation is the gateway route, then a minimal registry write, API-family/config resolution, registry refresh, best-effort config/hash sync, matching-session update, and audit. An OpenClaw API-family change can then restart the managed gateway after the shields lock is released but while the outer sandbox lock remains held. No sandbox deletion. | Registry and matching session store logical provider/model/credential-environment metadata. Audit records the action, sandbox, and reason rather than credentials; raw values remain gateway-bound. | Forward-only; no rollback. `rebuild` is the repair path for degraded state. Covered by `inference-set*.test.ts`. Gap: several stores can diverge after a mid-sequence failure. | | **Credential rotation** — `configRotateToken` in `src/lib/sandbox/config.ts`; `rotateSandboxToken` in `src/lib/sandbox/config-rotate-token.ts` | A session with `credentialEnv` selects the provider and binding. A non-null different `sandboxName` is rejected, but a legacy/null session name is accepted for the requested sandbox. The new value comes from a named environment variable, stdin, or a secret prompt; it is trimmed, then rejected when empty or still containing internal whitespace. | An OpenAI provider profile is validated before credential staging. When that profile is missing, its import is the first external mutation. `saveCredential` then stages the value in the current process. OpenShell provider update follows, with provider create as a fallback; audit is last. Other provider types begin with `saveCredential`. No sandbox deletion. | The logical binding is unchanged, so session and registry are not rewritten. The raw value exists only in process memory/environment and the gateway provider; audit records action/sandbox/reason without the value. | Profile validation or import failure stops credential staging and provider mutation. No rollback follows a successful profile import or provider update; an audit failure can report failure after the credential is already active. Covered by `test/security/config-rotate-token-provider-profile.test.ts` and the rotate-token case in `test/security/config-set-nested-ssrf.test.ts`. Gap: a null-name legacy session is not strongly bound to the requested sandbox. | | **Config, policy, resource, port-forward, and runtime setup contributions** — `configSet`; `prepareInitialSandboxCreatePolicy`; `selectResourceProfileForSandbox`; manifest compiler/runtime appliers; dashboard and channel forward helpers | Config uses validated dotpaths and SSRF-safe URL rewriting. Create/rebuild contributions are assembled by `sandbox-create-plan.ts` and `MessagingWorkflowPlanner`: policy presets/keys, resource flags, package/build steps, `hostForward`, runtime node preloads, env aliases, and secret scans. | Config’s first effect is a compare-and-swap sandbox write. Build-time contributions inherit the enclosing create/recreate boundary. Forward helpers can stop an existing forward and start its replacement in place after readiness, without recreating the sandbox. | Durable owners are compact registry messaging/policy/inference metadata, current manifests used for plan rehydration, onboard session, sandbox config/hash, gateway provider state, and shields audit. An interrupted onboarding session records the selected resource values or an explicit OpenShell-default choice; the resolved create intent remains process-local. Logical bindings are serializable; raw provider values are not. | CAS rejects stale config writes; OpenClaw/Hermes commit config and integrity hashes together, while other agents may refresh a path hash afterward. Audit and optional restart are post-commit and forward-only. Forward recovery can re-establish declared forwards. Gaps: no cross-contribution effect transaction/checkpoint. | @@ -204,7 +204,7 @@ The raw state layer still rejects a managed manifest unless both content authori Cross-provider clone and rebind, durable interrupted-restore recovery, and provider expansion remain separately reviewable units tracked by the [incremental runtime epic](https://github.com/NVIDIA/NemoClaw/issues/7744). If provider proof fails after filesystem restoration, NemoClaw reports that state changed and requires the operator to retry the same selected snapshot after the runtime stabilizes. -## Dormant Podman managed-bootstrap authority +## Podman managed-bootstrap authority The Podman candidate owns a separate `managed-bootstrap` command scope bound to one rootless engine authority. Before a bootstrap mutation, it @@ -224,10 +224,12 @@ The image transaction accepts only that prepared authority. It stages one protected root-apply request, starts the replacement, and authenticates the image-owned completion for OpenClaw, Hermes, or LangChain Deep Agents Code. The watcher stays stopped and the journal remains authoritative throughout. -These provider-owned modules remain disconnected from production runtime -selection; persisted post-commit recovery, GPU and local inference, installer -qualification, and supported activation remain later gates in -[`#7744`](https://github.com/NVIDIA/NemoClaw/issues/7744). +The registered native Podman provider consumes this authority when +`NEMOCLAW_GATEWAY_RUNTIME=podman` selects it for standard managed-image +onboarding. Persisted post-commit recovery, provider-owned cleanup, state-root +preparation, and the supported E2E matrix fail closed on ambiguous or changed +authority. The portable experimental profile remains an independent lifecycle +and does not consume this selection. ## Agent-specific differences diff --git a/src/lib/onboard/machine/handlers/preflight.test.ts b/src/lib/onboard/machine/handlers/preflight.test.ts index fad06beedcc..de5cf84b4d0 100644 --- a/src/lib/onboard/machine/handlers/preflight.test.ts +++ b/src/lib/onboard/machine/handlers/preflight.test.ts @@ -225,14 +225,14 @@ describe("handlePreflightState", () => { it("rejects changed gateway ownership before cached resume probe effects (#7411)", async () => { const session = createSession(); session.steps.preflight.status = "complete"; - const assertDockerBridgeAndContainerDnsHealthy = vi.fn(); + const assertRuntimeProviderHealthy = vi.fn(); const detectGpu = vi.fn(() => ({ type: "nvidia" }) as Gpu); const harness = createDeps({ assertGatewayReadiness: vi.fn(async () => { throw new Error("gateway ownership changed"); }), detectGpu, - assertDockerBridgeAndContainerDnsHealthy, + assertRuntimeProviderHealthy, }); await expect( @@ -242,7 +242,7 @@ describe("handlePreflightState", () => { }), ).rejects.toThrow("gateway ownership changed"); expect(detectGpu).not.toHaveBeenCalled(); - expect(assertDockerBridgeAndContainerDnsHealthy).not.toHaveBeenCalled(); + expect(assertRuntimeProviderHealthy).not.toHaveBeenCalled(); }); it("admits live host and gateway facts and presents advisories before a cached resume GPU proof (#7411)", async () => { @@ -271,10 +271,8 @@ describe("handlePreflightState", () => { calls.push("gpu-runtime-proof"); return { type: "nvidia" } as Gpu; }, - validateSandboxGpuPreflight: () => { + assertRuntimeProviderHealthy: () => { calls.push("gpu-validation"); - }, - assertDockerBridgeAndContainerDnsHealthy: () => { calls.push("bridge-dns"); }, }); @@ -425,7 +423,7 @@ describe("handlePreflightState", () => { throw new Error("host observations are stale"); }, detectGpu, - assertDockerBridgeAndContainerDnsHealthy: bridge, + assertRuntimeProviderHealthy: bridge, }); await expect( diff --git a/src/lib/onboard/machine/handlers/preflight.ts b/src/lib/onboard/machine/handlers/preflight.ts index b8066802641..2b96bd96171 100644 --- a/src/lib/onboard/machine/handlers/preflight.ts +++ b/src/lib/onboard/machine/handlers/preflight.ts @@ -65,13 +65,11 @@ export interface PreflightStateOptions< assertGatewayReadiness(): Promise; now?: () => Date; /** - * Resume backstop for #3508/#3630. Runs the same bridge+DNS fatal - * gate that `preflight()` does, so a cached preflight step cannot - * skip the new fatal checks for hosts where Docker bridge networking - * or container DNS is broken. Optional for back-compat with callers - * that haven't been updated yet. + * Resume backstop for #3508/#3630. Runs the selected provider's host, + * bridge, and DNS gate so cached preflight cannot skip live runtime + * readiness checks. Optional for back-compat with older callers. */ - assertDockerBridgeAndContainerDnsHealthy?(host: Host): void; + assertRuntimeProviderHealthy?(host: Host, config: Config): void; resolveSandboxGpuConfig( gpu: Gpu, options: { @@ -207,11 +205,14 @@ export async function handlePreflightState< presentAdvisories: false, }); } - deps.validateSandboxGpuPreflight(resumeSandboxGpuConfig); // Resume backstop for #3508/#3630. Cached preflight does not capture - // host Docker/DNS state, and a session written by an older NemoClaw - // may have skipped the new bridge/DNS fatal checks. - deps.assertDockerBridgeAndContainerDnsHealthy?.(resumeHost); + // live runtime/DNS state, and a session written by an older NemoClaw + // may have skipped the provider-owned checks. + if (deps.assertRuntimeProviderHealthy) { + deps.assertRuntimeProviderHealthy(resumeHost, resumeSandboxGpuConfig); + } else { + deps.validateSandboxGpuPreflight(resumeSandboxGpuConfig); + } } else { await deps.startRecordedStep("preflight"); gpu = await withPreflightTrace(() => deps.runPreflight({ optedOutGpuPassthrough: noGpu })); diff --git a/src/lib/onboard/machine/initial-flow-phases.test.ts b/src/lib/onboard/machine/initial-flow-phases.test.ts index 9a64a56666c..84788071e94 100644 --- a/src/lib/onboard/machine/initial-flow-phases.test.ts +++ b/src/lib/onboard/machine/initial-flow-phases.test.ts @@ -148,7 +148,6 @@ describe("initial onboard flow phases", () => { runPreflight: async () => (preflightFailure ? Promise.reject(preflightFailure) : gpu), assessHost: () => ({}), assertOnboardHostReadiness: vi.fn(), - assertDockerBridgeAndContainerDnsHealthy: vi.fn(), resolveSandboxGpuConfig: config, validateSandboxGpuPreflight: vi.fn(), skippedStepMessage: vi.fn(), @@ -403,9 +402,6 @@ describe("initial onboard flow phases", () => { assertOnboardHostReadiness: vi.fn(() => { calls.push("assert-host-readiness"); }), - assertDockerBridgeAndContainerDnsHealthy: vi.fn(() => { - calls.push("assert-bridge-dns"); - }), resolveSandboxGpuConfig: vi.fn((detectedGpu) => { calls.push("resolve-gpu-config"); return config(detectedGpu); @@ -538,7 +534,6 @@ describe("initial onboard flow phases", () => { "assert-gateway-readiness", "assert-host-readiness", "validate-gpu-preflight", - "assert-bridge-dns", "resolve-gpu-config", "ensure-resume-preflight-port", "commit-agent-transition", diff --git a/src/lib/onboard/machine/runtime-effectful-preflight.test.ts b/src/lib/onboard/machine/runtime-effectful-preflight.test.ts new file mode 100644 index 00000000000..d70228ffb18 --- /dev/null +++ b/src/lib/onboard/machine/runtime-effectful-preflight.test.ts @@ -0,0 +1,190 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { HostAssessment } from "../preflight"; +import type { SandboxGpuConfig } from "../sandbox-gpu-mode"; +import type { + RuntimeProviderBundle, + RuntimeProviderGatewayHostRuntime, +} from "../runtime-provider/contract"; +import { assertConfiguredRuntimeProviderHealthy } from "./runtime-effectful-preflight"; + +const host = { platform: "linux", isWsl: false } as HostAssessment; +const sandboxGpuConfig = { + sandboxGpuEnabled: false, + errors: [], +} as unknown as SandboxGpuConfig; + +function gatewayRuntime( + run: RuntimeProviderGatewayHostRuntime["network"]["run"], + ensureProbeImageCached: RuntimeProviderGatewayHostRuntime["network"]["ensureProbeImageCached"], +): RuntimeProviderGatewayHostRuntime { + return { + providerId: "candidate", + openShellDriver: "candidate", + bindAddress: "127.0.0.1", + grpcHost: "127.0.0.1", + sshGatewayHost: "127.0.0.1", + portCheckHost: "127.0.0.1", + socketPath: null, + requiredServerIpSans: [], + sandboxHostAddress: null, + usesHostGatewayRoute: false, + resourceOwnership: { label: "managed-by", value: "test" }, + gatewayConfig: { + sandboxNamespace: "scoped", + hostGatewayIp: null, + includeSupervisorBin: true, + processOwnership: "scoped-namespace", + }, + network: { + sandboxSourceCidrs: vi.fn(() => ["172.18.0.0/16"]), + inspect: vi.fn(), + usesHostGatewayRoute: vi.fn(() => false), + run, + ensureProbeImageCached, + }, + }; +} + +function providerBundle( + inspectHost: RuntimeProviderBundle["preflightDoctor"]["inspectHost"], + prepareHostRuntime: RuntimeProviderBundle["gateway"]["prepareHostRuntime"], +): RuntimeProviderBundle { + return { + identity: { + contractVersion: 1, + id: "candidate", + displayName: "Candidate Runtime", + }, + preflightDoctor: { + providerId: "candidate", + supported: true, + inspectHost, + validateSandboxGpu: vi.fn(), + preflightLifecycle: vi.fn(() => null), + }, + gateway: { + providerId: "candidate", + supported: true, + launcher: "nemoclaw", + inspectLegacyContainer: false, + prepareHostRuntime, + }, + } as unknown as RuntimeProviderBundle; +} + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe("configured runtime provider effectful preflight", () => { + it("resolves one provider and runs doctor, bridge, and DNS through its gateway network", () => { + vi.spyOn(console, "log").mockImplementation(() => undefined); + vi.spyOn(console, "warn").mockImplementation(() => undefined); + const run = vi + .fn() + .mockReturnValueOnce({ status: 0 }) + .mockReturnValueOnce({ + status: 1, + stdout: + "Server: 10.0.0.1\nAddress: 10.0.0.1:53\n** server can't find test.invalid: NXDOMAIN\n", + }); + const ensureProbeImageCached = vi.fn(() => ({ + ok: true, + alreadyCached: true, + })); + const runtime = gatewayRuntime(run, ensureProbeImageCached); + const inspectHost = vi.fn(() => ({ + group: "Host" as const, + label: "Candidate runtime", + status: "ok" as const, + detail: "ready", + })); + const prepareHostRuntime = vi.fn(() => runtime); + const provider = providerBundle(inspectHost, prepareHostRuntime); + const resolveProvider = vi.fn(() => provider); + + assertConfiguredRuntimeProviderHealthy(host, sandboxGpuConfig, false, process.exit, { + environment: {}, + platform: "linux", + architecture: "x64", + isPortableProfile: () => false, + resolveProvider, + }); + + expect(resolveProvider).toHaveBeenCalledOnce(); + expect(resolveProvider).toHaveBeenCalledWith("linux", "x64", {}); + expect(inspectHost).toHaveBeenCalledOnce(); + expect(provider.preflightDoctor.validateSandboxGpu).toHaveBeenCalledWith( + sandboxGpuConfig, + process.exit, + ); + expect(prepareHostRuntime).toHaveBeenCalledOnce(); + expect(ensureProbeImageCached).toHaveBeenCalledOnce(); + expect(run).toHaveBeenCalledTimes(2); + expect(run.mock.calls[0]?.[0]).toEqual([ + "run", + "--rm", + "--pull=missing", + "--network", + "bridge", + expect.stringContaining("busybox@sha256:"), + "true", + ]); + expect(run.mock.calls[1]?.[0]).toEqual([ + "run", + "--rm", + "--pull=missing", + "--network", + "bridge", + expect.stringContaining("busybox@sha256:"), + "nslookup", + expect.stringMatching(/^nemoclaw-dns-probe-[a-f0-9]+\.invalid$/u), + ]); + }); + + it("preserves the portable profile Docker compatibility preflight", () => { + const assertPortableRuntimeHealthy = vi.fn(); + const resolveProvider = vi.fn(); + + assertConfiguredRuntimeProviderHealthy(host, sandboxGpuConfig, true, process.exit, { + environment: {}, + isPortableProfile: () => true, + assertPortableRuntimeHealthy, + resolveProvider, + }); + + expect(assertPortableRuntimeHealthy).toHaveBeenCalledWith(host, true, process.exit); + expect(resolveProvider).not.toHaveBeenCalled(); + }); + + it("stops on provider doctor failure before preparing gateway networking", () => { + vi.spyOn(console, "error").mockImplementation(() => undefined); + const exitProcess = vi.fn((_code: number): never => { + throw new Error("exit"); + }); + const prepareHostRuntime = vi.fn(); + const provider = providerBundle( + () => ({ + group: "Host", + label: "Candidate runtime", + status: "fail", + detail: "unavailable", + hint: "start it", + }), + prepareHostRuntime, + ); + + expect(() => + assertConfiguredRuntimeProviderHealthy(host, sandboxGpuConfig, false, exitProcess, { + environment: {}, + isPortableProfile: () => false, + resolveProvider: () => provider, + }), + ).toThrow("exit"); + expect(exitProcess).toHaveBeenCalledWith(1); + expect(prepareHostRuntime).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/onboard/machine/runtime-effectful-preflight.ts b/src/lib/onboard/machine/runtime-effectful-preflight.ts new file mode 100644 index 00000000000..caedddadbc3 --- /dev/null +++ b/src/lib/onboard/machine/runtime-effectful-preflight.ts @@ -0,0 +1,205 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + assertDockerBridgeAndContainerDnsHealthy, + assertHostDnsHealthy, +} from "../bridge-dns-preflight"; +import { isPortableExperimentalProfile } from "../docker-driver-platform"; +import { + BUSYBOX_PROBE_IMAGE, + dnsProbeName, + type DockerBridgeContainerStartProbeResult, + type DnsProbeResult, + type HostAssessment, + isFatalContainerDnsProbeFailure, + probeContainerDns, + probeDockerBridgeContainerStart, +} from "../preflight"; +import type { + RuntimeProviderBundle, + RuntimeProviderDoctorCheck, + RuntimeProviderGatewayHostRuntime, +} from "../runtime-provider/contract"; +import { resolveConfiguredRuntimeProvider } from "../runtime-provider/selection"; +import type { SandboxGpuConfig } from "../sandbox-gpu-mode"; +import { validateSandboxGpuPreflight } from "../sandbox-gpu-preflight"; + +const RUNTIME_NETWORK_PROBE_TIMEOUT_MS = 20_000; + +type ExitProcess = (code: number) => never; +type RuntimeProviderResolver = ( + platform?: NodeJS.Platform, + architecture?: NodeJS.Architecture, + environment?: NodeJS.ProcessEnv, +) => RuntimeProviderBundle; + +export interface RuntimeEffectfulPreflightDependencies { + environment?: NodeJS.ProcessEnv; + platform?: NodeJS.Platform; + architecture?: NodeJS.Architecture; + isPortableProfile?: typeof isPortableExperimentalProfile; + resolveProvider?: RuntimeProviderResolver; + assertPortableRuntimeHealthy?: typeof assertDockerBridgeAndContainerDnsHealthy; + validatePortableSandboxGpuPreflight?: typeof validateSandboxGpuPreflight; +} + +export function bindConfiguredRuntimeProviderHealth( + isNonInteractive: () => boolean, +): (host: HostAssessment, sandboxGpuConfig: SandboxGpuConfig) => void { + return (host, sandboxGpuConfig) => + assertConfiguredRuntimeProviderHealthy(host, sandboxGpuConfig, isNonInteractive()); +} + +function printProbeDetails(details: string | undefined, output: typeof console.warn): void { + if (!details) return; + for (const line of details.split("\n").slice(-4)) { + if (line.trim()) output(` ${line.trim()}`); + } +} + +function presentProviderDoctor(check: RuntimeProviderDoctorCheck, exitProcess: ExitProcess): void { + if (check.status === "fail") { + console.error(` ✗ ${check.label}: ${check.detail}`); + if (check.hint) console.error(` ${check.hint}`); + exitProcess(1); + } + if (check.status === "warn") { + console.warn(` ⚠ ${check.label}: ${check.detail}`); + if (check.hint) console.warn(` ${check.hint}`); + return; + } + const marker = check.status === "ok" ? "✓" : "ⓘ"; + console.log(` ${marker} ${check.label}: ${check.detail}`); +} + +function probeProviderBridge( + runtime: RuntimeProviderGatewayHostRuntime, + cached: ReturnType, +): DockerBridgeContainerStartProbeResult { + return probeDockerBridgeContainerStart({ + command: ["run", "--rm", "--pull=missing", "--network", "bridge", BUSYBOX_PROBE_IMAGE, "true"], + ensureImageCachedOverride: cached, + runProbeImpl: (command, options) => + runtime.network.run(command, options?.timeout ?? RUNTIME_NETWORK_PROBE_TIMEOUT_MS), + }); +} + +function probeProviderDns( + runtime: RuntimeProviderGatewayHostRuntime, + cached: ReturnType, +): DnsProbeResult { + const probeName = dnsProbeName(); + return probeContainerDns({ + probeName, + command: [ + "run", + "--rm", + "--pull=missing", + "--network", + "bridge", + BUSYBOX_PROBE_IMAGE, + "nslookup", + probeName, + ], + ensureImageCachedOverride: cached, + runProbeImpl: (command, options) => + runtime.network.run(command, options?.timeout ?? RUNTIME_NETWORK_PROBE_TIMEOUT_MS), + }); +} + +function assertProviderNetworkHealthy( + providerDisplayName: string, + runtime: RuntimeProviderGatewayHostRuntime, + host: HostAssessment, + nonInteractive: boolean, + exitProcess: ExitProcess, +): void { + const cached = runtime.network.ensureProbeImageCached(BUSYBOX_PROBE_IMAGE); + const bridgeStart = probeProviderBridge(runtime, cached); + if (bridgeStart.ok) { + console.log(` ✓ ${providerDisplayName} can start bridge containers`); + } else if ( + bridgeStart.reason === "veth_unsupported" || + bridgeStart.reason === "timeout" || + bridgeStart.reason === "killed" || + bridgeStart.reason === "docker_daemon_unreachable" + ) { + console.error(` ✗ ${providerDisplayName} could not start a bridge-network test container.`); + printProbeDetails(bridgeStart.details, console.error); + console.error(` Verify ${providerDisplayName} bridge networking and retry onboarding.`); + exitProcess(1); + } else { + console.warn( + ` ⚠ ${providerDisplayName} bridge container start probe inconclusive (reason: ${bridgeStart.reason ?? "unknown"}).`, + ); + printProbeDetails(bridgeStart.details, console.warn); + console.warn(" Continuing to DNS probe for more specific diagnosis."); + } + + assertHostDnsHealthy(host, { nonInteractive, exit: exitProcess }); + + const dns = probeProviderDns(runtime, cached); + if (dns.ok) { + console.log(` ✓ ${providerDisplayName} container DNS resolution works`); + return; + } + if (!isFatalContainerDnsProbeFailure(dns)) { + console.warn( + ` ⚠ ${providerDisplayName} container DNS probe inconclusive (reason: ${dns.reason ?? "unknown"}).`, + ); + printProbeDetails(dns.details, console.warn); + console.warn(" Proceeding; a later sandbox operation will surface a definitive failure."); + return; + } + + console.error(` ✗ ${providerDisplayName} container DNS resolution failed.`); + printProbeDetails(dns.details, console.error); + console.error( + ` Verify ${providerDisplayName} bridge networking and container DNS, then retry onboarding.`, + ); + exitProcess(1); +} + +/** + * Resolve the configured native provider once, then execute its owned host and + * network preflight surfaces. The portable compatibility profile deliberately + * retains its existing Docker-specific preflight path. + */ +export function assertConfiguredRuntimeProviderHealthy( + host: HostAssessment, + sandboxGpuConfig: SandboxGpuConfig, + nonInteractive = false, + exitProcess: ExitProcess = (code) => process.exit(code), + dependencies: RuntimeEffectfulPreflightDependencies = {}, +): void { + const environment = dependencies.environment ?? process.env; + const isPortableProfile = dependencies.isPortableProfile ?? isPortableExperimentalProfile; + if (isPortableProfile(environment)) { + const validatePortableSandboxGpuPreflight = + dependencies.validatePortableSandboxGpuPreflight ?? validateSandboxGpuPreflight; + validatePortableSandboxGpuPreflight(sandboxGpuConfig, {}, exitProcess); + const assertPortableRuntimeHealthy = + dependencies.assertPortableRuntimeHealthy ?? assertDockerBridgeAndContainerDnsHealthy; + assertPortableRuntimeHealthy(host, nonInteractive, exitProcess); + return; + } + + const platform = dependencies.platform ?? process.platform; + const architecture = dependencies.architecture ?? process.arch; + const resolveProvider = dependencies.resolveProvider ?? resolveConfiguredRuntimeProvider; + const provider = resolveProvider(platform, architecture, environment); + presentProviderDoctor(provider.preflightDoctor.inspectHost(), exitProcess); + provider.preflightDoctor.validateSandboxGpu(sandboxGpuConfig, exitProcess); + const runtime = provider.gateway.prepareHostRuntime({ + environment, + platform, + }); + assertProviderNetworkHealthy( + provider.identity.displayName, + runtime, + host, + nonInteractive, + exitProcess, + ); +} diff --git a/src/lib/onboard/managed-bootstrap/adapter.test.ts b/src/lib/onboard/managed-bootstrap/adapter.test.ts index ba60c389078..f842d9e4cb8 100644 --- a/src/lib/onboard/managed-bootstrap/adapter.test.ts +++ b/src/lib/onboard/managed-bootstrap/adapter.test.ts @@ -65,6 +65,7 @@ function planFor(request: ReturnType) { }, profile: { agent: request.agent, fingerprint: request.profileFingerprint }, agentIdentity: { uid: 1000, gid: 1000, workdir: "/sandbox" }, + managedStateRoots: [], intendedWorkloadArgv: ["env", "A=1", "/usr/local/bin/nemoclaw-start"], expectedSupervisorArgv: ["/runtime/sandbox-supervisor", "supervise", "--foreground"], metadata: { "nemoclaw.ai/managed-profile": request.profileFingerprint }, @@ -1061,6 +1062,7 @@ describe("managed bootstrap adapter contract", () => { : { sandboxName, sandboxId: `mxc-${sandboxName}`, driverId: "mxc" }, bootstrapIdentity, code: "provider-owned-retry", + blockingScope: "sandbox" as const, retryable: true, detail: "opaque MXC recovery evidence", }); @@ -1103,6 +1105,7 @@ describe("managed bootstrap adapter contract", () => { sandbox: receipt.sandbox, bootstrapIdentity: IDENTITY, code: "retry", + blockingScope: "sandbox", retryable: true, detail: "retained", }, @@ -1123,6 +1126,7 @@ describe("managed bootstrap adapter contract", () => { sandbox: null, bootstrapIdentity: IDENTITY, code: "provider-owned-retry", + blockingScope: "sandbox", retryable: true, detail: "opaque MXC recovery evidence", } as const; @@ -1136,8 +1140,12 @@ describe("managed bootstrap adapter contract", () => { ); }); - it.each([{ scenario: "same-name failure" }, { scenario: "unknown-identity failure" }])( - "blocks same-name and identity-unknown failures while warning for unrelated sandboxes [$scenario]", + it.each([ + { scenario: "same-name failure" }, + { scenario: "unknown-identity failure" }, + { scenario: "provider-wide failure" }, + ])( + "blocks same-name, identity-unknown, and provider-wide failures while warning for unrelated sandboxes [$scenario]", ({ scenario }) => { const failure = (bootstrapIdentity: string, sandboxName: string | null) => Object.freeze({ @@ -1150,6 +1158,7 @@ describe("managed bootstrap adapter contract", () => { : Object.freeze({ sandboxName, sandboxId: `mxc-${sandboxName}`, driverId: "mxc" }), bootstrapIdentity, code: "provider-owned-retry", + blockingScope: "sandbox", retryable: true, detail: "opaque provider detail", }); @@ -1157,6 +1166,10 @@ describe("managed bootstrap adapter contract", () => { const unrelated = failure("a".repeat(64), "bravo"); const sameName = failure("b".repeat(64), "alpha"); const identityUnknown = failure("c".repeat(64), null); + const providerWide = Object.freeze({ + ...failure("d".repeat(64), "bravo"), + blockingScope: "provider" as const, + }); expect( enforceManagedBootstrapRecoveryForSandbox( @@ -1168,7 +1181,11 @@ describe("managed bootstrap adapter contract", () => { expect(warn).toHaveBeenCalledWith(expect.stringContaining("unrelated sandbox 'bravo'")); const blocking = ( - { "same-name failure": sameName, "unknown-identity failure": identityUnknown } as const + { + "same-name failure": sameName, + "unknown-identity failure": identityUnknown, + "provider-wide failure": providerWide, + } as const )[scenario]!; expect(() => enforceManagedBootstrapRecoveryForSandbox( diff --git a/src/lib/onboard/managed-bootstrap/adapter.ts b/src/lib/onboard/managed-bootstrap/adapter.ts index 9517d65ae44..3a315c08e2d 100644 --- a/src/lib/onboard/managed-bootstrap/adapter.ts +++ b/src/lib/onboard/managed-bootstrap/adapter.ts @@ -9,6 +9,7 @@ import { MANAGED_STARTUP_HOLD_EXECUTABLE, } from "../managed-startup/hold"; import type { ManagedStartupAgent } from "../managed-startup/profile"; +import type { ManagedStartupStateRoot } from "../managed-startup/state-roots"; import { type ManagedStartupRootApplyRequest, parseManagedStartupRootApplyRequest, @@ -65,6 +66,7 @@ export interface ManagedBootstrapExpectedPlan { readonly fingerprint: string; }; readonly agentIdentity: ManagedBootstrapAgentIdentity; + readonly managedStateRoots: readonly ManagedStartupStateRoot[]; readonly intendedWorkloadArgv: readonly string[]; readonly expectedSupervisorArgv: readonly string[]; readonly metadata: Readonly>; @@ -270,6 +272,8 @@ export interface ManagedBootstrapRecoveryFailure { readonly bootstrapIdentity: string; /** Provider-owned diagnostic code. Central orchestration must not branch on this value. */ readonly code: string; + /** Provider-wide when recovery may still own shared provider authority. */ + readonly blockingScope: "provider" | "sandbox"; readonly retryable: boolean; readonly detail: string; } @@ -511,6 +515,7 @@ function normalizeRecoveryFailure( candidate === null || Array.isArray(candidate) || candidate.schemaVersion !== MANAGED_BOOTSTRAP_SCHEMA_VERSION || + (candidate.blockingScope !== "provider" && candidate.blockingScope !== "sandbox") || typeof candidate.retryable !== "boolean" ) { protocolFail("recovery failure has an invalid schema"); @@ -537,6 +542,7 @@ function normalizeRecoveryFailure( sandbox: candidate.sandbox === null ? null : Object.freeze({ ...candidate.sandbox }), bootstrapIdentity: candidate.bootstrapIdentity, code: candidate.code, + blockingScope: candidate.blockingScope, retryable: candidate.retryable, detail: candidate.detail, }); @@ -578,7 +584,7 @@ export async function recoverManagedBootstrapTransactions( }); } -/** Block only failures that can own the requested name; warn for exact unrelated sandboxes. */ +/** Block failures that own the requested name or retain provider-wide shared authority. */ export function enforceManagedBootstrapRecoveryForSandbox( report: ManagedBootstrapRecoveryReport, sandboxName: string, @@ -586,10 +592,18 @@ export function enforceManagedBootstrapRecoveryForSandbox( ): ManagedBootstrapRecoveryReport { assertOpaqueString(sandboxName, "recovery target sandbox name"); const blocking = report.failures.filter( - (failure) => failure.sandbox === null || failure.sandbox.sandboxName === sandboxName, + (failure) => + failure.blockingScope === "provider" || + failure.sandbox === null || + failure.sandbox.sandboxName === sandboxName, ); for (const failure of report.failures) { - if (failure.sandbox === null || failure.sandbox.sandboxName === sandboxName) continue; + if ( + failure.blockingScope === "provider" || + failure.sandbox === null || + failure.sandbox.sandboxName === sandboxName + ) + continue; warn( `Managed bootstrap recovery retained unrelated sandbox '${failure.sandbox.sandboxName}' ` + `(${failure.bootstrapIdentity}, ${failure.code}).`, @@ -768,11 +782,66 @@ function assertExpectedPlan( protocolFail("planned profile does not match the root application request"); } assertAgentIdentity(plan.agentIdentity); + assertManagedStateRoots(plan.managedStateRoots); assertArgv(plan.intendedWorkloadArgv, "intended workload"); assertArgv(plan.expectedSupervisorArgv, "expected supervisor"); assertMetadata(plan.metadata); } +function assertManagedStateRoots(roots: readonly ManagedStartupStateRoot[]): void { + if (!Array.isArray(roots)) protocolFail("managed state roots must be one exact list"); + const targets = new Set(); + const resources = new Set(); + for (const root of roots) { + if ( + typeof root !== "object" || + root === null || + Array.isArray(root) || + typeof root.mountTarget !== "string" || + !root.mountTarget.startsWith("/") || + root.mountTarget === "/" || + root.mountTarget.includes("\0") || + typeof root.resourceIdentity !== "string" || + root.resourceIdentity.length === 0 || + root.resourceIdentity.includes("\0") || + targets.has(root.mountTarget) || + resources.has(root.resourceIdentity) || + typeof root.ownershipLabels !== "object" || + root.ownershipLabels === null || + Array.isArray(root.ownershipLabels) || + Object.entries(root.ownershipLabels).some( + ([name, value]) => name.length === 0 || typeof value !== "string", + ) || + !Number.isSafeInteger(root.uid) || + root.uid < 0 || + !Number.isSafeInteger(root.gid) || + root.gid < 0 || + !Number.isSafeInteger(root.mode) || + root.mode < 0 || + root.mode > 0o7777 || + typeof root.readWrite !== "boolean" + ) { + protocolFail("managed state-root declaration is invalid"); + } + targets.add(root.mountTarget); + resources.add(root.resourceIdentity); + } +} + +function freezeManagedStateRoots( + roots: readonly ManagedStartupStateRoot[], +): readonly ManagedStartupStateRoot[] { + assertManagedStateRoots(roots); + return Object.freeze( + roots.map((root) => + Object.freeze({ + ...root, + ownershipLabels: freezeMetadata(root.ownershipLabels), + }), + ), + ); +} + function freezeArgv(argv: readonly string[], label: string): readonly string[] { assertArgv(argv, label); return Object.freeze([...argv]); @@ -822,6 +891,7 @@ function normalizeExpectedPlan( gid: plan.agentIdentity.gid, workdir: plan.agentIdentity.workdir, }), + managedStateRoots: freezeManagedStateRoots(plan.managedStateRoots), intendedWorkloadArgv: freezeArgv(plan.intendedWorkloadArgv, "intended workload"), expectedSupervisorArgv: freezeArgv(plan.expectedSupervisorArgv, "expected supervisor"), metadata: freezeMetadata(plan.metadata), diff --git a/src/lib/onboard/managed-bootstrap/docker-recovery.test.ts b/src/lib/onboard/managed-bootstrap/docker-recovery.test.ts index 2d6fb3c1110..7097aee35ff 100644 --- a/src/lib/onboard/managed-bootstrap/docker-recovery.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker-recovery.test.ts @@ -93,6 +93,7 @@ describe("Docker managed bootstrap restart recovery", () => { sourcePhase: null, sandbox: authority().handle.sandbox, code: "legacy-agent-required", + blockingScope: "sandbox", retryable: true, detail: expect.stringContaining(OLD_ID), }, @@ -155,6 +156,7 @@ describe("Docker managed bootstrap restart recovery", () => { bootstrapIdentity: IDENTITY, sourcePhase: "owner-cleanup-required", code: "owner-cleanup-required", + blockingScope: "sandbox", retryable: true, }, ], @@ -282,6 +284,7 @@ describe("Docker managed bootstrap restart recovery", () => { { sourcePhase: "owner-cleanup-required", code: "commit-state-indeterminate", + blockingScope: "sandbox", retryable: true, }, ], @@ -339,6 +342,7 @@ describe("Docker managed bootstrap restart recovery", () => { { sourcePhase: "shared-state-committed", code: "durable-cleanup-pending", + blockingScope: "sandbox", retryable: true, }, ], diff --git a/src/lib/onboard/managed-bootstrap/docker-runtime.test.ts b/src/lib/onboard/managed-bootstrap/docker-runtime.test.ts index 319dbef2bb5..48a3ddf9ee2 100644 --- a/src/lib/onboard/managed-bootstrap/docker-runtime.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker-runtime.test.ts @@ -94,11 +94,18 @@ function compatibilityLifecycleInput( temporaryStateRoots.push(stateRoot); return { providerId: "docker", + environment: {}, stateRoot, bootstrapIdentity: IDENTITY, request: seed.request, image: seed.plan.image, agentIdentity: seed.plan.agentIdentity, + workspaceRoot: { + uid: seed.plan.agentIdentity.uid, + gid: seed.plan.agentIdentity.gid, + mode: 0o755, + }, + managedStateRoots: seed.plan.managedStateRoots, intendedWorkloadArgv: seed.plan.intendedWorkloadArgv, expectedSupervisorArgv: seed.plan.expectedSupervisorArgv, launchArgv: ["openshell", "sandbox", "create", "--name", "alpha"], @@ -127,6 +134,7 @@ function compatibilityLifecycleInput( inferenceProvider: "openai", gatewayUsesContainerBridge: true, gatewayPort: 8080, + reverifyBridgeReachability: vi.fn(), }, dependencies, }; @@ -514,11 +522,18 @@ describe("Docker managed-bootstrap lifecycle composition", () => { }); const lifecycle = createDockerManagedBootstrapSurface().createLifecycle({ providerId: "docker", + environment: {}, stateRoot, bootstrapIdentity: IDENTITY, request: seed.request, image: seed.plan.image, agentIdentity: seed.plan.agentIdentity, + workspaceRoot: { + uid: seed.plan.agentIdentity.uid, + gid: seed.plan.agentIdentity.gid, + mode: 0o755, + }, + managedStateRoots: seed.plan.managedStateRoots, intendedWorkloadArgv: seed.plan.intendedWorkloadArgv, expectedSupervisorArgv: seed.plan.expectedSupervisorArgv, launchArgv: ["openshell", "sandbox", "create", "--name", "alpha"], @@ -544,6 +559,7 @@ describe("Docker managed-bootstrap lifecycle composition", () => { inferenceProvider: "openai", gatewayUsesContainerBridge: false, gatewayPort: 0, + reverifyBridgeReachability: () => undefined, }, dependencies: {}, }); @@ -612,11 +628,18 @@ describe("Docker managed-bootstrap lifecycle composition", () => { }); const lifecycle = createDockerManagedBootstrapSurface().createLifecycle({ providerId: "docker", + environment: {}, stateRoot, bootstrapIdentity: IDENTITY, request: seed.request, image: seed.plan.image, agentIdentity: seed.plan.agentIdentity, + workspaceRoot: { + uid: seed.plan.agentIdentity.uid, + gid: seed.plan.agentIdentity.gid, + mode: 0o755, + }, + managedStateRoots: seed.plan.managedStateRoots, intendedWorkloadArgv: seed.plan.intendedWorkloadArgv, expectedSupervisorArgv: seed.plan.expectedSupervisorArgv, launchArgv: ["openshell", "sandbox", "create", "--name", "alpha"], @@ -643,6 +666,7 @@ describe("Docker managed-bootstrap lifecycle composition", () => { inferenceProvider: "openai", gatewayUsesContainerBridge: false, gatewayPort: 0, + reverifyBridgeReachability: () => undefined, }, dependencies: {}, }); diff --git a/src/lib/onboard/managed-bootstrap/docker-runtime.ts b/src/lib/onboard/managed-bootstrap/docker-runtime.ts index 61c6433f93d..b4e028e5dc3 100644 --- a/src/lib/onboard/managed-bootstrap/docker-runtime.ts +++ b/src/lib/onboard/managed-bootstrap/docker-runtime.ts @@ -247,6 +247,7 @@ function createDockerLifecycle( fingerprint: input.request.profileFingerprint, }, agentIdentity: input.agentIdentity, + managedStateRoots: input.managedStateRoots, intendedWorkloadArgv: input.intendedWorkloadArgv, expectedSupervisorArgv: input.expectedSupervisorArgv, metadata: {}, @@ -287,6 +288,7 @@ function createDockerLifecycle( selectedRoute: input.route, gatewayPort: input.network.gatewayPort, log: console.log, + reverifyBridgeReachability: input.network.reverifyBridgeReachability, }, ); }, diff --git a/src/lib/onboard/managed-bootstrap/docker-spec.test.ts b/src/lib/onboard/managed-bootstrap/docker-spec.test.ts index 68f47514bca..ff22377a3e6 100644 --- a/src/lib/onboard/managed-bootstrap/docker-spec.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker-spec.test.ts @@ -12,17 +12,77 @@ import { describe("managed bootstrap Docker launch spec", () => { it("hashes reproducible launch state while excluding runtime ID, phase, IP, and gateway", () => { const first = createDockerGpuInspectFixture(); + first.Id = "a".repeat(64); + first.Mounts = [ + { + Type: "image", + Source: "ghcr.io/nvidia/openshell/sandbox:v0.0.106", + Destination: "/opt/openshell/bin", + RW: false, + }, + ]; + first.HostConfig!.Binds = [ + ...first.HostConfig!.Binds!, + "/run/podman/old/merge:/opt/openshell/bin:lowerdir=/run/podman/old/lower,private", + ]; + first.HostConfig!.PidMode = "private"; + first.HostConfig!.PidsLimit = 2048; + first.HostConfig!.CpuPeriod = 100_000; + first.HostConfig!.CpuQuota = 250_000; + first.HostConfig!.NetworkMode = "bridge"; + first.HostConfig!.Annotations = { "io.container.manager": "libpod" }; + first.HostConfig!.OomScoreAdj = 0; + first.HostConfig!.Tmpfs = { + "/run/netns": "rw,nosuid,nodev,rprivate,tmpcopyup", + }; + first.NetworkSettings!.Networks!["openshell-docker"]!.Aliases = [ + first.Id.slice(0, 12), + "openshell-alpha", + ]; const second = structuredClone(first); - second.Id = "another-runtime-id"; + second.Id = "b".repeat(64); + second.HostConfig!.Binds = second.HostConfig!.Binds!.map((bind) => + bind.includes("/opt/openshell/bin") + ? "/run/podman/new/merge:/opt/openshell/bin:lowerdir=/run/podman/new/lower,private" + : bind, + ); Object.assign(second, { State: { Running: false, Dead: true } }); + second.HostConfig!.Annotations = { + ...second.HostConfig!.Annotations, + "io.podman.annotations.pids-limit": "2048", + }; + second.HostConfig!.Tmpfs = {}; + second.HostConfig!.OomScoreAdj = 500; second.NetworkSettings!.Networks!["openshell-docker"]!.IPAddress = "172.18.0.99"; second.NetworkSettings!.Networks!["openshell-docker"]!.Gateway = "172.18.0.254"; + second.NetworkSettings!.Networks!["openshell-docker"]!.Aliases = [ + second.Id.slice(0, 12), + "openshell-alpha", + "openshell-alpha", + ]; const expected = normalizeDockerManagedBootstrapLaunchSpec(first); const observed = normalizeDockerManagedBootstrapLaunchSpec(second); expect(observed.hash).toBe(expected.hash); expect(observed.canonicalJson).toBe(expected.canonicalJson); + expect(expected.spec.inspect.HostConfig?.Binds).not.toContainEqual( + expect.stringContaining("/opt/openshell/bin"), + ); + expect(expected.spec.inspect.HostConfig?.Mounts).toContainEqual({ + Type: "image", + Source: "ghcr.io/nvidia/openshell/sandbox:v0.0.106", + Target: "/opt/openshell/bin", + ReadOnly: true, + }); + expect(expected.spec.inspect.HostConfig?.PidMode).toBe(""); + expect(expected.spec.inspect.HostConfig?.CpuPeriod).toBe(0); + expect(expected.spec.inspect.HostConfig?.CpuQuota).toBe(0); + expect(expected.spec.inspect.HostConfig?.NetworkMode).toBe("openshell-docker"); + expect(expected.spec.inspect.HostConfig?.Tmpfs).toEqual({}); + expect(expected.spec.inspect.NetworkSettings?.Networks?.["openshell-docker"]?.Aliases).toEqual([ + "openshell-alpha", + ]); expect(parseDockerManagedBootstrapLaunchSpec(expected.canonicalJson)).toEqual(expected.spec); }); @@ -30,10 +90,15 @@ describe("managed bootstrap Docker launch spec", () => { const first = createDockerGpuInspectFixture(); const second = structuredClone(first); Object.assign(second.Config!, { StopTimeout: 45 }); + const annotated = structuredClone(first); + annotated.HostConfig!.Annotations = { "io.container.manager": "libpod" }; expect(normalizeDockerManagedBootstrapLaunchSpec(second).hash).not.toBe( normalizeDockerManagedBootstrapLaunchSpec(first).hash, ); + expect(normalizeDockerManagedBootstrapLaunchSpec(annotated).hash).not.toBe( + normalizeDockerManagedBootstrapLaunchSpec(first).hash, + ); }); it("hash-binds Docker-derived console and protected-path defaults", () => { diff --git a/src/lib/onboard/managed-bootstrap/docker-spec.ts b/src/lib/onboard/managed-bootstrap/docker-spec.ts index 9a05ef46abe..e56395d8652 100644 --- a/src/lib/onboard/managed-bootstrap/docker-spec.ts +++ b/src/lib/onboard/managed-bootstrap/docker-spec.ts @@ -34,6 +34,7 @@ const CONFIG_KEYS = new Set([ ]); const HOST_CONFIG_KEYS = new Set([ + "Annotations", "AutoRemove", "Binds", "BlkioDeviceReadBps", @@ -169,6 +170,8 @@ const NULLABLE_HOST_CONFIG_ARRAY_KEYS = [ ] as const; const DOCKER_DEFAULT_TMPFS_OPTIONS = new Set(["noexec", "nosuid", "nodev"]); +const PODMAN_PIDS_LIMIT_ANNOTATION = "io.podman.annotations.pids-limit"; +const PODMAN_RUNTIME_NETNS_TMPFS = "rw,nosuid,nodev,rprivate,tmpcopyup"; // Docker derives ConsoleSize, MaskedPaths, and ReadonlyPaths when it creates a // container. They have no corresponding create flags, but the adapter inspects @@ -231,8 +234,10 @@ function byCodeUnit(left: string, right: string): number { function normalizedNetworkSettings( value: DockerContainerInspect["NetworkSettings"], + runtimeId: string | undefined, ): DockerContainerInspect["NetworkSettings"] { const networks = value?.Networks ?? {}; + const normalizedRuntimeId = String(runtimeId ?? "").trim().toLowerCase(); return { Networks: Object.fromEntries( Object.entries(networks) @@ -240,7 +245,19 @@ function normalizedNetworkSettings( .map(([name, network]) => [ name, { - Aliases: [...(network.Aliases ?? [])].sort(), + Aliases: [ + ...new Set( + (network.Aliases ?? []).filter((alias) => { + const normalizedAlias = alias.trim().toLowerCase(); + return !( + /^[0-9a-f]{64}$/u.test(normalizedRuntimeId) && + /^[0-9a-f]{12,64}$/u.test(normalizedAlias) && + (normalizedRuntimeId.startsWith(normalizedAlias) || + normalizedAlias.startsWith(normalizedRuntimeId)) + ); + }), + ), + ].sort(), }, ]), ), @@ -342,7 +359,36 @@ function normalizedStructuredMounts(value: unknown): unknown { }); } -function normalizedHostConfig(hostConfig: Record): Record { +function normalizedImageMounts(value: DockerContainerInspect["Mounts"]): Array<{ + Type: "image"; + Source: string; + Target: string; + ReadOnly: boolean; +}> { + return (value ?? []) + .filter((mount) => mount.Type === "image") + .map((mount) => { + const source = String(mount.Source ?? "").trim(); + const target = String(mount.Destination ?? "").trim(); + if (!source || !target.startsWith("/") || typeof mount.RW !== "boolean") { + throw new Error("Managed bootstrap Docker image mount is invalid."); + } + return { Type: "image", Source: source, Target: target, ReadOnly: !mount.RW }; + }); +} + +function dockerBindTarget(bind: string): string { + const sourceDelimiter = bind.indexOf(":"); + if (sourceDelimiter < 0) return ""; + const optionsDelimiter = bind.indexOf(":", sourceDelimiter + 1); + return bind.slice(sourceDelimiter + 1, optionsDelimiter < 0 ? undefined : optionsDelimiter); +} + +function normalizedHostConfig( + hostConfig: Record, + imageMounts: ReturnType, + networkSettings: DockerContainerInspect["NetworkSettings"], +): Record { const normalized = { ...hostConfig }; for (const key of NULLABLE_HOST_CONFIG_ARRAY_KEYS) { if (normalized[key] === null) normalized[key] = []; @@ -354,9 +400,89 @@ function normalizedHostConfig(hostConfig: Record): Record) } + : null; + if ( + annotations && + Number.isSafeInteger(normalized.PidsLimit) && + String(annotations[PODMAN_PIDS_LIMIT_ANNOTATION] ?? "") === String(normalized.PidsLimit) + ) { + delete annotations[PODMAN_PIDS_LIMIT_ANNOTATION]; + normalized.Annotations = annotations; + } + // The native Podman provider is rootless. Podman preserves an Engine API + // request of zero while stopped, then rewrites it to the current user's + // effective floor (500) on start. Both values describe that same enforced + // runtime setting; use the stable effective value in the launch contract. + if ( + annotations?.["io.container.manager"] === "libpod" && + normalized.OomScoreAdj === 0 + ) { + normalized.OomScoreAdj = 500; + } + const tmpfs = + typeof normalized.Tmpfs === "object" && + normalized.Tmpfs !== null && + !Array.isArray(normalized.Tmpfs) + ? { ...(normalized.Tmpfs as Record) } + : {}; + if (tmpfs["/run/netns"] === PODMAN_RUNTIME_NETNS_TMPFS) delete tmpfs["/run/netns"]; + normalized.Tmpfs = tmpfs; + const attachedNetworks = Object.keys(networkSettings?.Networks ?? {}); + const configuredNetworkMode = String(normalized.NetworkMode ?? "").trim(); + if ( + attachedNetworks.length === 1 && + ["", "bridge", "default", "podman"].includes(configuredNetworkMode) && + !["bridge", "default", "podman"].includes(attachedNetworks[0]!) + ) { + normalized.NetworkMode = attachedNetworks[0]; + } + // Podman reports its default private PID namespace as `private`, while the + // Docker CLI represents the same default by omitting `--pid` and rejects + // `--pid private`. + if (normalized.PidMode === "private") normalized.PidMode = ""; + if ( + typeof normalized.NanoCpus === "number" && + normalized.NanoCpus > 0 && + ((typeof normalized.CpuPeriod === "number" && normalized.CpuPeriod !== 0) || + (typeof normalized.CpuQuota === "number" && normalized.CpuQuota !== 0)) + ) { + if ( + !Number.isSafeInteger(normalized.NanoCpus) || + !Number.isSafeInteger(normalized.CpuPeriod) || + !Number.isSafeInteger(normalized.CpuQuota) || + (normalized.CpuPeriod as number) <= 0 || + (normalized.CpuQuota as number) <= 0 || + normalized.NanoCpus * (normalized.CpuPeriod as number) !== + (normalized.CpuQuota as number) * 1_000_000_000 + ) { + throw new Error("Managed bootstrap Docker CPU limit representations conflict."); + } + // Podman exposes the quota derived from NanoCpus in all three fields. The + // Docker create API rejects receiving NanoCpus together with that exact + // derived period/quota pair, so retain the canonical NanoCpus form only. + normalized.CpuPeriod = 0; + normalized.CpuQuota = 0; + } for (const key of ["Binds", "MaskedPaths", "ReadonlyPaths"] as const) { if (key in normalized) normalized[key] = canonicalStringSet(normalized[key], key); } + if (imageMounts.length > 0) { + const imageTargets = new Set(imageMounts.map((mount) => mount.Target)); + const binds = Array.isArray(normalized.Binds) ? (normalized.Binds as string[]) : []; + normalized.Binds = binds.filter( + (bind) => !imageTargets.has(dockerBindTarget(bind)), + ); + const existingMounts = normalizedStructuredMounts(normalized.Mounts ?? []); + if (!Array.isArray(existingMounts)) { + throw new Error("Managed bootstrap Docker HostConfig.Mounts must be an array."); + } + normalized.Mounts = [...existingMounts, ...imageMounts]; + } for (const key of ["CapAdd", "CapDrop"] as const) { if (key in normalized) normalized[key] = canonicalCapabilities(normalized[key], key); } @@ -391,6 +517,7 @@ export function normalizeDockerManagedBootstrapLaunchSpec(inspect: DockerContain const raw = inspect as DockerContainerInspect & Record; const config = exactObject(raw.Config, "Config"); const hostConfig = exactObject(raw.HostConfig, "HostConfig"); + const imageMounts = normalizedImageMounts(inspect.Mounts); assertKnownKeys(config, CONFIG_KEYS, "Config"); assertKnownKeys(hostConfig, HOST_CONFIG_KEYS, "HostConfig"); const unsupportedConfig = [...UNSUPPORTED_CONFIG_KEYS].filter( @@ -426,8 +553,12 @@ export function normalizeDockerManagedBootstrapLaunchSpec(inspect: DockerContain inspect: { Name: inspect.Name, Config: normalizedConfig(config) as DockerContainerInspect["Config"], - HostConfig: normalizedHostConfig(hostConfig) as DockerContainerInspect["HostConfig"], - NetworkSettings: normalizedNetworkSettings(inspect.NetworkSettings), + HostConfig: normalizedHostConfig( + hostConfig, + imageMounts, + inspect.NetworkSettings, + ) as DockerContainerInspect["HostConfig"], + NetworkSettings: normalizedNetworkSettings(inspect.NetworkSettings, inspect.Id), ...("Platform" in raw && typeof raw.Platform === "string" ? { Platform: raw.Platform } : {}), }, }; diff --git a/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts b/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts index e1341c0a039..4cabe3df064 100644 --- a/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts +++ b/src/lib/onboard/managed-bootstrap/docker-test-fixture.ts @@ -171,6 +171,7 @@ export function authority(agent: ManagedStartupAgent = "hermes") { image: { repository: REPOSITORY, manifestDigest: MANIFEST }, profile: { agent, fingerprint: inputs.request.profileFingerprint }, agentIdentity: { uid: 1000, gid: 1000, workdir: "/sandbox" }, + managedStateRoots: [], intendedWorkloadArgv: ["env", "A=1", "/usr/local/bin/nemoclaw-start"], expectedSupervisorArgv: SUPERVISOR, metadata: inputs.metadata, diff --git a/src/lib/onboard/managed-bootstrap/docker.test.ts b/src/lib/onboard/managed-bootstrap/docker.test.ts index 27f28465ed1..2f51469ce35 100644 --- a/src/lib/onboard/managed-bootstrap/docker.test.ts +++ b/src/lib/onboard/managed-bootstrap/docker.test.ts @@ -185,7 +185,7 @@ describe("Docker managed bootstrap adapter", () => { it("preserves signed and accepts Docker-normalized required ulimits before cutover", async () => { const fake = fixture(); fake.original!.HostConfig!.Ulimits = [ - { Name: "nofile", Soft: 65_536, Hard: 65_536 }, + { Name: "RLIMIT_NOFILE", Soft: 65_536, Hard: 65_536 }, { Name: "memlock", Soft: -1, Hard: -1 }, ]; const adapter = createDockerManagedBootstrapAdapter(fake.deps); diff --git a/src/lib/onboard/managed-bootstrap/docker.ts b/src/lib/onboard/managed-bootstrap/docker.ts index f790a507acf..8c281feb18c 100644 --- a/src/lib/onboard/managed-bootstrap/docker.ts +++ b/src/lib/onboard/managed-bootstrap/docker.ts @@ -23,6 +23,7 @@ import { import { buildDockerGpuCloneRunArgs, dockerContainerName, + normalizeDockerUlimitName, shouldOmitOpenShellOciImageUser, } from "../docker-gpu-patch-clone"; import { @@ -42,11 +43,11 @@ import { } from "../docker-gpu-supervisor-reconnect"; import { openshellSandboxCommandEnvValue } from "../docker-startup-command-env"; import { - OPENSHELL_MANAGED_BY_LABEL, - OPENSHELL_MANAGED_BY_VALUE, + hasOpenShellSandboxOwnership, OPENSHELL_SANDBOX_ID_LABEL, OPENSHELL_SANDBOX_NAME_LABEL, queryOpenShellDockerSandboxContainers, + resolveOpenShellSandboxOwnershipLabel, } from "../openshell-docker-sandbox-containers"; import { cleanupTempDir, secureTempFile } from "../temp-files"; import { @@ -113,6 +114,7 @@ import { parseManagedBootstrapImageCompletion, serializeManagedBootstrapEnvelopeTar, } from "./envelope"; +import { prepareManagedBootstrapStateRoots } from "./state-root-authority"; const FULL_CONTAINER_ID_RE = /^[a-f0-9]{64}$/u; const FULL_SHA256_RE = /^sha256:[a-f0-9]{64}$/u; @@ -167,6 +169,7 @@ function dockerManagedBootstrapRecoveryFailure( sandbox: journal?.sandbox ?? legacyJournalContext?.sandbox ?? null, bootstrapIdentity, code: classified.code, + blockingScope: "sandbox", retryable: classified.retryable, detail: boundedRecoveryFailureDetail(error), }); @@ -493,7 +496,7 @@ function assertMetadata( ): void { const labels = inspect.Config?.Labels ?? {}; if ( - labels[OPENSHELL_MANAGED_BY_LABEL] !== OPENSHELL_MANAGED_BY_VALUE || + !hasOpenShellSandboxOwnership(labels) || labels[OPENSHELL_SANDBOX_NAME_LABEL] !== sandbox.sandboxName || labels[OPENSHELL_SANDBOX_ID_LABEL] !== sandbox.sandboxId ) { @@ -792,6 +795,16 @@ function objectField(record: Record, key: string): Record; } +function soleNetworkAliases(inspect: Record): unknown { + const networkSettings = objectField(inspect, "NetworkSettings"); + const networks = objectField(networkSettings, "Networks"); + const entries = Object.values(networks); + if (entries.length !== 1 || typeof entries[0] !== "object" || entries[0] === null) { + return null; + } + return (entries[0] as Record).Aliases; +} + function exactJson(value: unknown): string { return JSON.stringify(value ?? null); } @@ -904,7 +917,7 @@ function canonicalUlimits(value: unknown, label: string): string { throw new Error(`Managed bootstrap Docker ${label} is invalid.`); } const record = entry as Record; - const name = String(record.Name ?? ""); + const name = normalizeDockerUlimitName(record.Name); const soft = record.Soft; const hard = record.Hard; if (!name || !Number.isSafeInteger(soft) || !Number.isSafeInteger(hard)) { @@ -1125,8 +1138,12 @@ function assertReplacementMatchesIntent( ) .sort() .slice(0, 16); + const aliasDetail = + changedPaths.length === 1 && changedPaths[0]?.endsWith(".Aliases") + ? ` Expected aliases ${exactJson(soleNetworkAliases(originalInspect))}; observed aliases ${exactJson(soleNetworkAliases(observedInspect))}.` + : ""; throw new Error( - `Managed bootstrap Docker replacement normalized spec changed outside declared deltas: ${changedPaths.join(", ")}.`, + `Managed bootstrap Docker replacement normalized spec changed outside declared deltas: ${changedPaths.join(", ")}.${aliasDetail}`, ); } return replacementSpec.hash; @@ -1197,6 +1214,28 @@ function assertTransactionReplacement( } } +function assertPreparedReplacementSpec( + transaction: DockerBootstrapTransaction, + inspect: DockerContainerInspect, + expectedCanonicalJson: string, + phase: string, +): void { + const normalized = normalizeDockerManagedBootstrapLaunchSpec({ + ...inspect, + Name: `/${transaction.originalName}`, + }); + if (normalized.hash === transaction.replacementSpecHash) return; + const changedPaths = differingJsonPaths( + JSON.parse(expectedCanonicalJson), + JSON.parse(normalized.canonicalJson), + ) + .sort() + .slice(0, 16); + throw new Error( + `Managed bootstrap refused mutation because the exact replacement launch spec changed during ${phase}: ${changedPaths.join(", ")}.`, + ); +} + function assertCompletedCutoverRuntimeState( transaction: DockerBootstrapTransaction, deps: ResolvedDeps, @@ -1385,6 +1424,7 @@ function retainOwnedWorkloadForOwnerCleanup( ? `sandbox ${sandbox.sandboxId} with no previously resolved runtime ID` : `sandbox ${sandbox.sandboxId} expected runtime ${expectedRuntimeId}`; let containers: DockerCommandResult; + const ownership = resolveOpenShellSandboxOwnershipLabel(); try { containers = deps.dockerRun( [ @@ -1392,7 +1432,7 @@ function retainOwnedWorkloadForOwnerCleanup( "-a", "--no-trunc", "--filter", - `label=${OPENSHELL_MANAGED_BY_LABEL}=${OPENSHELL_MANAGED_BY_VALUE}`, + `label=${ownership.label}=${ownership.value}`, "--filter", `label=${OPENSHELL_SANDBOX_ID_LABEL}=${sandbox.sandboxId}`, "--format", @@ -1447,7 +1487,7 @@ function retainOwnedWorkloadForOwnerCleanup( } const labels = inspect.Config?.Labels ?? {}; if ( - labels[OPENSHELL_MANAGED_BY_LABEL] !== OPENSHELL_MANAGED_BY_VALUE || + !hasOpenShellSandboxOwnership(labels) || labels[OPENSHELL_SANDBOX_NAME_LABEL] !== sandbox.sandboxName || labels[OPENSHELL_SANDBOX_ID_LABEL] !== sandbox.sandboxId ) { @@ -3426,6 +3466,7 @@ export function createDockerManagedBootstrapAdapter( containerEntrypoint: MANAGED_BOOTSTRAP_TRAMPOLINE_EXECUTABLE, containerCommand: trampolineCommand, containerName: stagingName, + preserveManagedLaunchSpec: true, }); const options = { ignoreError: true, @@ -3633,6 +3674,16 @@ export function createDockerManagedBootstrapAdapter( assertTransactionReplacement(authority, preparedBeforeJournal); assertStableRunning(originalBeforeJournal, "pre-activation original"); assertExplicitlyStopped(preparedBeforeJournal, "pre-activation replacement"); + prepareManagedBootstrapStateRoots({ + inspect: originalBeforeJournal as Record, + roots: handle.plan.managedStateRoots, + captureVolume: (args) => + deps.dockerCapture(["volume", ...args], { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }), + }); if ( dockerContainerName(originalBeforeJournal) !== authority.originalName || dockerContainerName(preparedBeforeJournal) !== authority.replacementStagingName || @@ -3695,6 +3746,12 @@ export function createDockerManagedBootstrapAdapter( options, ); const afterReplacementRename = inspectExact(prepared.preparedRuntimeId, deps); + assertPreparedReplacementSpec( + journal, + afterReplacementRename, + prepared.expectedActivatedSpecCanonicalJson, + "Podman rename", + ); assertTransactionReplacement(journal, afterReplacementRename); if (dockerContainerName(afterReplacementRename) !== journal.originalName) { throw new Error( @@ -3705,6 +3762,12 @@ export function createDockerManagedBootstrapAdapter( const started = deps.dockerStart(prepared.preparedRuntimeId, options); const running = inspectExact(prepared.preparedRuntimeId, deps); + assertPreparedReplacementSpec( + journal, + running, + prepared.expectedActivatedSpecCanonicalJson, + "Podman start", + ); assertTransactionReplacement(journal, running); if (!isStableRunning(running)) { throw replacementNotStableError( diff --git a/src/lib/onboard/managed-bootstrap/podman-bootstrap-journal.test.ts b/src/lib/onboard/managed-bootstrap/podman-bootstrap-journal.test.ts index 3fd64cdb539..a80932d6ced 100644 --- a/src/lib/onboard/managed-bootstrap/podman-bootstrap-journal.test.ts +++ b/src/lib/onboard/managed-bootstrap/podman-bootstrap-journal.test.ts @@ -54,6 +54,13 @@ function journalFile(root: string): string { return path.join(root, PODMAN_BOOTSTRAP_JOURNAL_DIRECTORY, `${BOOTSTRAP_IDENTITY}.json`); } +function recordOriginalStopped(store: ReturnType) { + store.create(journal); + store.recordStateVolume(BOOTSTRAP_IDENTITY, STATE_VOLUME_MOUNTPOINT); + store.recordReplacement(BOOTSTRAP_IDENTITY, REPLACEMENT_RUNTIME_ID); + return store.recordOriginalStopped(BOOTSTRAP_IDENTITY); +} + afterEach(() => { vi.restoreAllMocks(); for (const root of roots.splice(0)) fs.rmSync(root, { force: true, recursive: true }); @@ -121,6 +128,35 @@ describe("Podman bootstrap phase journal", () => { expect(store.load(BOOTSTRAP_IDENTITY)).toBeNull(); }); + it("makes commit durable before original removal and compacts it idempotently", () => { + const root = temporaryRoot(); + const store = createFilePodmanBootstrapJournalStore(root); + recordOriginalStopped(store); + + const authorized = store.authorizeCommit(BOOTSTRAP_IDENTITY, ["original-stopped"]); + expect(authorized.phase).toBe("commit-authorized"); + expect(fs.readFileSync(`${journalFile(root)}.decision`, "utf8")).toBe("commit-authorized\n"); + expect(store.recordCommitted(BOOTSTRAP_IDENTITY).phase).toBe("committed"); + + // Simulate a crash after decision-file removal but before committed-journal + // removal. The committed journal alone is sufficient terminal authority. + fs.unlinkSync(`${journalFile(root)}.decision`); + store.removeAfterCommit(BOOTSTRAP_IDENTITY); + expect(store.load(BOOTSTRAP_IDENTITY)).toBeNull(); + }); + + it("recovers a commit decision that survived its journal acknowledgement", () => { + const root = temporaryRoot(); + const store = createFilePodmanBootstrapJournalStore(root); + recordOriginalStopped(store); + fs.writeFileSync(`${journalFile(root)}.decision`, "commit-authorized\n", { + flag: "wx", + mode: 0o600, + }); + + expect(store.load(BOOTSTRAP_IDENTITY)?.phase).toBe("commit-authorized"); + }); + it("recovers a rollback decision that survived its journal acknowledgement", () => { const root = temporaryRoot(); const store = createFilePodmanBootstrapJournalStore(root); diff --git a/src/lib/onboard/managed-bootstrap/podman-bootstrap-journal.ts b/src/lib/onboard/managed-bootstrap/podman-bootstrap-journal.ts index 0fbca210ebc..d35e6b19cdc 100644 --- a/src/lib/onboard/managed-bootstrap/podman-bootstrap-journal.ts +++ b/src/lib/onboard/managed-bootstrap/podman-bootstrap-journal.ts @@ -23,7 +23,9 @@ export type PodmanBootstrapJournalPhase = | "state-volume-created" | "replacement-created" | "original-stopped" - | "rollback-authorized"; + | "rollback-authorized" + | "commit-authorized" + | "committed"; export interface PodmanBootstrapJournal { readonly schemaVersion: typeof PODMAN_BOOTSTRAP_JOURNAL_SCHEMA_VERSION; @@ -65,10 +67,25 @@ export interface PodmanBootstrapJournalStore { /** Make rollback the only legal pre-commit outcome. */ readonly authorizeRollback: ( bootstrapIdentity: string, - expected: readonly Exclude[], + expected: readonly Exclude< + PodmanBootstrapJournalPhase, + "rollback-authorized" | "commit-authorized" | "committed" + >[], ) => PodmanBootstrapJournal; + /** Durably make commit the only legal terminal outcome before original removal. */ + readonly authorizeCommit: ( + bootstrapIdentity: string, + expected: readonly Exclude< + PodmanBootstrapJournalPhase, + "rollback-authorized" | "commit-authorized" | "committed" + >[], + ) => PodmanBootstrapJournal; + /** Record that exact final runtime identity and canonical name were independently proven. */ + readonly recordCommitted: (bootstrapIdentity: string) => PodmanBootstrapJournal; /** Remove the journal only after exact rollback state is independently proven. */ readonly removeAfterRollback: (bootstrapIdentity: string) => void; + /** Remove commit authority only after the committed runtime was independently proven. */ + readonly removeAfterCommit: (bootstrapIdentity: string) => void; } class PodmanBootstrapJournalExistsError extends Error { @@ -122,6 +139,8 @@ function exactPhase(value: unknown): PodmanBootstrapJournalPhase { "replacement-created", "original-stopped", "rollback-authorized", + "commit-authorized", + "committed", ].includes(String(value)) ) { fail("phase is unsupported"); @@ -245,8 +264,13 @@ export function normalizePodmanBootstrapJournal(value: unknown): PodmanBootstrap (replacementStateVolumeMountpoint !== null || replacementRuntimeId !== null)) || (normalized.phase === "state-volume-created" && (replacementStateVolumeMountpoint === null || replacementRuntimeId !== null)) || - (["replacement-created", "original-stopped"].includes(normalized.phase) && - (replacementStateVolumeMountpoint === null || replacementRuntimeId === null)) + (["replacement-created", "original-stopped", "commit-authorized", "committed"].includes( + normalized.phase, + ) && + (replacementStateVolumeMountpoint === null || replacementRuntimeId === null)) || + (normalized.phase === "rollback-authorized" && + replacementRuntimeId !== null && + replacementStateVolumeMountpoint === null) ) { fail("phase does not match the replacement runtime identity"); } @@ -438,15 +462,24 @@ export function createFilePodmanBootstrapJournalStore( const contents = readPrivateFile(target, "journal"); if (contents === null) return null; const journal = parsePodmanBootstrapJournal(contents); - const decision = readPrivateFile(decisionPath(target), "rollback decision"); + const decision = readPrivateFile(decisionPath(target), "terminal decision"); if (decision === null) return journal; - if (decision !== "rollback-authorized\n") { - fail("rollback decision is invalid"); + if (decision !== "rollback-authorized\n" && decision !== "commit-authorized\n") { + fail("terminal decision is invalid"); + } + const decidedPhase = decision.trim() as "rollback-authorized" | "commit-authorized"; + if ( + journal.phase === decidedPhase || + (decidedPhase === "commit-authorized" && journal.phase === "committed") + ) { + return journal; + } + if (journal.phase === "rollback-authorized" || journal.phase === "committed") { + fail("journal terminal phase conflicts with its durable decision"); } - if (journal.phase === "rollback-authorized") return journal; const decided = normalizePodmanBootstrapJournal({ ...journal, - phase: "rollback-authorized", + phase: decidedPhase, }); atomicWrite(directory, target, serializePodmanBootstrapJournal(decided), false); return decided; @@ -460,8 +493,8 @@ export function createFilePodmanBootstrapJournalStore( } assertDirectory(directory); const target = journalPath(directory, normalized.bootstrapIdentity); - if (readPrivateFile(decisionPath(target), "rollback decision") !== null) { - fail("stale rollback decision exists for this bootstrap identity"); + if (readPrivateFile(decisionPath(target), "terminal decision") !== null) { + fail("stale terminal decision exists for this bootstrap identity"); } atomicWrite(directory, target, serializePodmanBootstrapJournal(normalized), true); }, @@ -487,7 +520,7 @@ export function createFilePodmanBootstrapJournalStore( fail(`journal directory contains an unsupported entry: ${name}`); } for (const identity of decisions) { - if (!identities.has(identity)) fail(`rollback decision ${identity} has no journal`); + if (!identities.has(identity)) fail(`terminal decision ${identity} has no journal`); } return Object.freeze( [...identities].sort().map((identity) => { @@ -576,7 +609,10 @@ export function createFilePodmanBootstrapJournalStore( }, authorizeRollback( bootstrapIdentity: string, - expected: readonly Exclude[], + expected: readonly Exclude< + PodmanBootstrapJournalPhase, + "rollback-authorized" | "commit-authorized" | "committed" + >[], ) { if (!Array.isArray(expected) || expected.length === 0) { fail("rollback authorization requires at least one expected phase"); @@ -594,7 +630,7 @@ export function createFilePodmanBootstrapJournalStore( } catch (error) { if ( !(error instanceof PodmanBootstrapJournalExistsError) || - readPrivateFile(decision, "rollback decision") !== "rollback-authorized\n" + readPrivateFile(decision, "terminal decision") !== "rollback-authorized\n" ) { throw error; } @@ -606,6 +642,57 @@ export function createFilePodmanBootstrapJournalStore( atomicWrite(directory, target, serializePodmanBootstrapJournal(updated), false); return updated; }, + authorizeCommit( + bootstrapIdentity: string, + expected: readonly Exclude< + PodmanBootstrapJournalPhase, + "rollback-authorized" | "commit-authorized" | "committed" + >[], + ) { + if (!Array.isArray(expected) || expected.length === 0) { + fail("commit authorization requires at least one expected phase"); + } + assertDirectory(directory); + const target = journalPath(directory, bootstrapIdentity); + const current = load(bootstrapIdentity); + if (current?.phase === "commit-authorized" || current?.phase === "committed") return current; + if (!current || !expected.includes(current.phase as (typeof expected)[number])) { + fail(`commit authorization is not allowed from ${current?.phase ?? "absent"}`); + } + const decision = decisionPath(target); + try { + atomicWrite(directory, decision, "commit-authorized\n", true); + } catch (error) { + if ( + !(error instanceof PodmanBootstrapJournalExistsError) || + readPrivateFile(decision, "terminal decision") !== "commit-authorized\n" + ) { + throw error; + } + } + const updated = normalizePodmanBootstrapJournal({ + ...current, + phase: "commit-authorized", + }); + atomicWrite(directory, target, serializePodmanBootstrapJournal(updated), false); + return updated; + }, + recordCommitted(bootstrapIdentity: string) { + assertDirectory(directory); + const target = journalPath(directory, bootstrapIdentity); + const current = load(bootstrapIdentity); + if (current?.phase === "committed") return current; + if (!current || current.phase !== "commit-authorized") { + fail(`commit recording requires commit-authorized, found ${current?.phase ?? "absent"}`); + } + const updated = normalizePodmanBootstrapJournal({ ...current, phase: "committed" }); + atomicWrite(directory, target, serializePodmanBootstrapJournal(updated), false); + const persisted = load(bootstrapIdentity); + if (!persisted || !sameJournal(persisted, updated)) { + fail("committed runtime identity was not durably re-readable"); + } + return persisted; + }, removeAfterRollback(bootstrapIdentity: string) { assertDirectory(directory); const target = journalPath(directory, bootstrapIdentity); @@ -614,7 +701,29 @@ export function createFilePodmanBootstrapJournalStore( fail(`rollback removal requires rollback-authorized, found ${current?.phase ?? "absent"}`); } const decision = decisionPath(target); - if (readPrivateFile(decision, "rollback decision") !== null) { + if (readPrivateFile(decision, "terminal decision") !== null) { + fs.unlinkSync(decision); + fsyncDirectory(directory); + } + fs.unlinkSync(target); + fsyncDirectory(directory); + }, + removeAfterCommit(bootstrapIdentity: string) { + assertDirectory(directory); + const target = journalPath(directory, bootstrapIdentity); + const current = load(bootstrapIdentity); + if (!current || current.phase !== "committed") { + fail(`commit removal requires committed, found ${current?.phase ?? "absent"}`); + } + const decision = decisionPath(target); + const terminalDecision = readPrivateFile(decision, "terminal decision"); + if (terminalDecision !== null && terminalDecision !== "commit-authorized\n") { + fail("commit removal requires its durable terminal decision"); + } + // A committed journal is itself sufficient terminal authority. This + // makes compaction retryable after a crash between removing the decision + // file and removing the committed journal. + if (terminalDecision !== null) { fs.unlinkSync(decision); fsyncDirectory(directory); } diff --git a/src/lib/onboard/managed-bootstrap/podman-bootstrap-replacement.test.ts b/src/lib/onboard/managed-bootstrap/podman-bootstrap-replacement.test.ts index fe919a58155..bf681464fff 100644 --- a/src/lib/onboard/managed-bootstrap/podman-bootstrap-replacement.test.ts +++ b/src/lib/onboard/managed-bootstrap/podman-bootstrap-replacement.test.ts @@ -26,6 +26,8 @@ import { } from "./podman-bootstrap-replacement"; import { PODMAN_MANAGED_LABEL, + PODMAN_OPENSHELL_MANAGED_BY_LABEL, + PODMAN_OPENSHELL_MANAGED_BY_VALUE, PODMAN_SANDBOX_CONTAINER_PREFIX, PODMAN_SANDBOX_ID_LABEL, PODMAN_SANDBOX_NAME_LABEL, @@ -66,6 +68,10 @@ const LABELS = Object.freeze({ [PODMAN_SANDBOX_NAMESPACE_LABEL]: "", [PODMAN_SANDBOX_WORKSPACE_LABEL]: PODMAN_SANDBOX_WORKSPACE, }); +const REPLACEMENT_LABELS = Object.freeze({ + ...LABELS, + [PODMAN_OPENSHELL_MANAGED_BY_LABEL]: PODMAN_OPENSHELL_MANAGED_BY_VALUE, +}); const STATE_VOLUME_LABELS = Object.freeze({ [PODMAN_BOOTSTRAP_IDENTITY_LABEL]: BOOTSTRAP_IDENTITY, [PODMAN_BOOTSTRAP_STATE_VOLUME_LABEL]: "true", @@ -273,13 +279,21 @@ class PodmanHarness { this.capturedEnvironmentContents = fs.readFileSync(environmentFile, "utf8"); this.capturedEnvironmentMode = fs.statSync(environmentFile).mode & 0o777; const configuredResult = this.createResult; + const labels = Object.fromEntries( + args + .map((argument, index) => ({ argument, label: args[index + 1] ?? "" })) + .filter(({ argument }) => argument === "--label") + .map(({ label }) => ({ label, separator: label.indexOf("=") })) + .filter(({ separator }) => separator > 0) + .map(({ label, separator }) => [label.slice(0, separator), label.slice(separator + 1)]), + ); switch (configuredResult) { case null: this.replacement = { id: REPLACEMENT_RUNTIME_ID, name: STAGING_NAME, image: REPLACEMENT_IMAGE_ID, - labels: LABELS, + labels, entrypoint: ENTRYPOINT_ARGV, command: COMMAND_ARGV, environment: this.replacementEnvironment, @@ -335,6 +349,8 @@ function journalStore(): PodmanBootstrapJournalStore { function watcherLease() { const assertStillStopped = vi.fn(); const resumeAndProve = vi.fn(); + const resumeForObservationAndProve = vi.fn(); + const requiesceAndProve = vi.fn(); const lease: PodmanGatewayWatcherLease = { record: { schemaVersion: PODMAN_WATCHER_LEASE_SCHEMA_VERSION, @@ -349,7 +365,10 @@ function watcherLease() { pid: 1234, processStartIdentity: "pid-start-1234", }, + assertStillHeld: assertStillStopped, assertStillStopped, + resumeForObservationAndProve, + requiesceAndProve, resumeAndProve, }; return { assertStillStopped, lease, resumeAndProve }; @@ -392,7 +411,7 @@ describe("Podman bootstrap stopped replacement", () => { id: REPLACEMENT_RUNTIME_ID, name: STAGING_NAME, image: REPLACEMENT_IMAGE_ID, - labels: LABELS, + labels: REPLACEMENT_LABELS, running: false, }); expect(harness.stateVolume).toEqual({ @@ -415,6 +434,12 @@ describe("Podman bootstrap stopped replacement", () => { `${PODMAN_SANDBOX_NAME_LABEL}=${SANDBOX_NAME}`, STATE_VOLUME_NAME, ]); + expect(harness.calls).toContainEqual( + expect.arrayContaining([ + "--volume", + `${STATE_VOLUME_NAME}:${PODMAN_BOOTSTRAP_STATE_DIRECTORY}:rw,z,copy`, + ]), + ); expect(harness.capturedEnvironmentMode).toBe(0o600); expect(harness.capturedEnvironmentContents).toBe(`${ENVIRONMENT.join("\n")}\n`); expect(fs.existsSync(harness.capturedEnvironmentFile as string)).toBe(false); @@ -525,26 +550,25 @@ describe("Podman bootstrap stopped replacement", () => { expect(harness.calls).toEqual([]); }); - it.each([ - "-eSECRET=1", - "-lcom.nvidia.nemoclaw.override=true", - "-d=true", - ])("rejects attached protected shorthand %s before invoking Podman", (argument) => { - const harness = new PodmanHarness(); - const store = journalStore(); - const watcher = watcherLease(); - - expect(() => - prepareStoppedPodmanBootstrapReplacement({ - engine: harness.engine, - journalStore: store, - watcherLease: watcher.lease, - plan: { ...plan, runtimeArgs: [argument] }, - }), - ).toThrow("cannot set"); - expect(store.load(BOOTSTRAP_IDENTITY)).toBeNull(); - expect(harness.calls).toEqual([]); - }); + it.each(["-eSECRET=1", "-lcom.nvidia.nemoclaw.override=true", "-d=true"])( + "rejects attached protected shorthand %s before invoking Podman", + (argument) => { + const harness = new PodmanHarness(); + const store = journalStore(); + const watcher = watcherLease(); + + expect(() => + prepareStoppedPodmanBootstrapReplacement({ + engine: harness.engine, + journalStore: store, + watcherLease: watcher.lease, + plan: { ...plan, runtimeArgs: [argument] }, + }), + ).toThrow("cannot set"); + expect(store.load(BOOTSTRAP_IDENTITY)).toBeNull(); + expect(harness.calls).toEqual([]); + }, + ); it("does not confuse supported long options with protected shorthand", () => { const harness = new PodmanHarness(); @@ -688,6 +712,7 @@ describe("Podman bootstrap stopped replacement", () => { it("stops only the exact original after the stopped replacement remains stable", () => { const harness = new PodmanHarness(); + const capture = vi.spyOn(harness.engine, "capture"); const store = journalStore(); const watcher = watcherLease(); const prepared = prepare(harness, store, watcher.lease); @@ -704,6 +729,7 @@ describe("Podman bootstrap stopped replacement", () => { expect(harness.original.running).toBe(false); expect(harness.replacement?.running).toBe(false); expect(harness.calls).toContainEqual(["container", "stop", ORIGINAL_RUNTIME_ID]); + expect(capture).toHaveBeenCalledWith(["container", "stop", ORIGINAL_RUNTIME_ID], 60_000); expect(watcher.resumeAndProve).not.toHaveBeenCalled(); }); diff --git a/src/lib/onboard/managed-bootstrap/podman-bootstrap-replacement.ts b/src/lib/onboard/managed-bootstrap/podman-bootstrap-replacement.ts index 97f0f6f6592..95e1de31e19 100644 --- a/src/lib/onboard/managed-bootstrap/podman-bootstrap-replacement.ts +++ b/src/lib/onboard/managed-bootstrap/podman-bootstrap-replacement.ts @@ -19,6 +19,8 @@ import { } from "./podman-bootstrap-journal"; import { PODMAN_MANAGED_LABEL, + PODMAN_OPENSHELL_MANAGED_BY_LABEL, + PODMAN_OPENSHELL_MANAGED_BY_VALUE, PODMAN_SANDBOX_CONTAINER_PREFIX, PODMAN_SANDBOX_ID_LABEL, PODMAN_SANDBOX_NAME_LABEL, @@ -45,6 +47,7 @@ const MAX_ARGUMENTS = 512; const MAX_ARGUMENT_BYTES = 16 * 1024; const MAX_ENVIRONMENT_BYTES = 256 * 1024; const CREATE_TIMEOUT_MS = 300_000; +const STOP_TIMEOUT_MS = 60_000; const FORBIDDEN_RUNTIME_FLAGS = new Set([ "--cidfile", @@ -144,6 +147,7 @@ interface NormalizedReplacementPlan extends PodmanBootstrapReplacementPlan { readonly entrypointArgv: readonly string[]; readonly commandArgv: readonly string[]; readonly replacementImageContentId: string; + readonly replacementLabels: Readonly>; readonly replacementStagingName: string; readonly replacementStateVolumeName: string; readonly replacementStateVolumeLabels: Readonly>; @@ -278,6 +282,15 @@ function canonicalLabels( return labels; } +function replacementLabels( + labels: Readonly>, +): Readonly> { + return Object.freeze({ + ...labels, + [PODMAN_OPENSHELL_MANAGED_BY_LABEL]: PODMAN_OPENSHELL_MANAGED_BY_VALUE, + }); +} + function environmentEntries( value: unknown, label = "Podman replacement environment", @@ -438,6 +451,7 @@ function normalizePlan(plan: PodmanBootstrapReplacementPlan): NormalizedReplacem "Podman held-workload image content ID", ); const labels = canonicalLabels(held); + const managedReplacementLabels = replacementLabels(labels); const originalContainerName = safeString( held.containerName, "Podman held-workload container name", @@ -481,6 +495,7 @@ function normalizePlan(plan: PodmanBootstrapReplacementPlan): NormalizedReplacem entrypointArgv, commandArgv, replacementImageContentId, + replacementLabels: managedReplacementLabels, replacementStagingName, replacementStateVolumeName, replacementStateVolumeLabels, @@ -495,7 +510,7 @@ function normalizePlan(plan: PodmanBootstrapReplacementPlan): NormalizedReplacem replacementSpecFingerprint: stableHash({ replacementStagingName, replacementImageContentId, - labels, + labels: managedReplacementLabels, runtimeArgs, environment, entrypointArgv, @@ -515,19 +530,20 @@ function assertAuthority(authority: PodmanBootstrapReplacementAuthority): void { failure("Podman bootstrap requires one authority-bound managed-bootstrap engine.", false); } if ( - authority.watcherLease.record.phase !== "stopped" || - typeof authority.watcherLease.assertStillStopped !== "function" + (authority.watcherLease.record.phase !== "stopped" && + authority.watcherLease.record.phase !== "observing") || + typeof authority.watcherLease.assertStillHeld !== "function" ) { - failure("Podman bootstrap requires one durable stopped-watcher lease.", false); + failure("Podman bootstrap requires one durable watcher transaction lease.", false); } } -function captureWhileWatcherStopped( +function captureWhileWatcherHeld( authority: PodmanBootstrapReplacementAuthority, args: readonly string[], timeoutMs?: number, ): ContainerEngineCommandResult { - authority.watcherLease.assertStillStopped(); + authority.watcherLease.assertStillHeld(); let result: ContainerEngineCommandResult | undefined; let commandFailure: unknown; try { @@ -536,7 +552,7 @@ function captureWhileWatcherStopped( commandFailure = error; } try { - authority.watcherLease.assertStillStopped(); + authority.watcherLease.assertStillHeld(); } catch (error) { if (commandFailure === undefined) commandFailure = error; } @@ -584,7 +600,7 @@ function sameMap( } function volumeExists(authority: PodmanBootstrapReplacementAuthority, volumeName: string): boolean { - const result = captureWhileWatcherStopped(authority, ["volume", "exists", volumeName]); + const result = captureWhileWatcherHeld(authority, ["volume", "exists", volumeName]); if (result.status === 0) return true; if (result.status === 1) return false; requireZero(result, "Podman bootstrap state-volume existence check"); @@ -595,7 +611,7 @@ function inspectExactStateVolume( authority: PodmanBootstrapReplacementAuthority, expected: ExactStateVolumeExpectation, ): ExactStateVolumeObservation { - const result = captureWhileWatcherStopped(authority, ["volume", "inspect", expected.name]); + const result = captureWhileWatcherHeld(authority, ["volume", "inspect", expected.name]); requireZero(result, "Podman bootstrap state-volume inspect"); const entries = parseJson(result.stdout, "Podman bootstrap state-volume inspect"); if (!Array.isArray(entries) || entries.length !== 1) { @@ -689,7 +705,7 @@ function inspectExactContainer( expected: ExactContainerExpectation, ): ExactContainerObservation { const runtimeId = fullRuntimeId(expected.runtimeId, "Expected Podman runtime ID"); - const result = captureWhileWatcherStopped(authority, ["container", "inspect", runtimeId]); + const result = captureWhileWatcherHeld(authority, ["container", "inspect", runtimeId]); requireZero(result, "Podman bootstrap container inspect"); const entries = parseJson(result.stdout, "Podman bootstrap container inspect"); if (!Array.isArray(entries) || entries.length !== 1) { @@ -818,13 +834,13 @@ function createArgs(plan: NormalizedReplacementPlan, environmentFile: string): r "--env-file", environmentFile, ]; - for (const [key, value] of Object.entries(plan.heldWorkload.labels)) { + for (const [key, value] of Object.entries(plan.replacementLabels)) { args.push("--label", `${key}=${value}`); } args.push( ...plan.runtimeArgs, - "--mount", - `type=volume,source=${plan.replacementStateVolumeName},destination=${PODMAN_BOOTSTRAP_STATE_DIRECTORY},readonly=false,relabel=shared`, + "--volume", + `${plan.replacementStateVolumeName}:${PODMAN_BOOTSTRAP_STATE_DIRECTORY}:rw,z,copy`, "--entrypoint", JSON.stringify(plan.entrypointArgv), plan.replacementImageContentId, @@ -930,7 +946,7 @@ function expectedReplacement( runtimeId, name: plan.replacementStagingName, imageContentId: plan.replacementImageContentId, - labels: plan.heldWorkload.labels, + labels: plan.replacementLabels, running: false, entrypointArgv: plan.entrypointArgv, commandArgv: plan.commandArgv, @@ -949,7 +965,7 @@ function replacementExpectationFromJournal( runtimeId, name: journal.replacementStagingName, imageContentId: journal.replacementImageContentId, - labels: held.labels, + labels: replacementLabels(canonicalLabels(held)), running: false, stateVolume, }; @@ -972,7 +988,7 @@ function listStagingRuntimeIds( authority: PodmanBootstrapReplacementAuthority, stagingContainerName: string, ): readonly string[] { - const result = captureWhileWatcherStopped(authority, [ + const result = captureWhileWatcherHeld(authority, [ "container", "ls", "--all", @@ -999,7 +1015,7 @@ function containerExists( authority: PodmanBootstrapReplacementAuthority, runtimeId: string, ): boolean { - const result = captureWhileWatcherStopped(authority, ["container", "exists", runtimeId]); + const result = captureWhileWatcherHeld(authority, ["container", "exists", runtimeId]); if (result.status === 0) return true; if (result.status === 1) return false; requireZero(result, "Podman bootstrap container existence check"); @@ -1023,12 +1039,12 @@ export function prepareStoppedPodmanBootstrapReplacement( ): PodmanBootstrapPreparedReplacement { assertAuthority(input); const plan = normalizePlan(input.plan); - input.watcherLease.assertStillStopped(); + input.watcherLease.assertStillHeld(); if (volumeExists(input, plan.replacementStateVolumeName)) { failure("Podman bootstrap state-volume name is already in use.", false); } input.journalStore.create(createJournal(input, plan)); - const volumeCreate = captureWhileWatcherStopped(input, createStateVolumeArgs(plan)); + const volumeCreate = captureWhileWatcherHeld(input, createStateVolumeArgs(plan)); requireZero(volumeCreate, "Podman bootstrap state-volume creation"); parseCreatedStateVolumeName(volumeCreate.stdout, plan.replacementStateVolumeName); const stateVolume = inspectStableStateVolume(input, { @@ -1037,7 +1053,7 @@ export function prepareStoppedPodmanBootstrapReplacement( }); input.journalStore.recordStateVolume(plan.bootstrapIdentity, stateVolume.mountpoint); const result = privateEnvironmentFile(plan.environment, (environmentFile) => - captureWhileWatcherStopped(input, createArgs(plan, environmentFile), CREATE_TIMEOUT_MS), + captureWhileWatcherHeld(input, createArgs(plan, environmentFile), CREATE_TIMEOUT_MS), ); requireZero(result, "Podman stopped bootstrap replacement creation"); const replacementRuntimeId = parseCreatedRuntimeId(result.stdout); @@ -1092,7 +1108,11 @@ export function stopExactPodmanBootstrapOriginal( stateVolume, ), ); - const stop = captureWhileWatcherStopped(input, ["container", "stop", journal.originalRuntimeId]); + const stop = captureWhileWatcherHeld( + input, + ["container", "stop", journal.originalRuntimeId], + STOP_TIMEOUT_MS, + ); requireZero(stop, "Podman bootstrap original-container stop"); inspectStableContainer(input, expectedOriginal(journal, input.heldWorkload, false)); inspectStableContainer( @@ -1161,7 +1181,7 @@ export function rollbackPodmanBootstrapBeforeCommit( stateVolume, ), ); - const remove = captureWhileWatcherStopped(input, ["container", "rm", replacementRuntimeId]); + const remove = captureWhileWatcherHeld(input, ["container", "rm", replacementRuntimeId]); requireZero(remove, "Podman bootstrap replacement rollback removal"); if (containerExists(input, replacementRuntimeId)) { failure("Podman bootstrap replacement remained after exact rollback removal."); @@ -1174,7 +1194,7 @@ export function rollbackPodmanBootstrapBeforeCommit( let replacementStateVolumeRemoved = false; if (stateVolume) { - const removeVolume = captureWhileWatcherStopped(input, ["volume", "rm", stateVolume.name]); + const removeVolume = captureWhileWatcherHeld(input, ["volume", "rm", stateVolume.name]); requireZero(removeVolume, "Podman bootstrap state-volume rollback removal"); if (volumeExists(input, stateVolume.name)) { failure("Podman bootstrap state volume remained after exact rollback removal."); @@ -1188,11 +1208,7 @@ export function rollbackPodmanBootstrapBeforeCommit( ).running; let originalStarted = false; if (!originalWasRunning) { - const start = captureWhileWatcherStopped(input, [ - "container", - "start", - journal.originalRuntimeId, - ]); + const start = captureWhileWatcherHeld(input, ["container", "start", journal.originalRuntimeId]); requireZero(start, "Podman bootstrap original-container rollback start"); originalStarted = true; } diff --git a/src/lib/onboard/managed-bootstrap/podman-held-workload.test.ts b/src/lib/onboard/managed-bootstrap/podman-held-workload.test.ts index 4dbdabdba1d..b24777dcbf2 100644 --- a/src/lib/onboard/managed-bootstrap/podman-held-workload.test.ts +++ b/src/lib/onboard/managed-bootstrap/podman-held-workload.test.ts @@ -317,6 +317,19 @@ describe("Podman managed bootstrap held-workload inspection", () => { expect(() => inspect(fake.engine)).toThrow("image-owned root supervisor boundary"); }); + it.each(["0:0", "root:root"])( + "accepts the canonical OpenShell root supervisor identity %s", + (user) => { + const fake = engineWith([ + result(listOutput()), + result(inspectOutput({ user })), + result(inspectOutput({ user })), + ]); + + expect(inspect(fake.engine).runtimeId).toBe(RUNTIME_ID); + }, + ); + it("rejects a stopped held workload before replacement preparation", () => { const fake = engineWith([result(listOutput()), result(inspectOutput({ running: false }))]); diff --git a/src/lib/onboard/managed-bootstrap/podman-held-workload.ts b/src/lib/onboard/managed-bootstrap/podman-held-workload.ts index d941851a010..d5639cc7684 100644 --- a/src/lib/onboard/managed-bootstrap/podman-held-workload.ts +++ b/src/lib/onboard/managed-bootstrap/podman-held-workload.ts @@ -11,6 +11,8 @@ import { MANAGED_BOOTSTRAP_IDENTITY_ENV } from "./adapter"; // but bind sandbox identity to the same labels and default-workspace name that // the pinned OpenShell release emits. export const PODMAN_MANAGED_LABEL = "openshell.managed"; +export const PODMAN_OPENSHELL_MANAGED_BY_LABEL = "openshell.ai/managed-by"; +export const PODMAN_OPENSHELL_MANAGED_BY_VALUE = "openshell"; export const PODMAN_SANDBOX_ID_LABEL = "openshell.ai/sandbox-id"; export const PODMAN_SANDBOX_NAME_LABEL = "openshell.ai/sandbox-name"; export const PODMAN_SANDBOX_NAMESPACE_LABEL = "openshell.ai/sandbox-namespace"; @@ -252,8 +254,10 @@ function parseObservation( if (state.Running !== true || state.Paused === true || state.Restarting === true) { throw new Error("Podman held workload must be stably running before bootstrap preparation."); } - const configuredUser = String(config.User ?? ""); - if (configuredUser !== "" && configuredUser !== "0" && configuredUser !== "root") { + const configuredUser = String(config.User ?? "") + .trim() + .toLowerCase(); + if (!["", "0", "0:0", "root", "root:root"].includes(configuredUser)) { throw new Error("Podman held workload does not use the image-owned root supervisor boundary."); } const supervisorArgv = Object.freeze([ diff --git a/src/lib/onboard/managed-bootstrap/podman-image-transaction.test.ts b/src/lib/onboard/managed-bootstrap/podman-image-transaction.test.ts index 1c350bb566d..4641a6a9a30 100644 --- a/src/lib/onboard/managed-bootstrap/podman-image-transaction.test.ts +++ b/src/lib/onboard/managed-bootstrap/podman-image-transaction.test.ts @@ -70,7 +70,10 @@ function watcherLease(): PodmanGatewayWatcherLease { pid: 42, processStartIdentity: "pid-start-1", }, + assertStillHeld: vi.fn(), assertStillStopped: vi.fn(), + resumeForObservationAndProve: vi.fn(), + requiesceAndProve: vi.fn(), resumeAndProve: vi.fn(), }; } @@ -116,17 +119,24 @@ function preparedReplacement( } interface HarnessOptions { + readonly bootstrapLog?: string; + readonly bootstrapStartLog?: string; readonly completionAgent?: ManagedStartupAgent; + readonly completionMissingAfterSuccessfulCopyCount?: number; readonly completionMode?: number; readonly completionUnavailableCount?: number; readonly inspectImage?: string; readonly inspectName?: string; readonly inspectRuntimeId?: string; + readonly inspectExitCode?: number; + readonly inspectError?: string; + readonly inspectStatus?: string; readonly inspectStateVolumeMountpoint?: string; readonly inspectStateVolumeMode?: string; readonly inspectStateVolumeName?: string; readonly journal?: PodmanBootstrapJournal | null; readonly startsRunning?: boolean; + readonly startsRunningAfterStart?: boolean; } function harness(agent: ManagedStartupAgent, options: HarnessOptions = {}) { @@ -159,12 +169,21 @@ function harness(agent: ManagedStartupAgent, options: HarnessOptions = {}) { Type: "volume", }, ], - State: { Dead: false, Paused: false, Restarting: false, Running: running }, + State: { + Dead: false, + Error: options.inspectError ?? "", + ExitCode: options.inspectExitCode ?? 0, + OOMKilled: false, + Paused: false, + Restarting: false, + Running: running, + Status: options.inspectStatus ?? (running ? "running" : "created"), + }, }, ]), }); const start = (): ContainerEngineCommandResult => { - running = true; + running = options.startsRunningAfterStart ?? true; return result({ stdout: RUNTIME_ID }); }; const stageEnvelope = (archive: Buffer | undefined): ContainerEngineCommandResult => { @@ -189,20 +208,44 @@ function harness(agent: ManagedStartupAgent, options: HarnessOptions = {}) { }; const copyCompletion = (destination: string): ContainerEngineCommandResult => { completionAttempts += 1; - return completionAttempts <= (options.completionUnavailableCount ?? 0) - ? result({ status: 1, stderr: "completion not found" }) - : publishCompletion(destination); + const unavailable = options.completionUnavailableCount ?? 0; + const deferredResults = [ + [unavailable, () => result({ status: 1, stderr: "completion not found" })], + [unavailable + (options.completionMissingAfterSuccessfulCopyCount ?? 0), () => result()], + ] as const; + return ( + deferredResults.find(([throughAttempt]) => completionAttempts <= throughAttempt)?.[1]() ?? + publishCompletion(destination) + ); }; const copy = (args: readonly string[], input?: Buffer): ContainerEngineCommandResult => { const source = args[2] as string; const destination = args[3] as string; - return source.startsWith(`${RUNTIME_ID}:`) ? copyCompletion(destination) : stageEnvelope(input); + const publishStartLog = (): ContainerEngineCommandResult => + options.bootstrapStartLog === undefined + ? result({ status: 1, stderr: "start log unavailable" }) + : (() => { + fs.writeFileSync(destination, options.bootstrapStartLog, { + flag: "wx", + mode: 0o600, + }); + return result(); + })(); + const copiedSource = new Map ContainerEngineCommandResult>([ + [ + `${RUNTIME_ID}:/run/nemoclaw/managed-bootstrap-completion.json`, + () => copyCompletion(destination), + ], + [`${RUNTIME_ID}:/tmp/nemoclaw-start.log`, publishStartLog], + ]).get(source); + return copiedSource?.() ?? stageEnvelope(input); }; const handlers: Readonly< Record ContainerEngineCommandResult> > = { "container cp": copy, "container inspect": inspect, + "container logs": () => result({ stderr: options.bootstrapLog ?? "" }), "container start": start, }; const capture = vi.fn( @@ -253,13 +296,81 @@ function startInput(agent: ManagedStartupAgent, fake: ReturnType } describe("Podman image-owned bootstrap transaction", () => { - it.each( - MANAGED_STARTUP_AGENTS.filter((agent) => agent !== "pi"), - )("stages, starts, and authenticates one protected %s completion without exec", (agent) => { - const fake = harness(agent); - const transaction = startPodmanBootstrapImageTransaction(startInput(agent, fake), { - now: () => new Date("2026-08-01T12:00:00.000Z"), - }); + it.each(MANAGED_STARTUP_AGENTS.filter((agent) => agent !== "pi"))( + "stages, starts, and authenticates one protected %s completion without exec", + (agent) => { + const fake = harness(agent); + const transaction = startPodmanBootstrapImageTransaction(startInput(agent, fake), { + now: () => new Date("2026-08-01T12:00:00.000Z"), + }); + const completion = awaitPodmanBootstrapImageTransaction( + { + engine: fake.engine, + journalStore: fake.journalStore, + prepared: fake.prepared, + watcherLease: fake.watcher, + transaction, + timeoutSecs: 30, + }, + { now: () => new Date("2026-08-01T12:00:01.000Z") }, + ); + + expect(parseManagedBootstrapEnvelope(fake.stagedEnvelope())).toEqual({ + schemaVersion: 1, + bootstrapIdentity: BOOTSTRAP_IDENTITY, + rootApplyRequest: fake.request, + }); + expect(transaction).toMatchObject({ + agent, + bootstrapIdentity: BOOTSTRAP_IDENTITY, + engineAuthorityId: AUTHORITY_ID, + originalRuntimeId: ORIGINAL_RUNTIME_ID, + replacementRuntimeId: RUNTIME_ID, + replacementImageContentId: IMAGE_ID, + replacementSpecFingerprint: SPEC_FINGERPRINT, + replacementStagingName: STAGING_NAME, + replacementStateVolumeMountpoint: STATE_VOLUME_MOUNTPOINT, + replacementStateVolumeName: STATE_VOLUME_NAME, + watcherLeaseId: LEASE_ID, + }); + expect(completion).toMatchObject({ + agent, + bootstrapIdentity: BOOTSTRAP_IDENTITY, + engineAuthorityId: AUTHORITY_ID, + originalRuntimeId: ORIGINAL_RUNTIME_ID, + profileFingerprint: fake.request.profileFingerprint, + replacementRuntimeId: RUNTIME_ID, + replacementImageContentId: IMAGE_ID, + replacementSpecFingerprint: SPEC_FINGERPRINT, + replacementStagingName: STAGING_NAME, + replacementStateVolumeMountpoint: STATE_VOLUME_MOUNTPOINT, + replacementStateVolumeName: STATE_VOLUME_NAME, + transactionPending: true, + watcherLeaseId: LEASE_ID, + }); + expect(fake.commands).toContainEqual(["container", "start", RUNTIME_ID]); + expect(fake.commands).toContainEqual(["container", "cp", "-", `${RUNTIME_ID}:/`]); + expect( + fake.commandInputs.some( + (input) => input?.subarray(257, 263).toString("ascii") === "ustar\0", + ), + ).toBe(true); + expect(fake.commands).toContainEqual([ + "container", + "cp", + `${RUNTIME_ID}:/run/nemoclaw/managed-bootstrap-completion.json`, + expect.any(String), + ]); + expect(fake.commands.every((command) => !command.includes("exec"))).toBe(true); + expect(fake.commands.every((command) => !command.includes("--user"))).toBe(true); + expect(fake.watcher.assertStillHeld).toHaveBeenCalled(); + }, + ); + + it("retries an unpublished completion while retaining the stopped watcher lease", () => { + const fake = harness("openclaw", { completionUnavailableCount: 1 }); + const transaction = startPodmanBootstrapImageTransaction(startInput("openclaw", fake)); + let milliseconds = 0; const completion = awaitPodmanBootstrapImageTransaction( { engine: fake.engine, @@ -267,63 +378,28 @@ describe("Podman image-owned bootstrap transaction", () => { prepared: fake.prepared, watcherLease: fake.watcher, transaction, - timeoutSecs: 30, + timeoutSecs: 1, + }, + { + now: () => new Date(milliseconds), + pollIntervalMs: 25, + sleep: (duration) => { + milliseconds += duration; + }, }, - { now: () => new Date("2026-08-01T12:00:01.000Z") }, ); - expect(parseManagedBootstrapEnvelope(fake.stagedEnvelope())).toEqual({ - schemaVersion: 1, - bootstrapIdentity: BOOTSTRAP_IDENTITY, - rootApplyRequest: fake.request, - }); - expect(transaction).toMatchObject({ - agent, - bootstrapIdentity: BOOTSTRAP_IDENTITY, - engineAuthorityId: AUTHORITY_ID, - originalRuntimeId: ORIGINAL_RUNTIME_ID, - replacementRuntimeId: RUNTIME_ID, - replacementImageContentId: IMAGE_ID, - replacementSpecFingerprint: SPEC_FINGERPRINT, - replacementStagingName: STAGING_NAME, - replacementStateVolumeMountpoint: STATE_VOLUME_MOUNTPOINT, - replacementStateVolumeName: STATE_VOLUME_NAME, - watcherLeaseId: LEASE_ID, - }); - expect(completion).toMatchObject({ - agent, - bootstrapIdentity: BOOTSTRAP_IDENTITY, - engineAuthorityId: AUTHORITY_ID, - originalRuntimeId: ORIGINAL_RUNTIME_ID, - profileFingerprint: fake.request.profileFingerprint, - replacementRuntimeId: RUNTIME_ID, - replacementImageContentId: IMAGE_ID, - replacementSpecFingerprint: SPEC_FINGERPRINT, - replacementStagingName: STAGING_NAME, - replacementStateVolumeMountpoint: STATE_VOLUME_MOUNTPOINT, - replacementStateVolumeName: STATE_VOLUME_NAME, - transactionPending: true, - watcherLeaseId: LEASE_ID, - }); - expect(fake.commands).toContainEqual(["container", "start", RUNTIME_ID]); - expect(fake.commands).toContainEqual(["container", "cp", "-", `${RUNTIME_ID}:/`]); - expect( - fake.commandInputs.some((input) => input?.subarray(257, 263).toString("ascii") === "ustar\0"), - ).toBe(true); - expect(fake.commands).toContainEqual([ - "container", - "cp", - `${RUNTIME_ID}:/run/nemoclaw/managed-bootstrap-completion.json`, - expect.any(String), - ]); - expect(fake.commands.every((command) => !command.includes("exec"))).toBe(true); - expect(fake.commands.every((command) => !command.includes("--user"))).toBe(true); - expect(fake.watcher.assertStillStopped).toHaveBeenCalled(); + expect(completion.agent).toBe("openclaw"); + expect(fake.completionAttempts()).toBe(2); }); - it("retries an unpublished completion while retaining the stopped watcher lease", () => { - const fake = harness("openclaw", { completionUnavailableCount: 1 }); - const transaction = startPodmanBootstrapImageTransaction(startInput("openclaw", fake)); + it("retries when Podman reports copy success before publishing the destination", () => { + const fake = harness("langchain-deepagents-code", { + completionMissingAfterSuccessfulCopyCount: 1, + }); + const transaction = startPodmanBootstrapImageTransaction( + startInput("langchain-deepagents-code", fake), + ); let milliseconds = 0; const completion = awaitPodmanBootstrapImageTransaction( { @@ -343,7 +419,7 @@ describe("Podman image-owned bootstrap transaction", () => { }, ); - expect(completion.agent).toBe("openclaw"); + expect(completion.agent).toBe("langchain-deepagents-code"); expect(fake.completionAttempts()).toBe(2); }); @@ -411,6 +487,124 @@ describe("Podman image-owned bootstrap transaction", () => { expect(fake.commands.some((command) => command[1] === "cp")).toBe(false); }); + it("reports the bounded Podman exit state when a replacement does not stay running", () => { + const fake = harness("hermes", { + bootstrapLog: "[SECURITY] Managed bootstrap trampoline: agent identity mismatch", + inspectError: "bootstrap rejected", + inspectExitCode: 126, + inspectStatus: "exited", + startsRunningAfterStart: false, + }); + + expect(() => startPodmanBootstrapImageTransaction(startInput("hermes", fake))).toThrow( + "not stably running (status exited; exit 126; oom false; error bootstrap rejected; bootstrap [SECURITY] Managed bootstrap trampoline: agent identity mismatch)", + ); + expect(fake.commands).toContainEqual(["container", "logs", "--tail", "80", RUNTIME_ID]); + }); + + it("reports the bounded managed startup application failure", () => { + const fake = harness("hermes", { + bootstrapLog: + "Managed startup image application failed: required root-owned directory is missing: /var/lib/nemoclaw/runtime-state-mutation", + inspectExitCode: 1, + inspectStatus: "exited", + startsRunningAfterStart: false, + }); + + expect(() => startPodmanBootstrapImageTransaction(startInput("hermes", fake))).toThrow( + "not stably running (status exited; exit 1; oom false; bootstrap Managed startup image application failed: required root-owned directory is missing: /var/lib/nemoclaw/runtime-state-mutation)", + ); + }); + + it("reports the bounded managed startup shared-state failure", () => { + const fake = harness("hermes", { + bootstrapLog: + "Managed startup shared-state transaction failed: managed output directory crosses a nested filesystem mount: /sandbox/.hermes", + inspectExitCode: 1, + inspectStatus: "exited", + startsRunningAfterStart: false, + }); + + expect(() => startPodmanBootstrapImageTransaction(startInput("hermes", fake))).toThrow( + "not stably running (status exited; exit 1; oom false; bootstrap Managed startup shared-state transaction failed: managed output directory crosses a nested filesystem mount: /sandbox/.hermes)", + ); + }); + + it("truncates an allowlisted managed startup failure instead of dropping it", () => { + const detail = "x".repeat(600); + const fake = harness("hermes", { + bootstrapLog: `Managed startup image application failed: ${detail}`, + inspectExitCode: 1, + inspectStatus: "exited", + startsRunningAfterStart: false, + }); + + expect(() => startPodmanBootstrapImageTransaction(startInput("hermes", fake))).toThrow( + `bootstrap ${`Managed startup image application failed: ${detail}`.slice(0, 400)}`, + ); + }); + + it("reports a bounded Hermes startup refusal after managed profile application", () => { + const fake = harness("hermes", { + bootstrapLog: + "[SECURITY] Refusing Hermes startup because /sandbox/.hermes is not a safe directory", + inspectExitCode: 1, + inspectStatus: "exited", + startsRunningAfterStart: false, + }); + + expect(() => startPodmanBootstrapImageTransaction(startInput("hermes", fake))).toThrow( + "bootstrap [SECURITY] Refusing Hermes startup because /sandbox/.hermes is not a safe directory", + ); + }); + + it("reports the bounded Hermes runtime-state startup refusal", () => { + const fake = harness("hermes", { + bootstrapLog: + "runtime-state-mutation-startup-gate: held\n[SECURITY] Runtime state mutation startup gate failed.", + inspectExitCode: 1, + inspectStatus: "exited", + startsRunningAfterStart: false, + }); + + expect(() => startPodmanBootstrapImageTransaction(startInput("hermes", fake))).toThrow( + "not stably running (status exited; exit 1; oom false; bootstrap [SECURITY] Runtime state mutation startup gate failed.)", + ); + }); + + it("reports a bounded startup refusal from the protected temp-file copy fallback", () => { + const fake = harness("hermes", { + bootstrapLog: "", + bootstrapStartLog: "[SECURITY] Required entrypoint env-wrapper normalizer is missing.\n", + inspectExitCode: 1, + inspectStatus: "exited", + startsRunningAfterStart: false, + }); + + expect(() => startPodmanBootstrapImageTransaction(startInput("hermes", fake))).toThrow( + "not stably running (status exited; exit 1; oom false; bootstrap [SECURITY] Required entrypoint env-wrapper normalizer is missing.)", + ); + expect(fake.commands).toContainEqual([ + "container", + "cp", + `${RUNTIME_ID}:/tmp/nemoclaw-start.log`, + expect.any(String), + ]); + }); + + it("does not surface non-bootstrap container output in a replacement failure", () => { + const fake = harness("hermes", { + bootstrapLog: "secret-looking application output", + inspectExitCode: 1, + inspectStatus: "exited", + startsRunningAfterStart: false, + }); + + expect(() => startPodmanBootstrapImageTransaction(startInput("hermes", fake))).toThrow( + "not stably running (status exited; exit 1; oom false)", + ); + }); + it("rejects runtime and image drift before request staging", () => { const runtimeDrift = harness("openclaw", { inspectRuntimeId: "5".repeat(64) }); const imageDrift = harness("openclaw", { inspectImage: `sha256:${"6".repeat(64)}` }); @@ -514,7 +708,7 @@ describe("Podman image-owned bootstrap transaction", () => { transaction, timeoutSecs: 1, }), - ).toThrow("exact stopped OpenShell watcher lease"); + ).toThrow("exact OpenShell watcher transaction lease"); }); it("times out deterministically when the protected completion never appears", () => { @@ -544,13 +738,13 @@ describe("Podman image-owned bootstrap transaction", () => { expect(fake.completionAttempts()).toBe(3); }); - it("refuses to bootstrap a release candidate on Podman (#7927)", () => { + it("does not duplicate managed-agent support policy inside the Podman transaction", () => { const fake = harness("pi"); - expect(() => + expect( startPodmanBootstrapImageTransaction(startInput("pi", fake), { now: () => new Date("2026-08-01T12:00:00.000Z"), }), - ).toThrow("agent 'pi' is not supported on Podman; onboard it through the Docker compute runtime"); + ).toMatchObject({ agent: "pi" }); }); }); diff --git a/src/lib/onboard/managed-bootstrap/podman-image-transaction.ts b/src/lib/onboard/managed-bootstrap/podman-image-transaction.ts index 82a37c1ec54..ad10a938ba7 100644 --- a/src/lib/onboard/managed-bootstrap/podman-image-transaction.ts +++ b/src/lib/onboard/managed-bootstrap/podman-image-transaction.ts @@ -33,10 +33,14 @@ import type { PodmanGatewayWatcherLease } from "./podman-watcher-lease"; export const PODMAN_BOOTSTRAP_IMAGE_TRANSACTION_SCHEMA_VERSION = 1 as const; const COMPLETION_TEMP_PREFIX = "nemoclaw-podman-bootstrap-completion"; +const START_LOG_TEMP_PREFIX = "nemoclaw-podman-bootstrap-start-log"; +const START_LOG_PATH = "/tmp/nemoclaw-start.log"; +const START_LOG_MAX_BYTES = 64 * 1024; const FULL_RUNTIME_ID = /^[a-f0-9]{64}$/u; const IMAGE_CONTENT_ID = /^sha256:[a-f0-9]{64}$/u; const SHA256 = /^[a-f0-9]{64}$/u; const SAFE_NAME = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,252}$/u; +const SAFE_AGENT_ID = /^[a-z0-9][a-z0-9-]{0,127}$/u; const DEFAULT_COMMAND_TIMEOUT_MS = 30_000; const DEFAULT_START_TIMEOUT_MS = 90_000; const DEFAULT_POLL_INTERVAL_MS = 250; @@ -45,13 +49,6 @@ const MAX_TIMEOUT_SECONDS = 3_600; type BootstrapEngine = ContainerEngine & { readonly authorityId: string }; type ManagedStartupAgent = ManagedStartupRootApplyRequest["agent"]; -const PODMAN_BOOTSTRAP_AGENTS = Object.freeze({ - openclaw: true, - hermes: true, - "langchain-deepagents-code": true, - pi: false, -} satisfies Record); - export interface PodmanBootstrapImageTransactionInput { readonly engine: BootstrapEngine; readonly journalStore: Pick; @@ -99,10 +96,14 @@ export interface PodmanBootstrapImageTransactionDeps { } interface ExactPodmanContainerState { + readonly error: string; + readonly exitCode: number | null; readonly imageContentId: string; readonly name: string; + readonly oomKilled: boolean | null; readonly runtimeId: string; readonly running: boolean; + readonly status: string; readonly stateVolumeMountpoint: string; readonly stateVolumeName: string; } @@ -127,15 +128,8 @@ function exactEngine(engine: BootstrapEngine, expectedAuthorityId?: string): Boo } function exactAgent(value: string): ManagedStartupAgent { - if (Object.prototype.hasOwnProperty.call(PODMAN_BOOTSTRAP_AGENTS, value)) { - if (PODMAN_BOOTSTRAP_AGENTS[value as ManagedStartupAgent]) { - return value as ManagedStartupAgent; - } - return fail( - `agent '${value}' is not supported on Podman; onboard it through the Docker compute runtime`, - ); - } - return fail("the managed agent is unsupported"); + if (!SAFE_AGENT_ID.test(value)) fail("the managed agent identity is invalid"); + return value as ManagedStartupAgent; } function exactSha256(value: string, label: string): string { @@ -223,12 +217,12 @@ function exactWatcherLease(lease: PodmanGatewayWatcherLease, expectedLeaseId?: s if ( !lease || typeof lease !== "object" || - lease.record?.phase !== "stopped" || + (lease.record?.phase !== "stopped" && lease.record?.phase !== "observing") || (expectedLeaseId !== undefined && lease.record.leaseId !== expectedLeaseId) ) { - fail("the exact stopped OpenShell watcher lease is unavailable"); + fail("the exact OpenShell watcher transaction lease is unavailable"); } - lease.assertStillStopped(); + lease.assertStillHeld(); } function commandFailure(result: ContainerEngineCommandResult, action: string): never { @@ -358,7 +352,28 @@ function parseInspect( ) { return fail("the exact replacement is not in a stable running or stopped state"); } - return Object.freeze({ imageContentId, name, runtimeId, running: state.Running, ...stateVolume }); + const exitCode = + typeof state.ExitCode === "number" && Number.isSafeInteger(state.ExitCode) + ? state.ExitCode + : null; + const oomKilled = typeof state.OOMKilled === "boolean" ? state.OOMKilled : null; + const status = + typeof state.Status === "string" && state.Status.length <= 64 && !/[\r\n\0]/u.test(state.Status) + ? state.Status + : "unknown"; + const error = + typeof state.Error === "string" ? state.Error.replace(/\s+/gu, " ").trim().slice(-300) : ""; + return Object.freeze({ + error, + exitCode, + imageContentId, + name, + oomKilled, + runtimeId, + running: state.Running, + status, + ...stateVolume, + }); } function inspectExact( @@ -381,11 +396,82 @@ function sameState(left: ExactPodmanContainerState, right: ExactPodmanContainerS left.imageContentId === right.imageContentId && left.name === right.name && left.running === right.running && + left.status === right.status && + left.exitCode === right.exitCode && + left.error === right.error && + left.oomKilled === right.oomKilled && left.stateVolumeMountpoint === right.stateVolumeMountpoint && left.stateVolumeName === right.stateVolumeName ); } +function safeBootstrapFailureLine(output: string): string | null { + const allowed = output.split(/\r?\n/u).flatMap((line) => { + if (!/^[\x20-\x7e]+$/u.test(line)) return []; + const boundedPrefix = + /^(?:(?:\[SECURITY\] Managed bootstrap (?:entrypoint|trampoline)|Managed startup (?:image application|shared-state transaction) failed): |\[SECURITY\] (?:Refusing Hermes startup because |Config integrity check failed|HERMES_[A-Z0-9_]+: ))/u.test( + line, + ); + const exactFixedFailure = + /^(?:\[SECURITY\] (?:Required entrypoint env-wrapper normalizer is missing|Managed startup env wrapper has too many assignments|Managed startup env wrapper contains a malformed assignment|Required runtime state mutation startup gate is unavailable|Runtime state mutation startup gate failed|Managed DCode login profile is missing or unsafe|Could not protect the managed DCode login profile|DCode login profile is not protected; rebuild this sandbox)\.|runtime-state-mutation-startup-gate: held)$/u.test( + line, + ); + if (!boundedPrefix && !exactFixedFailure) { + return []; + } + return [line.slice(0, 400)]; + }); + return allowed.at(-1) ?? null; +} + +function boundedBootstrapStartLogFailure( + engine: BootstrapEngine, + runtimeId: string, +): string | null { + const file = secureTempFile(START_LOG_TEMP_PREFIX, ".log"); + let descriptor: number | null = null; + try { + const copied = engine.capture( + ["container", "cp", `${runtimeId}:${START_LOG_PATH}`, file], + DEFAULT_COMMAND_TIMEOUT_MS, + ); + if (copied.status !== 0 || copied.error) return null; + descriptor = fs.openSync(file, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); + const stat = fs.fstatSync(descriptor); + if (!stat.isFile() || stat.isSymbolicLink() || stat.nlink !== 1 || stat.size > START_LOG_MAX_BYTES) { + return null; + } + const uid = process.getuid?.(); + if (uid !== undefined && stat.uid !== uid) return null; + const output = fs.readFileSync(descriptor, "utf8"); + return Buffer.byteLength(output) === stat.size ? safeBootstrapFailureLine(output) : null; + } catch { + return null; + } finally { + if (descriptor !== null) fs.closeSync(descriptor); + cleanupTempDir(file, START_LOG_TEMP_PREFIX); + } +} + +function boundedBootstrapSecurityFailure( + engine: BootstrapEngine, + runtimeId: string, +): string | null { + for (let attempt = 0; attempt < 3; attempt += 1) { + const result = engine.capture( + ["container", "logs", "--tail", "80", runtimeId], + DEFAULT_COMMAND_TIMEOUT_MS, + ); + const containerLogFailure = + result.status === 0 && !result.error + ? safeBootstrapFailureLine(`${result.stdout}\n${result.stderr}`) + : null; + if (containerLogFailure) return containerLogFailure; + if (attempt < 2) defaultSleep(100); + } + return boundedBootstrapStartLogFailure(engine, runtimeId); +} + function inspectStable( engine: BootstrapEngine, prepared: PodmanBootstrapPreparedReplacement, @@ -394,7 +480,20 @@ function inspectStable( const first = inspectExact(engine, prepared); const second = inspectExact(engine, prepared); if (!sameState(first, second) || second.running !== expectedRunning) { - fail(`the exact replacement is not stably ${expectedRunning ? "running" : "stopped"}`); + const bootstrapFailure = + expectedRunning && !second.running + ? boundedBootstrapSecurityFailure(engine, prepared.replacementRuntimeId) + : null; + const detail = [ + `status ${second.status}`, + `exit ${second.exitCode === null ? "unknown" : String(second.exitCode)}`, + `oom ${second.oomKilled === null ? "unknown" : String(second.oomKilled)}`, + ...(second.error ? [`error ${second.error}`] : []), + ...(bootstrapFailure ? [`bootstrap ${bootstrapFailure}`] : []), + ].join("; "); + fail( + `the exact replacement is not stably ${expectedRunning ? "running" : "stopped"} (${detail})`, + ); } return second; } @@ -484,7 +583,18 @@ function tryCopyCompletion( DEFAULT_COMMAND_TIMEOUT_MS, ); if (result.status !== 0) return { completion: null, status: result.status }; - return { completion: readProtectedCompletion(file), status: result.status }; + try { + return { completion: readProtectedCompletion(file), status: result.status }; + } catch (error) { + // Podman can report a successful archive copy before its destination is + // visible to the host caller. Treat only that absent destination as an + // unpublished receipt and retain the existing bounded poll. All unsafe + // metadata and unstable-read failures remain fatal. + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return { completion: null, status: result.status }; + } + throw error; + } } finally { cleanupTempDir(file, COMPLETION_TEMP_PREFIX); } @@ -572,7 +682,7 @@ export function startPodmanBootstrapImageTransaction( }); } -/** Poll one protected image-owned completion while the exact watcher stays stopped. */ +/** Poll one protected image-owned completion while exact watcher authority stays held. */ export function awaitPodmanBootstrapImageTransaction( input: { readonly engine: BootstrapEngine; diff --git a/src/lib/onboard/managed-bootstrap/podman-runtime.test.ts b/src/lib/onboard/managed-bootstrap/podman-runtime.test.ts new file mode 100644 index 00000000000..212a2a03680 --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/podman-runtime.test.ts @@ -0,0 +1,965 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it, vi } from "vitest"; + +import { managedStartupE2eProfile } from "../../../../scripts/checks/generate-managed-startup-profile-fixture.mts"; +import type { PodmanBoundContainerEngine } from "../../adapters/podman"; +import { encodeManagedStartupProfile } from "../managed-startup/profile"; +import { createManagedStartupRootApplyRequest } from "../managed-startup/root-apply"; + +const coordinator = vi.hoisted(() => ({ + activate: vi.fn(), + finalize: vi.fn(), + prepare: vi.fn(), +})); + +vi.mock("./adapter", async (importOriginal) => ({ + ...(await importOriginal()), + activateManagedBootstrapSequence: coordinator.activate, + finalizeManagedBootstrapSequence: coordinator.finalize, + prepareManagedBootstrapSequence: coordinator.prepare, +})); + +import type { + ManagedBootstrapActivatedTransaction, + ManagedBootstrapAdapter, + ManagedBootstrapPreparedTransaction, +} from "./adapter"; +import { + createFilePodmanBootstrapJournalStore, + PODMAN_BOOTSTRAP_JOURNAL_SCHEMA_VERSION, + type PodmanBootstrapJournal, +} from "./podman-bootstrap-journal"; +import { + buildPodmanStandaloneGatewayEnvironmentAuthority, + createPodmanManagedBootstrapAdapter, + createPodmanManagedBootstrapSurface, + finishCommittedPodmanBootstrap, + observePodmanBootstrapReplacementReady, + preparePodmanManagedWorkspaceAuthority, + resolvePodmanManagedGatewayAuthority, + renderPodmanReplacementEnvironment, + renderPodmanReplacementHealthArgs, + renderPodmanReplacementMountArgs, + renderPodmanReplacementRuntimeArgs, + renderPodmanReplacementSecretArgs, +} from "./podman-runtime"; +import { prepareManagedBootstrapStateRoots } from "./state-root-authority"; +import type { PodmanGatewayWatcherLease } from "./podman-watcher-lease"; +import { PODMAN_WATCHER_LEASE_SCHEMA_VERSION } from "./podman-watcher-lease"; + +const IDENTITY = "1".repeat(64); +const MANIFEST_DIGEST = `sha256:${"2".repeat(64)}` as const; +const ORIGINAL_RUNTIME_ID = "2".repeat(64); +const REPLACEMENT_RUNTIME_ID = "3".repeat(64); +const SUPERVISOR_IMAGE = "ghcr.io/nvidia/openshell/supervisor:0.0.106"; +const LEASE_ID = "123e4567-e89b-42d3-a456-426614174000"; +const ENGINE_AUTHORITY_ID = `podman-sha256:${"4".repeat(64)}`; +const STORAGE_GRAPH_ROOT = "/run/user/1000/containers/storage"; + +function committedJournal(): PodmanBootstrapJournal { + return { + schemaVersion: PODMAN_BOOTSTRAP_JOURNAL_SCHEMA_VERSION, + phase: "preparing-replacement", + bootstrapIdentity: IDENTITY, + engineAuthorityId: ENGINE_AUTHORITY_ID, + watcherLeaseId: LEASE_ID, + sandboxName: "alpha", + sandboxId: "sandbox-alpha", + originalRuntimeId: ORIGINAL_RUNTIME_ID, + originalContainerName: "openshell-default--alpha-sandbox-alpha", + originalImageContentId: `sha256:${"5".repeat(64)}`, + originalSpecFingerprint: "6".repeat(64), + replacementStateVolumeName: "openshell-alpha-bootstrap-state", + replacementStateVolumeMountpoint: null, + replacementRuntimeId: null, + replacementStagingName: "openshell-alpha-bootstrap", + replacementImageContentId: `sha256:${"7".repeat(64)}`, + replacementSpecFingerprint: "8".repeat(64), + }; +} + +function runtimeInspect(runtimeId: string, name: string, image: string) { + return { + Id: runtimeId, + Name: name, + Image: image, + Config: { + Labels: { + "openshell.managed": "true", + "openshell.ai/managed-by": "openshell", + "openshell.ai/sandbox-id": "sandbox-alpha", + "openshell.ai/sandbox-name": "alpha", + "openshell.ai/sandbox-namespace": "", + "openshell.ai/sandbox-workspace": "default", + }, + }, + }; +} + +function engine(): PodmanBoundContainerEngine { + return { + operation: "managed-bootstrap", + engineId: "podman", + displayName: "Podman", + authorityId: "test:podman-authority", + endpointAuthorityId: "test:podman-endpoint", + capture: vi.fn(), + captureHost: vi.fn(), + assertAuthority: vi.fn(), + }; +} + +function adapter() { + const recoverUnfinishedTransactions = vi.fn(async () => ({ receipts: [], failures: [] })); + return { + value: { recoverUnfinishedTransactions } as unknown as ManagedBootstrapAdapter, + recoverUnfinishedTransactions, + }; +} + +function lifecycleInput(adapterOverride: ManagedBootstrapAdapter) { + const request = createManagedStartupRootApplyRequest({ + agent: "hermes", + encodedProfile: encodeManagedStartupProfile(managedStartupE2eProfile("hermes", false, false)), + }); + return { + providerId: "podman", + environment: {}, + dockerClientEnv: {}, + stateRoot: "/unused/provider-state", + bootstrapIdentity: IDENTITY, + request, + image: { + repository: "registry.example/nemoclaw/hermes", + manifestDigest: MANIFEST_DIGEST, + }, + agentIdentity: { uid: 1000, gid: 1000, workdir: "/sandbox" }, + workspaceRoot: { uid: 1000, gid: 1000, mode: 0o755 as const }, + managedStateRoots: [], + intendedWorkloadArgv: ["/usr/local/bin/nemoclaw-start"], + expectedSupervisorArgv: ["/opt/openshell/bin/supervisor"], + launchArgv: ["openshell", "sandbox", "create", "--name", "alpha"], + heldWorkloadArgv: ["/usr/local/bin/nemoclaw-managed-startup-hold"], + authorityStore: { recordPreparedAuthority: vi.fn() }, + adapterOverride, + route: "none" as const, + persistStartupCommand: false, + sandboxName: "alpha", + sandboxGpuConfig: { + mode: "0" as const, + hostGpuDetected: false, + hostGpuPlatform: null, + sandboxGpuEnabled: false, + sandboxGpuDevice: null, + errors: [], + }, + requiredLimits: [], + timeoutSecs: 30, + network: { + inferenceProvider: "openai", + gatewayUsesContainerBridge: false, + gatewayPort: 8080, + reverifyBridgeReachability: () => undefined, + }, + dependencies: {}, + }; +} + +function installCoordinatorMocks() { + const prepared = Object.freeze({}) as ManagedBootstrapPreparedTransaction; + const activated = Object.freeze({}) as ManagedBootstrapActivatedTransaction; + coordinator.prepare.mockImplementation(async (_adapter, input) => { + await input.create.launch({ + heldWorkloadArgv: ["/usr/local/bin/nemoclaw-managed-startup-hold"], + bootstrapIdentity: IDENTITY, + }); + return prepared; + }); + coordinator.activate.mockResolvedValue(activated); + coordinator.finalize.mockResolvedValue({} as never); + return { activated, prepared }; +} + +describe("Podman managed-bootstrap runtime surface", () => { + it.each([ + [8080, "nemoclaw", "openshell-docker-gateway"], + [18080, "nemoclaw-18080", "openshell-docker-gateway-18080"], + ])("binds gateway port %i to its runtime authority", (gatewayPort, gatewayName, stateDirName) => { + expect(resolvePodmanManagedGatewayAuthority({ HOME: "/home/test" }, gatewayPort)).toEqual({ + gatewayName, + stateDir: `/home/test/.local/state/nemoclaw/${stateDirName}`, + }); + }); + + it("preserves an explicit native gateway state-directory authority", () => { + expect( + resolvePodmanManagedGatewayAuthority( + { + HOME: "/home/test", + NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR: " /run/user/1000/nemoclaw-gateway ", + }, + 18080, + ), + ).toEqual({ + gatewayName: "nemoclaw-18080", + stateDir: "/run/user/1000/nemoclaw-gateway", + }); + }); + + it("emits exact replacement health until OpenShell observes the sandbox as Ready", () => { + let time = 0; + const capture = vi.fn(() => ({ + status: 0, + stdout: "healthy", + stderr: "", + error: undefined, + })); + const phases = ["Error", "Ready"]; + const runCaptureOpenshell = vi.fn(() => + JSON.stringify({ + id: "sandbox-alpha", + name: "alpha", + phase: phases.shift() ?? "Ready", + }), + ); + + observePodmanBootstrapReplacementReady({ + engine: { ...engine(), capture }, + runtimeId: REPLACEMENT_RUNTIME_ID, + sandboxName: "alpha", + sandboxId: "sandbox-alpha", + gatewayName: "nemoclaw", + runCaptureOpenshell, + sleep: (seconds) => { + time += seconds * 1000; + }, + now: () => time, + }); + + expect(capture).toHaveBeenCalledTimes(2); + expect(capture).toHaveBeenNthCalledWith( + 1, + ["healthcheck", "run", REPLACEMENT_RUNTIME_ID], + 15_000, + ); + expect(runCaptureOpenshell).toHaveBeenCalledTimes(2); + expect(runCaptureOpenshell).toHaveBeenLastCalledWith( + ["sandbox", "get", "-g", "nemoclaw", "alpha", "--output", "json"], + { ignoreError: true, timeout: 5_000 }, + ); + }); + + it("reproduces the OpenShell Podman health contract on the replacement", () => { + expect( + renderPodmanReplacementHealthArgs({ + Config: { + Healthcheck: { + Test: ["CMD-SHELL", "test -S /var/run/openshell.sock"], + Interval: 30_000_000_000, + Timeout: 2_000_000_000, + Retries: 10, + StartPeriod: 5_000_000_000, + }, + }, + }), + ).toEqual([ + "--health-cmd", + "test -S /var/run/openshell.sock", + "--health-interval", + "30000000000ns", + "--health-timeout", + "2000000000ns", + "--health-retries", + "10", + "--health-start-period", + "5000000000ns", + ]); + }); + + it("preserves NemoClaw workspace ownership across the managed replacement", () => { + expect( + renderPodmanReplacementEnvironment( + { + Config: { + User: "0:0", + WorkingDir: "/sandbox", + Entrypoint: ["/opt/openshell/bin/openshell-sandbox"], + Cmd: ["--workdir", "/sandbox"], + Labels: { "openshell.managed": "true" }, + Env: [ + "OPENSHELL_OCI_IMAGE_USER=1000:1000", + "OPENSHELL_SANDBOX_UID=", + "OPENSHELL_SANDBOX_GID=", + "OPENSHELL_SANDBOX_COMMAND=stale", + "PATH=/usr/bin", + ], + }, + }, + { + plan: { profile: { agent: "hermes", fingerprint: "a".repeat(64) } }, + bootstrapIdentity: IDENTITY, + intendedWorkloadArgv: ["/usr/local/bin/nemoclaw-start"], + } as never, + ), + ).toEqual([ + "OPENSHELL_SANDBOX_UID=", + "OPENSHELL_SANDBOX_GID=", + "PATH=/usr/bin", + "OPENSHELL_SANDBOX_COMMAND=/usr/local/bin/nemoclaw-start", + "NEMOCLAW_MANAGED_BOOTSTRAP_DROP_CAPABILITIES=0x32", + ]); + }); + + it("reproduces the OpenShell supervisor image mount on the bootstrap replacement", () => { + expect( + renderPodmanReplacementMountArgs( + { + Id: ORIGINAL_RUNTIME_ID, + Mounts: [ + { + Type: "image", + Source: SUPERVISOR_IMAGE, + Destination: "/opt/openshell/bin", + RW: false, + }, + ], + }, + STORAGE_GRAPH_ROOT, + ), + ).toEqual([ + "--mount", + `type=image,source=${SUPERVISOR_IMAGE},destination=/opt/openshell/bin,rw=false`, + ]); + }); + + it("collapses Podman's materialized supervisor bind into its image-mount identity", () => { + const image = SUPERVISOR_IMAGE; + expect( + renderPodmanReplacementMountArgs( + { + Id: ORIGINAL_RUNTIME_ID, + Mounts: [ + { + Type: "image", + Source: image, + Destination: "/opt/openshell/bin", + RW: false, + }, + { + Type: "bind", + Source: + `${STORAGE_GRAPH_ROOT}/overlay-containers/${ORIGINAL_RUNTIME_ID}` + + "/userdata/overlay/example/merge", + Destination: "/opt/openshell/bin", + RW: true, + }, + ], + }, + STORAGE_GRAPH_ROOT, + ), + ).toEqual(["--mount", `type=image,source=${image},destination=/opt/openshell/bin,rw=false`]); + }); + + it("rejects unrelated Podman mounts with the same destination", () => { + expect(() => + renderPodmanReplacementMountArgs( + { + Id: ORIGINAL_RUNTIME_ID, + Mounts: [ + { + Type: "bind", + Source: "/srv/first", + Destination: "/sandbox/state", + RW: true, + }, + { + Type: "bind", + Source: "/srv/second", + Destination: "/sandbox/state", + RW: true, + }, + ], + }, + STORAGE_GRAPH_ROOT, + ), + ).toThrow("mount destination resolves to ambiguous runtime mounts"); + }); + + it("rejects an image mount paired with another container's storage bind", () => { + expect(() => + renderPodmanReplacementMountArgs( + { + Id: ORIGINAL_RUNTIME_ID, + Mounts: [ + { + Type: "image", + Source: SUPERVISOR_IMAGE, + Destination: "/opt/openshell/bin", + RW: false, + }, + { + Type: "bind", + Source: + `${STORAGE_GRAPH_ROOT}/overlay-containers/${"f".repeat(64)}` + + "/userdata/overlay/example/merge", + Destination: "/opt/openshell/bin", + RW: true, + }, + ], + }, + STORAGE_GRAPH_ROOT, + ), + ).toThrow("mount destination resolves to ambiguous runtime mounts"); + }); + + it("restores the exact OpenShell named-volume workspace authority before replacement start", () => { + const mountpoint = + "/home/test/.local/share/containers/storage/volumes/openshell-sandbox-sandbox-alpha-workspace/_data"; + const prepareManagedWorkspaceRoot = vi.fn(() => ({ + path: mountpoint, + device: "8", + inode: "9001", + uid: 0, + gid: 999, + mode: 0o1775 as const, + })); + const capture = vi.fn(() => ({ + status: 0, + stdout: `openshell-sandbox-sandbox-alpha-workspace\n${mountpoint}\n`, + stderr: "", + error: undefined, + })); + + preparePodmanManagedWorkspaceAuthority({ + engine: { ...engine(), capture, prepareManagedWorkspaceRoot }, + inspect: { + Mounts: [ + { + Type: "volume", + Name: "openshell-sandbox-sandbox-alpha-workspace", + Driver: "local", + Source: mountpoint, + Destination: "/sandbox", + RW: true, + }, + ], + }, + sandboxId: "sandbox-alpha", + workspaceRoot: { uid: 0, gid: 999, mode: 0o1775 }, + }); + + expect(capture).toHaveBeenCalledExactlyOnceWith( + [ + "volume", + "inspect", + "--format", + "{{.Name}}\n{{.Mountpoint}}", + "openshell-sandbox-sandbox-alpha-workspace", + ], + 15_000, + ); + expect(prepareManagedWorkspaceRoot).toHaveBeenCalledExactlyOnceWith({ + path: mountpoint, + uid: 0, + gid: 999, + mode: 0o1775, + }); + }); + + it("prepares a synthetic declared state root without provider-specific agent logic", () => { + const mountpoint = + "/home/test/.local/share/containers/storage/volumes/synthetic-state-alpha/_data"; + const prepareManagedVolumeRoot = vi.fn(() => ({ + path: mountpoint, + device: "8", + inode: "9002", + uid: 1000, + gid: 1000, + mode: 0o3770 as const, + })); + const labels = { + "io.nvidia.nemoclaw.synthetic-state.managed": "true", + "io.nvidia.nemoclaw.synthetic-state.sandbox": "alpha", + }; + const captureVolume = vi.fn( + () => `synthetic-state-alpha\n${mountpoint}\n${JSON.stringify(labels)}\n`, + ); + + prepareManagedBootstrapStateRoots({ + inspect: { + Mounts: [ + { + Type: "volume", + Name: "synthetic-state-alpha", + Driver: "local", + Source: mountpoint, + Destination: "/sandbox/.synthetic", + RW: true, + }, + ], + }, + roots: [ + { + mountTarget: "/sandbox/.synthetic", + resourceIdentity: "synthetic-state-alpha", + ownershipLabels: labels, + uid: 1000, + gid: 1000, + mode: 0o3770, + readWrite: true, + }, + ], + captureVolume, + prepareRoot: prepareManagedVolumeRoot, + }); + + expect(captureVolume).toHaveBeenCalledExactlyOnceWith([ + "inspect", + "--format", + "{{.Name}}\n{{.Mountpoint}}\n{{json .Labels}}", + "synthetic-state-alpha", + ]); + expect(prepareManagedVolumeRoot).toHaveBeenCalledExactlyOnceWith({ + path: mountpoint, + uid: 1000, + gid: 1000, + mode: 0o3770, + }); + }); + + it("reproduces the exact OpenShell Podman token secret on the replacement", () => { + const secretId = "secret-identity"; + const capture = vi.fn(() => ({ + status: 0, + stdout: `${secretId}\n`, + stderr: "", + error: undefined, + })); + + expect( + renderPodmanReplacementSecretArgs( + { ...engine(), capture }, + { + Config: { + Cmd: ["--workdir", "/sandbox"], + Env: ["OPENSHELL_SANDBOX_TOKEN_FILE=/run/secrets/openshell-token"], + Secrets: [ + { + Name: "openshell-token-sandbox-alpha", + ID: secretId, + UID: 0, + GID: 0, + Mode: 0o400, + }, + ], + }, + }, + "sandbox-alpha", + ), + ).toEqual([ + "--secret", + "openshell-token-sandbox-alpha,target=/run/secrets/openshell-token,uid=0,gid=0,mode=0400", + ]); + expect(capture).toHaveBeenCalledExactlyOnceWith( + ["secret", "inspect", "--format", "{{.ID}}", "openshell-token-sandbox-alpha"], + 15_000, + ); + }); + + it("reproduces OpenShell's provider-owned Podman launch authority", () => { + expect( + renderPodmanReplacementRuntimeArgs({ + Config: { + User: "0:0", + Hostname: "sandbox-alpha", + WorkingDir: "/", + StopTimeout: 10, + }, + HostConfig: { + CapAdd: ["CAP_SYS_ADMIN", "NET_ADMIN"], + CapDrop: [ + "KILL", + "CHOWN", + "DAC_OVERRIDE", + "FOWNER", + "FSETID", + "SETGID", + "SETUID", + "NET_RAW", + ], + SecurityOpt: ["no-new-privileges", "seccomp=unconfined"], + ExtraHosts: ["host.openshell.internal:10.89.0.1"], + GroupAdd: ["44"], + Tmpfs: { "/run/netns": "rw,nosuid,nodev" }, + PortBindings: { + "22/tcp": [{ HostIp: "127.0.0.1", HostPort: "32122" }], + }, + Ulimits: [{ Name: "nofile", Soft: 1024, Hard: 2048 }], + Memory: 2_147_483_648, + PidsLimit: 2048, + OomScoreAdj: 500, + }, + }), + ).toEqual([ + "--user", + "0:0", + "--hostname", + "sandbox-alpha", + "--workdir", + "/", + "--stop-timeout", + "10", + "--cap-add", + "SYS_ADMIN", + "--cap-add", + "NET_ADMIN", + "--cap-add", + "DAC_OVERRIDE", + "--cap-add", + "FSETID", + "--cap-add", + "KILL", + "--cap-drop", + "CHOWN", + "--cap-drop", + "FOWNER", + "--cap-drop", + "SETGID", + "--cap-drop", + "SETUID", + "--cap-drop", + "NET_RAW", + "--security-opt", + "no-new-privileges", + "--security-opt", + "seccomp=unconfined", + "--add-host", + "host.openshell.internal:10.89.0.1", + "--group-add", + "44", + "--tmpfs", + "/run/netns:rw,nosuid,nodev", + "--publish", + "127.0.0.1:32122:22/tcp", + "--ulimit", + "nofile=1024:2048", + "--memory", + "2147483648", + "--pids-limit", + "2048", + "--oom-score-adj", + "500", + ]); + }); + + it("persists only non-secret native gateway launch environment", () => { + const first = buildPodmanStandaloneGatewayEnvironmentAuthority({ + PATH: "/usr/bin", + OPENSHELL_DRIVERS: "podman", + OPENSHELL_PODMAN_SOCKET: "/run/user/1000/podman/podman.sock", + DOCKER_HOST: "unix:///var/run/docker.sock", + NVIDIA_API_KEY: "first-secret", + GH_TOKEN: "first-token", + NEMOCLAW_BOOTSTRAP_PAYLOAD: "first-bootstrap-payload", + }); + const changedSecrets = buildPodmanStandaloneGatewayEnvironmentAuthority({ + PATH: "/usr/bin", + OPENSHELL_DRIVERS: "podman", + OPENSHELL_PODMAN_SOCKET: "/run/user/1000/podman/podman.sock", + DOCKER_HOST: "tcp://unrelated-docker:2375", + NVIDIA_API_KEY: "changed-secret", + GH_TOKEN: "changed-token", + NEMOCLAW_BOOTSTRAP_PAYLOAD: "changed-bootstrap-payload", + }); + + expect(changedSecrets).toEqual(first); + expect(first.map((entry) => entry.key)).toEqual([ + "OPENSHELL_DRIVERS", + "OPENSHELL_PODMAN_SOCKET", + "PATH", + ]); + expect(JSON.stringify(first)).not.toContain("secret"); + expect(JSON.stringify(first)).not.toContain("token"); + expect(JSON.stringify(first)).not.toContain("DOCKER_HOST"); + expect(JSON.stringify(first)).not.toContain("NEMOCLAW_BOOTSTRAP_PAYLOAD"); + expect(JSON.stringify(first)).not.toContain( + createHash("sha256").update("first-bootstrap-payload", "utf8").digest("hex"), + ); + }); + + it.each([ + ["after original removal", "openshell-alpha-bootstrap", true], + ["after replacement rename", "openshell-default--alpha-sandbox-alpha", false], + ])("finishes an authorized commit crash %s", (_label, replacementName, expectsRename) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-podman-runtime-commit-")); + try { + const store = createFilePodmanBootstrapJournalStore(root); + const initial = committedJournal(); + store.create(initial); + store.recordStateVolume(IDENTITY, "/var/lib/containers/storage/volumes/alpha/_data"); + store.recordReplacement(IDENTITY, REPLACEMENT_RUNTIME_ID); + store.recordOriginalStopped(IDENTITY); + const authorized = store.authorizeCommit(IDENTITY, ["original-stopped"]); + const runtimes = new Map([ + [ + REPLACEMENT_RUNTIME_ID, + runtimeInspect(REPLACEMENT_RUNTIME_ID, replacementName, `sha256:${"7".repeat(64)}`), + ], + ]); + const capture = vi.fn((args: readonly string[]) => { + const [kind, command, runtimeId, renameTarget] = args; + const exactRuntimeId = typeof runtimeId === "string" ? runtimeId : ""; + const unsupported = () => ({ + status: 2, + stdout: "", + stderr: "unsupported", + error: undefined, + }); + const handlers: Record ReturnType> = { + exists: () => ({ + status: runtimes.has(exactRuntimeId) ? 0 : 1, + stdout: "", + stderr: "", + error: undefined, + }), + inspect: () => { + const inspected = runtimes.get(exactRuntimeId); + return { + status: inspected ? 0 : 1, + stdout: inspected ? JSON.stringify([inspected]) : "", + stderr: "", + error: undefined, + }; + }, + rm: () => { + runtimes.delete(exactRuntimeId); + return { status: 0, stdout: "", stderr: "", error: undefined }; + }, + rename: () => { + const inspected = runtimes.get(exactRuntimeId); + const target = typeof renameTarget === "string" ? renameTarget : ""; + inspected && (inspected.Name = target); + return { + status: inspected && target ? 0 : 1, + stdout: "", + stderr: "", + error: undefined, + }; + }, + }; + return kind === "container" && exactRuntimeId + ? (handlers[command ?? ""] ?? unsupported)() + : unsupported(); + }); + const commitEngine = { + ...engine(), + authorityId: ENGINE_AUTHORITY_ID, + capture, + } as PodmanBoundContainerEngine; + const lease = { + record: { + schemaVersion: PODMAN_WATCHER_LEASE_SCHEMA_VERSION, + gatewayName: "nemoclaw", + gatewayPort: 8080, + launchIdentity: "launch", + ownerIdentity: "owner", + ownerKind: "standalone", + pid: 4100, + processStartIdentity: "start", + holder: { pid: 9100, processStartIdentity: "holder" }, + leaseId: LEASE_ID, + phase: "stopped", + }, + assertStillHeld: vi.fn(), + assertStillStopped: vi.fn(), + resumeForObservationAndProve: vi.fn(), + requiesceAndProve: vi.fn(), + resumeAndProve: vi.fn(), + } satisfies PodmanGatewayWatcherLease; + + expect( + finishCommittedPodmanBootstrap({ + engine: commitEngine, + journalStore: store, + journal: authorized, + watcherLease: lease, + }).phase, + ).toBe("committed"); + expect(store.load(IDENTITY)).toBeNull(); + expect(runtimes.get(REPLACEMENT_RUNTIME_ID)?.Name).toBe( + "openshell-default--alpha-sandbox-alpha", + ); + expect(capture.mock.calls.some(([args]) => Array.isArray(args) && args[1] === "rename")).toBe( + expectsRename, + ); + } finally { + fs.rmSync(root, { force: true, recursive: true }); + } + }); + + it("recovers a terminal cleanup crash that left only the watcher lease", async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-podman-runtime-recovery-")); + const recoverUnfinishedLease = vi.fn(); + try { + const adapter = createPodmanManagedBootstrapAdapter({ + engine: engine(), + stateRoot: root, + environment: {}, + gatewayPort: 8080, + workspaceRoot: { uid: 998, gid: 999, mode: 0o755 }, + watcherController: { + recoverUnfinishedLease, + reclaimStoppedLease: vi.fn(), + quiesceAndProve: vi.fn(), + }, + }); + + await expect(adapter.recoverUnfinishedTransactions()).resolves.toEqual({ + receipts: [], + failures: [], + }); + expect(recoverUnfinishedLease).toHaveBeenCalledOnce(); + } finally { + fs.rmSync(root, { force: true, recursive: true }); + } + }); + + it("selects the Podman provider and keeps compatibility routing disabled", () => { + const operationEngine = engine(); + const surface = createPodmanManagedBootstrapSurface(operationEngine); + const routing = surface.createOnboardRouting({ + sandboxName: "alpha", + openshellArgv: (args) => args, + nativeFallbackEnabled: true, + }); + + expect(surface.providerId).toBe("podman"); + expect(surface.supported).toBe(true); + expect(routing.nativeFallbackHasCleanBaseline).toBe(false); + expect(routing.inspectNativeRuntime()).toBeNull(); + expect(routing.isNativeCreateRoutingFailure("failure", true)).toBe(false); + expect(() => + routing.prepareCompatibilityLaunch({ + createArgs: [], + currentRegistryImageRef: null, + managedImageReference: + "nvcr.io/nvidia/nemoclaw/hermes@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + prebuildImageId: null, + allowUnbuiltSource: false, + compatibilityPolicyPath: "/unused/policy.yaml", + startupCommand: [], + runtimeSnapshot: null, + }), + ).toThrow("does not use Docker compatibility fallback"); + expect(operationEngine.capture).not.toHaveBeenCalled(); + }); + + it("wires recovery and commit through the selected provider without eager engine dispatch", async () => { + vi.clearAllMocks(); + installCoordinatorMocks(); + const operationEngine = engine(); + const injected = adapter(); + const lifecycle = createPodmanManagedBootstrapSurface(operationEngine).createLifecycle( + lifecycleInput(injected.value), + ); + + await expect(lifecycle.recoverUnfinished()).resolves.toEqual({ receipts: [], failures: [] }); + await expect( + lifecycle.runCreate(async ({ bootstrapIdentity }) => ({ + value: "created", + receipt: { + sandbox: { sandboxName: "alpha", sandboxId: "sandbox-alpha", driverId: "podman" }, + ready: true, + readyAt: "2026-08-22T00:00:00.000Z", + }, + bootstrapIdentity, + })), + ).resolves.toBe("created"); + await lifecycle.patch.commitAfterReady(); + + expect(injected.recoverUnfinishedTransactions).toHaveBeenCalledOnce(); + expect(coordinator.prepare).toHaveBeenCalledOnce(); + expect(coordinator.activate).toHaveBeenCalledOnce(); + expect(coordinator.finalize).toHaveBeenCalledExactlyOnceWith( + injected.value, + expect.objectContaining({ outcome: "commit" }), + ); + expect(operationEngine.capture).not.toHaveBeenCalled(); + }); + + it("uses Podman's all-GPU CDI authority when no exact device was selected", async () => { + vi.clearAllMocks(); + installCoordinatorMocks(); + const injected = adapter(); + const input = lifecycleInput(injected.value); + const lifecycle = createPodmanManagedBootstrapSurface(engine()).createLifecycle({ + ...input, + sandboxGpuConfig: { + ...input.sandboxGpuConfig, + mode: "1", + hostGpuDetected: true, + hostGpuPlatform: "linux", + sandboxGpuEnabled: true, + }, + }); + + await lifecycle.runCreate(async () => ({ + value: "created", + receipt: { + sandbox: { sandboxName: "alpha", sandboxId: "sandbox-alpha", driverId: "podman" }, + ready: true, + readyAt: "2026-08-22T00:00:00.000Z", + }, + })); + + expect(coordinator.prepare).toHaveBeenCalledWith( + injected.value, + expect.objectContaining({ + replacementOptions: { values: expect.objectContaining({ gpuModeArgs: ["--gpus", "all"] }) }, + }), + ); + }); + + it("wires rollback through the same provider-owned terminal transaction", async () => { + vi.clearAllMocks(); + installCoordinatorMocks(); + const operationEngine = engine(); + const injected = adapter(); + const lifecycle = createPodmanManagedBootstrapSurface(operationEngine).createLifecycle( + lifecycleInput(injected.value), + ); + + await lifecycle.runCreate(async () => ({ + value: "created", + receipt: { + sandbox: { sandboxName: "alpha", sandboxId: "sandbox-alpha", driverId: "podman" }, + ready: true, + readyAt: "2026-08-22T00:00:00.000Z", + }, + })); + await lifecycle.patch.rollbackManagedStartupAfterCreateFailure(); + + expect(coordinator.finalize).toHaveBeenCalledExactlyOnceWith( + injected.value, + expect.objectContaining({ outcome: "rollback" }), + ); + expect(operationEngine.capture).not.toHaveBeenCalled(); + }); + + it("rejects lifecycle construction for another provider identity", () => { + const injected = adapter(); + expect(() => + createPodmanManagedBootstrapSurface(engine()).createLifecycle({ + ...lifecycleInput(injected.value), + providerId: "docker", + }), + ).toThrow("another provider identity"); + }); +}); diff --git a/src/lib/onboard/managed-bootstrap/podman-runtime.ts b/src/lib/onboard/managed-bootstrap/podman-runtime.ts new file mode 100644 index 00000000000..a1f2a38ef04 --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/podman-runtime.ts @@ -0,0 +1,2318 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawn, spawnSync } from "node:child_process"; +import { createHash, randomUUID } from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import type { PodmanBoundContainerEngine } from "../../adapters/podman"; +import type { SandboxGpuProofResult } from "../../state/registry"; +import { + getDockerDriverGatewayRuntimeMarkerPath, + parseDockerDriverGatewayRuntimeMarker, + readDockerDriverGatewayRuntimeMarker, + writeDockerDriverGatewayPidFile, + writeDockerDriverGatewayRuntimeMarker, + type DockerDriverGatewayRuntimeMarker, +} from "../docker-driver-gateway-runtime-marker"; +import { + getTrustedActiveOpenShellGatewayUserServiceIdentity, + startOpenShellGatewayUserService, + stopOpenShellGatewayUserService, + waitForOpenShellGatewayRetry, +} from "../docker-driver-gateway-service"; +import { shouldOmitOpenShellOciImageUser } from "../docker-gpu-patch-clone"; +import type { DockerContainerInspect } from "../docker-gpu-patch-types"; +import { openshellSandboxCommandEnvValue } from "../docker-startup-command-env"; +import { resolveGatewayName, resolveGatewayStateDirName } from "../gateway-binding/identity"; +import type { ManagedStartupRootApplyRequest } from "../managed-startup/root-apply"; +import type { ManagedStartupWorkspaceRoot } from "../managed-startup/state-roots"; +import type { RuntimeProviderManagedImageBootstrapSurface } from "../runtime-provider/contract"; +import { + normalizePodmanLogicalMounts, + resolvePodmanStorageGraphRoot, +} from "../runtime-provider/podman-runtime-surfaces"; +import { + activateManagedBootstrapSequence, + finalizeManagedBootstrapSequence, + MANAGED_BOOTSTRAP_SCHEMA_VERSION, + prepareManagedBootstrapSequence, + renderManagedBootstrapHeldCommand, + type ManagedBootstrapAdapter, + type ManagedBootstrapAuthorityStore, + type ManagedBootstrapCompletionReceipt, + type ManagedBootstrapDiscoveredWorkload, + type ManagedBootstrapFinalizationReceipt, + type ManagedBootstrapHeldWorkloadHandle, + type ManagedBootstrapObservedSnapshot, + type ManagedBootstrapPreparedAuthority, + type ManagedBootstrapPreparedReplacementHandle, + type ManagedBootstrapRecoveryFailure, + type ManagedBootstrapRecoveryReceipt, + type ManagedBootstrapReplacementHandle, +} from "./adapter"; +import { MANAGED_BOOTSTRAP_REQUEST_FILE } from "./envelope"; +import { + createFilePodmanBootstrapJournalStore, + type PodmanBootstrapJournal, + type PodmanBootstrapJournalStore, + serializePodmanBootstrapJournal, +} from "./podman-bootstrap-journal"; +import { + PODMAN_BOOTSTRAP_REPLACEMENT_SCHEMA_VERSION, + prepareStoppedPodmanBootstrapReplacement, + rollbackPodmanBootstrapBeforeCommit, + stopExactPodmanBootstrapOriginal, + type PodmanBootstrapPreparedReplacement, +} from "./podman-bootstrap-replacement"; +import { + awaitPodmanBootstrapImageTransaction, + startPodmanBootstrapImageTransaction, + type PodmanBootstrapImageTransaction, + type PodmanBootstrapImageTransactionCompletion, +} from "./podman-image-transaction"; +import { + inspectExactPodmanHeldWorkload, + PODMAN_MANAGED_LABEL, + PODMAN_OPENSHELL_MANAGED_BY_LABEL, + PODMAN_OPENSHELL_MANAGED_BY_VALUE, + PODMAN_SANDBOX_ID_LABEL, + PODMAN_SANDBOX_NAME_LABEL, + PODMAN_SANDBOX_NAMESPACE, + PODMAN_SANDBOX_NAMESPACE_LABEL, + PODMAN_SANDBOX_WORKSPACE, + PODMAN_SANDBOX_WORKSPACE_LABEL, + type PodmanHeldWorkloadObservation, +} from "./podman-held-workload"; +import type { + PodmanGatewayWatcherLease, + PodmanGatewayWatcherLeaseHolder, + PodmanGatewayWatcherLeaseRecord, + PodmanGatewayWatcherLeaseStore, + PodmanGatewayWatcherSnapshot, + PodmanManagedGatewayWatcherController, +} from "./podman-watcher-lease"; +import { createPodmanManagedGatewayWatcherController } from "./podman-watcher-lease"; +import type { + ManagedBootstrapRuntimeCreateLifecycle, + ManagedBootstrapRuntimeCreateLifecycleInput, + ManagedBootstrapRuntimeOnboardRouting, + ManagedBootstrapRuntimeOnboardRoutingInput, +} from "./runtime-create"; +import { createManagedBootstrapTerminalFinalizer } from "./runtime-create"; +import { prepareManagedBootstrapStateRoots } from "./state-root-authority"; + +const PROVIDER_ID = "podman"; +const BOOTSTRAP_EXECUTABLE = "/usr/local/bin/nemoclaw-managed-bootstrap"; +const FULL_ID = /^[a-f0-9]{64}$/u; +const SHA256 = /^[a-f0-9]{64}$/u; +const SAFE_ENV = /^[A-Za-z_][A-Za-z0-9_]*=/u; +const SAFE_RESOURCE_NAME = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,252}$/u; +const LEASE_FILE = "managed-bootstrap-podman-watcher.json"; +const STANDALONE_LAUNCH_FILE = "managed-bootstrap-podman-gateway-launch.json"; +const MANAGED_BOOTSTRAP_TIMEOUT_MS = 300_000; +const REPLACEMENT_OBSERVATION_TIMEOUT_MS = 30_000; +const REPLACEMENT_OBSERVATION_INTERVAL_SECONDS = 0.25; +const OPENSHELL_TOKEN_SECRET_PREFIX = "openshell-token-"; +const OPENSHELL_PROXY_SECRET_PREFIX = "openshell-proxy-auth-"; +const OPENSHELL_WORKSPACE_VOLUME_PREFIX = "openshell-sandbox-"; +const OPENSHELL_WORKSPACE_VOLUME_SUFFIX = "-workspace"; +const OPENSHELL_WORKSPACE_DIRECTORY = "/sandbox"; +const PODMAN_BOOTSTRAP_CAPABILITY_DROP_ENV = "NEMOCLAW_MANAGED_BOOTSTRAP_DROP_CAPABILITIES=0x32"; +const PERSISTABLE_ENVIRONMENT_KEYS = new Set([ + "HOME", + "LANG", + "LC_ALL", + "LC_CTYPE", + "LD_LIBRARY_PATH", + "LOGNAME", + "NEMOCLAW_DOCKER_ENABLE_BIND_MOUNTS", + "NEMOCLAW_OPENSHELL_SANDBOX_NAMESPACE", + "NEMOCLAW_RUNTIME_PROVIDER_ID", + "NETAVARK_FW", + "OPENSHELL_BIND_ADDRESS", + "OPENSHELL_DB_URL", + "OPENSHELL_DOCKER_NETWORK_NAME", + "OPENSHELL_DOCKER_SUPERVISOR_BIN", + "OPENSHELL_DOCKER_SUPERVISOR_IMAGE", + "OPENSHELL_DRIVERS", + "OPENSHELL_GATEWAY_CONFIG", + "OPENSHELL_GRPC_ENDPOINT", + "OPENSHELL_LOCAL_TLS_DIR", + "OPENSHELL_PODMAN_SOCKET", + "OPENSHELL_SERVER_PORT", + "OPENSHELL_SSH_GATEWAY_HOST", + "OPENSHELL_SSH_GATEWAY_PORT", + "PATH", + "SHELL", + "SSL_CERT_DIR", + "SSL_CERT_FILE", + "TMPDIR", + "USER", + "XDG_CONFIG_HOME", + "XDG_DATA_HOME", + "XDG_RUNTIME_DIR", + "XDG_STATE_HOME", +]); + +type JsonRecord = Record; + +interface PodmanManagedBootstrapAdapterOptions { + readonly engine: PodmanBoundContainerEngine; + readonly stateRoot: string; + readonly environment: NodeJS.ProcessEnv; + readonly gatewayPort: number; + readonly gatewayName?: string; + readonly workspaceRoot: ManagedStartupWorkspaceRoot; + readonly watcherController?: PodmanManagedGatewayWatcherController; + readonly runCaptureOpenshell?: (args: string[], options?: Record) => string; + readonly sleep?: (seconds: number) => void; +} + +interface TransactionState { + readonly held: PodmanHeldWorkloadObservation; + readonly rawInspect: JsonRecord; + readonly request: ManagedStartupRootApplyRequest; + watcherLease?: PodmanGatewayWatcherLease; + prepared?: PodmanBootstrapPreparedReplacement; + imageTransaction?: PodmanBootstrapImageTransaction; + completion?: PodmanBootstrapImageTransactionCompletion; + contractReplacementSpecCanonical?: string; + contractReplacementSpecHash?: string; +} + +function record(value: unknown, label: string): JsonRecord { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error(`Managed bootstrap Podman ${label} must be an object.`); + } + return value as JsonRecord; +} + +function stringArray(value: unknown, label: string): readonly string[] { + if ( + !Array.isArray(value) || + value.some((entry) => typeof entry !== "string" || entry.includes("\0")) + ) { + throw new Error(`Managed bootstrap Podman ${label} must be a bounded string array.`); + } + return Object.freeze([...value]); +} + +function commandFailure( + action: string, + result: ReturnType, +): never { + const detail = (result.stderr || result.stdout || result.error?.message || "unknown failure") + .replace(/\s+/gu, " ") + .trim() + .slice(-600); + throw new Error( + `Managed bootstrap Podman ${action} failed (exit ${String(result.status)}): ${detail}`, + ); +} + +function capture( + engine: PodmanBoundContainerEngine, + args: readonly string[], + action: string, + timeout = MANAGED_BOOTSTRAP_TIMEOUT_MS, +) { + const result = engine.capture(args, timeout); + if (result.status !== 0 || result.error) commandFailure(action, result); + return result; +} + +function inspectRuntime(engine: PodmanBoundContainerEngine, runtimeId: string): JsonRecord { + if (!FULL_ID.test(runtimeId)) throw new Error("Managed bootstrap Podman runtime ID is invalid."); + const result = capture(engine, ["container", "inspect", runtimeId], "container inspect", 15_000); + let parsed: unknown; + try { + parsed = JSON.parse(result.stdout); + } catch { + throw new Error("Managed bootstrap Podman inspect returned unreadable JSON."); + } + if (!Array.isArray(parsed) || parsed.length !== 1) { + throw new Error("Managed bootstrap Podman inspect must resolve exactly one container."); + } + const inspected = record(parsed[0], "inspect entry"); + if (String(inspected.Id ?? "").toLowerCase() !== runtimeId) { + throw new Error("Managed bootstrap Podman inspect returned another runtime identity."); + } + return inspected; +} + +export function observePodmanBootstrapReplacementReady(input: { + readonly engine: PodmanBoundContainerEngine; + readonly runtimeId: string; + readonly sandboxName: string; + readonly sandboxId: string; + readonly gatewayName: string; + readonly runCaptureOpenshell: NonNullable< + PodmanManagedBootstrapAdapterOptions["runCaptureOpenshell"] + >; + readonly sleep?: (seconds: number) => void; + readonly now?: () => number; + readonly timeoutMs?: number; +}): void { + if (!FULL_ID.test(input.runtimeId)) { + throw new Error("Managed bootstrap Podman replacement runtime ID is invalid."); + } + const now = input.now ?? Date.now; + const sleep = input.sleep ?? waitForOpenShellGatewayRetry; + const timeoutMs = input.timeoutMs ?? REPLACEMENT_OBSERVATION_TIMEOUT_MS; + const deadline = now() + timeoutMs; + const maxAttempts = Math.ceil(timeoutMs / (REPLACEMENT_OBSERVATION_INTERVAL_SECONDS * 1000)); + let lastPhase = "unobserved"; + let lastHealthFailure = "none"; + + for (let attempt = 0; attempt <= maxAttempts; attempt += 1) { + const health = input.engine.capture(["healthcheck", "run", input.runtimeId], 15_000); + if (health.error || (health.status !== 0 && health.status !== 1)) { + commandFailure("replacement health observation", health); + } + if (health.status !== 0) { + lastHealthFailure = (health.stderr || health.stdout || "unhealthy").trim().slice(-300); + } + + const output = input.runCaptureOpenshell( + ["sandbox", "get", "-g", input.gatewayName, input.sandboxName, "--output", "json"], + { ignoreError: true, timeout: 5_000 }, + ); + if (output.trim()) { + let observed: JsonRecord; + try { + observed = record(JSON.parse(output), "OpenShell replacement observation"); + } catch { + throw new Error("Managed bootstrap Podman OpenShell observation returned unreadable JSON."); + } + if (observed.id !== input.sandboxId) { + throw new Error("Managed bootstrap Podman OpenShell sandbox identity changed."); + } + lastPhase = typeof observed.phase === "string" ? observed.phase : "unknown"; + if (lastPhase === "Ready" && health.status === 0) return; + } + if (attempt < maxAttempts && now() < deadline) { + sleep(REPLACEMENT_OBSERVATION_INTERVAL_SECONDS); + } else { + break; + } + } + + throw new Error( + `Managed bootstrap Podman replacement health was not observed by OpenShell before timeout (phase ${lastPhase}; health ${lastHealthFailure}).`, + ); +} + +function sha256(value: string): string { + return createHash("sha256").update(value, "utf8").digest("hex"); +} + +function canonicalInspect(inspect: JsonRecord): string { + // Podman emits its inspect object in a stable field order. Persist only the + // provider-owned launch facets used to reproduce the exact replacement. + const config = record(inspect.Config, "Config"); + const hostConfig = record(inspect.HostConfig ?? {}, "HostConfig"); + const networkSettings = record(inspect.NetworkSettings ?? {}, "NetworkSettings"); + const canonical = { + Config: { + Cmd: stringArray(config.Cmd ?? [], "Config.Cmd"), + Entrypoint: stringArray(config.Entrypoint ?? [], "Config.Entrypoint"), + Env: stringArray(config.Env ?? [], "Config.Env"), + Healthcheck: record(config.Healthcheck, "Config.Healthcheck"), + Labels: record(config.Labels ?? {}, "Config.Labels"), + Secrets: Array.isArray(config.Secrets) ? config.Secrets : [], + WorkingDir: String(config.WorkingDir ?? ""), + }, + HostConfig: hostConfig, + Mounts: Array.isArray(inspect.Mounts) ? inspect.Mounts : [], + NetworkSettings: { Networks: networkSettings.Networks ?? {} }, + }; + return JSON.stringify(canonical); +} + +function healthDuration(value: unknown, label: string, minimum: number): number { + if (typeof value !== "number" || !Number.isSafeInteger(value) || value < minimum) { + throw new Error(`Managed bootstrap Podman ${label} is invalid.`); + } + return value; +} + +/** Reproduce OpenShell's exact Podman health contract on the managed replacement. */ +export function renderPodmanReplacementHealthArgs(inspect: JsonRecord): readonly string[] { + const config = record(inspect.Config, "Config"); + const health = record(config.Healthcheck, "Config.Healthcheck"); + const test = stringArray(health.Test, "Config.Healthcheck.Test"); + if (test.length !== 2 || test[0] !== "CMD-SHELL" || test[1]?.trim().length === 0) { + throw new Error("Managed bootstrap Podman health check must be one CMD-SHELL command."); + } + const interval = healthDuration(health.Interval, "health-check interval", 1); + const timeout = healthDuration(health.Timeout, "health-check timeout", 1_000_000_000); + const retries = healthDuration(health.Retries, "health-check retries", 1); + const startPeriod = healthDuration(health.StartPeriod, "health-check start period", 0); + return Object.freeze([ + "--health-cmd", + test[1] as string, + "--health-interval", + `${String(interval)}ns`, + "--health-timeout", + `${String(timeout)}ns`, + "--health-retries", + String(retries), + "--health-start-period", + `${String(startPeriod)}ns`, + ]); +} + +function boundedRuntimeValue(value: unknown, label: string): string { + if ( + typeof value !== "string" || + value.length === 0 || + value.length > 4_096 || + value.includes("\0") || + /[\r\n]/u.test(value) + ) { + throw new Error(`Managed bootstrap Podman ${label} is invalid.`); + } + return value; +} + +function optionalNonNegativeIntegerFlag( + args: string[], + flag: string, + value: unknown, + label: string, +): void { + if (value === undefined || value === null || value === 0 || value === "") return; + if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0) { + throw new Error(`Managed bootstrap Podman ${label} is invalid.`); + } + args.push(flag, String(value)); +} + +function optionalStringFlag(args: string[], flag: string, value: unknown, label: string): void { + if (value === undefined || value === null || value === "") return; + args.push(flag, boundedRuntimeValue(value, label)); +} + +/** Reproduce the provider-owned OpenShell Podman launch contract. */ +export function renderPodmanReplacementRuntimeArgs(inspect: JsonRecord): readonly string[] { + const config = record(inspect.Config, "Config"); + const host = record(inspect.HostConfig ?? {}, "HostConfig"); + const args: string[] = []; + + const user = String(config.User ?? "").trim(); + if (!["0", "0:0", "root", "root:root"].includes(user)) { + throw new Error("Managed bootstrap Podman supervisor user must remain root."); + } + args.push("--user", "0:0"); + optionalStringFlag(args, "--hostname", config.Hostname, "hostname"); + const workingDirectory = String(config.WorkingDir ?? ""); + if (workingDirectory) { + if ( + !path.isAbsolute(workingDirectory) || + path.normalize(workingDirectory) !== workingDirectory + ) { + throw new Error("Managed bootstrap Podman working directory is invalid."); + } + args.push("--workdir", workingDirectory); + } + optionalNonNegativeIntegerFlag(args, "--stop-timeout", config.StopTimeout, "stop timeout"); + + const capability = (value: unknown, label: string): string => { + const normalized = boundedRuntimeValue(value, label).replace(/^CAP_/u, ""); + if (!/^[A-Z][A-Z0-9_]{0,63}$/u.test(normalized)) { + throw new Error(`Managed bootstrap Podman ${label} is invalid.`); + } + return normalized; + }; + const addedCapabilities = stringArray(host.CapAdd ?? [], "HostConfig.CapAdd").map((value) => + capability(value, "added capability"), + ); + // OpenShell's long-running Podman supervisor deliberately drops these + // capabilities. The short-lived managed bootstrap replacement needs them + // to mutate the agent-owned workspace, preserve setgid state roots, and + // signal only its exact pidfd-pinned child before resuming that supervisor. + const bootstrapCapabilities: ReadonlySet = new Set(["DAC_OVERRIDE", "FSETID", "KILL"]); + for (const value of new Set([...addedCapabilities, ...bootstrapCapabilities])) { + args.push("--cap-add", value); + } + for (const value of stringArray(host.CapDrop ?? [], "HostConfig.CapDrop")) { + const dropped = capability(value, "dropped capability"); + if (!bootstrapCapabilities.has(dropped)) { + args.push("--cap-drop", dropped); + } + } + for (const value of stringArray(host.SecurityOpt ?? [], "HostConfig.SecurityOpt")) { + args.push("--security-opt", boundedRuntimeValue(value, "security option")); + } + for (const value of stringArray(host.ExtraHosts ?? [], "HostConfig.ExtraHosts")) { + args.push("--add-host", boundedRuntimeValue(value, "host alias")); + } + for (const value of stringArray(host.GroupAdd ?? [], "HostConfig.GroupAdd")) { + args.push("--group-add", boundedRuntimeValue(value, "supplementary group")); + } + for (const value of stringArray(host.Dns ?? [], "HostConfig.Dns")) { + args.push("--dns", boundedRuntimeValue(value, "DNS server")); + } + for (const value of stringArray(host.DnsSearch ?? [], "HostConfig.DnsSearch")) { + args.push("--dns-search", boundedRuntimeValue(value, "DNS search domain")); + } + + if (host.Tmpfs !== undefined && host.Tmpfs !== null) { + const tmpfs = record(host.Tmpfs, "HostConfig.Tmpfs"); + for (const destination of Object.keys(tmpfs).sort()) { + if (!path.isAbsolute(destination) || path.normalize(destination) !== destination) { + throw new Error("Managed bootstrap Podman tmpfs destination is invalid."); + } + const options = tmpfs[destination]; + const rendered = options + ? `${destination}:${boundedRuntimeValue(options, "tmpfs options")}` + : destination; + args.push("--tmpfs", rendered); + } + } + + if (host.PortBindings !== undefined && host.PortBindings !== null) { + const bindings = record(host.PortBindings, "HostConfig.PortBindings"); + for (const containerEndpoint of Object.keys(bindings).sort()) { + const match = /^(\d{1,5})\/(tcp|udp|sctp)$/u.exec(containerEndpoint); + const containerPort = Number(match?.[1]); + if (!match || containerPort < 1 || containerPort > 65_535) { + throw new Error("Managed bootstrap Podman published container endpoint is invalid."); + } + const rows = bindings[containerEndpoint]; + if (!Array.isArray(rows) || rows.length !== 1) { + throw new Error("Managed bootstrap Podman published host endpoint is ambiguous."); + } + const binding = record(rows[0], "HostConfig.PortBindings entry"); + const hostPort = Number(binding.HostPort); + if (!Number.isSafeInteger(hostPort) || hostPort < 1 || hostPort > 65_535) { + throw new Error("Managed bootstrap Podman published host port is invalid."); + } + const hostIp = String(binding.HostIp ?? ""); + if (hostIp.includes("\0") || /[\r\n]/u.test(hostIp)) { + throw new Error("Managed bootstrap Podman published host address is invalid."); + } + const endpoint = `${hostIp ? `${hostIp}:` : ""}${String(hostPort)}:${String(containerPort)}/${match[2]}`; + args.push("--publish", endpoint); + } + } + + if (host.Ulimits !== undefined && host.Ulimits !== null) { + if (!Array.isArray(host.Ulimits)) { + throw new Error("Managed bootstrap Podman ulimits are invalid."); + } + for (const value of host.Ulimits) { + const limit = record(value, "HostConfig.Ulimits entry"); + const name = boundedRuntimeValue(limit.Name, "ulimit name"); + const soft = Number(limit.Soft); + const hard = Number(limit.Hard); + if (![soft, hard].every((entry) => Number.isSafeInteger(entry) && entry >= -1)) { + throw new Error("Managed bootstrap Podman ulimit bounds are invalid."); + } + args.push("--ulimit", `${name}=${String(soft)}:${String(hard)}`); + } + } + + optionalNonNegativeIntegerFlag(args, "--memory", host.Memory, "memory limit"); + optionalNonNegativeIntegerFlag( + args, + "--memory-reservation", + host.MemoryReservation, + "memory reservation", + ); + optionalNonNegativeIntegerFlag(args, "--memory-swap", host.MemorySwap, "memory swap limit"); + optionalNonNegativeIntegerFlag(args, "--cpu-shares", host.CpuShares, "CPU shares"); + optionalNonNegativeIntegerFlag(args, "--cpu-quota", host.CpuQuota, "CPU quota"); + optionalNonNegativeIntegerFlag(args, "--cpu-period", host.CpuPeriod, "CPU period"); + optionalNonNegativeIntegerFlag(args, "--pids-limit", host.PidsLimit, "PID limit"); + optionalNonNegativeIntegerFlag(args, "--oom-score-adj", host.OomScoreAdj, "OOM score adjustment"); + optionalStringFlag(args, "--cpuset-cpus", host.CpusetCpus, "CPU set"); + optionalStringFlag(args, "--cpuset-mems", host.CpusetMems, "memory-node set"); + return Object.freeze(args); +} + +function replacementCommand( + handle: ManagedBootstrapHeldWorkloadHandle, + snapshot: ManagedBootstrapObservedSnapshot, +): readonly string[] { + return Object.freeze([ + "--agent", + handle.plan.profile.agent, + "--profile-fingerprint", + handle.plan.profile.fingerprint, + "--bootstrap-identity", + handle.bootstrapIdentity, + "--agent-uid", + String(snapshot.agentIdentity.uid), + "--agent-gid", + String(snapshot.agentIdentity.gid), + "--agent-workdir", + snapshot.agentIdentity.workdir, + "--request-file", + MANAGED_BOOTSTRAP_REQUEST_FILE, + "--", + ...snapshot.supervisorArgv, + ]); +} + +export function renderPodmanReplacementEnvironment( + inspect: JsonRecord, + handle: ManagedBootstrapHeldWorkloadHandle, +): readonly string[] { + const config = record(inspect.Config, "Config"); + const intended = openshellSandboxCommandEnvValue(handle.intendedWorkloadArgv); + if (!intended) throw new Error("Managed bootstrap Podman intended workload argv is invalid."); + const inspectedUser = String(config.User ?? "").trim(); + const labels = record(config.Labels ?? {}, "Config.Labels"); + const exactPodmanBoundary = labels[PODMAN_MANAGED_LABEL] === "true"; + const workspaceInspect = exactPodmanBoundary + ? { + ...inspect, + Config: { + ...config, + User: ["0:0", "root", "root:root"].includes(inspectedUser) ? "0" : inspectedUser, + WorkingDir: "/", + Labels: { ...labels, "openshell.ai/managed-by": "openshell" }, + }, + } + : inspect; + const omitOciImageUser = shouldOmitOpenShellOciImageUser( + workspaceInspect as unknown as DockerContainerInspect, + handle.intendedWorkloadArgv, + ); + const values = stringArray(config.Env ?? [], "Config.Env").filter( + (entry) => + !entry.startsWith("OPENSHELL_SANDBOX_COMMAND=") && + !entry.startsWith("NEMOCLAW_MANAGED_BOOTSTRAP_DROP_CAPABILITIES=") && + (!omitOciImageUser || !entry.startsWith("OPENSHELL_OCI_IMAGE_USER=")), + ); + if (values.some((entry) => !SAFE_ENV.test(entry))) { + throw new Error("Managed bootstrap Podman environment contains an invalid assignment."); + } + values.push(`OPENSHELL_SANDBOX_COMMAND=${intended}`); + values.push(PODMAN_BOOTSTRAP_CAPABILITY_DROP_ENV); + return Object.freeze(values); +} + +function networkArgs(inspect: JsonRecord): string[] { + const settings = record(inspect.NetworkSettings ?? {}, "NetworkSettings"); + const networks = record(settings.Networks ?? {}, "NetworkSettings.Networks"); + const names = Object.keys(networks).sort(); + if (names.length > 1) { + throw new Error( + "Managed bootstrap Podman cannot reproduce an ambiguous multi-network runtime.", + ); + } + return names[0] ? ["--network", names[0]] : []; +} + +export function renderPodmanReplacementMountArgs( + inspect: JsonRecord, + storageGraphRoot: string, +): string[] { + if (!Array.isArray(inspect.Mounts)) return []; + const args: string[] = []; + const mounts = normalizePodmanLogicalMounts( + inspect.Mounts.map((value) => record(value, "mount")), + storageGraphRoot, + String(inspect.Id ?? ""), + ) as readonly JsonRecord[]; + const destinations = new Set(); + for (const mount of mounts) { + const destination = String(mount.Destination ?? ""); + if (!destination || !path.isAbsolute(destination)) { + throw new Error("Managed bootstrap Podman mount cannot be reproduced exactly."); + } + if (destinations.has(destination)) { + throw new Error( + "Managed bootstrap Podman mount destination resolves to ambiguous runtime mounts.", + ); + } + destinations.add(destination); + const type = String(mount.Type ?? ""); + const name = String(mount.Name ?? ""); + const source = name || String(mount.Source ?? ""); + if (!source) { + throw new Error("Managed bootstrap Podman mount cannot be reproduced exactly."); + } + if (type !== "volume" && type !== "bind" && type !== "image") { + throw new Error(`Managed bootstrap Podman mount type '${type}' is unsupported.`); + } + const options = + type === "image" + ? `,rw=${mount.RW === true ? "true" : "false"}` + : mount.RW === false + ? ",ro" + : ""; + args.push("--mount", `type=${type},source=${source},destination=${destination}${options}`); + } + return args; +} + +function exactPodmanManagedWorkspaceVolume( + inspect: JsonRecord, + sandboxId: string, +): { readonly name: string; readonly mountpoint: string } { + const expectedName = `${OPENSHELL_WORKSPACE_VOLUME_PREFIX}${sandboxId}${OPENSHELL_WORKSPACE_VOLUME_SUFFIX}`; + if (!SAFE_RESOURCE_NAME.test(expectedName) || !Array.isArray(inspect.Mounts)) { + throw new Error("Managed bootstrap Podman workspace-volume identity is invalid."); + } + const matches = inspect.Mounts.map((value) => record(value, "mount")).filter( + (mount) => mount.Destination === OPENSHELL_WORKSPACE_DIRECTORY, + ); + if (matches.length !== 1) { + throw new Error("Managed bootstrap Podman workspace must resolve to one exact mount."); + } + const mount = matches[0] as JsonRecord; + const mountpoint = String(mount.Source ?? ""); + if ( + mount.Type !== "volume" || + mount.Name !== expectedName || + mount.RW !== true || + (mount.Driver !== undefined && mount.Driver !== "" && mount.Driver !== "local") || + !path.isAbsolute(mountpoint) || + path.normalize(mountpoint) !== mountpoint || + mountpoint === path.parse(mountpoint).root + ) { + throw new Error("Managed bootstrap Podman workspace-volume authority is invalid."); + } + return Object.freeze({ name: expectedName, mountpoint }); +} + +/** Restore the image contract on OpenShell's exact persistent Podman workspace root. */ +export function preparePodmanManagedWorkspaceAuthority(input: { + readonly engine: PodmanBoundContainerEngine; + readonly inspect: JsonRecord; + readonly sandboxId: string; + readonly workspaceRoot: ManagedStartupWorkspaceRoot; +}): void { + const workspace = exactPodmanManagedWorkspaceVolume(input.inspect, input.sandboxId); + const inspected = capture( + input.engine, + ["volume", "inspect", "--format", "{{.Name}}\n{{.Mountpoint}}", workspace.name], + "workspace-volume inspection", + 15_000, + ).stdout.trimEnd(); + if (inspected !== `${workspace.name}\n${workspace.mountpoint}`) { + throw new Error("Managed bootstrap Podman workspace-volume mountpoint identity changed."); + } + const prepare = input.engine.prepareManagedWorkspaceRoot; + if (!prepare) { + throw new Error("Managed bootstrap Podman workspace-root preparation is unavailable."); + } + const receipt = prepare({ + path: workspace.mountpoint, + uid: input.workspaceRoot.uid, + gid: input.workspaceRoot.gid, + mode: input.workspaceRoot.mode, + }); + if ( + receipt.path !== workspace.mountpoint || + receipt.uid !== input.workspaceRoot.uid || + receipt.gid !== input.workspaceRoot.gid || + receipt.mode !== input.workspaceRoot.mode || + !/^\d+$/u.test(receipt.device) || + !/^\d+$/u.test(receipt.inode) + ) { + throw new Error("Managed bootstrap Podman workspace-root receipt is invalid."); + } +} + +function exactSecretTarget(value: string, label: string): string { + if ( + !path.isAbsolute(value) || + path.normalize(value) !== value || + value === path.parse(value).root || + value.includes(",") + ) { + throw new Error(`Managed bootstrap Podman ${label} secret target is invalid.`); + } + return value; +} + +export function renderPodmanReplacementSecretArgs( + engine: PodmanBoundContainerEngine, + inspect: JsonRecord, + sandboxId: string, +): string[] { + const config = record(inspect.Config, "Config"); + const environmentEntries = stringArray(config.Env ?? [], "Config.Env"); + if (environmentEntries.some((entry) => !SAFE_ENV.test(entry))) { + throw new Error("Managed bootstrap Podman environment contains an invalid assignment."); + } + const environment = new Map( + environmentEntries.map((entry) => { + const separator = entry.indexOf("="); + return [entry.slice(0, separator), entry.slice(separator + 1)] as const; + }), + ); + const command = stringArray(config.Cmd ?? [], "Config.Cmd"); + const proxyFlag = command.indexOf("--upstream-proxy-auth-file"); + const targets = new Map(); + const tokenName = `${OPENSHELL_TOKEN_SECRET_PREFIX}${sandboxId}`; + const tokenTarget = environment.get("OPENSHELL_SANDBOX_TOKEN_FILE"); + if (!tokenTarget) { + throw new Error("Managed bootstrap Podman token secret target is unavailable."); + } + targets.set(tokenName, exactSecretTarget(tokenTarget, "token")); + const proxyName = `${OPENSHELL_PROXY_SECRET_PREFIX}${sandboxId}`; + const proxyTarget = proxyFlag >= 0 ? command[proxyFlag + 1] : undefined; + if (proxyTarget) targets.set(proxyName, exactSecretTarget(proxyTarget, "proxy")); + + const secrets = Array.isArray(config.Secrets) + ? config.Secrets.map((value) => record(value, "Config.Secrets entry")) + : []; + const args: string[] = []; + for (const secret of secrets) { + const name = String(secret.Name ?? ""); + const id = String(secret.ID ?? ""); + const target = targets.get(name); + if (!SAFE_RESOURCE_NAME.test(name) || !id || !target) { + throw new Error("Managed bootstrap Podman cannot reproduce an unknown runtime secret."); + } + const uid = Number(secret.UID); + const gid = Number(secret.GID); + const mode = Number(secret.Mode); + if ( + !Number.isSafeInteger(uid) || + uid < 0 || + !Number.isSafeInteger(gid) || + gid < 0 || + !Number.isSafeInteger(mode) || + mode < 0 || + mode > 0o777 + ) { + throw new Error("Managed bootstrap Podman runtime secret ownership is invalid."); + } + const inspected = capture( + engine, + ["secret", "inspect", "--format", "{{.ID}}", name], + "runtime secret inspection", + 15_000, + ).stdout.trim(); + if (inspected !== id) { + throw new Error("Managed bootstrap Podman runtime secret identity changed."); + } + args.push( + "--secret", + `${name},target=${target},uid=${String(uid)},gid=${String(gid)},mode=${mode.toString(8).padStart(4, "0")}`, + ); + targets.delete(name); + } + if (targets.size !== 0) { + throw new Error("Managed bootstrap Podman required runtime secret is unavailable."); + } + return args; +} + +function optionArgs( + values: Readonly>, +): string[] { + const args: string[] = []; + const gpu = values.gpuModeArgs; + if (Array.isArray(gpu)) { + for (let index = 0; index < gpu.length; index += 1) { + const current = String(gpu[index]); + if (current === "--gpus") { + const selector = String(gpu[index + 1] ?? ""); + if (selector === "all") args.push("--device", "nvidia.com/gpu=all"); + index += 1; + } else if (current === "--device") { + args.push(current, String(gpu[index + 1] ?? "")); + index += 1; + } + } + } + const limits = values.requiredUlimits; + if (Array.isArray(limits)) { + for (const limit of limits) args.push("--ulimit", String(limit)); + } + const groups = values.extraGroupGids; + if (Array.isArray(groups)) { + for (const group of groups) args.push("--group-add", String(group)); + } + return args; +} + +function processStartIdentity(pid: number): string { + const stat = fs.readFileSync(`/proc/${String(pid)}/stat`, "utf8"); + const end = stat.lastIndexOf(")"); + const fields = stat.slice(end + 2).split(" "); + const start = fields[19]; + if (!start || !/^\d+$/u.test(start)) + throw new Error("Podman gateway process start identity is unavailable."); + return `linux:${start}`; +} + +function processState(pid: number): string | null { + try { + const status = fs.readFileSync(`/proc/${String(pid)}/status`, "utf8"); + return status.match(/^State:\s+([A-Z])/mu)?.[1] ?? null; + } catch { + return null; + } +} + +function processInstanceAlive(snapshot: PodmanGatewayWatcherSnapshot): boolean { + const state = processState(snapshot.pid); + if (state === null || state === "X" || state === "Z") return false; + try { + return processStartIdentity(snapshot.pid) === snapshot.processStartIdentity; + } catch { + return false; + } +} + +function processInstanceSuspended(snapshot: PodmanGatewayWatcherSnapshot): boolean { + if (processState(snapshot.pid) !== "T") return false; + try { + return processStartIdentity(snapshot.pid) === snapshot.processStartIdentity; + } catch { + return false; + } +} + +function atomicLeaseWrite(file: string, recordValue: PodmanGatewayWatcherLeaseRecord): void { + const directory = path.dirname(file); + fs.mkdirSync(directory, { recursive: true, mode: 0o700 }); + const temporary = `${file}.${randomUUID()}.tmp`; + const descriptor = fs.openSync( + temporary, + fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL | fs.constants.O_NOFOLLOW, + 0o600, + ); + try { + fs.writeFileSync(descriptor, `${JSON.stringify(recordValue)}\n`, "utf8"); + fs.fsyncSync(descriptor); + } finally { + fs.closeSync(descriptor); + } + fs.renameSync(temporary, file); + fs.chmodSync(file, 0o600); + const directoryDescriptor = fs.openSync(directory, "r"); + try { + fs.fsyncSync(directoryDescriptor); + } finally { + fs.closeSync(directoryDescriptor); + } +} + +function createFileWatcherLeaseStore(stateRoot: string): PodmanGatewayWatcherLeaseStore { + const file = path.join(stateRoot, LEASE_FILE); + const read = (): PodmanGatewayWatcherLeaseRecord | null => { + try { + return JSON.parse(fs.readFileSync(file, "utf8")) as PodmanGatewayWatcherLeaseRecord; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw error; + } + }; + return Object.freeze({ + read, + acquire(recordValue: PodmanGatewayWatcherLeaseRecord) { + if (read() !== null) + throw new Error("Managed bootstrap Podman watcher lease already exists."); + const directory = path.dirname(file); + fs.mkdirSync(directory, { recursive: true, mode: 0o700 }); + const descriptor = fs.openSync( + file, + fs.constants.O_WRONLY | + fs.constants.O_CREAT | + fs.constants.O_EXCL | + fs.constants.O_NOFOLLOW, + 0o600, + ); + try { + fs.writeFileSync(descriptor, `${JSON.stringify(recordValue)}\n`, "utf8"); + fs.fsyncSync(descriptor); + } finally { + fs.closeSync(descriptor); + } + const directoryDescriptor = fs.openSync(directory, "r"); + try { + fs.fsyncSync(directoryDescriptor); + } finally { + fs.closeSync(directoryDescriptor); + } + }, + advance(expectedLeaseId: string, recordValue: PodmanGatewayWatcherLeaseRecord) { + const current = read(); + if (!current || current.leaseId !== expectedLeaseId) { + throw new Error("Managed bootstrap Podman watcher lease changed before advance."); + } + atomicLeaseWrite(file, recordValue); + }, + clear(expectedLeaseId: string) { + const current = read(); + if (!current || current.leaseId !== expectedLeaseId) { + throw new Error("Managed bootstrap Podman watcher lease changed before release."); + } + fs.unlinkSync(file); + const descriptor = fs.openSync(path.dirname(file), "r"); + try { + fs.fsyncSync(descriptor); + } finally { + fs.closeSync(descriptor); + } + }, + }); +} + +interface StandaloneGatewayLaunch { + readonly executable: string; + readonly argv0: string; + readonly args: readonly string[]; + readonly environment: NodeJS.ProcessEnv; + readonly cwd: string; + readonly marker: DockerDriverGatewayRuntimeMarker; + readonly pidFile: string; + readonly markerFile: string; +} + +export interface PersistedStandaloneGatewayEnvironmentEntry { + readonly key: string; + readonly valueHash: string; + readonly literalValue?: string; +} + +export function buildPodmanStandaloneGatewayEnvironmentAuthority( + environment: NodeJS.ProcessEnv, +): readonly PersistedStandaloneGatewayEnvironmentEntry[] { + return Object.freeze( + Object.entries(environment) + .filter((entry): entry is [string, string] => typeof entry[1] === "string") + .filter(([key]) => PERSISTABLE_ENVIRONMENT_KEYS.has(key)) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, value]) => Object.freeze({ key, valueHash: sha256(value), literalValue: value })), + ); +} + +interface PersistedStandaloneGatewayLaunch { + readonly schemaVersion: 1; + readonly launchIdentity: string; + readonly executable: string; + readonly argv0: string; + readonly args: readonly string[]; + readonly environment: readonly PersistedStandaloneGatewayEnvironmentEntry[]; + readonly cwd: string; + readonly marker: DockerDriverGatewayRuntimeMarker; + readonly pidFile: string; + readonly markerFile: string; +} + +function writePrivateJson(file: string, value: unknown): void { + const directory = path.dirname(file); + fs.mkdirSync(directory, { recursive: true, mode: 0o700 }); + const temporary = `${file}.${randomUUID()}.tmp`; + let descriptor: number | null = null; + try { + descriptor = fs.openSync( + temporary, + fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL | fs.constants.O_NOFOLLOW, + 0o600, + ); + fs.writeFileSync(descriptor, `${JSON.stringify(value)}\n`, "utf8"); + fs.fsyncSync(descriptor); + fs.closeSync(descriptor); + descriptor = null; + fs.renameSync(temporary, file); + fs.chmodSync(file, 0o600); + const directoryDescriptor = fs.openSync(directory, "r"); + try { + fs.fsyncSync(directoryDescriptor); + } finally { + fs.closeSync(directoryDescriptor); + } + } finally { + if (descriptor !== null) fs.closeSync(descriptor); + try { + fs.unlinkSync(temporary); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + } +} + +function persistedStandaloneGatewayLaunch( + launchIdentity: string, + launch: StandaloneGatewayLaunch, +): PersistedStandaloneGatewayLaunch { + const environment = buildPodmanStandaloneGatewayEnvironmentAuthority(launch.environment); + return Object.freeze({ + schemaVersion: 1 as const, + launchIdentity, + executable: launch.executable, + argv0: launch.argv0, + args: launch.args, + environment: Object.freeze(environment), + cwd: launch.cwd, + marker: launch.marker, + pidFile: launch.pidFile, + markerFile: launch.markerFile, + }); +} + +function loadPersistedStandaloneGatewayLaunch( + file: string, + expectedLaunchIdentity: string, +): StandaloneGatewayLaunch { + const parsed = record(JSON.parse(fs.readFileSync(file, "utf8")), "gateway launch authority") as + | JsonRecord + | PersistedStandaloneGatewayLaunch; + if ( + parsed.schemaVersion !== 1 || + parsed.launchIdentity !== expectedLaunchIdentity || + !Array.isArray(parsed.args) || + !Array.isArray(parsed.environment) + ) { + throw new Error("Managed Podman standalone gateway launch authority is invalid."); + } + const executable = String(parsed.executable ?? ""); + const argv0 = String(parsed.argv0 ?? ""); + const cwd = String(parsed.cwd ?? ""); + const pidFile = String(parsed.pidFile ?? ""); + const markerFile = String(parsed.markerFile ?? ""); + if ( + argv0.length === 0 || + argv0.includes("\0") || + ![executable, cwd, pidFile, markerFile].every( + (value) => path.isAbsolute(value) && path.normalize(value) === value && !value.includes("\0"), + ) + ) { + throw new Error("Managed Podman standalone gateway launch paths are invalid."); + } + const args = stringArray(parsed.args, "gateway launch argv"); + const environment: NodeJS.ProcessEnv = {}; + for (const rawEntry of parsed.environment) { + const entry = record(rawEntry, "gateway launch environment entry"); + const key = String(entry.key ?? ""); + const expectedHash = String(entry.valueHash ?? ""); + if (!PERSISTABLE_ENVIRONMENT_KEYS.has(key) || !SHA256.test(expectedHash)) { + throw new Error("Managed Podman standalone gateway environment authority is invalid."); + } + const value = entry.literalValue; + if (typeof value !== "string" || sha256(value) !== expectedHash) { + throw new Error( + `Managed Podman standalone gateway environment value '${key}' is unavailable for exact recovery.`, + ); + } + environment[key] = value; + } + const marker = parseDockerDriverGatewayRuntimeMarker(JSON.stringify(parsed.marker)); + if ( + !marker || + marker.driver !== PROVIDER_ID || + (marker.gatewayBin !== null && marker.gatewayBin !== executable) + ) { + throw new Error("Managed Podman standalone gateway runtime marker authority changed."); + } + return Object.freeze({ + executable, + argv0, + args, + environment: Object.freeze(environment), + cwd, + marker, + pidFile, + markerFile, + }); +} + +function removeStandaloneGatewayLaunchAuthority( + file: string, + expectedLaunchIdentity: string | null, +): void { + let contents: unknown; + try { + contents = JSON.parse(fs.readFileSync(file, "utf8")); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return; + throw error; + } + const persisted = record(contents, "gateway launch authority"); + if (expectedLaunchIdentity !== null && persisted.launchIdentity !== expectedLaunchIdentity) { + throw new Error("Managed Podman standalone gateway launch authority changed before cleanup."); + } + fs.unlinkSync(file); + const descriptor = fs.openSync(path.dirname(file), "r"); + try { + fs.fsyncSync(descriptor); + } finally { + fs.closeSync(descriptor); + } +} + +function readProcessEnvironment(pid: number): NodeJS.ProcessEnv { + return Object.fromEntries( + fs + .readFileSync(`/proc/${String(pid)}/environ`, "utf8") + .split("\0") + .filter(Boolean) + .map((entry) => { + const separator = entry.indexOf("="); + if (separator <= 0) + throw new Error("Managed bootstrap Podman gateway environment is invalid."); + return [entry.slice(0, separator), entry.slice(separator + 1)]; + }), + ); +} + +function readStandaloneGatewayLaunch( + pid: number, + marker: DockerDriverGatewayRuntimeMarker, + pidFile: string, + markerFile: string, +): StandaloneGatewayLaunch { + const argv = fs + .readFileSync(`/proc/${String(pid)}/cmdline`, "utf8") + .split("\0") + .filter(Boolean); + if (argv.length === 0) throw new Error("Managed bootstrap Podman gateway argv is unavailable."); + const executable = fs.realpathSync(`/proc/${String(pid)}/exe`); + if (marker.gatewayBin && fs.realpathSync(marker.gatewayBin) !== executable) { + throw new Error("Managed bootstrap Podman gateway executable changed from its runtime marker."); + } + return Object.freeze({ + executable, + argv0: argv[0] as string, + args: Object.freeze(argv.slice(1)), + environment: Object.freeze(readProcessEnvironment(pid)), + cwd: fs.realpathSync(`/proc/${String(pid)}/cwd`), + marker, + pidFile, + markerFile, + }); +} + +function waitForProcessExit(snapshot: PodmanGatewayWatcherSnapshot): boolean { + for (let attempt = 0; attempt < 300; attempt += 1) { + if (!processInstanceAlive(snapshot)) return true; + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 10); + } + return !processInstanceAlive(snapshot); +} + +function listenerPids(port: number): readonly number[] { + const result = spawnSync("lsof", ["-ti", `:${String(port)}`, "-sTCP:LISTEN"], { + encoding: "utf8", + timeout: 5_000, + }); + if (result.error || (result.status !== 0 && result.status !== 1)) { + throw new Error( + "Managed bootstrap Podman could not enumerate the complete gateway listener set.", + ); + } + if (result.status === 1) return Object.freeze([]); + return Object.freeze( + String(result.stdout ?? "") + .split(/\r?\n/u) + .map((value) => Number(value.trim())) + .filter((pid) => Number.isSafeInteger(pid) && pid > 0), + ); +} + +/** Resolve the name and state directory that own one native Podman gateway port. */ +export function resolvePodmanManagedGatewayAuthority( + environment: NodeJS.ProcessEnv, + gatewayPort: number, + gatewayName?: string, + homeDir: string = environment.HOME || os.homedir(), +): { readonly gatewayName: string; readonly stateDir: string } { + const configured = environment.NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR; + const stateDir = + configured && configured.trim() + ? path.resolve(configured.trim()) + : path.join(homeDir, ".local", "state", "nemoclaw", resolveGatewayStateDirName(gatewayPort)); + return Object.freeze({ + gatewayName: gatewayName ?? resolveGatewayName(gatewayPort), + stateDir, + }); +} + +function createProductionWatcherController( + options: PodmanManagedBootstrapAdapterOptions, + authority: ReturnType, +): PodmanManagedGatewayWatcherController { + const { gatewayName, stateDir } = authority; + const pidFile = path.join(stateDir, "openshell-gateway.pid"); + const markerFile = getDockerDriverGatewayRuntimeMarkerPath(stateDir); + const standaloneLaunchFile = path.join(options.stateRoot, STANDALONE_LAUNCH_FILE); + const standaloneLaunches = new Map(); + const watcherStore = createFileWatcherLeaseStore(options.stateRoot); + + const snapshotForKnownPid = (pid: number): PodmanGatewayWatcherSnapshot => { + const service = getTrustedActiveOpenShellGatewayUserServiceIdentity({ + env: options.environment, + home: options.environment.HOME, + platform: "linux", + }); + const processStart = processStartIdentity(pid); + if (service?.pid === pid) { + const ownerIdentity = `managed-service:${service.executablePath ?? "openshell-gateway"}`; + return Object.freeze({ + gatewayName, + gatewayPort: options.gatewayPort, + launchIdentity: sha256(`${gatewayName}\0${String(options.gatewayPort)}\0${ownerIdentity}`), + ownerIdentity, + ownerKind: "managed-service" as const, + pid, + processStartIdentity: processStart, + }); + } + const marker = readDockerDriverGatewayRuntimeMarker(markerFile); + const recordedPid = Number(fs.readFileSync(pidFile, "utf8").trim()); + if (!marker || marker.driver !== PROVIDER_ID || marker.pid !== pid || recordedPid !== pid) { + throw new Error("Managed bootstrap Podman gateway runtime marker does not own its listener."); + } + const endpoint = new URL(marker.endpoint); + if (Number(endpoint.port) !== options.gatewayPort || marker.platform !== "linux") { + throw new Error("Managed bootstrap Podman gateway runtime marker targets another gateway."); + } + const ownerIdentity = `standalone:${stateDir}:${marker.gatewayBin ?? "gateway"}`; + const launchIdentity = sha256( + `${gatewayName}\0${String(options.gatewayPort)}\0${ownerIdentity}\0${marker.desiredEnvHash}\0${marker.createdAt}`, + ); + if (!standaloneLaunches.has(launchIdentity)) { + const launch = readStandaloneGatewayLaunch(pid, marker, pidFile, markerFile); + standaloneLaunches.set(launchIdentity, launch); + writePrivateJson( + standaloneLaunchFile, + persistedStandaloneGatewayLaunch(launchIdentity, launch), + ); + } + return Object.freeze({ + gatewayName, + gatewayPort: options.gatewayPort, + launchIdentity, + ownerIdentity, + ownerKind: "standalone" as const, + pid, + processStartIdentity: processStart, + }); + }; + + const listTargetWatchers = (): readonly PodmanGatewayWatcherSnapshot[] => + listenerPids(options.gatewayPort).map((pid) => { + try { + return snapshotForKnownPid(pid); + } catch { + return Object.freeze({ + gatewayName, + gatewayPort: options.gatewayPort, + launchIdentity: `unproven-launch:${String(pid)}`, + ownerIdentity: `unproven-owner:${String(pid)}`, + ownerKind: "standalone" as const, + pid, + processStartIdentity: processStartIdentity(pid), + }); + } + }); + + const controller = createPodmanManagedGatewayWatcherController({ + store: watcherStore, + captureCurrent() { + const listeners = listenerPids(options.gatewayPort); + if (listeners.length !== 1) { + throw new Error("Managed bootstrap Podman requires exactly one gateway listener."); + } + return snapshotForKnownPid(listeners[0] as number); + }, + listTargetWatchers, + isProcessInstanceAlive: processInstanceAlive, + captureLeaseHolder: (): PodmanGatewayWatcherLeaseHolder => ({ + pid: process.pid, + processStartIdentity: processStartIdentity(process.pid), + }), + isLeaseHolderAlive: (holder) => + processInstanceAlive({ + gatewayName, + gatewayPort: options.gatewayPort, + launchIdentity: "lease-holder", + ownerIdentity: "lease-holder", + ownerKind: "standalone", + pid: holder.pid, + processStartIdentity: holder.processStartIdentity, + }), + isOwnerStopped(snapshot) { + if (snapshot.ownerKind === "managed-service") { + return ( + getTrustedActiveOpenShellGatewayUserServiceIdentity({ + env: options.environment, + home: options.environment.HOME, + platform: "linux", + }) === null + ); + } + if (processInstanceSuspended(snapshot)) return true; + return !listTargetWatchers().some( + (candidate) => + candidate.ownerKind === snapshot.ownerKind && + candidate.ownerIdentity === snapshot.ownerIdentity && + candidate.launchIdentity === snapshot.launchIdentity, + ); + }, + stopExactOwner(snapshot) { + if (snapshot.ownerKind === "managed-service") { + const stopped = stopOpenShellGatewayUserService({ + env: options.environment, + home: options.environment.HOME, + platform: "linux", + }); + if (!stopped.attempted || !stopped.stopped) { + throw new Error(stopped.reason ?? "Managed Podman gateway service did not stop."); + } + return; + } + if (!processInstanceAlive(snapshot)) { + throw new Error( + "Managed bootstrap Podman standalone gateway identity changed before stop.", + ); + } + process.kill(snapshot.pid, "SIGTERM"); + if (waitForProcessExit(snapshot)) return; + if (!processInstanceAlive(snapshot)) return; + process.kill(snapshot.pid, "SIGKILL"); + if (!waitForProcessExit(snapshot)) { + throw new Error("Managed bootstrap Podman standalone gateway did not stop."); + } + }, + resumeSameOwner(snapshot) { + if (snapshot.ownerKind === "managed-service") { + const started = startOpenShellGatewayUserService({ + env: options.environment, + home: options.environment.HOME, + platform: "linux", + }); + if (!started.attempted || !started.started) { + throw new Error(started.reason ?? "Managed Podman gateway service did not resume."); + } + return; + } + if (processInstanceSuspended(snapshot)) { + process.kill(snapshot.pid, "SIGCONT"); + return; + } + const launch = + standaloneLaunches.get(snapshot.launchIdentity) ?? + loadPersistedStandaloneGatewayLaunch(standaloneLaunchFile, snapshot.launchIdentity); + const child = spawn(launch.executable, [...launch.args], { + argv0: launch.argv0, + cwd: launch.cwd, + detached: true, + env: launch.environment, + stdio: "ignore", + }); + child.unref(); + if (!child.pid) throw new Error("Managed Podman standalone gateway did not return a pid."); + writeDockerDriverGatewayPidFile(launch.pidFile, child.pid); + writeDockerDriverGatewayRuntimeMarker(launch.markerFile, { + ...launch.marker, + pid: child.pid, + }); + }, + isHealthy(snapshot) { + return ( + processInstanceAlive(snapshot) && + !processInstanceSuspended(snapshot) && + listenerPids(options.gatewayPort).includes(snapshot.pid) + ); + }, + }); + const wrapLease = (lease: PodmanGatewayWatcherLease): PodmanGatewayWatcherLease => + Object.freeze({ + get record() { + return lease.record; + }, + assertStillHeld: lease.assertStillHeld, + assertStillStopped: lease.assertStillStopped, + resumeForObservationAndProve: lease.resumeForObservationAndProve, + requiesceAndProve: lease.requiesceAndProve, + resumeAndProve() { + lease.resumeAndProve(); + if (lease.record.ownerKind === "standalone") { + removeStandaloneGatewayLaunchAuthority(standaloneLaunchFile, lease.record.launchIdentity); + standaloneLaunches.delete(lease.record.launchIdentity); + } + }, + }); + return Object.freeze({ + recoverUnfinishedLease() { + const recordValue = watcherStore.read(); + controller.recoverUnfinishedLease(); + if (recordValue?.ownerKind === "standalone") { + removeStandaloneGatewayLaunchAuthority(standaloneLaunchFile, recordValue.launchIdentity); + standaloneLaunches.delete(recordValue.launchIdentity); + } + }, + reclaimStoppedLease: (expectedLeaseId: string) => + wrapLease(controller.reclaimStoppedLease(expectedLeaseId)), + quiesceAndProve() { + try { + return wrapLease(controller.quiesceAndProve()); + } catch (error) { + // When quiescence restored the exact owner and cleared its lease, its + // launch authority must not remain consumable by a later transaction. + if (watcherStore.read() === null) { + removeStandaloneGatewayLaunchAuthority(standaloneLaunchFile, null); + standaloneLaunches.clear(); + } + throw error; + } + }, + }); +} + +function sandboxIdentity(journal: PodmanBootstrapJournal) { + return Object.freeze({ + sandboxName: journal.sandboxName, + sandboxId: journal.sandboxId, + driverId: PROVIDER_ID, + }); +} + +function heldFromJournal( + journal: PodmanBootstrapJournal, + engine: PodmanBoundContainerEngine, +): PodmanHeldWorkloadObservation { + const inspect = inspectRuntime(engine, journal.originalRuntimeId); + const config = record(inspect.Config, "Config"); + return Object.freeze({ + containerName: journal.originalContainerName, + heldWorkloadArgv: [], + imageContentId: journal.originalImageContentId, + labels: record(config.Labels ?? {}, "Config.Labels") as Readonly>, + runtimeId: journal.originalRuntimeId, + // The journal records the original after stable running capture. Recovery + // separately re-inspects its current state before deciding whether to start it. + running: true, + sandboxId: journal.sandboxId, + sandboxName: journal.sandboxName, + supervisorArgv: Object.freeze([ + ...stringArray(config.Entrypoint ?? [], "Config.Entrypoint"), + ...stringArray(config.Cmd ?? [], "Config.Cmd"), + ]), + }); +} + +function runtimeExists(engine: PodmanBoundContainerEngine, runtimeId: string): boolean { + const result = engine.capture(["container", "exists", runtimeId], 15_000); + if (!result.error && result.status === 0) return true; + if (!result.error && result.status === 1) return false; + return commandFailure("container existence proof", result); +} + +function exactImageContentId(value: unknown): string { + const normalized = String(value ?? "").toLowerCase(); + const match = normalized.match(/^(?:sha256:)?([a-f0-9]{64})$/u); + if (!match?.[1]) { + throw new Error("Managed bootstrap Podman inspect image identity is invalid."); + } + return `sha256:${match[1]}`; +} + +function proveJournalRuntime( + engine: PodmanBoundContainerEngine, + journal: PodmanBootstrapJournal, + runtimeId: string, + allowedNames: readonly string[], + expectedImageContentId: string, + requireCurrentOwnership = false, +): JsonRecord { + const first = inspectRuntime(engine, runtimeId); + const second = inspectRuntime(engine, runtimeId); + if (JSON.stringify(first) !== JSON.stringify(second)) { + throw new Error("Managed bootstrap Podman runtime changed during stable inspection."); + } + const name = String(second.Name ?? "").replace(/^\//u, ""); + const config = record(second.Config, "Config"); + const labels = record(config.Labels ?? {}, "Config.Labels") as Readonly>; + if ( + !allowedNames.includes(name) || + exactImageContentId(second.Image) !== expectedImageContentId || + labels[PODMAN_MANAGED_LABEL] !== "true" || + labels[PODMAN_SANDBOX_ID_LABEL] !== journal.sandboxId || + labels[PODMAN_SANDBOX_NAME_LABEL] !== journal.sandboxName || + labels[PODMAN_SANDBOX_NAMESPACE_LABEL] !== PODMAN_SANDBOX_NAMESPACE || + labels[PODMAN_SANDBOX_WORKSPACE_LABEL] !== PODMAN_SANDBOX_WORKSPACE || + (requireCurrentOwnership && + labels[PODMAN_OPENSHELL_MANAGED_BY_LABEL] !== PODMAN_OPENSHELL_MANAGED_BY_VALUE) + ) { + throw new Error( + "Managed bootstrap Podman runtime does not match its exact durable ownership authority.", + ); + } + return second; +} + +interface FinishCommittedPodmanBootstrapInput { + readonly engine: PodmanBoundContainerEngine; + readonly journalStore: PodmanBootstrapJournalStore; + readonly journal: PodmanBootstrapJournal; + readonly watcherLease: PodmanGatewayWatcherLease; +} + +/** Finish a durably authorized Podman replacement without guessing a runtime identity. */ +export function finishCommittedPodmanBootstrap( + input: FinishCommittedPodmanBootstrapInput, +): PodmanBootstrapJournal { + const { engine, journalStore, watcherLease } = input; + const journal = journalStore.load(input.journal.bootstrapIdentity); + if ( + !journal || + (journal.phase !== "commit-authorized" && journal.phase !== "committed") || + journal.engineAuthorityId !== engine.authorityId || + journal.watcherLeaseId !== watcherLease.record.leaseId || + journal.replacementRuntimeId === null + ) { + throw new Error("Managed bootstrap Podman commit authority changed before finalization."); + } + watcherLease.assertStillStopped(); + + if (runtimeExists(engine, journal.originalRuntimeId)) { + proveJournalRuntime( + engine, + journal, + journal.originalRuntimeId, + [journal.originalContainerName], + journal.originalImageContentId, + ); + capture(engine, ["container", "rm", journal.originalRuntimeId], "original cleanup"); + } + if (runtimeExists(engine, journal.originalRuntimeId)) { + throw new Error("Managed bootstrap Podman original remained after exact commit removal."); + } + if (!runtimeExists(engine, journal.replacementRuntimeId)) { + throw new Error("Managed bootstrap Podman replacement disappeared after commit authorization."); + } + const replacement = proveJournalRuntime( + engine, + journal, + journal.replacementRuntimeId, + [journal.replacementStagingName, journal.originalContainerName], + journal.replacementImageContentId, + true, + ); + const currentName = String(replacement.Name ?? "").replace(/^\//u, ""); + if (currentName === journal.replacementStagingName) { + capture( + engine, + ["container", "rename", journal.replacementRuntimeId, journal.originalContainerName], + "replacement activation rename", + ); + } + proveJournalRuntime( + engine, + journal, + journal.replacementRuntimeId, + [journal.originalContainerName], + journal.replacementImageContentId, + true, + ); + watcherLease.assertStillStopped(); + const committed = journalStore.recordCommitted(journal.bootstrapIdentity); + journalStore.removeAfterCommit(journal.bootstrapIdentity); + return committed; +} + +function completionReceipt( + handle: ManagedBootstrapHeldWorkloadHandle, + snapshot: ManagedBootstrapObservedSnapshot, + replacement: PodmanBootstrapPreparedReplacement, + completion: PodmanBootstrapImageTransactionCompletion, + replacementSpecHash: string, +): ManagedBootstrapCompletionReceipt { + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + runtimeId: replacement.replacementRuntimeId, + image: snapshot.image, + runtimeImageContentId: replacement.replacementImageContentId, + originalSpecHash: snapshot.specHash, + replacementSpecHash, + profileFingerprint: handle.plan.profile.fingerprint, + bootstrapIdentity: handle.bootstrapIdentity, + transactionPending: completion.transactionPending, + completedAt: completion.completedAt, + }); +} + +export function createPodmanManagedBootstrapAdapter( + options: PodmanManagedBootstrapAdapterOptions, +): ManagedBootstrapAdapter { + if (options.engine.operation !== "managed-bootstrap" || options.engine.engineId !== PROVIDER_ID) { + throw new Error("Managed bootstrap Podman requires an operation-scoped engine."); + } + const journalStore = createFilePodmanBootstrapJournalStore(options.stateRoot); + const gatewayAuthority = resolvePodmanManagedGatewayAuthority( + options.environment, + options.gatewayPort, + options.gatewayName, + ); + const watcherController = + options.watcherController ?? createProductionWatcherController(options, gatewayAuthority); + const transactions = new Map(); + + return Object.freeze({ + async recoverUnfinishedTransactions() { + const receipts: ManagedBootstrapRecoveryReceipt[] = []; + const failures: ManagedBootstrapRecoveryFailure[] = []; + const unfinished = journalStore.listUnfinished(); + if (unfinished.length === 0) { + // Commit/rollback compacts its journal before resuming the gateway. A + // crash in that final window leaves only the durable watcher lease. + watcherController.recoverUnfinishedLease(); + } + for (const journal of unfinished) { + try { + const lease = watcherController.reclaimStoppedLease(journal.watcherLeaseId); + const committed = journal.phase === "commit-authorized" || journal.phase === "committed"; + if (committed) { + finishCommittedPodmanBootstrap({ + engine: options.engine, + journalStore, + journal, + watcherLease: lease, + }); + } else { + const held = heldFromJournal(journal, options.engine); + rollbackPodmanBootstrapBeforeCommit({ + bootstrapIdentity: journal.bootstrapIdentity, + engine: options.engine, + heldWorkload: held, + journalStore, + watcherLease: lease, + }); + } + lease.resumeAndProve(); + receipts.push( + Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + providerId: PROVIDER_ID, + sourcePhase: journal.phase, + sandbox: sandboxIdentity(journal), + bootstrapIdentity: journal.bootstrapIdentity, + outcome: committed ? ("committed" as const) : ("rolled-back" as const), + finalization: Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: sandboxIdentity(journal), + bootstrapIdentity: journal.bootstrapIdentity, + outcome: committed ? ("committed" as const) : ("rolled-back" as const), + restoredRuntimeId: committed ? null : journal.originalRuntimeId, + restoredSpecHash: committed ? null : journal.originalSpecFingerprint, + heldWorkloadRemoved: committed, + alreadyRolledBack: false, + finalizedAt: new Date().toISOString(), + }), + }), + ); + } catch (error) { + failures.push( + Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + providerId: PROVIDER_ID, + sourcePhase: journal.phase, + sandbox: sandboxIdentity(journal), + bootstrapIdentity: journal.bootstrapIdentity, + code: + journal.phase === "commit-authorized" || journal.phase === "committed" + ? "podman-commit-incomplete" + : "podman-rollback-incomplete", + blockingScope: "provider", + retryable: true, + detail: error instanceof Error ? error.message : String(error), + }), + ); + } + } + return Object.freeze({ + receipts: Object.freeze(receipts), + failures: Object.freeze(failures), + }); + }, + + async createHeldWorkload(input: Parameters[0]) { + if (input.plan.driverId !== PROVIDER_ID) { + throw new Error("Managed bootstrap Podman received another provider plan."); + } + const bootstrapIdentity = input.bootstrapIdentity; + if (!bootstrapIdentity || !SHA256.test(bootstrapIdentity)) { + throw new Error("Managed bootstrap Podman bootstrap identity is invalid."); + } + const heldWorkloadArgv = renderManagedBootstrapHeldCommand( + input.request, + bootstrapIdentity, + input.plan.intendedWorkloadArgv, + ); + const createReceipt = await input.launch({ heldWorkloadArgv, bootstrapIdentity }); + const held = inspectExactPodmanHeldWorkload({ + engine: options.engine, + sandboxName: input.plan.sandboxName, + sandboxId: createReceipt.sandbox.sandboxId, + sandboxNamespace: "", + bootstrapIdentity, + expectedHeldWorkloadArgv: heldWorkloadArgv, + expectedSupervisorArgv: input.plan.expectedSupervisorArgv, + }); + transactions.set(bootstrapIdentity, { + held, + rawInspect: inspectRuntime(options.engine, held.runtimeId), + request: input.request, + }); + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: createReceipt.sandbox, + bootstrapIdentity, + heldWorkloadArgv: Object.freeze(heldWorkloadArgv), + intendedWorkloadArgv: Object.freeze([...input.plan.intendedWorkloadArgv]), + plan: input.plan, + createReceipt, + }); + }, + + async cleanupIncompleteCreate( + input: Parameters[0], + ) { + const observation = inspectExactPodmanHeldWorkload({ + engine: options.engine, + sandboxName: input.plan.sandboxName, + sandboxId: input.createReceipt.sandbox.sandboxId, + sandboxNamespace: "", + bootstrapIdentity: input.bootstrapIdentity, + expectedHeldWorkloadArgv: input.heldWorkloadArgv, + expectedSupervisorArgv: input.plan.expectedSupervisorArgv, + }); + capture( + options.engine, + ["container", "rm", "--force", observation.runtimeId], + "incomplete create cleanup", + ); + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: input.createReceipt.sandbox, + bootstrapIdentity: input.bootstrapIdentity, + outcome: "rolled-back", + restoredRuntimeId: null, + restoredSpecHash: null, + heldWorkloadRemoved: true, + alreadyRolledBack: false, + finalizedAt: new Date().toISOString(), + }); + }, + + async discoverHeldWorkload( + input: Parameters[0], + ): Promise { + const pending = transactions.get(input.bootstrapIdentity); + const handle = pending?.held; + if (handle) { + return Object.freeze({ + sandbox: input.sandbox, + runtimeId: handle.runtimeId, + bootstrapIdentity: input.bootstrapIdentity, + }); + } + throw new Error("Managed bootstrap Podman discovery has no exact create authority."); + }, + + async inspectHeldWorkload({ + handle, + discovered, + }: Parameters[0]) { + const existing = transactions.get(handle.bootstrapIdentity); + if (!existing) { + throw new Error("Managed bootstrap Podman lost its exact create authority."); + } + const held = existing.held; + if (held.runtimeId !== discovered.runtimeId) { + throw new Error("Managed bootstrap Podman discovery identity changed."); + } + const rawInspect = inspectRuntime(options.engine, held.runtimeId); + const canonical = canonicalInspect(rawInspect); + transactions.set(handle.bootstrapIdentity, { + held, + rawInspect, + request: existing.request, + }); + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + runtimeId: held.runtimeId, + bootstrapIdentity: handle.bootstrapIdentity, + image: handle.plan.image, + runtimeImageContentId: held.imageContentId, + specHash: sha256(canonical), + specCanonicalJson: canonical, + agentIdentity: handle.plan.agentIdentity, + supervisorArgv: held.supervisorArgv, + heldWorkloadArgv: handle.heldWorkloadArgv, + metadata: handle.plan.metadata, + }); + }, + + async prepareBootstrapReplacement({ + handle, + snapshot, + request, + replacementOptions, + }: Parameters[0]) { + const current = transactions.get(handle.bootstrapIdentity); + if (!current || current.held.runtimeId !== snapshot.runtimeId) { + throw new Error("Managed bootstrap Podman lost its exact held workload authority."); + } + const watcherLease = watcherController.quiesceAndProve(); + current.watcherLease = watcherLease; + let prepared: PodmanBootstrapPreparedReplacement; + try { + watcherLease.resumeForObservationAndProve(); + prepared = prepareStoppedPodmanBootstrapReplacement({ + engine: options.engine, + journalStore, + watcherLease, + plan: { + schemaVersion: PODMAN_BOOTSTRAP_REPLACEMENT_SCHEMA_VERSION, + bootstrapIdentity: handle.bootstrapIdentity, + heldWorkload: current.held, + runtimeArgs: Object.freeze([ + ...renderPodmanReplacementRuntimeArgs(current.rawInspect), + ...networkArgs(current.rawInspect), + ...renderPodmanReplacementMountArgs( + current.rawInspect, + resolvePodmanStorageGraphRoot(options.engine), + ), + ...renderPodmanReplacementSecretArgs( + options.engine, + current.rawInspect, + current.held.sandboxId, + ), + ...renderPodmanReplacementHealthArgs(current.rawInspect), + ...optionArgs(replacementOptions.values), + ]), + environment: renderPodmanReplacementEnvironment(current.rawInspect, handle), + entrypointArgv: [BOOTSTRAP_EXECUTABLE], + commandArgv: replacementCommand(handle, snapshot), + replacementImageContentId: snapshot.runtimeImageContentId, + }, + }); + } catch (error) { + try { + watcherLease.requiesceAndProve(); + if (journalStore.load(handle.bootstrapIdentity)) { + rollbackPodmanBootstrapBeforeCommit({ + bootstrapIdentity: handle.bootstrapIdentity, + engine: options.engine, + heldWorkload: current.held, + journalStore, + watcherLease, + }); + } + } finally { + watcherLease.resumeAndProve(); + } + throw error; + } + current.prepared = prepared; + const contractReplacementSpecCanonical = JSON.stringify({ + providerId: PROVIDER_ID, + originalRuntimeId: snapshot.runtimeId, + replacementRuntimeId: prepared.replacementRuntimeId, + replacementImageContentId: prepared.replacementImageContentId, + replacementSpecFingerprint: prepared.replacementSpecFingerprint, + }); + const contractReplacementSpecHash = sha256(contractReplacementSpecCanonical); + current.contractReplacementSpecCanonical = contractReplacementSpecCanonical; + current.contractReplacementSpecHash = contractReplacementSpecHash; + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + bootstrapIdentity: handle.bootstrapIdentity, + originalRuntimeId: snapshot.runtimeId, + preparedRuntimeId: prepared.replacementRuntimeId, + image: snapshot.image, + runtimeImageContentId: prepared.replacementImageContentId, + originalSpecHash: snapshot.specHash, + preparedSpecHash: contractReplacementSpecHash, + preparedSpecCanonicalJson: contractReplacementSpecCanonical, + expectedActivatedSpecHash: contractReplacementSpecHash, + expectedActivatedSpecCanonicalJson: contractReplacementSpecCanonical, + profileFingerprint: request.profileFingerprint, + rollbackAuthority: serializePodmanBootstrapJournal(prepared.journal), + }); + }, + + async activateBootstrapReplacement({ + handle, + snapshot, + prepared, + }: Parameters[0]) { + const current = transactions.get(handle.bootstrapIdentity); + if ( + !current?.prepared || + !current.watcherLease || + !current.contractReplacementSpecHash || + !current.contractReplacementSpecCanonical || + current.prepared.replacementRuntimeId !== prepared.preparedRuntimeId + ) { + throw new Error("Managed bootstrap Podman prepared authority changed before activation."); + } + current.watcherLease.requiesceAndProve(); + current.prepared = stopExactPodmanBootstrapOriginal({ + engine: options.engine, + heldWorkload: current.held, + journalStore, + prepared: current.prepared, + watcherLease: current.watcherLease, + }); + preparePodmanManagedWorkspaceAuthority({ + engine: options.engine, + inspect: current.rawInspect, + sandboxId: current.held.sandboxId, + workspaceRoot: options.workspaceRoot, + }); + const prepareStateRoot = options.engine.prepareManagedVolumeRoot; + if (handle.plan.managedStateRoots.length > 0 && !prepareStateRoot) { + throw new Error("Managed bootstrap Podman volume-root preparation is unavailable."); + } + prepareManagedBootstrapStateRoots({ + inspect: current.rawInspect, + roots: handle.plan.managedStateRoots, + captureVolume: (args) => + capture(options.engine, ["volume", ...args], "state-volume inspection", 15_000).stdout, + ...(prepareStateRoot + ? { + prepareRoot: (input) => prepareStateRoot(input), + } + : {}), + }); + current.watcherLease.resumeForObservationAndProve(); + current.imageTransaction = startPodmanBootstrapImageTransaction({ + engine: options.engine, + journalStore, + watcherLease: current.watcherLease, + agent: handle.plan.profile.agent, + prepared: current.prepared, + profileFingerprint: handle.plan.profile.fingerprint, + request: current.request, + }); + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: handle.sandbox, + bootstrapIdentity: handle.bootstrapIdentity, + originalRuntimeId: snapshot.runtimeId, + replacementRuntimeId: current.prepared.replacementRuntimeId, + image: snapshot.image, + runtimeImageContentId: current.prepared.replacementImageContentId, + originalSpecHash: snapshot.specHash, + replacementSpecHash: current.contractReplacementSpecHash, + replacementSpecCanonicalJson: current.contractReplacementSpecCanonical, + profileFingerprint: handle.plan.profile.fingerprint, + }); + }, + + async awaitBootstrap({ + handle, + snapshot, + replacement: _replacement, + timeoutSecs, + }: Parameters[0]) { + const current = transactions.get(handle.bootstrapIdentity); + if ( + !current?.prepared || + !current.imageTransaction || + !current.watcherLease || + !current.contractReplacementSpecHash + ) { + throw new Error("Managed bootstrap Podman image transaction is unavailable."); + } + const completion = awaitPodmanBootstrapImageTransaction({ + engine: options.engine, + journalStore, + prepared: current.prepared, + watcherLease: current.watcherLease, + transaction: current.imageTransaction, + timeoutSecs, + }); + const runCaptureOpenshell = options.runCaptureOpenshell; + if (!runCaptureOpenshell) { + throw new Error("Managed bootstrap Podman requires OpenShell observation authority."); + } + observePodmanBootstrapReplacementReady({ + engine: options.engine, + runtimeId: current.prepared.replacementRuntimeId, + sandboxName: handle.sandbox.sandboxName, + sandboxId: handle.sandbox.sandboxId, + gatewayName: gatewayAuthority.gatewayName, + runCaptureOpenshell, + ...(options.sleep ? { sleep: options.sleep } : {}), + }); + current.completion = completion; + return completionReceipt( + handle, + snapshot, + current.prepared, + completion, + current.contractReplacementSpecHash, + ); + }, + + async finalizeBootstrap( + input: Parameters[0], + ): Promise { + const current = transactions.get(input.handle.bootstrapIdentity); + if (!current?.prepared || !current.watcherLease) + throw new Error("Managed bootstrap Podman transaction is unavailable."); + current.watcherLease.requiesceAndProve(); + if (input.outcome === "rollback") { + const receipt = rollbackPodmanBootstrapBeforeCommit({ + bootstrapIdentity: input.handle.bootstrapIdentity, + engine: options.engine, + heldWorkload: current.held, + journalStore, + watcherLease: current.watcherLease, + }); + current.watcherLease.resumeAndProve(); + transactions.delete(input.handle.bootstrapIdentity); + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: input.handle.sandbox, + bootstrapIdentity: input.handle.bootstrapIdentity, + outcome: "rolled-back", + restoredRuntimeId: receipt.originalRuntimeId, + restoredSpecHash: input.snapshot?.specHash ?? null, + heldWorkloadRemoved: false, + alreadyRolledBack: false, + finalizedAt: new Date().toISOString(), + }); + } + if (!current.completion || !input.completion) { + throw new Error("Managed bootstrap Podman commit requires image completion authority."); + } + const journal = journalStore.authorizeCommit(input.handle.bootstrapIdentity, [ + "original-stopped", + ]); + finishCommittedPodmanBootstrap({ + engine: options.engine, + journalStore, + journal, + watcherLease: current.watcherLease, + }); + current.watcherLease.resumeAndProve(); + const runCaptureOpenshell = options.runCaptureOpenshell; + if (!runCaptureOpenshell) { + throw new Error("Managed bootstrap Podman requires OpenShell observation authority."); + } + observePodmanBootstrapReplacementReady({ + engine: options.engine, + runtimeId: current.prepared.replacementRuntimeId, + sandboxName: input.handle.sandbox.sandboxName, + sandboxId: input.handle.sandbox.sandboxId, + gatewayName: gatewayAuthority.gatewayName, + runCaptureOpenshell, + ...(options.sleep ? { sleep: options.sleep } : {}), + }); + transactions.delete(input.handle.bootstrapIdentity); + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: input.handle.sandbox, + bootstrapIdentity: input.handle.bootstrapIdentity, + outcome: "committed", + restoredRuntimeId: null, + restoredSpecHash: null, + heldWorkloadRemoved: true, + alreadyRolledBack: false, + finalizedAt: new Date().toISOString(), + }); + }, + }); +} + +export function createPodmanManagedBootstrapAuthorityStore( + stateRoot: string, +): ManagedBootstrapAuthorityStore { + const store = createFilePodmanBootstrapJournalStore(stateRoot); + return Object.freeze({ + async recordPreparedAuthority(authority: ManagedBootstrapPreparedAuthority) { + const journal = store.load(authority.bootstrapIdentity); + if (!journal || serializePodmanBootstrapJournal(journal) !== authority.rollbackAuthority) { + throw new Error( + "Managed bootstrap Podman prepared authority does not match its durable journal.", + ); + } + return Object.freeze({ + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandbox: authority.sandbox, + bootstrapIdentity: authority.bootstrapIdentity, + authorityFingerprint: authority.authorityFingerprint, + recordId: `podman-managed-bootstrap/${authority.bootstrapIdentity}`, + recordedAt: new Date().toISOString(), + }); + }, + }); +} + +function createPodmanRuntimePatch( + sandboxName: string, + input: ManagedBootstrapRuntimeCreateLifecycleInput, +) { + let finalizer: ReturnType | null = null; + const selectedMode = input.sandboxGpuConfig.sandboxGpuEnabled + ? Object.freeze({ + kind: "podman-cdi", + label: "Podman CDI", + device: input.sandboxGpuConfig.sandboxGpuDevice ?? "", + args: Object.freeze( + input.sandboxGpuConfig.sandboxGpuDevice + ? ["--device", input.sandboxGpuConfig.sandboxGpuDevice] + : [], + ), + }) + : null; + return { + attach(value: ReturnType) { + finalizer = value; + }, + patch: Object.freeze({ + maybeApplyDuringCreate: () => undefined, + replacementRuntimeId: () => null, + createFailureMessage: () => null, + exitOnPatchError: () => undefined, + rollbackManagedStartupAfterCreateFailure: async () => { + await finalizer?.rollback(); + }, + ensureApplied: () => undefined, + waitForSupervisorReconnectIfNeeded: () => undefined, + commitAfterReady: async () => { + await finalizer?.commit(); + }, + selectedMode: () => selectedMode, + printReadinessFailureIfEnabled: () => undefined, + verifyGpuOrExit: async ( + verify: (sandboxName: string) => SandboxGpuProofResult, + ): Promise => verify(sandboxName), + }), + }; +} + +function createLifecycle( + engine: PodmanBoundContainerEngine, + input: ManagedBootstrapRuntimeCreateLifecycleInput, +): ManagedBootstrapRuntimeCreateLifecycle { + if (input.providerId !== PROVIDER_ID) { + throw new Error("Managed bootstrap Podman received another provider identity."); + } + const adapter = + input.adapterOverride ?? + createPodmanManagedBootstrapAdapter({ + engine, + stateRoot: input.stateRoot, + environment: input.environment, + gatewayPort: input.network.gatewayPort, + workspaceRoot: input.workspaceRoot, + ...(input.dependencies.runCaptureOpenshell + ? { runCaptureOpenshell: input.dependencies.runCaptureOpenshell } + : {}), + ...(input.dependencies.sleep ? { sleep: input.dependencies.sleep } : {}), + }); + const authorityStore = input.authorityStore; + const runtimePatch = createPodmanRuntimePatch(input.sandboxName, input); + return Object.freeze({ + launchArgv: input.launchArgv, + patch: runtimePatch.patch, + recoverUnfinished: () => adapter.recoverUnfinishedTransactions(), + prepareNetwork: async () => undefined, + async runCreate( + launch: (value: { + readonly heldWorkloadArgv: readonly string[]; + readonly bootstrapIdentity: string; + }) => Promise<{ + readonly value: T; + readonly receipt: import("./adapter").ManagedBootstrapCreateReceipt; + }>, + ) { + const gpuDevice = input.sandboxGpuConfig.sandboxGpuDevice; + const plan = { + schemaVersion: MANAGED_BOOTSTRAP_SCHEMA_VERSION, + sandboxName: input.sandboxName, + driverId: PROVIDER_ID, + image: input.image, + profile: { agent: input.request.agent, fingerprint: input.request.profileFingerprint }, + agentIdentity: input.agentIdentity, + managedStateRoots: input.managedStateRoots, + intendedWorkloadArgv: input.intendedWorkloadArgv, + expectedSupervisorArgv: input.expectedSupervisorArgv, + metadata: {}, + } as const; + let launched: { + readonly value: T; + readonly receipt: import("./adapter").ManagedBootstrapCreateReceipt; + } | null = null; + const prepared = await prepareManagedBootstrapSequence(adapter, { + create: { + plan, + request: input.request, + bootstrapIdentity: input.bootstrapIdentity, + launch: async (value) => { + launched = await launch(value); + return launched.receipt; + }, + }, + request: input.request, + replacementOptions: { + values: { + gpuModeArgs: input.sandboxGpuConfig.sandboxGpuEnabled + ? gpuDevice + ? ["--device", gpuDevice] + : ["--gpus", "all"] + : [], + requiredUlimits: input.requiredLimits.map( + (limit) => `${limit.name}=${String(limit.soft)}:${String(limit.hard)}`, + ), + extraGroupGids: [], + }, + }, + }); + const activated = await activateManagedBootstrapSequence(adapter, { + transaction: prepared, + authorityStore, + timeoutSecs: input.timeoutSecs, + }); + runtimePatch.attach( + createManagedBootstrapTerminalFinalizer((outcome) => + finalizeManagedBootstrapSequence(adapter, { outcome, transaction: activated }).then( + () => undefined, + ), + ), + ); + if (!launched) throw new Error("Managed bootstrap Podman did not return its create receipt."); + return (launched as { readonly value: T }).value; + }, + }); +} + +export function createPodmanManagedBootstrapSurface( + engine: PodmanBoundContainerEngine, +): RuntimeProviderManagedImageBootstrapSurface { + return Object.freeze({ + providerId: PROVIDER_ID, + supported: true, + bootstrapKind: "managed-image", + createAuthorityStore: ({ stateRoot }: { readonly stateRoot: string }) => + createPodmanManagedBootstrapAuthorityStore(stateRoot), + createLifecycle: (input: ManagedBootstrapRuntimeCreateLifecycleInput) => + createLifecycle(engine, input), + createOnboardRouting: ( + _input: ManagedBootstrapRuntimeOnboardRoutingInput, + ): ManagedBootstrapRuntimeOnboardRouting => ({ + nativeFallbackHasCleanBaseline: false, + inspectNativeRuntime: () => null, + isNativeCreateRoutingFailure: () => false, + isTrustedNativeRuntimeError: () => false, + isNativeReadinessRoutingFailure: () => false, + prepareCompatibilityLaunch: () => { + throw new Error("Managed Podman onboarding does not use Docker compatibility fallback."); + }, + }), + }); +} diff --git a/src/lib/onboard/managed-bootstrap/podman-watcher-lease.test.ts b/src/lib/onboard/managed-bootstrap/podman-watcher-lease.test.ts index 9872e11a61e..c1f1645f1ca 100644 --- a/src/lib/onboard/managed-bootstrap/podman-watcher-lease.test.ts +++ b/src/lib/onboard/managed-bootstrap/podman-watcher-lease.test.ts @@ -22,11 +22,20 @@ const ORIGINAL = Object.freeze({ pid: 4_100, processStartIdentity: "proc-start-100", } as const satisfies PodmanGatewayWatcherSnapshot); +const STANDALONE_ORIGINAL = Object.freeze({ + ...ORIGINAL, + ownerKind: "standalone" as const, +}); const RESUMED = Object.freeze({ ...ORIGINAL, pid: 4_200, processStartIdentity: "proc-start-200", }); +const RESUMED_AGAIN = Object.freeze({ + ...ORIGINAL, + pid: 4_300, + processStartIdentity: "proc-start-300", +}); const HOLDER = Object.freeze({ pid: 9_100, processStartIdentity: "holder-start-100" }); function record( @@ -41,15 +50,23 @@ function record( }); } -function harness(overrides: Partial = {}) { +function harness( + overrides: Partial = {}, + options: { + readonly initial?: PodmanGatewayWatcherSnapshot; + readonly retainQuiescedProcess?: boolean; + } = {}, +) { + const initial = options.initial ?? ORIGINAL; let durable: PodmanGatewayWatcherLeaseRecord | null = null; let holderAlive = true; let ownerStopped = false; - let watchers: PodmanGatewayWatcherSnapshot[] = [ORIGINAL]; - const alive = new Set(["4100:proc-start-100"]); - const healthy = new Set(["4100:proc-start-100"]); + let resumeCount = 0; const key = (entry: PodmanGatewayWatcherSnapshot) => `${String(entry.pid)}:${entry.processStartIdentity}`; + let watchers: PodmanGatewayWatcherSnapshot[] = [initial]; + const alive = new Set([key(initial)]); + const healthy = new Set([key(initial)]); const writes: PodmanGatewayWatcherLeaseRecord[] = []; const store = { read: vi.fn(() => durable), @@ -68,21 +85,37 @@ function harness(overrides: Partial = durable = null; }), }; - const stopExactOwner = vi.fn(() => { - alive.delete(key(ORIGINAL)); - healthy.delete(key(ORIGINAL)); - watchers = []; + const stopExactOwner = vi.fn((entry: PodmanGatewayWatcherSnapshot = initial) => { + healthy.delete(key(entry)); + const stopByMode = { + remove: () => { + alive.delete(key(entry)); + watchers = []; + }, + retain: () => undefined, + } as const; + stopByMode[options.retainQuiescedProcess === true ? "retain" : "remove"](); ownerStopped = true; }); const resumeSameOwner = vi.fn(() => { + const resumeByMode = { + replace: () => { + const resumed = resumeCount++ === 0 ? RESUMED : RESUMED_AGAIN; + watchers = [resumed]; + alive.add(key(resumed)); + healthy.add(key(resumed)); + }, + retain: () => { + const retained = watchers[0] as PodmanGatewayWatcherSnapshot; + healthy.add(key(retained)); + }, + } as const; + resumeByMode[options.retainQuiescedProcess === true ? "retain" : "replace"](); ownerStopped = false; - watchers = [RESUMED]; - alive.add(key(RESUMED)); - healthy.add(key(RESUMED)); }); const deps: PodmanManagedGatewayWatcherControllerDeps = { store, - captureCurrent: () => ORIGINAL, + captureCurrent: () => initial, captureLeaseHolder: () => HOLDER, listTargetWatchers: () => watchers, isProcessInstanceAlive: (entry) => alive.has(key(entry)), @@ -104,6 +137,7 @@ function harness(overrides: Partial = return { alive, controller: createPodmanManagedGatewayWatcherController(deps), + deps, durable: () => durable, healthy, ownerStopped: () => ownerStopped, @@ -146,6 +180,47 @@ describe("durable Podman OpenShell watcher lease", () => { expect(fake.store.clear).toHaveBeenCalledWith(LEASE_ID); }); + it("retains durable authority while observation runs and requiesces before finalization", () => { + const fake = harness(); + const lease = fake.controller.quiesceAndProve(); + + lease.resumeForObservationAndProve(); + expect(fake.watchers()).toEqual([RESUMED]); + expect(fake.durable()).toEqual(expect.objectContaining({ ...RESUMED, phase: "observing" })); + lease.assertStillHeld(); + + lease.requiesceAndProve(); + expect(fake.watchers()).toEqual([]); + expect(fake.durable()).toEqual(expect.objectContaining({ ...RESUMED, phase: "stopped" })); + lease.assertStillStopped(); + + lease.resumeAndProve(); + expect(fake.resumeSameOwner).toHaveBeenCalledTimes(2); + expect(fake.stopExactOwner).toHaveBeenCalledTimes(2); + expect(fake.durable()).toBeNull(); + }); + + it("suspends and resumes one exact standalone watcher without replacing its process", () => { + const fake = harness({}, { initial: STANDALONE_ORIGINAL, retainQuiescedProcess: true }); + const lease = fake.controller.quiesceAndProve(); + + expect(fake.watchers()).toEqual([STANDALONE_ORIGINAL]); + expect(fake.alive).toContain("4100:proc-start-100"); + expect(fake.healthy).not.toContain("4100:proc-start-100"); + lease.assertStillStopped(); + + lease.resumeForObservationAndProve(); + expect(fake.watchers()).toEqual([STANDALONE_ORIGINAL]); + expect(fake.healthy).toContain("4100:proc-start-100"); + + lease.requiesceAndProve(); + lease.resumeAndProve(); + expect(fake.stopExactOwner).toHaveBeenCalledTimes(2); + expect(fake.resumeSameOwner).toHaveBeenCalledTimes(2); + expect(fake.watchers()).toEqual([STANDALONE_ORIGINAL]); + expect(fake.durable()).toBeNull(); + }); + it("clears an acquiring record when the crash preceded the stop request", () => { const fake = harness(); fake.setDurable(record("acquiring")); @@ -198,6 +273,25 @@ describe("durable Podman OpenShell watcher lease", () => { expect(fake.durable()).toEqual(record("stopped")); }); + it("lets a fresh controller reclaim the exact crash-left stopped lease", () => { + const fake = harness(); + fake.setDurable(record("stopped")); + fake.setHolderAlive(false); + fake.stopExactOwner(); + + const freshController = createPodmanManagedGatewayWatcherController(fake.deps); + const lease = freshController.reclaimStoppedLease(LEASE_ID); + + expect(fake.store.advance).toHaveBeenCalledWith( + LEASE_ID, + expect.objectContaining({ leaseId: LEASE_ID, phase: "stopped" }), + ); + lease.assertStillStopped(); + lease.resumeAndProve(); + expect(fake.resumeSameOwner).toHaveBeenCalledOnce(); + expect(fake.durable()).toBeNull(); + }); + it("refuses ambiguous watcher ownership before persisting or stopping", () => { const fake = harness({ listTargetWatchers: () => [ diff --git a/src/lib/onboard/managed-bootstrap/podman-watcher-lease.ts b/src/lib/onboard/managed-bootstrap/podman-watcher-lease.ts index 264a773e47b..b138f2df384 100644 --- a/src/lib/onboard/managed-bootstrap/podman-watcher-lease.ts +++ b/src/lib/onboard/managed-bootstrap/podman-watcher-lease.ts @@ -13,7 +13,7 @@ const CONTROL_CHARACTER = /[\u0000-\u001f\u007f-\u009f]/u; export const PODMAN_WATCHER_LEASE_SCHEMA_VERSION = 2; export type PodmanGatewayWatcherOwnerKind = "managed-service" | "standalone"; -export type PodmanGatewayWatcherLeasePhase = "acquiring" | "stopped"; +export type PodmanGatewayWatcherLeasePhase = "acquiring" | "stopped" | "observing"; /** * Immutable, non-secret evidence identifying one target-bound host gateway @@ -85,13 +85,21 @@ export interface PodmanManagedGatewayWatcherControllerDeps { export interface PodmanGatewayWatcherLease { readonly record: PodmanGatewayWatcherLeaseRecord; + /** Prove the transaction still owns either exact stopped or exact observed authority. */ + readonly assertStillHeld: () => void; readonly assertStillStopped: () => void; + /** Resume the exact owner while retaining durable transaction authority. */ + readonly resumeForObservationAndProve: () => void; + /** Stop the exact observed owner again before a terminal runtime mutation. */ + readonly requiesceAndProve: () => void; readonly resumeAndProve: () => void; } export interface PodmanManagedGatewayWatcherController { /** Recover a crash-left lease before another Podman transaction may start. */ readonly recoverUnfinishedLease: () => void; + /** Reclaim the exact stopped lease referenced by an unfinished bootstrap journal. */ + readonly reclaimStoppedLease: (expectedLeaseId: string) => PodmanGatewayWatcherLease; /** Durably acquire exclusive authority and prove the exact owner is stopped. */ readonly quiesceAndProve: () => PodmanGatewayWatcherLease; } @@ -202,7 +210,7 @@ function normalizeRecord(value: PodmanGatewayWatcherLeaseRecord): PodmanGatewayW typeof value !== "object" || value.schemaVersion !== PODMAN_WATCHER_LEASE_SCHEMA_VERSION || !SAFE_LEASE_ID.test(value.leaseId) || - (value.phase !== "acquiring" && value.phase !== "stopped") + (value.phase !== "acquiring" && value.phase !== "stopped" && value.phase !== "observing") ) { throw new PodmanGatewayWatcherLeaseError( "Durable Podman OpenShell watcher lease is invalid.", @@ -282,19 +290,31 @@ function assertStopped( receipt: Readonly, deps: PodmanManagedGatewayWatcherControllerDeps, ): void { - if (deps.isProcessInstanceAlive(receipt)) { - throw new PodmanGatewayWatcherLeaseError( - "The captured OpenShell watcher process instance is still alive.", - true, - ); - } if (!deps.isOwnerStopped(receipt)) { throw new PodmanGatewayWatcherLeaseError( "The captured OpenShell watcher lifecycle owner is not proven stopped.", true, ); } - if (readTargetWatchers(receipt, deps).length !== 0) { + const watchers = readTargetWatchers(receipt, deps); + if (watchers.length === 0) { + if (deps.isProcessInstanceAlive(receipt)) { + throw new PodmanGatewayWatcherLeaseError( + "The captured OpenShell watcher process instance is still alive.", + true, + ); + } + return; + } + const retained = watchers[0] as Readonly | undefined; + if ( + receipt.ownerKind !== "standalone" || + watchers.length !== 1 || + !retained || + !sameProcessInstance(receipt, retained) || + !deps.isProcessInstanceAlive(retained) || + deps.isHealthy(retained) + ) { throw new PodmanGatewayWatcherLeaseError( "A target-bound OpenShell watcher appeared while the durable stop lease was held.", true, @@ -351,7 +371,7 @@ function exactHealthyReplacement( receipt: Readonly, deps: PodmanManagedGatewayWatcherControllerDeps, ): Readonly | null { - if (deps.isProcessInstanceAlive(receipt) || deps.isOwnerStopped(receipt)) return null; + if (deps.isOwnerStopped(receipt)) return null; const watchers = readTargetWatchers(receipt, deps); if (watchers.length === 0) return null; if (watchers.length !== 1) { @@ -374,7 +394,7 @@ function exactHealthyReplacement( function resumeAndProve( receipt: Readonly, deps: PodmanManagedGatewayWatcherControllerDeps, -): void { +): Readonly { const existing = readTargetWatchers(receipt, deps); if (existing.length > 1) { throw new PodmanGatewayWatcherLeaseError( @@ -384,20 +404,25 @@ function resumeAndProve( } if (existing.length === 1) { const current = existing[0] as Readonly; + if (!sameLaunchOwner(receipt, current) || !deps.isProcessInstanceAlive(current)) { + throw new PodmanGatewayWatcherLeaseError( + "A target-bound OpenShell watcher exists without exact healthy owner proof.", + true, + ); + } + if (!deps.isOwnerStopped(receipt) && deps.isHealthy(current)) return current; if ( - !sameLaunchOwner(receipt, current) || - deps.isOwnerStopped(receipt) || - !deps.isProcessInstanceAlive(current) || - !deps.isHealthy(current) + receipt.ownerKind !== "standalone" || + !sameProcessInstance(receipt, current) || + !deps.isOwnerStopped(receipt) ) { throw new PodmanGatewayWatcherLeaseError( "A target-bound OpenShell watcher exists without exact healthy owner proof.", true, ); } - return; } - if (!deps.isOwnerStopped(receipt)) { + if (existing.length === 0 && !deps.isOwnerStopped(receipt)) { throw new PodmanGatewayWatcherLeaseError( "The captured owner is neither stopped nor serving an exact healthy watcher.", true, @@ -425,6 +450,7 @@ function resumeAndProve( true, ); } + return resumed; } function readLease(deps: PodmanManagedGatewayWatcherControllerDeps) { @@ -444,6 +470,164 @@ function sameLease( ); } +function createHeldLease( + stopped: PodmanGatewayWatcherLeaseRecord, + deps: PodmanManagedGatewayWatcherControllerDeps, +): PodmanGatewayWatcherLease { + let current = stopped; + let released = false; + return Object.freeze({ + get record() { + return current; + }, + assertStillHeld: () => { + if (released) { + throw new PodmanGatewayWatcherLeaseError("Podman watcher lease was already released."); + } + const durable = readLease(deps); + if (!durable || !sameLease(current, durable)) { + throw new PodmanGatewayWatcherLeaseError( + "Durable Podman watcher lease changed while transaction authority was held.", + true, + ); + } + if (durable.phase === "stopped") { + assertStopped(durable, deps); + } else if (durable.phase === "observing") { + requireExclusiveCurrent(durable, deps); + } else { + throw new PodmanGatewayWatcherLeaseError( + "Durable Podman watcher lease has no usable transaction authority.", + true, + ); + } + current = durable; + }, + assertStillStopped: () => { + if (released) { + throw new PodmanGatewayWatcherLeaseError("Podman watcher lease was already released."); + } + const durable = readLease(deps); + if (!durable || !sameLease(current, durable) || durable.phase !== "stopped") { + throw new PodmanGatewayWatcherLeaseError( + "Durable Podman watcher lease changed while it was held.", + true, + ); + } + assertStopped(current, deps); + }, + resumeForObservationAndProve: () => { + if (released) { + throw new PodmanGatewayWatcherLeaseError("Podman watcher lease was already released."); + } + const durable = readLease(deps); + if (!durable || !sameLease(current, durable) || durable.phase !== "stopped") { + throw new PodmanGatewayWatcherLeaseError( + "Durable Podman watcher lease changed before observation.", + true, + ); + } + const observing = Object.freeze({ ...current, phase: "observing" as const }); + deps.store.advance(current.leaseId, observing); + const resumed = resumeAndProve(observing, deps); + const observed = Object.freeze({ + ...observing, + ...resumed, + schemaVersion: PODMAN_WATCHER_LEASE_SCHEMA_VERSION, + holder: observing.holder, + leaseId: observing.leaseId, + phase: "observing" as const, + }); + deps.store.advance(current.leaseId, observed); + current = observed; + }, + requiesceAndProve: () => { + if (released) { + throw new PodmanGatewayWatcherLeaseError("Podman watcher lease was already released."); + } + const durable = readLease(deps); + if (!durable || !sameLease(current, durable)) { + throw new PodmanGatewayWatcherLeaseError( + "Durable Podman watcher lease changed before terminal quiescence.", + true, + ); + } + if (durable.phase === "stopped") { + assertStopped(durable, deps); + current = durable; + return; + } + if (durable.phase !== "observing") { + throw new PodmanGatewayWatcherLeaseError( + "Durable Podman watcher lease is not available for terminal quiescence.", + true, + ); + } + current = stopObservedOwnerAndProve(durable, deps); + }, + resumeAndProve: () => { + if (released) return; + const durable = readLease(deps); + if (!durable || !sameLease(current, durable) || durable.phase !== "stopped") { + throw new PodmanGatewayWatcherLeaseError( + "Durable Podman watcher lease changed before release.", + true, + ); + } + resumeAndProve(current, deps); + deps.store.clear(current.leaseId); + released = true; + }, + }); +} + +function stopObservedOwnerAndProve( + observing: PodmanGatewayWatcherLeaseRecord, + deps: PodmanManagedGatewayWatcherControllerDeps, +): PodmanGatewayWatcherLeaseRecord { + const watchers = readTargetWatchers(observing, deps); + let stoppedSnapshot: Readonly = observing; + if (watchers.length === 0) { + if (!deps.isOwnerStopped(observing)) { + throw new PodmanGatewayWatcherLeaseError( + "The observed Podman watcher disappeared without a stopped lifecycle owner.", + true, + ); + } + } else { + if (watchers.length !== 1) { + throw new PodmanGatewayWatcherLeaseError( + "Multiple target-bound OpenShell watchers appeared before terminal quiescence.", + true, + ); + } + const observed = watchers[0] as Readonly; + if ( + !sameLaunchOwner(observing, observed) || + !deps.isProcessInstanceAlive(observed) || + !deps.isHealthy(observed) + ) { + throw new PodmanGatewayWatcherLeaseError( + "The observed OpenShell watcher no longer has exact healthy owner proof.", + true, + ); + } + deps.stopExactOwner(observed); + assertStopped(observed, deps); + stoppedSnapshot = observed; + } + const stopped = Object.freeze({ + ...observing, + ...stoppedSnapshot, + schemaVersion: PODMAN_WATCHER_LEASE_SCHEMA_VERSION, + holder: observing.holder, + leaseId: observing.leaseId, + phase: "stopped" as const, + }); + deps.store.advance(observing.leaseId, stopped); + return stopped; +} + /** * Build the inert watcher authority needed by native Podman replacement. * @@ -470,6 +654,37 @@ export function createPodmanManagedGatewayWatcherController( return Object.freeze({ recoverUnfinishedLease, + reclaimStoppedLease: (expectedLeaseId: string) => { + if (!SAFE_LEASE_ID.test(expectedLeaseId)) { + throw new PodmanGatewayWatcherLeaseError("Podman watcher lease identity is invalid."); + } + const record = readLease(deps); + if ( + !record || + record.leaseId !== expectedLeaseId || + (record.phase !== "stopped" && record.phase !== "observing") + ) { + throw new PodmanGatewayWatcherLeaseError( + "The exact stopped Podman watcher lease referenced by bootstrap recovery is absent.", + true, + ); + } + if (deps.isLeaseHolderAlive(record.holder)) { + throw new PodmanGatewayWatcherLeaseError( + "Durable Podman watcher lease is still owned by a live transaction process.", + true, + ); + } + const quiesced = + record.phase === "observing" ? stopObservedOwnerAndProve(record, deps) : record; + assertStopped(quiesced, deps); + const reclaimed = Object.freeze({ + ...quiesced, + holder: normalizeHolder(deps.captureLeaseHolder()), + }); + deps.store.advance(expectedLeaseId, reclaimed); + return createHeldLease(reclaimed, deps); + }, quiesceAndProve: () => { recoverUnfinishedLease(); @@ -531,36 +746,7 @@ export function createPodmanManagedGatewayWatcherController( ); } - let released = false; - return Object.freeze({ - record: stopped, - assertStillStopped: () => { - if (released) { - throw new PodmanGatewayWatcherLeaseError("Podman watcher lease was already released."); - } - const durable = readLease(deps); - if (!durable || !sameLease(stopped, durable) || durable.phase !== "stopped") { - throw new PodmanGatewayWatcherLeaseError( - "Durable Podman watcher lease changed while it was held.", - true, - ); - } - assertStopped(stopped, deps); - }, - resumeAndProve: () => { - if (released) return; - const durable = readLease(deps); - if (!durable || !sameLease(stopped, durable)) { - throw new PodmanGatewayWatcherLeaseError( - "Durable Podman watcher lease changed before release.", - true, - ); - } - resumeAndProve(stopped, deps); - deps.store.clear(leaseId); - released = true; - }, - }); + return createHeldLease(stopped, deps); }, }); } diff --git a/src/lib/onboard/managed-bootstrap/runtime-create.ts b/src/lib/onboard/managed-bootstrap/runtime-create.ts index 94059ff97da..e68eef3fa98 100644 --- a/src/lib/onboard/managed-bootstrap/runtime-create.ts +++ b/src/lib/onboard/managed-bootstrap/runtime-create.ts @@ -3,6 +3,10 @@ import type { SandboxGpuProofResult } from "../../state/registry"; import type { ManagedStartupRootApplyRequest } from "../managed-startup/root-apply"; +import type { + ManagedStartupStateRoot, + ManagedStartupWorkspaceRoot, +} from "../managed-startup/state-roots"; import type { SandboxGpuConfig } from "../sandbox-gpu-mode"; import type { ManagedBootstrapAdapter, @@ -85,11 +89,14 @@ export interface ManagedBootstrapRuntimePatch { export interface ManagedBootstrapRuntimeCreateLifecycleInput { readonly providerId: string; + readonly environment: NodeJS.ProcessEnv; readonly stateRoot: string; readonly bootstrapIdentity: string; readonly request: ManagedStartupRootApplyRequest; readonly image: ManagedBootstrapImageIdentity; readonly agentIdentity: ManagedBootstrapAgentIdentity; + readonly workspaceRoot: ManagedStartupWorkspaceRoot; + readonly managedStateRoots: readonly ManagedStartupStateRoot[]; readonly intendedWorkloadArgv: readonly string[]; readonly expectedSupervisorArgv: readonly string[]; readonly launchArgv: readonly string[]; @@ -109,6 +116,7 @@ export interface ManagedBootstrapRuntimeCreateLifecycleInput { readonly inferenceProvider: string; readonly gatewayUsesContainerBridge: boolean; readonly gatewayPort: number; + readonly reverifyBridgeReachability: () => void | Promise; }; readonly dependencies: ManagedBootstrapRuntimeDependencies; } diff --git a/src/lib/onboard/managed-bootstrap/state-root-authority.ts b/src/lib/onboard/managed-bootstrap/state-root-authority.ts new file mode 100644 index 00000000000..6eb79eb807c --- /dev/null +++ b/src/lib/onboard/managed-bootstrap/state-root-authority.ts @@ -0,0 +1,162 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import path from "node:path"; + +import type { ManagedStartupStateRoot } from "../managed-startup/state-roots"; + +type JsonRecord = Record; + +export interface ManagedBootstrapStateRootReceipt { + readonly mountTarget: string; + readonly resourceIdentity: string; + readonly mountpoint: string; + readonly uid: number; + readonly gid: number; + readonly mode: number; + readonly readWrite: boolean; + readonly prepared: boolean; +} + +export interface ManagedBootstrapStateRootPreparation { + readonly path: string; + readonly uid: number; + readonly gid: number; + readonly mode: number; +} + +type ManagedBootstrapStateRootPreparationReceipt = ManagedBootstrapStateRootPreparation & { + readonly device: string; + readonly inode: string; +}; + +function record(value: unknown, label: string): JsonRecord { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error(`Managed bootstrap ${label} is invalid.`); + } + return value as JsonRecord; +} + +function exactMountpoint(value: unknown): string { + const mountpoint = typeof value === "string" ? value : ""; + if ( + !path.isAbsolute(mountpoint) || + path.normalize(mountpoint) !== mountpoint || + mountpoint === path.parse(mountpoint).root + ) { + throw new Error("Managed bootstrap state-root mountpoint authority is invalid."); + } + return mountpoint; +} + +function volumeEvidence( + root: ManagedStartupStateRoot, + captureVolume: (args: readonly string[]) => string, +): { + readonly mountpoint: string; + readonly labels: Readonly>; +} { + const output = captureVolume([ + "inspect", + "--format", + "{{.Name}}\n{{.Mountpoint}}\n{{json .Labels}}", + root.resourceIdentity, + ]).trimEnd(); + const separator = output.lastIndexOf("\n"); + if (separator < 0) { + throw new Error("Managed bootstrap state-root resource evidence is incomplete."); + } + const identity = output.slice(0, separator); + const identitySeparator = identity.indexOf("\n"); + if (identitySeparator < 0 || identity.slice(0, identitySeparator) !== root.resourceIdentity) { + throw new Error("Managed bootstrap state-root resource identity changed."); + } + const mountpoint = exactMountpoint(identity.slice(identitySeparator + 1)); + let labels: unknown; + try { + labels = JSON.parse(output.slice(separator + 1)); + } catch { + throw new Error("Managed bootstrap state-root ownership labels are invalid."); + } + const observedLabels = record(labels, "state-root ownership labels"); + if ( + !Object.entries(root.ownershipLabels).every(([name, value]) => observedLabels[name] === value) + ) { + throw new Error("Managed bootstrap state-root resource ownership changed."); + } + return Object.freeze({ + mountpoint, + labels: Object.freeze({ ...observedLabels }), + }); +} + +function exactContainerMount( + inspect: JsonRecord, + root: ManagedStartupStateRoot, + mountpoint: string, +): void { + if (!Array.isArray(inspect.Mounts)) { + throw new Error("Managed bootstrap state-root mount inventory is invalid."); + } + const matches = inspect.Mounts.map((value) => record(value, "state-root mount")).filter( + (mount) => mount.Destination === root.mountTarget, + ); + if (matches.length !== 1) { + throw new Error("Managed bootstrap state root must resolve to one exact mount."); + } + const mount = matches[0] as JsonRecord; + const driver = mount.Driver; + if ( + mount.Type !== "volume" || + mount.Name !== root.resourceIdentity || + mount.Source !== mountpoint || + mount.RW !== root.readWrite || + (driver !== undefined && driver !== "" && driver !== "local") + ) { + throw new Error("Managed bootstrap state-root mount authority changed."); + } +} + +export function prepareManagedBootstrapStateRoots(input: { + readonly inspect: JsonRecord; + readonly roots: readonly ManagedStartupStateRoot[]; + readonly captureVolume: (args: readonly string[]) => string; + readonly prepareRoot?: ( + input: ManagedBootstrapStateRootPreparation, + ) => ManagedBootstrapStateRootPreparationReceipt; +}): readonly ManagedBootstrapStateRootReceipt[] { + return Object.freeze( + input.roots.map((root) => { + const evidence = volumeEvidence(root, input.captureVolume); + exactContainerMount(input.inspect, root, evidence.mountpoint); + const preparation = Object.freeze({ + path: evidence.mountpoint, + uid: root.uid, + gid: root.gid, + mode: root.mode, + }); + const receipt = input.prepareRoot?.(preparation); + if ( + receipt && + (receipt.path !== preparation.path || + receipt.uid !== preparation.uid || + receipt.gid !== preparation.gid || + receipt.mode !== preparation.mode || + !/^\d+$/u.test(receipt.device) || + !/^\d+$/u.test(receipt.inode)) + ) { + throw new Error("Managed bootstrap state-root preparation receipt is invalid."); + } + return Object.freeze({ + mountTarget: root.mountTarget, + resourceIdentity: root.resourceIdentity, + mountpoint: evidence.mountpoint, + uid: root.uid, + gid: root.gid, + mode: root.mode, + readWrite: root.readWrite, + prepared: receipt !== undefined, + }); + }), + ); +} diff --git a/src/lib/onboard/managed-image/agents.ts b/src/lib/onboard/managed-image/agents.ts new file mode 100644 index 00000000000..df9d7e4d86e --- /dev/null +++ b/src/lib/onboard/managed-image/agents.ts @@ -0,0 +1,17 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export { + CANDIDATE_MANAGED_IMAGE_AGENTS, + isCandidateManagedImageAgent, + isManagedImageAgent, + isShippedManagedImageAgent, + MANAGED_IMAGE_AGENTS, + MANAGED_IMAGE_RUNTIME_IDENTITIES, + managedImageRuntimeIdentity, + SHIPPED_MANAGED_IMAGE_AGENTS, + type CandidateManagedImageAgent, + type ManagedImageAgent, + type ManagedImageRuntimeIdentity, + type ShippedManagedImageAgent, +} from "./contract"; diff --git a/src/lib/onboard/managed-startup-shared-state-transaction.test.ts b/src/lib/onboard/managed-startup-shared-state-transaction.test.ts index 7cc42889bc6..cd2b6297f61 100644 --- a/src/lib/onboard/managed-startup-shared-state-transaction.test.ts +++ b/src/lib/onboard/managed-startup-shared-state-transaction.test.ts @@ -325,16 +325,16 @@ describe("managed startup shared-state transaction", () => { ); }); - it("retains the no-mounted-state-root boundary for other agents", () => { + it("accepts the exact declared OpenClaw state root mount", () => { const root = agentRoot("openclaw"); fs.mkdirSync(root); fs.writeFileSync(path.join(root, "openclaw.json"), "{}\n"); simulateMountedStateRoot(root); - expect(() => + expect( beginManagedStartupSharedStateTransaction(managedStartupE2eProfile("openclaw"), options), - ).toThrow(/crosses a nested filesystem mount/u); - expect(fs.existsSync(transactionDirectory)).toBe(false); + ).toBe(true); + expect(fs.existsSync(transactionDirectory)).toBe(true); }); it("tracks only active post-install messaging outputs and leaves disabled targets alone", () => { diff --git a/src/lib/onboard/managed-startup/shared-state-transaction.ts b/src/lib/onboard/managed-startup/shared-state-transaction.ts index 32474bdab55..f6156d89c40 100644 --- a/src/lib/onboard/managed-startup/shared-state-transaction.ts +++ b/src/lib/onboard/managed-startup/shared-state-transaction.ts @@ -15,6 +15,7 @@ import { type ManagedStartupAgent, type ManagedStartupProfile, } from "./profile"; +import { managedStartupStateRootMountTargets } from "./state-roots"; const TRANSACTION_SCHEMA_VERSION = 1; const MAX_TRANSACTION_FILES = 128; @@ -312,17 +313,27 @@ function relativeTarget(target: string, options: ResolvedOptions): string { /** * SOURCE_OF_TRUTH_REVIEW - * invalidState: the authorized Hermes named-volume root appears on another filesystem and is - * rejected as a nested mount, while a broader exception could hide an unsafe descendant mount. - * sourceBoundary: the managed Hermes volume lifecycle authorizes only the exact `.hermes` root; - * these transaction validators remain authoritative for every descendant device boundary. - * whyNotSourceFix: a Docker named volume necessarily changes the root device, so the transaction - * must adopt that device at the exact Hermes root and continue rejecting later device changes. + * invalidState: an agent-declared managed state root appears on another filesystem and is rejected + * as a nested mount, while a broader exception could hide an unsafe descendant mount. + * sourceBoundary: the managed state-root declaration authorizes only its exact agent root; these + * transaction validators remain authoritative for every descendant device boundary. + * whyNotSourceFix: a managed volume necessarily changes the root device, so the transaction must + * adopt that device at the exact declared root and continue rejecting later device changes. * regressionTest: managed-startup-shared-state-transaction.test.ts proves exact-root prepare and * rollback acceptance, descendant-mount rejection in both paths, and rejection for other agents. - * removalCondition: remove the Hermes exception when its durable state root no longer arrives as - * a distinct filesystem mount, or when transaction storage moves wholly inside that mount. + * removalCondition: remove this adoption when managed roots no longer arrive as distinct + * filesystem mounts, or when transaction storage moves wholly inside each declared root. */ +function isDeclaredAgentStateRoot( + expectedAgent: ManagedStartupAgent, + outputRoot: string, + options: ResolvedOptions, +): boolean { + const relative = path.relative(options.sandboxRoot, outputRoot).split(path.sep).join("/"); + const canonicalTarget = path.posix.join("/sandbox", relative); + return managedStartupStateRootMountTargets(expectedAgent).includes(canonicalTarget); +} + function validateExistingAncestors( target: string, expectedAgent: ManagedStartupAgent, @@ -349,11 +360,9 @@ function validateExistingAncestors( if (stat.isSymbolicLink() || !stat.isDirectory()) { fail(`transaction path ancestor is unsafe: ${current}`); } - // Hermes owns one explicitly durable state root. Docker supplies that root - // as a named volume, so its exact mountpoint may cross from /sandbox onto - // another filesystem. Keep every descendant on that same device and keep - // the historical no-nested-mount contract for every other agent. - if (current === outputRoot && expectedAgent === "hermes") { + // An exact agent-declared state root may cross from /sandbox onto its + // managed volume. Every descendant must remain on that adopted device. + if (current === outputRoot && isDeclaredAgentStateRoot(expectedAgent, outputRoot, options)) { expectedDevice = stat.dev; } else if (stat.dev !== expectedDevice) { fail(`transaction path crosses a nested filesystem mount: ${current}`); @@ -374,7 +383,7 @@ function managedOutputDevice(expectedAgent: ManagedStartupAgent, options: Resolv if (stat.isSymbolicLink() || !stat.isDirectory()) { fail(`managed output root is unsafe: ${outputRoot}`); } - if (expectedAgent !== "hermes" && stat.dev !== sandboxStat.dev) { + if (!isDeclaredAgentStateRoot(expectedAgent, outputRoot, options) && stat.dev !== sandboxStat.dev) { fail(`managed output root crosses a nested filesystem mount: ${outputRoot}`); } return stat.dev; diff --git a/src/lib/onboard/managed-startup/state-roots.ts b/src/lib/onboard/managed-startup/state-roots.ts new file mode 100644 index 00000000000..27cb0313e10 --- /dev/null +++ b/src/lib/onboard/managed-startup/state-roots.ts @@ -0,0 +1,198 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import path from "node:path"; + +import type { ManagedStartupAgent } from "./profile"; + +export interface ManagedStartupStateRoot { + readonly mountTarget: string; + readonly resourceIdentity: string; + readonly ownershipLabels: Readonly>; + readonly uid: number; + readonly gid: number; + readonly mode: number; + readonly readWrite: boolean; +} + +export interface ManagedStartupWorkspaceRoot { + readonly uid: number; + readonly gid: number; + readonly mode: 0o755 | 0o1775; +} + +type ManagedStartupStateRootDeclaration = { + readonly mountTarget: string; + readonly resourceIdentity: (sandboxName: string) => string; + readonly ownershipLabels: ( + sandboxName: string, + mountTarget: string, + ) => Readonly>; + readonly uidAuthority: "agent"; + readonly gidAuthority: "agent"; + readonly mode: number; + readonly readWrite: true; +}; + +export const MANAGED_HERMES_STATE_ROOT = "/sandbox/.hermes" as const; +export const MANAGED_OPENCLAW_STATE_ROOT = "/sandbox/.openclaw" as const; +const HERMES_STATE_VOLUME_NAME_PREFIX = "nemoclaw-hermes-state-v1"; +const OPENCLAW_STATE_VOLUME_NAME_PREFIX = "nemoclaw-openclaw-state-v1"; + +/** + * RFC_9909_FOLLOW_UP + * + * Temporary core-owned registry for agent-specific managed state-root + * declarations. Runtime providers consume only the resulting generic + * ManagedStartupStateRoot list and must not compare or branch on agent IDs. + * + * Follow-up ownership: the RFC #9909 implementation epic, once created. + * Do not invent an issue number here. Replace this breadcrumb with the epic + * number when that epic exists. + * + * Removal condition: agent packages/manifests can declare validated managed + * state roots without adding agent IDs or production logic to NemoClaw core. + */ +const MANAGED_AGENT_STATE_ROOTS = Object.freeze({ + openclaw: Object.freeze([ + Object.freeze({ + mountTarget: MANAGED_OPENCLAW_STATE_ROOT, + resourceIdentity: (sandboxName: string) => + `${OPENCLAW_STATE_VOLUME_NAME_PREFIX}-${sandboxName}`, + ownershipLabels: (sandboxName: string, mountTarget: string) => + Object.freeze({ + "io.nvidia.nemoclaw.openclaw-state.managed": "true", + "io.nvidia.nemoclaw.openclaw-state.schema": "1", + "io.nvidia.nemoclaw.openclaw-state.sandbox": sandboxName, + "io.nvidia.nemoclaw.openclaw-state.target": mountTarget, + }), + uidAuthority: "agent", + gidAuthority: "agent", + mode: 0o2770, + readWrite: true, + } satisfies ManagedStartupStateRootDeclaration), + ]), + hermes: Object.freeze([ + Object.freeze({ + mountTarget: MANAGED_HERMES_STATE_ROOT, + resourceIdentity: (sandboxName: string) => + `${HERMES_STATE_VOLUME_NAME_PREFIX}-${sandboxName}`, + ownershipLabels: (sandboxName: string, mountTarget: string) => + Object.freeze({ + "io.nvidia.nemoclaw.hermes-state.managed": "true", + "io.nvidia.nemoclaw.hermes-state.schema": "1", + "io.nvidia.nemoclaw.hermes-state.sandbox": sandboxName, + "io.nvidia.nemoclaw.hermes-state.target": mountTarget, + }), + uidAuthority: "agent", + gidAuthority: "agent", + mode: 0o3770, + readWrite: true, + } satisfies ManagedStartupStateRootDeclaration), + ]), + "langchain-deepagents-code": Object.freeze([]), + pi: Object.freeze([]), +} satisfies Record); + +export function managedStartupStateRootMountTargets(agent: ManagedStartupAgent): readonly string[] { + return Object.freeze(MANAGED_AGENT_STATE_ROOTS[agent].map(({ mountTarget }) => mountTarget)); +} + +const MANAGED_AGENT_WORKSPACE_ROOTS = Object.freeze({ + openclaw: Object.freeze({ uidAuthority: "agent", gidAuthority: "agent", mode: 0o755 }), + hermes: Object.freeze({ uidAuthority: "agent", gidAuthority: "agent", mode: 0o755 }), + "langchain-deepagents-code": Object.freeze({ + uidAuthority: "root", + gidAuthority: "agent", + mode: 0o1775, + }), + pi: Object.freeze({ uidAuthority: "agent", gidAuthority: "agent", mode: 0o755 }), +} satisfies Record< + ManagedStartupAgent, + { + readonly uidAuthority: "agent" | "root"; + readonly gidAuthority: "agent"; + readonly mode: ManagedStartupWorkspaceRoot["mode"]; + } +>); + +function exactSandboxName(sandboxName: string): string { + if ( + sandboxName.length === 0 || + sandboxName.includes("\0") || + sandboxName.includes("/") || + sandboxName === "." || + sandboxName === ".." + ) { + throw new Error("Managed startup state-root sandbox identity is invalid."); + } + return sandboxName; +} + +function exactAgentIdentity(value: number, label: string): number { + if (!Number.isSafeInteger(value) || value < 0 || value > 2_147_483_647) { + throw new Error(`Managed startup state-root ${label} authority is invalid.`); + } + return value; +} + +export function managedStartupWorkspaceRoot(input: { + readonly agent: ManagedStartupAgent; + readonly agentIdentity: { readonly uid: number; readonly gid: number }; +}): ManagedStartupWorkspaceRoot { + const uid = exactAgentIdentity(input.agentIdentity.uid, "workspace UID"); + const gid = exactAgentIdentity(input.agentIdentity.gid, "workspace GID"); + const declaration = MANAGED_AGENT_WORKSPACE_ROOTS[input.agent]; + return Object.freeze({ + uid: declaration.uidAuthority === "root" ? 0 : uid, + gid, + mode: declaration.mode, + }); +} + +export function managedStartupStateRoots(input: { + readonly agent: ManagedStartupAgent; + readonly sandboxName: string; + readonly agentIdentity: { readonly uid: number; readonly gid: number }; +}): readonly ManagedStartupStateRoot[] { + const sandboxName = exactSandboxName(input.sandboxName); + const uid = exactAgentIdentity(input.agentIdentity.uid, "UID"); + const gid = exactAgentIdentity(input.agentIdentity.gid, "GID"); + return Object.freeze( + MANAGED_AGENT_STATE_ROOTS[input.agent].map((declaration) => { + const mountTarget: string = declaration.mountTarget; + if ( + !path.posix.isAbsolute(mountTarget) || + path.posix.normalize(mountTarget) !== mountTarget || + mountTarget === "/" + ) { + throw new Error("Managed startup state-root mount target is invalid."); + } + const resourceIdentity = declaration.resourceIdentity(sandboxName); + return Object.freeze({ + mountTarget, + resourceIdentity, + ownershipLabels: declaration.ownershipLabels(sandboxName, mountTarget), + uid, + gid, + mode: declaration.mode, + readWrite: declaration.readWrite, + }); + }), + ); +} + +export function managedHermesStateVolumeName(sandboxName: string): string { + return `${HERMES_STATE_VOLUME_NAME_PREFIX}-${exactSandboxName(sandboxName)}`; +} + +export function managedHermesStateVolumeLabels( + sandboxName: string, +): Readonly> { + const [root] = managedStartupStateRoots({ + agent: "hermes", + sandboxName, + agentIdentity: { uid: 0, gid: 0 }, + }); + return root?.ownershipLabels ?? Object.freeze({}); +} diff --git a/src/lib/onboard/managed-workload-rebuild-transaction.test.ts b/src/lib/onboard/managed-workload-rebuild-transaction.test.ts index 046ef5e8fe0..69c6ed45ed2 100644 --- a/src/lib/onboard/managed-workload-rebuild-transaction.test.ts +++ b/src/lib/onboard/managed-workload-rebuild-transaction.test.ts @@ -206,6 +206,7 @@ function bundle(providerId: string): RuntimeProviderBundle { status: "ok", detail: "socket-free", }), + validateSandboxGpu: () => undefined, preflightLifecycle: () => null, }, gateway: { @@ -213,6 +214,32 @@ function bundle(providerId: string): RuntimeProviderBundle { supported: true, launcher: "nemoclaw", inspectLegacyContainer: false, + prepareHostRuntime: () => ({ + providerId, + openShellDriver: "memory", + bindAddress: "127.0.0.1", + grpcHost: "127.0.0.1", + sshGatewayHost: "127.0.0.1", + portCheckHost: "127.0.0.1", + socketPath: null, + requiredServerIpSans: [], + sandboxHostAddress: null, + usesHostGatewayRoute: false, + resourceOwnership: { label: "test.managed", value: providerId }, + gatewayConfig: { + sandboxNamespace: "scoped", + hostGatewayIp: null, + includeSupervisorBin: true, + processOwnership: "scoped-namespace", + }, + network: { + sandboxSourceCidrs: () => [], + inspect: () => undefined, + usesHostGatewayRoute: () => false, + run: () => ({ status: 0 }), + ensureProbeImageCached: () => ({ ok: true, alreadyCached: true }), + }, + }), }, workload: { providerId, diff --git a/src/lib/onboard/managed-workload/hermes-state-volume.test.ts b/src/lib/onboard/managed-workload/hermes-state-volume.test.ts index 5014571e45a..cc04eaf096e 100644 --- a/src/lib/onboard/managed-workload/hermes-state-volume.test.ts +++ b/src/lib/onboard/managed-workload/hermes-state-volume.test.ts @@ -3,10 +3,23 @@ import { describe, expect, it, vi } from "vitest"; +import type { PodmanBoundContainerEngine, PodmanContainerEngine } from "../../adapters/podman"; import { createHermesStateVolumeDockerHarness as dockerHarness } from "../__test-helpers__/hermes-state-volume"; +import { + createDockerRuntimeProviderBundle, + createKubernetesRuntimeProviderBundle, +} from "../runtime-provider/docker"; +import { createPodmanRuntimeProviderBundle } from "../runtime-provider/podman"; import { MANAGED_HERMES_STATE_ROOT, + MANAGED_OPENCLAW_STATE_ROOT, + managedStartupStateRoots, +} from "../managed-startup/state-roots"; +import { managedImageRuntimeIdentity } from "../managed-image/agents"; +import { prepareManagedStateVolumes } from "./managed-state-volumes"; +import { managedHermesStateVolumeName, + removeManagedAgentStateVolumes, prepareManagedHermesStateVolume, removeManagedHermesStateVolume, } from "./hermes-state-volume"; @@ -54,6 +67,77 @@ describe("managed Hermes state volume", () => { expect(unregister).not.toHaveBeenCalled(); }); + it("uses the registered native provider for the same managed Hermes volume contract", () => { + const runtime = dockerHarness(); + const scope = prepareManagedHermesStateVolume( + { ...context, runtimeProviderId: "podman" }, + { + runDocker: runtime.runDocker as never, + registerExitCleanup: () => () => undefined, + }, + ); + + expect(scope?.mount).toMatchObject({ + source: "nemoclaw-hermes-state-v1-alpha", + target: MANAGED_HERMES_STATE_ROOT, + }); + }); + + it("dispatches native volume lifecycle through the selected provider operation", () => { + const runtime = dockerHarness(); + const workloadCleanupCapture = vi.fn((args: readonly string[]) => { + expect(args[0]).toBe("volume"); + const result = runtime.runDocker(args.slice(1)) as { + status: number | null; + stdout?: string | Buffer; + stderr?: string | Buffer; + error?: Error; + }; + return { + status: result.status ?? 1, + stdout: String(result.stdout ?? ""), + stderr: String(result.stderr ?? ""), + ...(result.error ? { error: result.error } : {}), + }; + }); + const engine = ( + operation: PodmanContainerEngine["operation"], + capture: PodmanContainerEngine["capture"] = vi.fn(() => ({ + status: 0, + stdout: "", + stderr: "", + })), + ): PodmanBoundContainerEngine => ({ + operation, + engineId: "podman", + displayName: "Podman", + authorityId: `podman:${operation}`, + endpointAuthorityId: "podman:test-endpoint", + capture, + captureHost: capture, + assertAuthority: vi.fn(), + }); + const provider = createPodmanRuntimeProviderBundle({ + engines: { + hostDoctor: engine("host-doctor"), + sandboxLifecycle: engine("sandbox-lifecycle"), + workloadCleanup: engine("workload-cleanup", workloadCleanupCapture), + }, + }); + + const scope = prepareManagedHermesStateVolume( + { ...context, runtimeProviderId: "podman" }, + { + runtimeProviders: { podman: provider }, + registerExitCleanup: () => () => undefined, + }, + ); + + expect(scope?.mount.source).toBe("nemoclaw-hermes-state-v1-alpha"); + expect(workloadCleanupCapture).toHaveBeenCalled(); + expect(workloadCleanupCapture.mock.calls.every(([args]) => args[0] === "volume")).toBe(true); + }); + it("commits a newly created volume after registration so exit cleanup preserves it", () => { const docker = dockerHarness(); let exitCleanup: (() => void) | null = null; @@ -128,16 +212,58 @@ describe("managed Hermes state volume", () => { expect(foreign.volume).not.toBeNull(); }); + it("projects and retires the declared OpenClaw state root through the generic volume path", () => { + const docker = dockerHarness(); + const roots = managedStartupStateRoots({ + agent: "openclaw", + sandboxName: "alpha", + agentIdentity: managedImageRuntimeIdentity("openclaw"), + }); + const scope = prepareManagedStateVolumes( + { roots }, + { + runContainerEngine: docker.runDocker as never, + registerExitCleanup: () => () => undefined, + }, + ); + + expect(scope?.mounts).toEqual([ + { + type: "volume", + source: "nemoclaw-openclaw-state-v1-alpha", + target: MANAGED_OPENCLAW_STATE_ROOT, + read_only: false, + }, + ]); + scope!.commit(); + expect( + removeManagedAgentStateVolumes( + { ...context, agentName: "openclaw" }, + { runDocker: docker.runDocker as never }, + ), + ).toEqual([{ status: "removed" }]); + expect(docker.volume).toBeNull(); + }); + it.each([ ["agent", { ...context, agentName: "openclaw" }], ["provider", { ...context, runtimeProviderId: "kubernetes" }], ["workload", { ...context, workloadKind: "legacy-dockerfile" }], - ])("does not provision outside the managed Docker Hermes %s boundary", (_boundary, input) => { - const docker = dockerHarness(); + ])( + "does not provision outside the managed container-engine Hermes %s boundary", + (_boundary, input) => { + const docker = dockerHarness(); - expect( - prepareManagedHermesStateVolume(input, { runDocker: docker.runDocker as never }), - ).toBeNull(); - expect(docker.calls).toEqual([]); - }); + expect( + prepareManagedHermesStateVolume(input, { + runDocker: docker.runDocker as never, + runtimeProviders: { + docker: createDockerRuntimeProviderBundle(), + kubernetes: createKubernetesRuntimeProviderBundle(), + }, + }), + ).toBeNull(); + expect(docker.calls).toEqual([]); + }, + ); }); diff --git a/src/lib/onboard/managed-workload/hermes-state-volume.ts b/src/lib/onboard/managed-workload/hermes-state-volume.ts index aadb803f7f7..99d4da05efc 100644 --- a/src/lib/onboard/managed-workload/hermes-state-volume.ts +++ b/src/lib/onboard/managed-workload/hermes-state-volume.ts @@ -1,31 +1,31 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import type { DockerRunOptions, DockerRunResult } from "../../adapters/docker/run"; - -/** - * SOURCE_OF_TRUTH_REVIEW - * invalidState: a managed-image Hermes sandbox starts without durable writable state, or a - * same-named foreign Docker volume is mistaken for NemoClaw-owned state during cleanup. - * sourceBoundary: the managed image does not declare a provider volume and OpenShell keeps - * state-mutation validation strict; this module is the sole owner of the Docker volume name, - * exact four-label ownership contract, create/reuse verification, and removal decision. - * whyNotSourceFix: the provider-neutral image cannot name a sandbox-scoped Docker volume, while - * weakening provider validation would accept absent, ambiguous, or read-only state mounts. - * regressionTest: hermes-state-volume.test.ts covers create, reuse, ownership refusal, and - * removal; sandbox-create-plan.test.ts and destroy-flow.test.ts cover lifecycle integration. - * removalCondition: remove when the managed-image or runtime-provider contract creates, - * reconciles, and ownership-gates an equivalent durable Hermes state volume end to end. - */ -export const MANAGED_HERMES_STATE_ROOT = "/sandbox/.hermes" as const; - -const VOLUME_NAME_PREFIX = "nemoclaw-hermes-state-v1"; -const MANAGED_LABEL = "io.nvidia.nemoclaw.hermes-state.managed"; -const SCHEMA_LABEL = "io.nvidia.nemoclaw.hermes-state.schema"; -const SANDBOX_LABEL = "io.nvidia.nemoclaw.hermes-state.sandbox"; -const TARGET_LABEL = "io.nvidia.nemoclaw.hermes-state.target"; -const MISSING_VOLUME_PATTERN = /\bno such volume\b/iu; -const COMMAND_TIMEOUT_MS = 30_000; +import { isManagedImageAgent, managedImageRuntimeIdentity } from "../managed-image/agents"; +import { + managedHermesStateVolumeLabels, + managedHermesStateVolumeName, + managedStartupStateRoots, + MANAGED_HERMES_STATE_ROOT, +} from "../managed-startup/state-roots"; +import type { + RuntimeProviderBundle, + RuntimeProviderBundleRegistry, + RuntimeProviderContainerEngineOperation, +} from "../runtime-provider/contract"; +import { + prepareManagedStateVolumes, + removeManagedStateVolumes, + type ManagedStateVolumeCleanupResult, + type ManagedStateVolumeDeps, + type ManagedStateVolumeMount, +} from "./managed-state-volumes"; + +export { + managedHermesStateVolumeLabels, + managedHermesStateVolumeName, + MANAGED_HERMES_STATE_ROOT, +}; export type ManagedHermesStateVolumeContext = { readonly agentName: string | null | undefined; @@ -34,27 +34,23 @@ export type ManagedHermesStateVolumeContext = { readonly workloadKind: string; }; -export type ManagedHermesStateVolumeMount = { - readonly type: "volume"; - readonly source: string; +export type ManagedHermesStateVolumeMount = ManagedStateVolumeMount & { readonly target: typeof MANAGED_HERMES_STATE_ROOT; readonly read_only: false; }; -export type ManagedHermesStateVolumeCleanupResult = - | { readonly status: "not-applicable" | "absent" | "removed" } - | { - readonly status: "not-owned" | "failed"; - readonly detail: string; - readonly volumeName: string; - }; +export type ManagedHermesStateVolumeCleanupResult = ManagedStateVolumeCleanupResult; +export type ManagedAgentStateVolumeCleanupResult = ManagedStateVolumeCleanupResult; -type DockerRunResultLike = Pick; -type DockerRun = (args: readonly string[], options?: DockerRunOptions) => DockerRunResultLike; +type LegacyContainerEngineRun = NonNullable; -export type ManagedHermesStateVolumeDeps = { - readonly runDocker?: DockerRun; - readonly registerExitCleanup?: (cleanup: () => void) => () => void; +export type ManagedHermesStateVolumeDeps = Omit< + ManagedStateVolumeDeps, + "runContainerEngine" | "runtimeProvider" +> & { + readonly runDocker?: LegacyContainerEngineRun; + readonly runtimeProvider?: RuntimeProviderBundle; + readonly runtimeProviders?: RuntimeProviderBundleRegistry; }; export type ManagedHermesStateVolumeScope = { @@ -65,209 +61,144 @@ export type ManagedHermesStateVolumeScope = { commit(): void; }; -type VolumeObservation = - | { readonly status: "absent" } - | { readonly status: "observed"; readonly labels: Readonly> } - | { readonly status: "failed"; readonly detail: string }; - -function defaultDockerRun(args: readonly string[], options?: DockerRunOptions): DockerRunResult { - const { dockerVolumeRun } = - require("../../adapters/docker/volume") as typeof import("../../adapters/docker/volume"); - return dockerVolumeRun(args, options); -} - -function defaultRegisterExitCleanup(cleanup: () => void): () => void { - process.on("exit", cleanup); - return () => process.removeListener("exit", cleanup); -} - -function commandOutput(result: DockerRunResultLike): string { - return `${String(result.stdout ?? "")}\n${String(result.stderr ?? "")}`.trim(); -} - -function boundedDetail(result: DockerRunResultLike): string { - return commandOutput(result).replace(/\s+/gu, " ").slice(0, 500) || "Docker command failed"; -} - -function expectedLabels(sandboxName: string): Readonly> { - return Object.freeze({ - [MANAGED_LABEL]: "true", - [SCHEMA_LABEL]: "1", - [SANDBOX_LABEL]: sandboxName, - [TARGET_LABEL]: MANAGED_HERMES_STATE_ROOT, +function hermesStateRoots(sandboxName: string) { + return managedStartupStateRoots({ + agent: "hermes", + sandboxName, + agentIdentity: managedImageRuntimeIdentity("hermes"), }); } -function labelsMatch( - observed: Readonly>, - expected: Readonly>, -): boolean { - return Object.entries(expected).every(([name, value]) => observed[name] === value); +function managedAgentStateRoots(context: ManagedHermesStateVolumeContext) { + if ( + context.workloadKind !== "managed-image" || + typeof context.agentName !== "string" || + !isManagedImageAgent(context.agentName) + ) { + return []; + } + return managedStartupStateRoots({ + agent: context.agentName, + sandboxName: context.sandboxName, + agentIdentity: managedImageRuntimeIdentity(context.agentName), + }); } -function inspectVolume(volumeName: string, runDocker: DockerRun): VolumeObservation { - const result = runDocker(["inspect", "--format", "{{json .}}", volumeName], { - ignoreError: true, - maxBuffer: 256 * 1024, - suppressOutput: true, - timeout: COMMAND_TIMEOUT_MS, - }); - if (result.status !== 0) { - return MISSING_VOLUME_PATTERN.test(commandOutput(result)) - ? { status: "absent" } - : { status: "failed", detail: boundedDetail(result) }; - } - const lines = String(result.stdout ?? "") - .split(/\r?\n/u) - .map((line) => line.trim()) - .filter(Boolean); - if (lines.length !== 1) { - return { status: "failed", detail: "Docker returned an ambiguous volume inspection." }; - } - try { - const value = JSON.parse(lines[0]!) as unknown; - if (!value || typeof value !== "object" || Array.isArray(value)) { - return { status: "failed", detail: "Docker returned a malformed volume inspection." }; - } - const record = value as Record; - if (record.Name !== volumeName) { - return { status: "failed", detail: "Docker returned the wrong volume identity." }; - } - const labelsValue = record.Labels; - if (!labelsValue || typeof labelsValue !== "object" || Array.isArray(labelsValue)) { - return { status: "observed", labels: Object.freeze({}) }; - } - const labels: Record = {}; - for (const [name, labelValue] of Object.entries(labelsValue)) { - if (typeof labelValue !== "string") { - return { status: "failed", detail: "Docker returned malformed volume labels." }; - } - labels[name] = labelValue; - } - return { status: "observed", labels: Object.freeze(labels) }; - } catch { - return { status: "failed", detail: "Docker returned invalid JSON for the volume inspection." }; - } +function selectedRuntimeProvider( + runtimeProviderId: string | null | undefined, + providers?: RuntimeProviderBundleRegistry, + runtimeProvider?: RuntimeProviderBundle, +): RuntimeProviderBundle | undefined { + if (runtimeProvider) return runtimeProvider; + const providerId = runtimeProviderId?.trim().toLowerCase(); + return providerId && providers ? providers[providerId] : undefined; } -export function managedHermesStateVolumeName(sandboxName: string): string { - return `${VOLUME_NAME_PREFIX}-${sandboxName}`; +function supportsContainerEngineOperation( + provider: RuntimeProviderBundle | undefined, + operation: RuntimeProviderContainerEngineOperation, +): boolean { + return ( + provider?.containerEngine.supported === true && + provider.containerEngine.identities.some((identity) => identity.operation === operation) + ); } export function requiresManagedHermesStateVolume( context: ManagedHermesStateVolumeContext, + providers?: RuntimeProviderBundleRegistry, + runtimeProvider?: RuntimeProviderBundle, ): boolean { + const hasLifecycleAuthority = + !providers && !runtimeProvider + ? true + : supportsContainerEngineOperation( + selectedRuntimeProvider(context.runtimeProviderId, providers, runtimeProvider), + "sandbox-lifecycle", + ); return ( context.agentName === "hermes" && - context.runtimeProviderId === "docker" && + hasLifecycleAuthority && context.workloadKind === "managed-image" ); } -function removeOwnedVolume( - sandboxName: string, - runDocker: DockerRun, -): ManagedHermesStateVolumeCleanupResult { - const volumeName = managedHermesStateVolumeName(sandboxName); - const observation = inspectVolume(volumeName, runDocker); - if (observation.status === "absent") return { status: "absent" }; - if (observation.status === "failed") { - return { status: "failed", detail: observation.detail, volumeName }; - } - if (!labelsMatch(observation.labels, expectedLabels(sandboxName))) { - return { - status: "not-owned", - detail: "the exact NemoClaw ownership labels are absent or changed", - volumeName, - }; - } - const result = runDocker(["rm", volumeName], { - ignoreError: true, - suppressOutput: true, - timeout: COMMAND_TIMEOUT_MS, - }); - return result.status === 0 - ? { status: "removed" } - : { status: "failed", detail: boundedDetail(result), volumeName }; +function genericDeps( + deps: ManagedHermesStateVolumeDeps, + runtimeProviderId: string | null | undefined, +): ManagedStateVolumeDeps { + const runtimeProvider = + deps.runtimeProvider ?? selectedRuntimeProvider(runtimeProviderId, deps.runtimeProviders); + return { + ...(deps.runDocker && !runtimeProvider ? { runContainerEngine: deps.runDocker } : {}), + ...(runtimeProvider ? { runtimeProvider } : {}), + ...(deps.registerExitCleanup ? { registerExitCleanup: deps.registerExitCleanup } : {}), + }; } export function prepareManagedHermesStateVolume( context: ManagedHermesStateVolumeContext, deps: ManagedHermesStateVolumeDeps = {}, ): ManagedHermesStateVolumeScope | null { - if (!requiresManagedHermesStateVolume(context)) return null; - const runDocker = deps.runDocker ?? defaultDockerRun; - const volumeName = managedHermesStateVolumeName(context.sandboxName); - const labels = expectedLabels(context.sandboxName); - const before = inspectVolume(volumeName, runDocker); - if (before.status === "failed") { - throw new Error(`Cannot inspect managed Hermes state volume '${volumeName}': ${before.detail}`); - } - let created = false; - if (before.status === "absent") { - const createArgs = ["create"]; - for (const [name, value] of Object.entries(labels).sort(([left], [right]) => - left.localeCompare(right), - )) { - createArgs.push("--label", `${name}=${value}`); - } - createArgs.push(volumeName); - const createdResult = runDocker(createArgs, { - ignoreError: true, - suppressOutput: true, - timeout: COMMAND_TIMEOUT_MS, - }); - if (createdResult.status !== 0) { - throw new Error( - `Cannot create managed Hermes state volume '${volumeName}': ${boundedDetail(createdResult)}`, - ); - } - created = true; + if (!requiresManagedHermesStateVolume(context, deps.runtimeProviders, deps.runtimeProvider)) { + return null; } - const verified = inspectVolume(volumeName, runDocker); - if (verified.status !== "observed" || !labelsMatch(verified.labels, labels)) { - if (created) removeOwnedVolume(context.sandboxName, runDocker); - const detail = - verified.status === "failed" - ? verified.detail - : verified.status === "absent" - ? "the volume disappeared after creation" - : "the exact NemoClaw ownership labels do not match"; - throw new Error(`Cannot use managed Hermes state volume '${volumeName}': ${detail}.`); + const root = hermesStateRoots(context.sandboxName)[0]; + if (!root) throw new Error("Hermes managed state-root declaration is unavailable."); + const scope = prepareManagedStateVolumes( + { roots: [root] }, + genericDeps(deps, context.runtimeProviderId), + ); + if (!scope || !scope.mounts[0]) { + throw new Error("Hermes managed state-volume scope is unavailable."); } - - let committed = false; - const cleanup = (): ManagedHermesStateVolumeCleanupResult => { - if (committed || !created) return { status: "not-applicable" }; - return removeOwnedVolume(context.sandboxName, runDocker); - }; - const unregisterExitCleanup = created - ? (deps.registerExitCleanup ?? defaultRegisterExitCleanup)(() => { - cleanup(); - }) - : () => undefined; - return { - mount: Object.freeze({ - type: "volume", - source: volumeName, - target: MANAGED_HERMES_STATE_ROOT, - read_only: false, - }), - reused: !created, - volumeName, - cleanupIncompleteCreate: cleanup, - commit() { - committed = true; - unregisterExitCleanup(); - }, + mount: scope.mounts[0] as ManagedHermesStateVolumeMount, + reused: scope.reused[0] === true, + volumeName: root.resourceIdentity, + cleanupIncompleteCreate: () => + scope.cleanupIncompleteCreate()[0] ?? { status: "not-applicable" }, + commit: () => scope.commit(), }; } export function removeManagedHermesStateVolume( context: ManagedHermesStateVolumeContext, - deps: Pick = {}, + deps: Pick< + ManagedHermesStateVolumeDeps, + "runDocker" | "runtimeProvider" | "runtimeProviders" + > = {}, ): ManagedHermesStateVolumeCleanupResult { - if (!requiresManagedHermesStateVolume(context)) return { status: "not-applicable" }; - return removeOwnedVolume(context.sandboxName, deps.runDocker ?? defaultDockerRun); + if (!requiresManagedHermesStateVolume(context, deps.runtimeProviders, deps.runtimeProvider)) { + return { status: "not-applicable" }; + } + return ( + removeManagedStateVolumes( + { roots: hermesStateRoots(context.sandboxName) }, + genericDeps(deps, context.runtimeProviderId), + )[0] ?? { status: "not-applicable" } + ); +} + +export function removeManagedAgentStateVolumes( + context: ManagedHermesStateVolumeContext, + deps: Pick< + ManagedHermesStateVolumeDeps, + "runDocker" | "runtimeProvider" | "runtimeProviders" + > = {}, +): readonly ManagedAgentStateVolumeCleanupResult[] { + const runtimeProvider = selectedRuntimeProvider( + context.runtimeProviderId, + deps.runtimeProviders, + deps.runtimeProvider, + ); + const hasCleanupAuthority = + !deps.runtimeProviders && !deps.runtimeProvider + ? true + : supportsContainerEngineOperation(runtimeProvider, "workload-cleanup"); + if (!hasCleanupAuthority) return []; + return removeManagedStateVolumes( + { roots: managedAgentStateRoots(context) }, + genericDeps(deps, context.runtimeProviderId), + ); } diff --git a/src/lib/onboard/managed-workload/managed-state-volumes.ts b/src/lib/onboard/managed-workload/managed-state-volumes.ts new file mode 100644 index 00000000000..b8ecb80ecdf --- /dev/null +++ b/src/lib/onboard/managed-workload/managed-state-volumes.ts @@ -0,0 +1,302 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { ManagedStartupStateRoot } from "../managed-startup/state-roots"; +import type { RuntimeProviderBundle } from "../runtime-provider/contract"; + +const MISSING_VOLUME_PATTERN = /\bno such volume\b/iu; +const COMMAND_TIMEOUT_MS = 30_000; + +export type ManagedStateVolumeMount = { + readonly type: "volume"; + readonly source: string; + readonly target: string; + readonly read_only: boolean; +}; + +export type ManagedStateVolumeCleanupResult = + | { readonly status: "not-applicable" | "absent" | "removed" } + | { + readonly status: "not-owned" | "failed"; + readonly detail: string; + readonly volumeName: string; + }; + +type ContainerEngineRunOptions = { + readonly ignoreError?: boolean; + readonly maxBuffer?: number; + readonly suppressOutput?: boolean; + readonly timeout?: number; +}; + +type ContainerEngineRunResult = { + readonly status: number | null; + readonly stdout?: string | Buffer; + readonly stderr?: string | Buffer; + readonly error?: Error; +}; + +type ContainerEngineRun = ( + args: readonly string[], + options?: ContainerEngineRunOptions, +) => ContainerEngineRunResult; + +export type ManagedStateVolumeDeps = { + readonly runContainerEngine?: ContainerEngineRun; + readonly runtimeProvider?: RuntimeProviderBundle; + readonly registerExitCleanup?: (cleanup: () => void) => () => void; +}; + +export type ManagedStateVolumeScope = { + readonly mounts: readonly ManagedStateVolumeMount[]; + readonly reused: readonly boolean[]; + cleanupIncompleteCreate(): readonly ManagedStateVolumeCleanupResult[]; + commit(): void; +}; + +type VolumeObservation = + | { readonly status: "absent" } + | { + readonly status: "observed"; + readonly labels: Readonly>; + } + | { readonly status: "failed"; readonly detail: string }; + +function defaultRuntimeVolumeRun(provider: RuntimeProviderBundle): ContainerEngineRun { + const containerEngine = provider.containerEngine; + if (containerEngine.supported !== true) { + throw new Error("The selected runtime provider does not expose container-engine authority."); + } + return (args, options) => + containerEngine.capture("workload-cleanup", ["volume", ...args], options?.timeout); +} + +function defaultRegisterExitCleanup(cleanup: () => void): () => void { + process.on("exit", cleanup); + return () => process.removeListener("exit", cleanup); +} + +function commandOutput(result: ContainerEngineRunResult): string { + return `${String(result.stdout ?? "")}\n${String(result.stderr ?? "")}`.trim(); +} + +function boundedDetail(result: ContainerEngineRunResult): string { + return commandOutput(result).replace(/\s+/gu, " ").slice(0, 500) || "runtime command failed"; +} + +function labelsMatch( + observed: Readonly>, + expected: Readonly>, +): boolean { + return Object.entries(expected).every(([name, value]) => observed[name] === value); +} + +function inspectVolume(root: ManagedStartupStateRoot, run: ContainerEngineRun): VolumeObservation { + const result = run(["inspect", "--format", "{{json .}}", root.resourceIdentity], { + ignoreError: true, + maxBuffer: 256 * 1024, + suppressOutput: true, + timeout: COMMAND_TIMEOUT_MS, + }); + if (result.status !== 0) { + return MISSING_VOLUME_PATTERN.test(commandOutput(result)) + ? { status: "absent" } + : { status: "failed", detail: boundedDetail(result) }; + } + const lines = String(result.stdout ?? "") + .split(/\r?\n/u) + .map((line) => line.trim()) + .filter(Boolean); + if (lines.length !== 1) { + return { + status: "failed", + detail: "Container engine returned an ambiguous volume inspection.", + }; + } + try { + const value = JSON.parse(lines[0]!) as unknown; + if (!value || typeof value !== "object" || Array.isArray(value)) { + return { + status: "failed", + detail: "Container engine returned a malformed volume inspection.", + }; + } + const record = value as Record; + if (record.Name !== root.resourceIdentity) { + return { + status: "failed", + detail: "Container engine returned the wrong volume identity.", + }; + } + const labelsValue = record.Labels; + if (!labelsValue || typeof labelsValue !== "object" || Array.isArray(labelsValue)) { + return { status: "observed", labels: Object.freeze({}) }; + } + const labels: Record = {}; + for (const [name, labelValue] of Object.entries(labelsValue)) { + if (typeof labelValue !== "string") { + return { + status: "failed", + detail: "Container engine returned malformed volume labels.", + }; + } + labels[name] = labelValue; + } + return { status: "observed", labels: Object.freeze(labels) }; + } catch { + return { + status: "failed", + detail: "Container engine returned invalid JSON for the volume inspection.", + }; + } +} + +function removeOwnedVolume( + root: ManagedStartupStateRoot, + run: ContainerEngineRun, +): ManagedStateVolumeCleanupResult { + const observation = inspectVolume(root, run); + if (observation.status === "absent") return { status: "absent" }; + if (observation.status === "failed") { + return { + status: "failed", + detail: observation.detail, + volumeName: root.resourceIdentity, + }; + } + if (!labelsMatch(observation.labels, root.ownershipLabels)) { + return { + status: "not-owned", + detail: "the exact NemoClaw ownership labels are absent or changed", + volumeName: root.resourceIdentity, + }; + } + const result = run(["rm", root.resourceIdentity], { + ignoreError: true, + suppressOutput: true, + timeout: COMMAND_TIMEOUT_MS, + }); + return result.status === 0 + ? { status: "removed" } + : { + status: "failed", + detail: boundedDetail(result), + volumeName: root.resourceIdentity, + }; +} + +function supportsManagedStateVolumes(provider?: RuntimeProviderBundle): boolean { + return provider?.containerEngine.supported !== false; +} + +export function prepareManagedStateVolumes( + input: { + readonly roots: readonly ManagedStartupStateRoot[]; + }, + deps: ManagedStateVolumeDeps = {}, +): ManagedStateVolumeScope | null { + if (input.roots.length === 0 || !supportsManagedStateVolumes(deps.runtimeProvider)) { + return null; + } + const provider = deps.runtimeProvider ?? null; + const run = + deps.runContainerEngine ?? + (provider + ? defaultRuntimeVolumeRun(provider) + : (() => { + throw new Error("Managed state volumes require runtime provider authority."); + })()); + const created: ManagedStartupStateRoot[] = []; + const reused: boolean[] = []; + try { + for (const root of input.roots) { + const before = inspectVolume(root, run); + if (before.status === "failed") { + throw new Error( + `Cannot inspect managed state volume '${root.resourceIdentity}': ${before.detail}`, + ); + } + if (before.status === "absent") { + const createArgs = ["create"]; + for (const [name, value] of Object.entries(root.ownershipLabels).sort(([left], [right]) => + left.localeCompare(right), + )) { + createArgs.push("--label", `${name}=${value}`); + } + createArgs.push(root.resourceIdentity); + const result = run(createArgs, { + ignoreError: true, + suppressOutput: true, + timeout: COMMAND_TIMEOUT_MS, + }); + if (result.status !== 0) { + throw new Error( + `Cannot create managed state volume '${root.resourceIdentity}': ${boundedDetail(result)}`, + ); + } + created.push(root); + } + const verified = inspectVolume(root, run); + if (verified.status !== "observed" || !labelsMatch(verified.labels, root.ownershipLabels)) { + const detail = + verified.status === "failed" + ? verified.detail + : verified.status === "absent" + ? "the volume disappeared after creation" + : "the exact NemoClaw ownership labels do not match"; + throw new Error(`Cannot use managed state volume '${root.resourceIdentity}': ${detail}.`); + } + reused.push(before.status === "observed"); + } + } catch (error) { + for (const root of [...created].reverse()) removeOwnedVolume(root, run); + throw error; + } + let committed = false; + const cleanup = (): readonly ManagedStateVolumeCleanupResult[] => + committed ? [] : [...created].reverse().map((root) => removeOwnedVolume(root, run)); + const unregisterExitCleanup = + created.length > 0 + ? (deps.registerExitCleanup ?? defaultRegisterExitCleanup)(() => { + cleanup(); + }) + : () => undefined; + return Object.freeze({ + mounts: Object.freeze( + input.roots.map((root) => + Object.freeze({ + type: "volume" as const, + source: root.resourceIdentity, + target: root.mountTarget, + read_only: !root.readWrite, + }), + ), + ), + reused: Object.freeze(reused), + cleanupIncompleteCreate: cleanup, + commit() { + committed = true; + unregisterExitCleanup(); + }, + }); +} + +export function removeManagedStateVolumes( + input: { + readonly roots: readonly ManagedStartupStateRoot[]; + }, + deps: Pick = {}, +): readonly ManagedStateVolumeCleanupResult[] { + if (input.roots.length === 0 || !supportsManagedStateVolumes(deps.runtimeProvider)) { + return Object.freeze([]); + } + const provider = deps.runtimeProvider ?? null; + const run = + deps.runContainerEngine ?? + (provider + ? defaultRuntimeVolumeRun(provider) + : (() => { + throw new Error("Managed state volumes require runtime provider authority."); + })()); + return Object.freeze(input.roots.map((root) => removeOwnedVolume(root, run))); +} diff --git a/src/lib/onboard/managed-workload/onboard-orchestration.test.ts b/src/lib/onboard/managed-workload/onboard-orchestration.test.ts index c6d0b7a7a1d..c3409afbaa9 100644 --- a/src/lib/onboard/managed-workload/onboard-orchestration.test.ts +++ b/src/lib/onboard/managed-workload/onboard-orchestration.test.ts @@ -8,6 +8,27 @@ import path from "node:path"; import { afterAll, describe, expect, it, vi } from "vitest"; import { createHermesStateVolumeDockerHarness } from "../__test-helpers__/hermes-state-volume"; +import { + managedStartupStateRoots, + managedStartupWorkspaceRoot, +} from "../managed-startup/state-roots"; + +describe("managed workspace-root declarations", () => { + it("preserves the DCode sticky root-owned login-profile boundary generically", () => { + expect( + managedStartupWorkspaceRoot({ + agent: "langchain-deepagents-code", + agentIdentity: { uid: 999, gid: 999 }, + }), + ).toEqual({ uid: 0, gid: 999, mode: 0o1775 }); + expect( + managedStartupWorkspaceRoot({ + agent: "openclaw", + agentIdentity: { uid: 998, gid: 998 }, + }), + ).toEqual({ uid: 998, gid: 998, mode: 0o755 }); + }); +}); const preparationState = vi.hoisted(() => ({ prepared: undefined as unknown, @@ -40,7 +61,7 @@ vi.mock("../../core/version", () => ({ import { mapManagedStartupProfileToAgentEnvironment } from "../managed-startup/agent-environment"; import { - createManagedHermesStateVolumeOnboardLifecycle, + createManagedStateVolumeOnboardLifecycle, createManagedWorkloadOnboardRuntime, prepareHermesPortableSandboxWorkloadForLifecycle, prepareOnboardSandboxWorkloadLaunch, @@ -327,15 +348,26 @@ describe("managed workload onboard orchestration", () => { const docker = createHermesStateVolumeDockerHarness(); let exitCleanup: (() => void) | null = null; - const lifecycle = createManagedHermesStateVolumeOnboardLifecycle( + const lifecycle = createManagedStateVolumeOnboardLifecycle( { - agentName: "hermes", - runtimeProvider: { identity: { id: "docker" } } as never, - sandboxName: "alpha", - workloadKind: "managed-image", + roots: managedStartupStateRoots({ + agent: "hermes", + sandboxName: "alpha", + agentIdentity: { uid: 1000, gid: 1000 }, + }), + runtimeProvider: { + identity: { id: "docker" }, + workload: { managedStateMountDriverId: "docker" }, + containerEngine: { + supported: true, + identities: [ + { operation: "sandbox-lifecycle", engineId: "docker", displayName: "Docker" }, + ], + }, + } as never, }, { - runDocker: docker.runDocker as never, + runContainerEngine: docker.runDocker as never, registerExitCleanup: (cleanup) => { exitCleanup = cleanup; return vi.fn(); @@ -343,8 +375,11 @@ describe("managed workload onboard orchestration", () => { }, ); - lifecycle!.materializeSandboxCreatePlan({} as never, (input) => { - expect(input.managedStateMount).toMatchObject({ target: "/sandbox/.hermes" }); + lifecycle.materializeSandboxCreatePlan({} as never, (input) => { + expect(input.managedStateMounts).toEqual([ + expect.objectContaining({ target: "/sandbox/.hermes" }), + ]); + expect(input.managedStateMountDriverId).toBe("docker"); return {} as never; }); exitCleanup!(); @@ -405,6 +440,33 @@ describe("managed workload onboard orchestration", () => { } }); + it("retains reused managed-image publication identity for live PR onboarding", async () => { + const candidateRevision = "b".repeat(40); + const publicationRevision = "a".repeat(40); + const fixtureRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-live-e2e-catalog-")); + const catalogPath = path.join(fixtureRoot, "catalog.json"); + fs.writeFileSync(catalogPath, "{}\n", { mode: 0o600 }); + try { + const { prepared, runtime } = createFreshOnboardingRuntime({ + GITHUB_ACTIONS: "true", + NEMOCLAW_RUN_LIVE_E2E: "1", + NEMOCLAW_E2E_EXPECTED_SHA: candidateRevision, + NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG: catalogPath, + NEMOCLAW_E2E_MANAGED_IMAGE_REVISION: publicationRevision, + }); + + await expect(runtime.ensurePreparedWorkload()).resolves.toBe(prepared); + expect(prepareSandboxWorkloadSource).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ + catalogPath, + expectedCatalogRevision: publicationRevision, + }), + ); + } finally { + fs.rmSync(fixtureRoot, { force: true, recursive: true }); + } + }); + it("omits the qualification catalog revision outside GitHub Actions (#9385)", async () => { const { prepared, runtime } = createFreshOnboardingRuntime({ E2E_MANAGED_IMAGE_REVISION: "a".repeat(40), diff --git a/src/lib/onboard/managed-workload/onboard-orchestration.ts b/src/lib/onboard/managed-workload/onboard-orchestration.ts index ed697934ce4..c294e29ac06 100644 --- a/src/lib/onboard/managed-workload/onboard-orchestration.ts +++ b/src/lib/onboard/managed-workload/onboard-orchestration.ts @@ -13,6 +13,10 @@ import type { import type { OpenShellComputePlan } from "../compute/plan"; import { resolveCorporateCa } from "../corporate-ca"; import { enforceDockerGpuPatchPreserveNetwork } from "../docker-gpu-local-inference"; +import { + isSandboxBridgeGatewayReachable, + verifySandboxBridgeGatewayReachableOrExit, +} from "../gateway-sandbox-reachability"; import { initialDockerGpuRoute, renderSandboxCreateArgsForGpuRoute, @@ -27,8 +31,14 @@ import { type ManagedStartupOnboardProfileInput, } from "../managed-startup/onboard-profile"; import { createManagedStartupRootApplyRequest } from "../managed-startup/root-apply"; -import { getChannelsFromPlan } from "../messaging-plan-session"; -import { getMessagingChannelConfigFromPlan } from "../messaging-config"; +import { + managedStartupStateRoots, + managedStartupWorkspaceRoot, +} from "../managed-startup/state-roots"; +import { + getChannelsFromPlan, + getMessagingChannelConfigFromPlan, +} from "../messaging-plan-session"; import type { MessagingTokenDef } from "../messaging-prep"; import { resolveSandboxBuildContext, resolveSandboxBuildPatch } from "../prepared-dcode-rebuild"; import { @@ -68,11 +78,17 @@ import { } from "../workload/rebuild"; import { resolveSandboxWorkloadRuntimeCapabilities } from "../workload/runtime"; import { - prepareManagedHermesStateVolume, - removeManagedHermesStateVolume, - type ManagedHermesStateVolumeContext, - type ManagedHermesStateVolumeDeps, -} from "./hermes-state-volume"; + prepareManagedStateVolumes, + removeManagedStateVolumes, + type ManagedStateVolumeDeps, +} from "./managed-state-volumes"; + +export { + managedStartupStateRoots, + managedStartupWorkspaceRoot, + prepareManagedStateVolumes, + removeManagedStateVolumes, +}; type ManagedProfileInput = Omit< ManagedStartupOnboardProfileInput, @@ -84,9 +100,10 @@ type BootstrapProvider = RuntimeProviderBundle & { readonly bootstrap: RuntimeProviderManagedImageBootstrapSurface; }; -export { normalizeRuntimeProviderIdentity, removeManagedHermesStateVolume }; +export { normalizeRuntimeProviderIdentity }; -export type ManagedHermesStateVolumeOnboardLifecycle = { +export type ManagedStateVolumeOnboardLifecycle = { + readonly roots: readonly import("../managed-startup/state-roots").ManagedStartupStateRoot[]; materializeSandboxCreatePlan( input: MaterializeSandboxCreatePlanInput, materialize: (input: MaterializeSandboxCreatePlanInput) => SandboxCreatePlan, @@ -94,24 +111,34 @@ export type ManagedHermesStateVolumeOnboardLifecycle = { commit(): void; }; -export function createManagedHermesStateVolumeOnboardLifecycle( - input: Omit & { +export function createManagedStateVolumeOnboardLifecycle( + input: { + readonly roots: readonly import("../managed-startup/state-roots").ManagedStartupStateRoot[]; readonly runtimeProvider: RuntimeProviderBundle | null; }, - deps: ManagedHermesStateVolumeDeps = {}, -): ManagedHermesStateVolumeOnboardLifecycle { - const scope = prepareManagedHermesStateVolume( + deps: ManagedStateVolumeDeps = {}, +): ManagedStateVolumeOnboardLifecycle { + const scope = prepareManagedStateVolumes( + { roots: input.roots }, { - agentName: input.agentName, - runtimeProviderId: input.runtimeProvider?.identity.id, - sandboxName: input.sandboxName, - workloadKind: input.workloadKind, + ...deps, + ...(input.runtimeProvider ? { runtimeProvider: input.runtimeProvider } : {}), }, - deps, ); + const managedStateMountDriverId = scope + ? input.runtimeProvider?.workload.managedStateMountDriverId + : undefined; + if (scope && !managedStateMountDriverId) { + throw new Error("Managed state volumes require provider-owned mount projection."); + } return { + roots: input.roots, materializeSandboxCreatePlan(input, materialize) { - return materialize({ ...input, managedStateMount: scope?.mount }); + return materialize({ + ...input, + managedStateMounts: scope?.mounts, + managedStateMountDriverId, + }); }, commit() { scope?.commit(); @@ -477,11 +504,23 @@ export async function prepareOnboardSandboxWorkloadLaunch( let dashboardRemoteBindPrepared = false; let launch: SandboxCreateLaunchWithPrebuild; if (input.workload.source.kind === "managed-image") { + const runtimeProvider = requireBootstrapProvider(input.runtime.runtimeProvider); + const gatewayRuntime = runtimeProvider.gateway.prepareHostRuntime({ + environment: process.env, + platform: process.platform, + }); await enforceDockerGpuPatchPreserveNetwork(input.gpu.provider, input.gpu.config, { dockerDriverGateway: input.gpu.dockerDriverGateway, selectedRoute: initialGpuRoute, gatewayPort: input.gpu.gatewayPort, log, + reverifyBridgeReachability: () => + verifySandboxBridgeGatewayReachableOrExit(true, { + skip: false, + port: input.gpu.gatewayPort, + reachabilityImpl: (options) => + isSandboxBridgeGatewayReachable({ ...options, gatewayRuntime }), + }), }); const profile = input.runtime.ensurePreparedProfile(input.workload); if (!profile) throw new Error("Managed sandbox workload is missing its startup profile."); @@ -583,6 +622,7 @@ export async function prepareSelectedOnboardSandboxWorkloadLaunch( export function resolveOnboardManagedBootstrapLaunch(input: { readonly runtime: ManagedWorkloadOnboardRuntime; readonly workload: PreparedSandboxWorkloadSource; + readonly sandboxName: string; readonly stateRoot: string; readonly bootstrapIdentity: string | null; readonly request: import("../managed-startup/root-apply").ManagedStartupRootApplyRequest | null; @@ -595,6 +635,7 @@ export function resolveOnboardManagedBootstrapLaunch(input: { "Managed image onboarding is missing its identity-bound bootstrap launch contract.", ); } + const agentIdentity = managedImageRuntimeIdentity(input.workload.source.contract.agent); return { bootstrapIdentity: input.bootstrapIdentity, stateRoot: input.stateRoot, @@ -605,7 +646,16 @@ export function resolveOnboardManagedBootstrapLaunch(input: { repository: input.workload.source.contract.image, manifestDigest: input.workload.source.contract.digest, }, - agentIdentity: managedImageRuntimeIdentity(input.workload.source.contract.agent), + agentIdentity, + workspaceRoot: managedStartupWorkspaceRoot({ + agent: input.workload.source.contract.agent, + agentIdentity, + }), + managedStateRoots: managedStartupStateRoots({ + agent: input.workload.source.contract.agent, + sandboxName: input.sandboxName, + agentIdentity, + }), intendedWorkloadArgv: input.intendedWorkloadArgv, expectedSupervisorArgv: OPENSHELL_SANDBOX_SUPERVISOR_ARGV, } as const; diff --git a/src/lib/onboard/openshell-docker-sandbox-containers.test.ts b/src/lib/onboard/openshell-docker-sandbox-containers.test.ts index 4516c985c46..66b4a65c46f 100644 --- a/src/lib/onboard/openshell-docker-sandbox-containers.test.ts +++ b/src/lib/onboard/openshell-docker-sandbox-containers.test.ts @@ -4,6 +4,7 @@ import { describe, expect, it, vi } from "vitest"; import { queryOpenShellDockerSandboxRuntimeSnapshot, + resolveOpenShellSandboxOwnershipLabel, removeExactOpenShellDockerSandboxContainers, } from "./openshell-docker-sandbox-containers"; @@ -13,6 +14,19 @@ const EMPTY_RUNTIME_FIELDS = [IMAGE_ID, BOOKKEEPING_IMAGE_REF, "", null, [], "ru const ACTIVATED_CONTAINER_ID = "b".repeat(64); const ROLLBACK_CONTAINER_ID = "c".repeat(64); +describe("resolveOpenShellSandboxOwnershipLabel", () => { + it("keeps Docker compatibility ownership independent of native provider selection", () => { + expect(resolveOpenShellSandboxOwnershipLabel({})).toEqual({ + label: "openshell.ai/managed-by", + value: "openshell", + }); + expect(resolveOpenShellSandboxOwnershipLabel({ NEMOCLAW_GATEWAY_RUNTIME: "podman" })).toEqual({ + label: "openshell.ai/managed-by", + value: "openshell", + }); + }); +}); + function observeContainerIds(ids: readonly string[], malformedRows = 0) { return { status: "observed" as const, @@ -285,21 +299,20 @@ describe("queryOpenShellDockerSandboxRuntimeSnapshot", () => { ); }); - it.each([ - "all,0", - "0,0", - "GPU-0 with-space", - ])("rejects ambiguous NVIDIA_VISIBLE_DEVICES value %s", (value) => { - const { result } = querySnapshot( - [IMAGE_ID, BOOKKEEPING_IMAGE_REF, "", null, [], "nvidia"], - value, - ); - - expect(result).toEqual({ - ok: false, - error: "docker inspect returned invalid NVIDIA_VISIBLE_DEVICES", - }); - }); + it.each(["all,0", "0,0", "GPU-0 with-space"])( + "rejects ambiguous NVIDIA_VISIBLE_DEVICES value %s", + (value) => { + const { result } = querySnapshot( + [IMAGE_ID, BOOKKEEPING_IMAGE_REF, "", null, [], "nvidia"], + value, + ); + + expect(result).toEqual({ + ok: false, + error: "docker inspect returned invalid NVIDIA_VISIBLE_DEVICES", + }); + }, + ); it.each([ ["unknown runtime", null, [], "nvidia-container-runtime"], @@ -329,21 +342,24 @@ describe("queryOpenShellDockerSandboxRuntimeSnapshot", () => { ], "runc", ], - ])("keeps well-formed open-world GPU configuration %s unknown", (_label, requests, devices, runtime) => { - const { result } = querySnapshot([ - IMAGE_ID, - BOOKKEEPING_IMAGE_REF, - "", - requests, - devices, - runtime, - ]); - - expect(result).toMatchObject({ - ok: true, - nativeGpuAttachmentState: "unknown", - }); - }); + ])( + "keeps well-formed open-world GPU configuration %s unknown", + (_label, requests, devices, runtime) => { + const { result } = querySnapshot([ + IMAGE_ID, + BOOKKEEPING_IMAGE_REF, + "", + requests, + devices, + runtime, + ]); + + expect(result).toMatchObject({ + ok: true, + nativeGpuAttachmentState: "unknown", + }); + }, + ); it.each([ ["zero", ""], diff --git a/src/lib/onboard/openshell-docker-sandbox-containers.ts b/src/lib/onboard/openshell-docker-sandbox-containers.ts index ef111b4a347..190dab3f992 100644 --- a/src/lib/onboard/openshell-docker-sandbox-containers.ts +++ b/src/lib/onboard/openshell-docker-sandbox-containers.ts @@ -15,6 +15,25 @@ export const OPENSHELL_SANDBOX_ID_LABEL = "openshell.ai/sandbox-id"; export const OPENSHELL_SANDBOX_NAMESPACE_LABEL = "openshell.ai/sandbox-namespace"; export const OPENSHELL_SANDBOX_WORKSPACE_LABEL = "openshell.ai/sandbox-workspace"; +export type OpenShellSandboxOwnershipLabel = { + readonly label: string; + readonly value: string; +}; + +export function resolveOpenShellSandboxOwnershipLabel( + _env: NodeJS.ProcessEnv = process.env, +): OpenShellSandboxOwnershipLabel { + return { label: OPENSHELL_MANAGED_BY_LABEL, value: OPENSHELL_MANAGED_BY_VALUE }; +} + +export function hasOpenShellSandboxOwnership( + labels: Readonly>, + env: NodeJS.ProcessEnv = process.env, +): boolean { + const ownership = resolveOpenShellSandboxOwnershipLabel(env); + return labels[ownership.label] === ownership.value; +} + const DOCKER_SANDBOX_QUERY_TIMEOUT_MS = 30_000; const STALE_DOCKER_ORPHAN_TIMEOUT_MS = 30_000; diff --git a/src/lib/onboard/preflight-docker-credential-store.test.ts b/src/lib/onboard/preflight-docker-credential-store.test.ts index 7da92378aeb..821a1a85dae 100644 --- a/src/lib/onboard/preflight-docker-credential-store.test.ts +++ b/src/lib/onboard/preflight-docker-credential-store.test.ts @@ -230,7 +230,7 @@ describe("onboard preflight credential-store warning (#9457)", () => { assessHost: headlessDockerDesktopHost, detectGpu: () => null, warnIfHostProxyMissesLoopback: vi.fn(), - assertDockerBridgeAndContainerDnsHealthy: bridge, + assertRuntimeProviderHealthy: bridge, validateSandboxGpuPreflight: vi.fn(), }; const result = runFatalOnboardRuntimePreflight({}, context); diff --git a/src/lib/onboard/reachability/host-service-message.ts b/src/lib/onboard/reachability/host-service-message.ts new file mode 100644 index 00000000000..4fd84a8eb77 --- /dev/null +++ b/src/lib/onboard/reachability/host-service-message.ts @@ -0,0 +1,41 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { cliName } from "../branding"; + +export interface HostServiceUnreachableResult { + readonly ok: boolean; + readonly reason: string; + readonly port?: number; + readonly networkName: string; + readonly subnet?: string; + readonly gatewayIp?: string; +} + +const HOST_INTERNAL_NAME = "host.openshell.internal"; + +export function formatHostServiceUnreachableMessage( + result: HostServiceUnreachableResult, + options: { serviceLabel: string; port?: number }, +): string { + if (result.ok || result.reason !== "tcp_failed") return ""; + + const port = options.port ?? result.port; + const allowCmd = + result.subnet && result.gatewayIp + ? ` sudo ufw allow from ${result.subnet} to ${result.gatewayIp} port ${port} proto tcp` + : result.subnet + ? ` sudo ufw allow from ${result.subnet} to any port ${port} proto tcp` + : [ + ` SUBNET=$(docker network inspect ${result.networkName} --format '{{(index .IPAM.Config 0).Subnet}}')`, + ` sudo ufw allow from "$SUBNET" to any port ${port} proto tcp`, + ].join("\n"); + + return [ + ` ✗ Sandbox containers cannot reach the ${options.serviceLabel} at ${HOST_INTERNAL_NAME}:${port}.`, + " A host firewall may be blocking traffic from the OpenShell Docker bridge.", + " To allow it:", + allowCmd, + ` Then rerun \`${cliName()} onboard\`.`, + ].join("\n"); +} diff --git a/src/lib/onboard/runtime-control-flow.ts b/src/lib/onboard/runtime-control-flow.ts index 2b16c8f3b92..0acada4916b 100644 --- a/src/lib/onboard/runtime-control-flow.ts +++ b/src/lib/onboard/runtime-control-flow.ts @@ -12,6 +12,10 @@ import { applyOnboardToolDisclosureRequest } from "./tool-disclosure-flow"; import type { OnboardOptions } from "./types"; export { clearAgentScopedResumeState }; +export { + resolveCurrentOpenShellComputePlan, + resolveCurrentOpenShellRuntimeSelection, +} from "./compute/plan"; export interface RuntimeControlAgentDeps { error(message: string): void; diff --git a/src/lib/onboard/runtime-provider/access.ts b/src/lib/onboard/runtime-provider/access.ts index d4b5a00d425..03546f08ee6 100644 --- a/src/lib/onboard/runtime-provider/access.ts +++ b/src/lib/onboard/runtime-provider/access.ts @@ -58,6 +58,7 @@ export { requireRuntimeProviderStateMutationSurface, resolveRuntimeProviderBundle, runtimeProviderContainerEngineIdentity, + runtimeProviderSupportsContainerEngineOperation, } from "./registry"; export { prepareAgentDefinitionProtectionTransitionPlan, diff --git a/src/lib/onboard/runtime-provider/activation.test.ts b/src/lib/onboard/runtime-provider/activation.test.ts index ac8187a0cc3..9952e01fdcd 100644 --- a/src/lib/onboard/runtime-provider/activation.test.ts +++ b/src/lib/onboard/runtime-provider/activation.test.ts @@ -152,6 +152,7 @@ function completeBundle(providerId: string): RuntimeProviderBundle { engineId: "contract-fixture", displayName: "Contract fixture", })), + capture: () => ({ status: 0, stdout: "", stderr: "" }), }, }; } @@ -276,13 +277,38 @@ describe("runtime provider activation catalog", () => { ).toEqual([true, true, true]); }); - it("leaves Docker and Kubernetes unchanged with no production candidate registration", () => { - expect(Object.keys(CURRENT_RUNTIME_PROVIDER_BUNDLES)).toEqual(["docker", "kubernetes"]); - expect(Object.keys(createCurrentRuntimeProviderBundles())).toEqual(["docker", "kubernetes"]); - expect(CURRENT_RUNTIME_PROVIDER_BUNDLES).not.toHaveProperty("podman"); + it("registers qualified Podman without changing the established providers", () => { + expect(Object.keys(CURRENT_RUNTIME_PROVIDER_BUNDLES)).toEqual([ + "docker", + "kubernetes", + "podman", + ]); + expect(Object.keys(createCurrentRuntimeProviderBundles())).toEqual([ + "docker", + "kubernetes", + "podman", + ]); + expect(CURRENT_RUNTIME_PROVIDER_BUNDLES.podman?.identity.id).toBe("podman"); expect(CURRENT_RUNTIME_PROVIDER_BUNDLES).not.toHaveProperty("mxc"); }); + it("exposes one stable read-only view of the lazily constructed current registry", () => { + const firstPodman = CURRENT_RUNTIME_PROVIDER_BUNDLES.podman; + + expect(firstPodman).toBeDefined(); + expect(CURRENT_RUNTIME_PROVIDER_BUNDLES.podman).toBe(firstPodman); + expect(() => { + (CURRENT_RUNTIME_PROVIDER_BUNDLES as Record).podman = + CURRENT_RUNTIME_PROVIDER_BUNDLES.docker!; + }).toThrow(TypeError); + expect(Object.keys(CURRENT_RUNTIME_PROVIDER_BUNDLES)).toEqual([ + "docker", + "kubernetes", + "podman", + ]); + expect(CURRENT_RUNTIME_PROVIDER_BUNDLES.podman).toBe(firstPodman); + }); + it("rejects native-artifact bootstrap from production activation (#8178)", () => { const candidate = CANDIDATE_TOPOLOGIES[2]; const complete = completeBundle(candidate.providerId); diff --git a/src/lib/onboard/runtime-provider/activation.ts b/src/lib/onboard/runtime-provider/activation.ts index 956fb97aee1..b84d1480bd8 100644 --- a/src/lib/onboard/runtime-provider/activation.ts +++ b/src/lib/onboard/runtime-provider/activation.ts @@ -1,10 +1,6 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { - MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION, - MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, -} from "../managed-image/contract"; import { NATIVE_RUNTIME_QUALIFICATION_PRODUCER_WORKFLOW, NATIVE_RUNTIME_QUALIFICATION_PROTECTED_REPOSITORY, @@ -12,6 +8,8 @@ import { type NativeRuntimeQualificationExpectedSource, } from "./native-qualification-authority"; import { + MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION, + MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, RUNTIME_PROVIDER_STATE_MUTATION_CONTRACT_VERSION, type RuntimeProviderBundle, type RuntimeProviderBundleRegistry, diff --git a/src/lib/onboard/runtime-provider/configured-runtime.ts b/src/lib/onboard/runtime-provider/configured-runtime.ts new file mode 100644 index 00000000000..5763138e8a6 --- /dev/null +++ b/src/lib/onboard/runtime-provider/configured-runtime.ts @@ -0,0 +1,29 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export const NEMOCLAW_GATEWAY_RUNTIME_ENV = "NEMOCLAW_GATEWAY_RUNTIME"; + +export type NemoClawGatewayRuntime = "docker" | "podman"; + +/** + * Restore the explicit native-runtime selector used by the original Podman + * experiment. Portable profile selection remains an independent authority and + * is resolved by its existing code path. + */ +export function resolveNemoClawGatewayRuntime( + env: NodeJS.ProcessEnv = process.env, +): NemoClawGatewayRuntime { + const raw = env[NEMOCLAW_GATEWAY_RUNTIME_ENV]; + const normalized = String(raw ?? "") + .trim() + .toLowerCase(); + if (!normalized || normalized === "docker") return "docker"; + if (normalized === "podman") return "podman"; + throw new Error( + `${NEMOCLAW_GATEWAY_RUNTIME_ENV} must be either "docker" or "podman"; got ${JSON.stringify(raw)}`, + ); +} + +export function isPodmanGatewayRuntimeEnabled(env: NodeJS.ProcessEnv = process.env): boolean { + return resolveNemoClawGatewayRuntime(env) === "podman"; +} diff --git a/src/lib/onboard/runtime-provider/contract.ts b/src/lib/onboard/runtime-provider/contract.ts index f50395c18ca..b5dae52c484 100644 --- a/src/lib/onboard/runtime-provider/contract.ts +++ b/src/lib/onboard/runtime-provider/contract.ts @@ -10,12 +10,20 @@ import type { } from "../managed-bootstrap/runtime-create"; import type { NativeArtifactWorkloadReceiptV1 } from "../workload/native-artifact"; import type { ManagedImageSelectionPolicy } from "../workload/source"; +import type { SandboxGpuConfig } from "../sandbox-gpu-mode"; import type { HostLocalInferenceOperation, HostLocalInferenceOperationInput, HostLocalInferenceService, } from "./host-local-inference"; +export { + MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION, + MANAGED_IMAGE_PLATFORMS, + MANAGED_IMAGE_REPOSITORIES, + MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, +} from "../managed-image/contract"; + export const RUNTIME_PROVIDER_BUNDLE_CONTRACT_VERSION = 1 as const; export const RUNTIME_PROVIDER_SNAPSHOT_CONTRACT_VERSION = 1 as const; export const RUNTIME_PROVIDER_SNAPSHOT_PREFLIGHT_SCHEMA_VERSION = 1 as const; @@ -71,6 +79,74 @@ export interface RuntimeProviderPlanDefinition { readonly gatewayLauncher: RuntimeProviderGatewayLauncher; } +export interface RuntimeProviderGatewayHostRuntimeInput { + readonly environment: NodeJS.ProcessEnv; + readonly platform: NodeJS.Platform; + /** Optional exact socket supplied by a caller that already prepared provider authority. */ + readonly socketPath?: string; +} + +export interface RuntimeProviderGatewayNetworkInfo { + readonly subnet?: string; + readonly gatewayIp?: string; +} + +export interface RuntimeProviderGatewayCommandResult { + readonly status: number | null; + readonly stdout?: string | Buffer | null; + readonly stderr?: string | Buffer | null; + readonly signal?: NodeJS.Signals | null; + readonly error?: string; + readonly errorCode?: string | null; + readonly timedOut?: boolean; +} + +export interface RuntimeProviderGatewayImageCacheResult { + readonly ok: boolean; + readonly alreadyCached?: boolean; + readonly reason?: "inspect_unavailable" | "pull_failed" | "pull_timeout"; + readonly details?: string; +} + +/** + * Provider-owned gateway behavior projected into generic orchestration. None of + * these values identify a provider; callers consume the behavior without + * branching on the bundle identity. + */ +export interface RuntimeProviderGatewayHostRuntime { + /** Opaque identity persisted for provider-owned runtime observation. */ + readonly providerId: string; + readonly openShellDriver: string; + readonly bindAddress: string; + /** Host advertised to the gateway's sandbox-facing gRPC transport. */ + readonly grpcHost: string; + /** Host used by the local SSH gateway client. */ + readonly sshGatewayHost: string; + readonly portCheckHost: string; + readonly socketPath: string | null; + readonly requiredServerIpSans: readonly string[]; + readonly sandboxHostAddress: string | null; + readonly usesHostGatewayRoute: boolean; + readonly resourceOwnership: { + readonly label: string; + readonly value: string; + }; + readonly gatewayConfig: { + readonly sandboxNamespace: "scoped" | "omitted"; + readonly hostGatewayIp: string | null; + readonly includeSupervisorBin: boolean; + readonly processOwnership: "scoped-namespace" | "runtime-marker"; + }; + readonly network: { + /** Provider-owned sandbox source ranges authorized to reach host services. */ + sandboxSourceCidrs(): readonly string[]; + inspect(networkName: string): RuntimeProviderGatewayNetworkInfo | undefined; + usesHostGatewayRoute(): boolean; + run(args: readonly string[], timeoutMs: number): RuntimeProviderGatewayCommandResult; + ensureProbeImageCached(image: string): RuntimeProviderGatewayImageCacheResult; + }; +} + export type RuntimeProviderReadOnlyHostMountCapability = | { readonly supported: true; @@ -260,6 +336,85 @@ export type RuntimeProviderLifecycleStopOutcome = RuntimeProviderLifecycleResult readonly state?: "already-stopped" | "stopped"; }; +/** Opaque provider-owned identity for one observed sandbox runtime resource. */ +export interface RuntimeProviderPrivilegedSandboxTarget { + readonly providerId: string; + readonly resourceHandle: string; +} + +export interface RuntimeProviderPrivilegedSandboxCommandInput { + readonly sandbox: SandboxEntry; + readonly sandboxName: string; + readonly registeredSandboxNames: readonly string[]; + readonly command: readonly string[]; + readonly input?: Buffer; + readonly sanitizeEnvironment: boolean; + readonly expectedResourceHandle?: string; + readonly timeoutMs: number; + readonly maxOutputBytes?: number; +} + +export interface RuntimeProviderPrivilegedSandboxCommandResult { + readonly status: number | null; + readonly signal: NodeJS.Signals | null; + readonly stdout: Buffer; + readonly stderr: Buffer; + readonly error?: Error; +} + +export type RuntimeProviderStoppedSandboxStateCleanupFailure = + | "sandbox-registry-unavailable" + | "provider-cleanup-unavailable" + | "state-paths-invalid" + | "runtime-discovery-failed" + | "no-eligible-stopped-runtime" + | "runtime-ownership-invalid" + | "runtime-inspection-failed" + | "runtime-not-stopped" + | "state-resource-unavailable" + | "cleanup-helper-image-unavailable" + | "cleanup-helper-ownership-invalid" + | "cleanup-helper-reconciliation-failed" + | "cleanup-state-tree-unsafe" + | "cleanup-deletion-unconfirmed" + | "cleanup-helper-failed" + | "runtime-revalidation-failed" + | "lifecycle-authority-unavailable"; + +export type RuntimeProviderStoppedSandboxStateCleanupResult = + | { readonly cleared: true } + | { + readonly cleared: false; + readonly failure: RuntimeProviderStoppedSandboxStateCleanupFailure; + readonly cleanupHelperName?: string; + }; + +export interface RuntimeProviderStoppedSandboxStateCleanupInput { + readonly sandbox: SandboxEntry; + readonly sandboxName: string; + readonly registeredSandboxNames: readonly string[]; + readonly paths: readonly string[]; +} + +export interface RuntimeProviderPrivilegedSandboxControl { + resolveTarget( + input: Pick< + RuntimeProviderPrivilegedSandboxCommandInput, + "registeredSandboxNames" | "sandbox" | "sandboxName" + >, + ): RuntimeProviderPrivilegedSandboxTarget; + execute( + input: RuntimeProviderPrivilegedSandboxCommandInput, + ): RuntimeProviderPrivilegedSandboxCommandResult; + clearStoppedStateRoots?( + input: RuntimeProviderStoppedSandboxStateCleanupInput, + ): RuntimeProviderStoppedSandboxStateCleanupResult; + /** Docker-only compatibility for E2E probes that invoke the Docker CLI directly. */ + buildLegacyDockerArgv?( + input: Omit, + ): string[]; +} + export interface RuntimeProviderLifecycleStopHooks { readonly beforeStop: () => void; } @@ -274,6 +429,14 @@ export interface RuntimeProviderCleanupInput { readonly sandboxName: string; } +/** Provider-owned proof for the exact runtime resource targeted by destroy. */ +export interface RuntimeProviderDestroyIdentityReceipt { + readonly schemaVersion: 1; + readonly providerId: string; + readonly resourceHandle: string | null; + readonly ownershipSha256: string | null; +} + export type RuntimeProviderWorkloadCleanupPlan = | { readonly action: "retain"; @@ -475,6 +638,7 @@ export interface RuntimeProviderSnapshotRestoreReceipt { export type RuntimeProviderPreflightDoctorSurface = RuntimeProviderSupportedSurface<{ inspectHost(): RuntimeProviderDoctorCheck; + validateSandboxGpu(config: SandboxGpuConfig, exitProcess: (code: number) => never): void; preflightLifecycle( action: RuntimeProviderLifecycleAction, input: RuntimeProviderLifecycleInput, @@ -484,10 +648,15 @@ export type RuntimeProviderPreflightDoctorSurface = RuntimeProviderSupportedSurf export type RuntimeProviderGatewaySurface = RuntimeProviderSupportedSurface<{ readonly launcher: RuntimeProviderGatewayLauncher; readonly inspectLegacyContainer: boolean; + prepareHostRuntime( + input: RuntimeProviderGatewayHostRuntimeInput, + ): RuntimeProviderGatewayHostRuntime; }>; export type RuntimeProviderWorkloadSurface = RuntimeProviderSupportedSurface<{ readonly profile: RuntimeProviderWorkloadProfile; + /** Provider-owned OpenShell driver-config key for managed state mounts. */ + readonly managedStateMountDriverId?: string; acceptsReceipt(receipt: SandboxWorkloadReceipt | undefined): boolean; }>; @@ -501,6 +670,9 @@ export type RuntimeProviderHostLocalInferenceSurface = export type RuntimeProviderLifecycleSurface = | RuntimeProviderSupportedSurface<{ readonly channelStopTransport: RuntimeProviderChannelStopTransport; + /** Provider-owned timeout for direct container lifecycle mutations. */ + readonly containerMutationTimeoutMs?: number; + readonly privilegedSandboxControl: RuntimeProviderPrivilegedSandboxControl; start(input: RuntimeProviderLifecycleInput): RuntimeProviderLifecycleResult; verifyStarted( input: RuntimeProviderLifecycleInput, @@ -628,6 +800,12 @@ export type RuntimeProviderRecoverySurface = export type RuntimeProviderCleanupSurface = | RuntimeProviderSupportedSurface<{ + /** Observe immutable runtime identity and ownership without mutation. */ + captureDestroyIdentity?( + input: RuntimeProviderCleanupInput, + ): RuntimeProviderDestroyIdentityReceipt; + /** Prove a provider-owned runtime created before registry finalization. */ + captureDestroyIdentityByName?(sandboxName: string): RuntimeProviderDestroyIdentityReceipt; prepareDestroy( input: RuntimeProviderCleanupInput, operations: RuntimeProviderCleanupOperations, @@ -651,6 +829,11 @@ export type RuntimeProviderContainerEngineSurface = readonly engineId: string; readonly displayName: string; }[]; + capture( + operation: RuntimeProviderContainerEngineOperation, + args: readonly string[], + timeoutMs?: number, + ): RuntimeProviderCommandCapture; }> | RuntimeProviderUnsupportedSurface; diff --git a/src/lib/onboard/runtime-provider/current.ts b/src/lib/onboard/runtime-provider/current.ts index 43c587512fa..b139e5c0cfd 100644 --- a/src/lib/onboard/runtime-provider/current.ts +++ b/src/lib/onboard/runtime-provider/current.ts @@ -3,37 +3,162 @@ import { composeActivatedRuntimeProviderBundles, + RUNTIME_PROVIDER_ACTIVATION_ACCELERATION_MODES, + RUNTIME_PROVIDER_ACTIVATION_AGENTS, + RUNTIME_PROVIDER_ACTIVATION_CONTRACT_VERSION, + RUNTIME_PROVIDER_ACTIVATION_INFERENCE_SERVICES, + RUNTIME_PROVIDER_ACTIVATION_JOURNEYS, + RUNTIME_PROVIDER_ACTIVATION_PLATFORMS, + RUNTIME_PROVIDER_ACTIVATION_ROOT_MODES, type RuntimeProviderActivationRegistration, } from "./activation"; import type { RuntimeProviderBundle, RuntimeProviderBundleRegistry } from "./contract"; import { createDockerRuntimeProviderBundle, createKubernetesRuntimeProviderBundle } from "./docker"; -import { createRuntimeProviderBundleRegistry, requireRuntimeProviderBundle } from "./registry"; +import { isPortableExperimentalProfile } from "../experimental/portable-profile"; +import { resolveNemoClawGatewayRuntime } from "./configured-runtime"; +import type { NativeRuntimeQualificationAuthority } from "./native-qualification-authority"; +import { createCurrentPodmanRuntimeProviderBundle } from "./podman"; +import { + createRuntimeProviderBundleRegistry, + requireRuntimeProviderBundle, + resolveRuntimeProviderBundle, +} from "./registry"; /** - * The production-selectable set remains intentionally limited to the two - * providers NemoClaw already ships. Future providers must land as one complete - * bundle and separately pass their activation gate. + * Established providers precede qualification-gated registrations so adding a + * provider does not add another selection branch to managed orchestration. */ -const ESTABLISHED_RUNTIME_PROVIDER_BUNDLES: RuntimeProviderBundleRegistry = - createRuntimeProviderBundleRegistry([ +let establishedRuntimeProviderBundles: RuntimeProviderBundleRegistry | null = null; + +function getEstablishedRuntimeProviderBundles(): RuntimeProviderBundleRegistry { + establishedRuntimeProviderBundles ??= createRuntimeProviderBundleRegistry([ ["docker", createDockerRuntimeProviderBundle()], ["kubernetes", createKubernetesRuntimeProviderBundle()], ]); + return establishedRuntimeProviderBundles; +} + +const PODMAN_QUALIFICATION_SOURCE = Object.freeze({ + repository: "NVIDIA/NemoClaw", + workflow: ".github/workflows/e2e.yaml", + pullRequestNumber: 9232, + candidateRepository: "NVIDIA/NemoClaw", + headSha: "504fcf718a8ece560c021c5ed4656851ef419e84", + baseRef: "main" as const, + baseSha: "146643bd71ee72cc0e1ce86ebf73a7756c0c4806", + runId: 31984240689, + attempt: 1, + jobId: 95256339031, + artifact: Object.freeze({ + id: 9273257568, + name: "e2e-dispatch-31984240689-1", + digest: "sha256:f75a7240e53eae1816216d0aa180dfbf0abfa6abedf9ac53791da749b07a66f8", + }), +}); + +const PODMAN_QUALIFICATION_AUTHORITY: NativeRuntimeQualificationAuthority = Object.freeze({ + schemaVersion: 1, + qualificationId: "podman-protected-host-local-inference", + providerId: "podman", + source: PODMAN_QUALIFICATION_SOURCE, +}); + +function createPodmanActivationRegistration(): RuntimeProviderActivationRegistration { + return { + declaration: { + contractVersion: RUNTIME_PROVIDER_ACTIVATION_CONTRACT_VERSION, + providerId: "podman", + topology: { + hostAuthority: "rootless", + transport: "operation-scoped", + }, + agents: RUNTIME_PROVIDER_ACTIVATION_AGENTS, + platforms: RUNTIME_PROVIDER_ACTIVATION_PLATFORMS, + qualificationRootModes: RUNTIME_PROVIDER_ACTIVATION_ROOT_MODES, + accelerationModes: RUNTIME_PROVIDER_ACTIVATION_ACCELERATION_MODES, + hostLocalInferenceServices: RUNTIME_PROVIDER_ACTIVATION_INFERENCE_SERVICES, + journeys: RUNTIME_PROVIDER_ACTIVATION_JOURNEYS, + installer: { releaseInstaller: true, dockerUnavailable: true }, + qualification: { + qualificationId: PODMAN_QUALIFICATION_AUTHORITY.qualificationId, + source: PODMAN_QUALIFICATION_SOURCE, + }, + }, + qualificationAuthority: PODMAN_QUALIFICATION_AUTHORITY, + bundle: createCurrentPodmanRuntimeProviderBundle(), + }; +} + +let currentRuntimeProviderActivations: readonly RuntimeProviderActivationRegistration[] | null = + null; + +function getCurrentRuntimeProviderActivations(): readonly RuntimeProviderActivationRegistration[] { + currentRuntimeProviderActivations ??= Object.freeze([createPodmanActivationRegistration()]); + return currentRuntimeProviderActivations; +} export function createCurrentRuntimeProviderBundles( - activations: readonly RuntimeProviderActivationRegistration[] = [], + activations: readonly RuntimeProviderActivationRegistration[] = getCurrentRuntimeProviderActivations(), ): RuntimeProviderBundleRegistry { - return composeActivatedRuntimeProviderBundles(ESTABLISHED_RUNTIME_PROVIDER_BUNDLES, activations); + return composeActivatedRuntimeProviderBundles( + getEstablishedRuntimeProviderBundles(), + activations, + ); } -export const CURRENT_RUNTIME_PROVIDER_BUNDLES: RuntimeProviderBundleRegistry = - createCurrentRuntimeProviderBundles(); +let currentRuntimeProviderBundles: RuntimeProviderBundleRegistry | null = null; + +function getCurrentRuntimeProviderBundles(): RuntimeProviderBundleRegistry { + currentRuntimeProviderBundles ??= createCurrentRuntimeProviderBundles(); + return currentRuntimeProviderBundles; +} + +/** + * Read-only lazy view of the qualification-backed current registry. Deferring + * bundle construction until the first lookup keeps provider implementations + * free to import provider-neutral orchestration without creating an ESM + * initialization cycle back through this registration boundary. + */ +export const CURRENT_RUNTIME_PROVIDER_BUNDLES: RuntimeProviderBundleRegistry = new Proxy( + Object.create(null) as RuntimeProviderBundleRegistry, + { + get: (_target, property) => Reflect.get(getCurrentRuntimeProviderBundles(), property), + getOwnPropertyDescriptor: (_target, property) => { + const descriptor = Reflect.getOwnPropertyDescriptor( + getCurrentRuntimeProviderBundles(), + property, + ); + return descriptor ? { ...descriptor, configurable: true } : undefined; + }, + has: (_target, property) => Reflect.has(getCurrentRuntimeProviderBundles(), property), + ownKeys: () => Reflect.ownKeys(getCurrentRuntimeProviderBundles()), + defineProperty: () => false, + deleteProperty: () => false, + set: () => false, + }, +); export function resolveCurrentRuntimeProviderBundle( platform: NodeJS.Platform = process.platform, arch: NodeJS.Architecture = process.arch, providers: RuntimeProviderBundleRegistry = CURRENT_RUNTIME_PROVIDER_BUNDLES, + env: NodeJS.ProcessEnv = process.env, ): RuntimeProviderBundle { const managedLocalGateway = platform === "linux" || (platform === "darwin" && arch === "arm64"); - return requireRuntimeProviderBundle(managedLocalGateway ? "docker" : "kubernetes", providers); + if (!managedLocalGateway) return requireRuntimeProviderBundle("kubernetes", providers); + if (isPortableExperimentalProfile(env)) { + return requireRuntimeProviderBundle("docker", providers); + } + const configured = resolveNemoClawGatewayRuntime(env); + if (configured === "podman" && platform !== "linux") { + throw new Error("Native Podman runtime provider is supported only on Linux."); + } + return requireRuntimeProviderBundle(configured, providers); +} + +export function resolveRegisteredRuntimeProviderBundle( + providerId: string | null | undefined, + providers: RuntimeProviderBundleRegistry = CURRENT_RUNTIME_PROVIDER_BUNDLES, +): RuntimeProviderBundle | null { + return resolveRuntimeProviderBundle(providerId, providers); } diff --git a/src/lib/onboard/runtime-provider/docker-llama-cpp-managed-lifecycle.ts b/src/lib/onboard/runtime-provider/docker-llama-cpp-managed-lifecycle.ts index baa47a89080..ae9a92d5f61 100644 --- a/src/lib/onboard/runtime-provider/docker-llama-cpp-managed-lifecycle.ts +++ b/src/lib/onboard/runtime-provider/docker-llama-cpp-managed-lifecycle.ts @@ -22,7 +22,7 @@ import { type LlamaCppHostLocalLaunchContract, type LlamaCppHostLocalRuntimeBindings, } from "../../inference/llama-cpp/host-local-runtime"; -import { formatHostServiceUnreachableMessage } from "../host-service-reachability"; +import { formatHostServiceUnreachableMessage } from "../reachability/host-service-message"; import { validateUfwRuleOperands } from "../ufw-auto-apply"; import { createDockerLlamaCppPrivateBridgeController, diff --git a/src/lib/onboard/runtime-provider/docker-privileged-sandbox-control.ts b/src/lib/onboard/runtime-provider/docker-privileged-sandbox-control.ts new file mode 100644 index 00000000000..aaf1e260be8 --- /dev/null +++ b/src/lib/onboard/runtime-provider/docker-privileged-sandbox-control.ts @@ -0,0 +1,254 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { dockerSpawnSync } from "../../adapters/docker/exec"; +import { dockerCapture } from "../../adapters/docker/run"; +import { resolvePortableDemoPrivilegedExecTarget } from "../experimental/portable-demo-lifecycle"; +import { compareAndSetSandboxLifecycleGeneration } from "../../state/registry/lifecycle-generation-cas"; +import type { + RuntimeProviderPrivilegedSandboxCommandInput, + RuntimeProviderPrivilegedSandboxCommandResult, + RuntimeProviderPrivilegedSandboxControl, + RuntimeProviderPrivilegedSandboxTarget, +} from "./contract"; +import { + DirectSandboxFallbackUnavailableError, + PinnedSandboxResourceIdentityChangedError, +} from "./privileged-sandbox-control-errors"; +import { selectDockerPrivilegedSandboxTarget } from "./docker-privileged-sandbox-identity"; +import { createDockerOperationAuthority } from "./docker-operation-authority"; +import { + clearStoppedSandboxStateWithEngine, + sandboxStateResourceFromMounts, + type StoppedSandboxStateObservation, +} from "./stopped-sandbox-state-cleanup"; + +const OPENSHELL_MANAGED_BY_LABEL = "openshell.ai/managed-by"; +const OPENSHELL_MANAGED_BY_VALUE = "openshell"; +const OPENSHELL_SANDBOX_NAME_LABEL = "openshell.ai/sandbox-name"; +const DIRECT_SANDBOX_DISCOVERY_TIMEOUT_MS = 5000; +const SANITIZED_PRIVILEGED_ENV = [ + "BASH_ENV=", + "ENV=", + "GCONV_PATH=", + "GLIBC_TUNABLES=", + "LD_AUDIT=", + "LD_LIBRARY_PATH=", + "LD_PRELOAD=", + "LOCPATH=", + "NODE_OPTIONS=", + "PERL5OPT=", + "PYTHONHOME=", + "PYTHONINSPECT=", + "PYTHONNOUSERSITE=1", + "PYTHONPATH=", + "PYTHONSTARTUP=", + "PYTHONUSERBASE=", + "RUBYOPT=", +] as const; + +type SandboxEntry = import("../../state/registry").SandboxEntry; + +function findDirectSandboxContainer( + sandboxName: string, + registeredSandboxNames: readonly string[], +): string | null { + let output: string; + try { + output = dockerCapture( + [ + "ps", + "--no-trunc", + "--filter", + `label=${OPENSHELL_MANAGED_BY_LABEL}=${OPENSHELL_MANAGED_BY_VALUE}`, + "--filter", + `label=${OPENSHELL_SANDBOX_NAME_LABEL}=${sandboxName}`, + "--format", + "{{.ID}}\t{{.Names}}", + ], + { timeout: DIRECT_SANDBOX_DISCOVERY_TIMEOUT_MS }, + ); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + throw new DirectSandboxFallbackUnavailableError( + `Direct sandbox container discovery failed for '${sandboxName}': ${detail}`, + { cause: error }, + ); + } + return selectDockerPrivilegedSandboxTarget(sandboxName, output, registeredSandboxNames); +} + +function expectedDirectContainerPattern(sandboxName: string): string { + return ( + `openshell-${sandboxName}, openshell-${sandboxName}-*, or ` + + `openshell-default--${sandboxName}-*` + ); +} + +function portableTarget(sandboxName: string, sandbox: SandboxEntry) { + if (sandbox.openshellDriver?.trim().toLowerCase() !== "docker") return null; + return resolvePortableDemoPrivilegedExecTarget(sandboxName, { + ...(sandbox.lifecycleGeneration ? { registryGeneration: sandbox.lifecycleGeneration } : {}), + backfillRegistryGeneration: (generation) => + compareAndSetSandboxLifecycleGeneration(sandbox, generation), + }); +} + +function resolveDockerTarget( + input: Pick< + RuntimeProviderPrivilegedSandboxCommandInput, + "registeredSandboxNames" | "sandbox" | "sandboxName" + >, +): RuntimeProviderPrivilegedSandboxTarget { + const portable = portableTarget(input.sandboxName, input.sandbox); + if (portable) { + portable.assertRuntimeAuthority(); + return Object.freeze({ providerId: "docker", resourceHandle: portable.containerId }); + } + const containerId = findDirectSandboxContainer(input.sandboxName, input.registeredSandboxNames); + if (!containerId) { + throw new DirectSandboxFallbackUnavailableError( + `No running direct OpenShell sandbox container found for '${input.sandboxName}' ` + + `(driver: ${input.sandbox.openshellDriver ?? "unspecified"}). Expected one ` + + `OpenShell-managed container labeled '${OPENSHELL_SANDBOX_NAME_LABEL}=` + + `${input.sandboxName}' and named ${expectedDirectContainerPattern(input.sandboxName)}. ` + + "Is the sandbox running?", + ); + } + return Object.freeze({ providerId: "docker", resourceHandle: containerId }); +} + +function executeDockerCommand( + input: RuntimeProviderPrivilegedSandboxCommandInput, +): RuntimeProviderPrivilegedSandboxCommandResult { + const argv = buildLegacyDockerArgv(input); + const result = dockerSpawnSync(argv, { + encoding: null, + input: input.input, + maxBuffer: input.maxOutputBytes, + stdio: input.input ? ["pipe", "pipe", "pipe"] : ["ignore", "pipe", "pipe"], + timeout: input.timeoutMs, + }); + return Object.freeze({ + status: result.status, + signal: result.signal, + stdout: Buffer.isBuffer(result.stdout) ? result.stdout : Buffer.from(result.stdout ?? ""), + stderr: Buffer.isBuffer(result.stderr) ? result.stderr : Buffer.from(result.stderr ?? ""), + ...(result.error ? { error: result.error } : {}), + }); +} + +function buildLegacyDockerArgv( + input: Omit, +): string[] { + const portable = portableTarget(input.sandboxName, input.sandbox); + const target = portable + ? (() => { + portable.assertRuntimeAuthority(); + return portable.containerId; + })() + : resolveDockerTarget(input).resourceHandle; + if (input.expectedResourceHandle !== undefined && input.expectedResourceHandle !== target) { + throw new PinnedSandboxResourceIdentityChangedError(input.sandboxName); + } + const environment = input.sanitizeEnvironment + ? SANITIZED_PRIVILEGED_ENV.flatMap((value) => ["--env", value]) + : []; + const argv = [ + ...(portable ? ["--host", portable.dockerHost] : []), + "exec", + ...(input.input ? ["-i"] : []), + ...environment, + "--user", + portable ? "0" : "root", + target, + ...input.command, + ]; + return argv; +} + +function observeStoppedDockerTarget( + engine: ReturnType["engine"], + input: Parameters< + NonNullable + >[0], +): StoppedSandboxStateObservation { + let lookup; + try { + lookup = engine.capture( + [ + "ps", + "--all", + "--no-trunc", + "--filter", + `label=${OPENSHELL_MANAGED_BY_LABEL}=${OPENSHELL_MANAGED_BY_VALUE}`, + "--filter", + `label=${OPENSHELL_SANDBOX_NAME_LABEL}=${input.sandboxName}`, + "--format", + "{{.ID}}\t{{.Names}}", + ], + DIRECT_SANDBOX_DISCOVERY_TIMEOUT_MS, + ); + } catch { + return { failure: "runtime-discovery-failed" }; + } + if (lookup.status !== 0 || lookup.error) return { failure: "runtime-discovery-failed" }; + let resourceHandle: string | null; + try { + resourceHandle = selectDockerPrivilegedSandboxTarget( + input.sandboxName, + lookup.stdout, + input.registeredSandboxNames, + ); + } catch { + return { failure: "runtime-ownership-invalid" }; + } + if (!resourceHandle || /-nemoclaw-gpu-backup-\d+$/u.test(lookup.stdout)) { + return { failure: "no-eligible-stopped-runtime" }; + } + const inspected = engine.capture( + ["inspect", "--format", "{{.Id}}\t{{.State.Running}}\t{{json .Mounts}}", resourceHandle], + 30_000, + ); + if (inspected.status !== 0 || inspected.error) return { failure: "runtime-inspection-failed" }; + const [id, running, mountsJson, ...unexpected] = inspected.stdout.trim().split("\t"); + if ( + unexpected.length > 0 || + id !== resourceHandle || + (running !== "true" && running !== "false") || + !mountsJson + ) { + return { failure: "runtime-ownership-invalid" }; + } + let mounts: unknown; + try { + mounts = JSON.parse(mountsJson); + } catch { + return { failure: "state-resource-unavailable" }; + } + const stateResource = sandboxStateResourceFromMounts(mounts, input.paths); + return stateResource + ? { target: { resourceHandle, running: running === "true", stateResource } } + : { failure: "state-resource-unavailable" }; +} + +function clearStoppedStateRoots( + input: Parameters< + NonNullable + >[0], +) { + const engine = createDockerOperationAuthority("sandbox-lifecycle").engine; + return clearStoppedSandboxStateWithEngine(input.sandboxName, input.paths, { + capture: (args, timeoutMs = 30_000) => engine.capture(args, timeoutMs), + observe: () => observeStoppedDockerTarget(engine, input), + }); +} + +export function createDockerPrivilegedSandboxControl(): RuntimeProviderPrivilegedSandboxControl { + return Object.freeze({ + resolveTarget: resolveDockerTarget, + execute: executeDockerCommand, + clearStoppedStateRoots, + buildLegacyDockerArgv, + }); +} diff --git a/src/lib/onboard/runtime-provider/docker-privileged-sandbox-identity.ts b/src/lib/onboard/runtime-provider/docker-privileged-sandbox-identity.ts new file mode 100644 index 00000000000..95efe432f88 --- /dev/null +++ b/src/lib/onboard/runtime-provider/docker-privileged-sandbox-identity.ts @@ -0,0 +1,69 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { resolveSandboxContainerOwner } from "../../domain/sandbox/container-owner"; + +type LabeledSandboxContainer = { + readonly id: string; + readonly name: string; +}; + +export function dockerContainerNameMatchesSandbox( + containerName: string, + sandboxName: string, +): boolean { + return resolveSandboxContainerOwner(containerName, sandboxName, [sandboxName]) === containerName; +} + +function owningRegisteredSandboxName( + containerName: string, + registeredNames: readonly string[], +): string | null { + return ( + registeredNames.find((name) => dockerContainerNameMatchesSandbox(containerName, name)) ?? null + ); +} + +function parseLabeledSandboxContainers(output: string): LabeledSandboxContainer[] { + return output + .split(/\r?\n/u) + .map((line) => line.trim()) + .filter(Boolean) + .map((line) => { + const [id, name, ...unexpected] = line.split("\t"); + if (!id || !name || unexpected.length > 0 || /\s/u.test(id)) { + throw new Error("Docker returned malformed OpenShell sandbox container metadata."); + } + return { id, name }; + }); +} + +export function selectDockerPrivilegedSandboxTarget( + sandboxName: string, + labeledContainerRows: string, + registeredNames: readonly string[] = [sandboxName], +): string | null { + const names = Array.from(new Set([...registeredNames, sandboxName])).sort( + (left, right) => right.length - left.length || left.localeCompare(right), + ); + const candidates = parseLabeledSandboxContainers(labeledContainerRows); + if ( + candidates.some( + ({ name }) => + !dockerContainerNameMatchesSandbox(name, sandboxName) || + owningRegisteredSandboxName(name, names) !== sandboxName, + ) + ) { + throw new Error( + `OpenShell container labels and names disagree for sandbox '${sandboxName}'; ` + + "refusing lifecycle execution.", + ); + } + if (candidates.length > 1) { + throw new Error( + `Multiple running OpenShell containers are labeled for sandbox '${sandboxName}'; ` + + "refusing ambiguous lifecycle execution.", + ); + } + return candidates[0]?.id ?? null; +} diff --git a/src/lib/onboard/runtime-provider/docker-state-mutation.ts b/src/lib/onboard/runtime-provider/docker-state-mutation.ts index 7067a2e20c4..aa8f4e8e04b 100644 --- a/src/lib/onboard/runtime-provider/docker-state-mutation.ts +++ b/src/lib/onboard/runtime-provider/docker-state-mutation.ts @@ -52,7 +52,6 @@ import { prepareRuntimeProviderStateMutationPlan } from "./state-mutation"; const DOCKER_PROVIDER_ID = "docker"; const SUPPORTED_STATE_ROOT = "/sandbox/.hermes"; const HELPER_PYTHON_PATH = "/opt/hermes/.venv/bin/python3"; -const HELPER_PATH = "/usr/local/lib/nemoclaw/runtime-state-mutation-control.py"; const HELPER_TRANSPORT_BROKER_PATH = "/usr/local/lib/nemoclaw/runtime-state-mutation-transport-broker.py"; const HELPER_FAST_TIMEOUT_MS = 30_000; @@ -90,12 +89,14 @@ const MAX_HELPER_TRANSPORT_BYTES = 128 * 1024; const MAX_INSPECTION_BYTES = 1024 * 1024; const MAX_MOUNTS = 256; const RUNTIME_QUERY_FORMAT = "{{.ID}}"; -const INSPECT_FORMAT = - '[{{json .Id}},{{json .State.Running}},{{json .State.Status}},{{json .State.Paused}},{{json .State.Restarting}},{{json .State.Dead}},{{json .State.Pid}},{{json (index .Config.Labels "openshell.ai/managed-by")}},{{json (index .Config.Labels "openshell.ai/sandbox-name")}},{{json (index .Config.Labels "openshell.ai/sandbox-id")}},{{json .HostConfig.PidMode}},{{json .HostConfig.Privileged}},{{json .Mounts}}]'; +const DEFAULT_MANAGED_LABEL_KEY = "openshell.ai/managed-by"; +const DEFAULT_MANAGED_LABEL_VALUE = "openshell"; const SHA256 = /^[a-f0-9]{64}$/u; const CONTAINER_ID = /^[a-f0-9]{64}$/u; const PROVIDER_ID = /^[a-z][a-z0-9-]{0,62}$/u; const SAFE_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/u; +const SAFE_LABEL_KEY = /^[A-Za-z0-9][A-Za-z0-9._/-]{0,127}$/u; +const SAFE_LABEL_VALUE = /^[A-Za-z0-9][A-Za-z0-9._/-]{0,127}$/u; const LIFECYCLE_GENERATION = /^[A-Za-z0-9][A-Za-z0-9._:/=+-]{0,511}$/u; const MOUNT_NAMESPACE = /^mnt:\[[1-9][0-9]*\]$/u; const POSITIVE_DECIMAL = /^[1-9][0-9]*$/u; @@ -143,7 +144,7 @@ interface DockerRuntimeObservation { readonly providerDisplayName: string; readonly runtimeId: string; readonly runtimePid: number; - readonly pidMode: ""; + readonly pidMode: string; readonly privileged: false; readonly sandboxIdentitySha256: string; readonly containerMountsSha256: string; @@ -203,6 +204,18 @@ export interface ContainerStateMutationOwnerOptions { readonly authority: ContainerStateMutationAuthority; readonly engineAuthorityStore: PersistedEngineAuthorityStore; readonly lifecycleStore: PersistedEngineLifecycleStore; + /** Provider-native Go-template field for the full immutable runtime ID. */ + readonly runtimeIdInspectField?: "Id" | "ID"; + /** Provider-native inspect value proving a private PID namespace. */ + readonly privatePidMode?: string; + /** Provider-owned label proving that the runtime belongs to OpenShell. */ + readonly managedLabelKey?: string; + readonly managedLabelValue?: string; + /** Provider-owned normalization for one logical mount reported as multiple inspect rows. */ + readonly normalizeInspectionMounts?: ( + mounts: readonly unknown[], + runtimeId: string, + ) => readonly unknown[]; } export type DockerStateMutationOwnerOptions = Omit< @@ -231,6 +244,14 @@ export interface ContainerStateMutationSurfaceOptions { readonly createAuthority: ( input: RuntimeProviderStateMutationContext, ) => ContainerStateMutationAuthority; + readonly runtimeIdInspectField?: "Id" | "ID"; + readonly privatePidMode?: string; + readonly managedLabelKey?: string; + readonly managedLabelValue?: string; + readonly normalizeInspectionMounts?: ( + mounts: readonly unknown[], + runtimeId: string, + ) => readonly unknown[]; readonly resolveStateDir?: (environment: NodeJS.ProcessEnv) => string; readonly withDirectSandboxExecutionExclusion?: ( sandboxName: string, @@ -478,6 +499,9 @@ function parseInspection( expectedRuntimeId: string, expectedSandboxName: string, providerDisplayName: string, + expectedPrivatePidMode: string, + expectedManagedLabelValue: string, + normalizeMounts: (mounts: readonly unknown[], runtimeId: string) => readonly unknown[], ): DockerRuntimeObservation { if ( output.length === 0 || @@ -527,10 +551,10 @@ function parseInspection( ) { fail(`${providerDisplayName} container is not one stable running runtime`); } - if (managedBy !== "openshell" || sandboxName !== expectedSandboxName) { + if (managedBy !== expectedManagedLabelValue || sandboxName !== expectedSandboxName) { fail(`${providerDisplayName} container does not belong to the exact OpenShell sandbox`); } - if (pidMode !== "" || privileged !== false) { + if (pidMode !== expectedPrivatePidMode || privileged !== false) { fail(`${providerDisplayName} container does not have one private unprivileged PID namespace`); } const sandboxId = exactText(sandboxIdInput, "OpenShell sandbox identity", 512); @@ -538,7 +562,7 @@ function parseInspection( if (!Array.isArray(mountsInput) || mountsInput.length > MAX_MOUNTS) { fail(`${providerDisplayName} container mounts are malformed`); } - const mounts = mountsInput + const mounts = normalizeMounts(mountsInput, runtimeId) .map(parseMount) .sort((left, right) => left.destination < right.destination @@ -565,7 +589,7 @@ function parseInspection( providerDisplayName, runtimeId, runtimePid: runtimePid as number, - pidMode: "", + pidMode: expectedPrivatePidMode, privileged: false, sandboxIdentitySha256, containerMountsSha256: mountsSha256(mounts), @@ -1025,31 +1049,25 @@ function requireCurrentEngineAuthority( ); } -function inspectCommand(runtimeId: string) { +function inspectCommand( + runtimeId: string, + runtimeIdField: "Id" | "ID" = "Id", + managedLabelKey = DEFAULT_MANAGED_LABEL_KEY, +) { + boundedString(managedLabelKey, SAFE_LABEL_KEY, "managed runtime label key"); + const format = + `[{{json .${runtimeIdField}}},{{json .State.Running}},{{json .State.Status}},` + + `{{json .State.Paused}},{{json .State.Restarting}},{{json .State.Dead}},` + + `{{json .State.Pid}},{{json (index .Config.Labels "${managedLabelKey}")}},` + + '{{json (index .Config.Labels "openshell.ai/sandbox-name")}},' + + '{{json (index .Config.Labels "openshell.ai/sandbox-id")}},' + + "{{json .HostConfig.PidMode}},{{json .HostConfig.Privileged}},{{json .Mounts}}]"; return Object.freeze({ - args: Object.freeze(["container", "inspect", "--format", INSPECT_FORMAT, runtimeId]), + args: Object.freeze(["container", "inspect", "--format", format, runtimeId]), targetIndex: 4, }); } -function helperCommand(runtimeId: string, action: HelperAction) { - return Object.freeze({ - args: Object.freeze([ - "container", - "exec", - "--interactive", - "--user", - "root", - runtimeId, - HELPER_PYTHON_PATH, - "-I", - HELPER_PATH, - action, - ]), - targetIndex: 5, - }); -} - type HelperTransportCapture = ( command: PersistedEngineLifecycleExactCommand, timeoutMs: number, @@ -1266,7 +1284,6 @@ function finishReleasedHelperTransport( bindingSha256: string, transactionId: string, ): void { - if (options.providerId !== DOCKER_PROVIDER_ID) return; const capture: HelperTransportCapture = (command, timeoutMs) => { requireCurrentEngineAuthority(options, bindingSha256); const result = options.authority.engine.capture(command.args, timeoutMs); @@ -1473,7 +1490,11 @@ function inspectDirect( ): DockerRuntimeObservation { requireCurrentEngineAuthority(options, bindingSha256); const result = options.authority.engine.capture( - inspectCommand(options.runtimeId).args, + inspectCommand( + options.runtimeId, + options.runtimeIdInspectField, + options.managedLabelKey ?? DEFAULT_MANAGED_LABEL_KEY, + ).args, INSPECT_TIMEOUT_MS, ); requireCurrentEngineAuthority(options, bindingSha256); @@ -1482,6 +1503,9 @@ function inspectDirect( options.runtimeId, options.sandboxName, options.providerDisplayName, + options.privatePidMode ?? "", + options.managedLabelValue ?? DEFAULT_MANAGED_LABEL_VALUE, + options.normalizeInspectionMounts ?? ((mounts) => mounts), ); requireRegistryLiveIdentity(options, observation); return observation; @@ -1491,12 +1515,24 @@ function inspectAuthorized( scope: AuthorizedPersistedEngineLifecycle, options: ContainerStateMutationOwnerOptions, ): DockerRuntimeObservation { - const result = scope.captureExact("target", inspectCommand, INSPECT_TIMEOUT_MS); + const result = scope.captureExact( + "target", + (runtimeId) => + inspectCommand( + runtimeId, + options.runtimeIdInspectField, + options.managedLabelKey ?? DEFAULT_MANAGED_LABEL_KEY, + ), + INSPECT_TIMEOUT_MS, + ); const observation = parseInspection( requireCommandSuccess(result, `${options.providerDisplayName} container inspection`), options.runtimeId, options.sandboxName, options.providerDisplayName, + options.privatePidMode ?? "", + options.managedLabelValue ?? DEFAULT_MANAGED_LABEL_VALUE, + options.normalizeInspectionMounts ?? ((mounts) => mounts), ); requireRegistryLiveIdentity(options, observation); return observation; @@ -1534,21 +1570,9 @@ function invokeHelperAuthorized( action: HelperAction, input: Buffer, ): DockerStateMutationHelperReceipt { - if (options.providerId === DOCKER_PROVIDER_ID) { - const capture: HelperTransportCapture = (command, timeoutMs) => - scope.captureExact("target", () => command, timeoutMs); - return invokeHelperTransport(capture, options, scope.record.transactionId, action, input); - } - const result = scope.captureExact( - "target", - (runtimeId) => helperCommand(runtimeId, action), - helperTimeoutMs(action), - input, - ); - return parseHelperReceipt( - requireCommandSuccess(result, `root helper ${action}`), - options.providerId, - ); + const capture: HelperTransportCapture = (command, timeoutMs) => + scope.captureExact("target", () => command, timeoutMs); + return invokeHelperTransport(capture, options, scope.record.transactionId, action, input); } function lifecycleInput( @@ -1951,9 +1975,7 @@ function acquireAuthorizedReceipt( ) { fail("persisted state mutation intent does not match the lifecycle transaction"); } - if (options.providerId === DOCKER_PROVIDER_ID) { - ensureHelperTransportAuthorized(scope, options, exactTransactionId); - } + ensureHelperTransportAuthorized(scope, options, exactTransactionId); signalSupervisorAuthorized(scope, options, "SIGSTOP"); const receipt = invokeHelperAuthorized( scope, @@ -2007,31 +2029,18 @@ function queryEstablishedReceipt( execution.transactionId, expectedFence?.providerHandle, ); - const result = - options.providerId === DOCKER_PROVIDER_ID - ? invokeHelperTransport( - (command, timeoutMs) => { - guard(); - const captured = options.authority.engine.capture(command.args, timeoutMs); - guard(); - return captured; - }, - options, - execution.transactionId, - action, - request, - ) - : parseHelperReceipt( - requireCommandSuccess( - options.authority.engine.capture( - helperCommand(options.runtimeId, action).args, - helperTimeoutMs(action), - request, - ), - `root helper ${action}`, - ), - options.providerId, - ); + const result = invokeHelperTransport( + (command, timeoutMs) => { + guard(); + const captured = options.authority.engine.capture(command.args, timeoutMs); + guard(); + return captured; + }, + options, + execution.transactionId, + action, + request, + ); guard(); const receipt = result; validateReceipt(receipt, options, bindingSha256, before, currentRecord); @@ -2134,7 +2143,11 @@ function releaseAuthorizedFence( export function createContainerStateMutationOwner( optionsInput: ContainerStateMutationOwnerOptions, ): ContainerStateMutationOwner { - const options = Object.freeze({ ...optionsInput }); + const options = Object.freeze({ + privatePidMode: "", + runtimeIdInspectField: "Id" as const, + ...optionsInput, + }); boundedString(options.providerId, PROVIDER_ID, "provider identity"); boundedString(options.providerDisplayName, SAFE_NAME, "provider display name"); boundedString(options.sandboxName, SAFE_NAME, "sandbox name"); @@ -2486,14 +2499,18 @@ function resolveExactLabeledRuntimeId( authority: ContainerStateMutationAuthority, sandboxName: string, providerDisplayName: string, + managedLabelKey = DEFAULT_MANAGED_LABEL_KEY, + managedLabelValue = DEFAULT_MANAGED_LABEL_VALUE, ): string { + boundedString(managedLabelKey, SAFE_LABEL_KEY, "managed runtime label key"); + boundedString(managedLabelValue, SAFE_LABEL_VALUE, "managed runtime label value"); const result = authority.engine.capture( [ "ps", "-a", "--no-trunc", "--filter", - "label=openshell.ai/managed-by=openshell", + `label=${managedLabelKey}=${managedLabelValue}`, "--filter", `label=openshell.ai/sandbox-name=${sandboxName}`, "--format", @@ -2553,6 +2570,8 @@ function createSurfaceOwnerOptions( authority, input.sandboxName, options.providerDisplayName, + options.managedLabelKey, + options.managedLabelValue, ); return { providerId: options.providerId, @@ -2570,6 +2589,19 @@ function createSurfaceOwnerOptions( authority, engineAuthorityStore, lifecycleStore: createFilePersistedEngineLifecycleStore(stateDir), + ...(options.runtimeIdInspectField + ? { runtimeIdInspectField: options.runtimeIdInspectField } + : {}), + ...(options.privatePidMode === undefined ? {} : { privatePidMode: options.privatePidMode }), + ...(options.managedLabelKey === undefined + ? {} + : { managedLabelKey: options.managedLabelKey }), + ...(options.managedLabelValue === undefined + ? {} + : { managedLabelValue: options.managedLabelValue }), + ...(options.normalizeInspectionMounts === undefined + ? {} + : { normalizeInspectionMounts: options.normalizeInspectionMounts }), }; } @@ -2593,7 +2625,6 @@ function retireReleasedSurfaceStateMutations( } | undefined; lifecycleStore.retireReleasedStateMutations(input.sandboxName, (record) => { - if (options.providerId !== DOCKER_PROVIDER_ID) return; if (!cleanup) { const ownerOptions = createSurfaceOwnerOptions(input, options, "existing"); cleanup = { diff --git a/src/lib/onboard/runtime-provider/docker.ts b/src/lib/onboard/runtime-provider/docker.ts index 1bc2511cbbe..884f9471241 100644 --- a/src/lib/onboard/runtime-provider/docker.ts +++ b/src/lib/onboard/runtime-provider/docker.ts @@ -2,6 +2,12 @@ // SPDX-License-Identifier: Apache-2.0 import { captureHostCommand } from "../../actions/sandbox/doctor-host-command"; +import { dockerCapture, dockerRun } from "../../adapters/docker/run"; +import { + DEFAULT_GATEWAY_BIND_ADDRESS, + getGatewayConnectHost, + parseGatewayBindAddress, +} from "../../core/gateway-address"; import { isDockerRuntimeDown, printDockerRuntimeDownGuidance, @@ -13,24 +19,29 @@ import { recoverDockerDriverSandbox, } from "../docker-driver-sandbox-recovery"; import { createDockerManagedBootstrapSurface } from "../managed-bootstrap/docker-runtime"; +import { + DOCKER_NETWORK_IPAM_INSPECT_FORMAT, + parseDockerNetworkIpamEntries, + resolveDockerDriverNetworkName, +} from "../experimental/docker-network-authority"; import { hasPortableAgentSandboxLifecycleReceipt, recoverPortableAgentSandboxLifecycle, stopPortableAgentSandboxLifecycle, } from "../experimental/portable-agent-lifecycle"; import { withMcpLifecycleLockSync } from "../../state/mcp-lifecycle-lock-acquisition"; +import { queryOpenShellDockerSandboxRuntimeSnapshot } from "../openshell-docker-sandbox-containers"; +import { validateSandboxGpuPreflight } from "../sandbox-gpu-preflight"; import { MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION, MANAGED_IMAGE_PLATFORMS, MANAGED_IMAGE_REPOSITORIES, MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, -} from "../managed-image/contract"; -import { queryOpenShellDockerSandboxRuntimeSnapshot } from "../openshell-docker-sandbox-containers"; -import { RUNTIME_PROVIDER_BUNDLE_CONTRACT_VERSION, type RuntimeProviderBundle, type RuntimeProviderCleanupInput, type RuntimeProviderCommandCapture, + type RuntimeProviderContainerEngineOperation, type RuntimeProviderDoctorCheck, type RuntimeProviderLifecycleInput, type RuntimeProviderLifecycleResult, @@ -42,6 +53,7 @@ import { } from "./contract"; import { createDockerLlamaCppHostLocalOperation } from "./docker-llama-cpp-operation"; import { createDockerStateMutationSurface } from "./docker-state-mutation"; +import { createDockerPrivilegedSandboxControl } from "./docker-privileged-sandbox-control"; import { createDockerRuntimeProviderSnapshotSurface } from "./snapshot"; type DockerOpResult = { status?: number | null }; @@ -75,6 +87,76 @@ export interface DockerRuntimeProviderDependencies { const DOCKER_OPERATION_TIMEOUT_MS = 30_000; const AT_REST_STATUS_PREFIXES = ["Exited", "Created", "Dead"] as const; +function inspectDockerGatewayNetwork(networkName: string) { + const raw = dockerCapture( + ["network", "inspect", "--format", DOCKER_NETWORK_IPAM_INSPECT_FORMAT, networkName], + { ignoreError: true }, + ); + for (const entry of parseDockerNetworkIpamEntries(raw) ?? []) { + if (entry.gatewayIp && !entry.gatewayIp.includes(":")) return entry; + } + return undefined; +} + +function dockerGatewayUsesHostGatewayRoute(): boolean { + if (process.platform !== "linux") return true; + const info = dockerCapture( + ["info", "--format", "{{.OperatingSystem}}\n{{range .Labels}}{{.}}\n{{end}}"], + { ignoreError: true }, + ); + return /Docker Desktop|com\.docker\.desktop\./iu.test(info); +} + +function runDockerGatewayCommand(args: readonly string[], timeoutMs: number) { + const result = dockerRun([...args], { + timeout: timeoutMs, + ignoreError: true, + suppressOutput: true, + }); + const error = result.error as NodeJS.ErrnoException | undefined; + return { + status: result.status ?? null, + signal: result.signal, + error: error?.message, + errorCode: error?.code ?? null, + timedOut: error?.code === "ETIMEDOUT", + stderr: result.stderr, + stdout: result.stdout, + }; +} + +function ensureDockerGatewayProbeImageCached(image: string) { + const inspect = runDockerGatewayCommand(["image", "inspect", image], 10_000); + if (inspect.status === 0) return { ok: true as const, alreadyCached: true }; + if (inspect.status === null || inspect.errorCode) { + return { + ok: false as const, + reason: "inspect_unavailable" as const, + details: inspect.error ?? String(inspect.stderr ?? "").trim(), + }; + } + const pull = runDockerGatewayCommand(["pull", image], 120_000); + if (pull.status === 0) return { ok: true as const, alreadyCached: false }; + return { + ok: false as const, + reason: pull.timedOut ? ("pull_timeout" as const) : ("pull_failed" as const), + details: pull.error ?? String(pull.stderr ?? "").trim(), + }; +} + +function captureDockerContainerEngineOperation( + deps: DockerRuntimeProviderDependencies, + supportedOperations: ReadonlySet, + operation: RuntimeProviderContainerEngineOperation, + args: readonly string[], + timeoutMs?: number, +): RuntimeProviderCommandCapture { + if (!supportedOperations.has(operation)) { + throw new Error(`Docker provider does not register the '${operation}' engine operation.`); + } + return deps.captureHostCommand("docker", [...args], timeoutMs); +} + function loadDockerStop(): DockerStop { return (require("../../adapters/docker") as { dockerStop: DockerStop }).dockerStop; } @@ -420,6 +502,13 @@ export function createDockerRuntimeProviderBundle( ): RuntimeProviderBundle { const providerId = "docker"; const deps = resolveDependencies(overrides); + const containerEngineOperations = new Set([ + "host-doctor", + "gateway-inspection", + "host-local-inference", + "sandbox-lifecycle", + "workload-cleanup", + ]); const futureReason = "This operation is intentionally deferred to a later provider slice."; return { identity: { @@ -441,6 +530,8 @@ export function createDockerRuntimeProviderBundle( providerId, supported: true, inspectHost: () => inspectDockerHost(deps), + validateSandboxGpu: (config, exitProcess) => + validateSandboxGpuPreflight(config, {}, exitProcess), preflightLifecycle: (action, input) => dockerLifecyclePreflight(action, input, deps), }, gateway: { @@ -448,11 +539,54 @@ export function createDockerRuntimeProviderBundle( supported: true, launcher: "nemoclaw", inspectLegacyContainer: false, + prepareHostRuntime: (input) => { + const bindAddress = parseGatewayBindAddress( + "NEMOCLAW_GATEWAY_BIND_ADDRESS", + DEFAULT_GATEWAY_BIND_ADDRESS, + input.environment, + ); + const connectHost = getGatewayConnectHost(bindAddress); + return { + providerId, + openShellDriver: "docker", + bindAddress, + grpcHost: connectHost, + sshGatewayHost: connectHost, + portCheckHost: bindAddress, + socketPath: null, + requiredServerIpSans: [], + sandboxHostAddress: null, + usesHostGatewayRoute: false, + resourceOwnership: { + label: "openshell.ai/managed-by", + value: "openshell", + }, + gatewayConfig: { + sandboxNamespace: "scoped", + hostGatewayIp: null, + includeSupervisorBin: true, + processOwnership: "scoped-namespace", + }, + network: { + sandboxSourceCidrs: () => { + const network = inspectDockerGatewayNetwork( + resolveDockerDriverNetworkName(input.environment), + ); + return network?.subnet ? [network.subnet] : []; + }, + inspect: inspectDockerGatewayNetwork, + usesHostGatewayRoute: dockerGatewayUsesHostGatewayRoute, + run: runDockerGatewayCommand, + ensureProbeImageCached: ensureDockerGatewayProbeImageCached, + }, + }; + }, }, workload: { providerId, supported: true, profile: COMPLETE_MANAGED_IMAGE_V1_PROFILE, + managedStateMountDriverId: "docker", acceptsReceipt: (receipt) => acceptsReceipt(COMPLETE_MANAGED_IMAGE_V1_PROFILE, receipt), }, hostLocalInference: { @@ -465,6 +599,8 @@ export function createDockerRuntimeProviderBundle( providerId, supported: true, channelStopTransport: "docker-kubectl-first", + containerMutationTimeoutMs: DOCKER_OPERATION_TIMEOUT_MS, + privilegedSandboxControl: createDockerPrivilegedSandboxControl(), start: (input) => startDockerSandbox(input, deps), verifyStarted: (input, verifyGateway) => verifyGateway(input.sandboxName), stop: (input, hooks) => stopDockerSandbox(input, hooks, deps), @@ -508,6 +644,14 @@ export function createDockerRuntimeProviderBundle( { operation: "sandbox-lifecycle", engineId: "docker", displayName: "Docker" }, { operation: "workload-cleanup", engineId: "docker", displayName: "Docker" }, ], + capture: (operation, args, timeoutMs) => + captureDockerContainerEngineOperation( + deps, + containerEngineOperations, + operation, + args, + timeoutMs, + ), }, }; } @@ -517,6 +661,11 @@ export function createKubernetesRuntimeProviderBundle( ): RuntimeProviderBundle { const providerId = "kubernetes"; const deps = resolveDependencies(overrides); + const containerEngineOperations = new Set([ + "host-doctor", + "gateway-inspection", + "workload-cleanup", + ]); const futureReason = "This operation is intentionally deferred to a later provider slice."; const profile = { support: null, @@ -548,6 +697,8 @@ export function createKubernetesRuntimeProviderBundle( providerId, supported: true, inspectHost: () => inspectDockerHost(deps), + validateSandboxGpu: (config, exitProcess) => + validateSandboxGpuPreflight(config, {}, exitProcess), preflightLifecycle: () => null, }, gateway: { @@ -555,6 +706,9 @@ export function createKubernetesRuntimeProviderBundle( supported: true, launcher: "openshell", inspectLegacyContainer: true, + prepareHostRuntime: () => { + throw new Error("The Kubernetes provider does not launch a host-managed gateway."); + }, }, workload: { providerId, @@ -604,6 +758,14 @@ export function createKubernetesRuntimeProviderBundle( { operation: "gateway-inspection", engineId: "docker", displayName: "Docker" }, { operation: "workload-cleanup", engineId: "docker", displayName: "Docker" }, ], + capture: (operation, args, timeoutMs) => + captureDockerContainerEngineOperation( + deps, + containerEngineOperations, + operation, + args, + timeoutMs, + ), }, }; } diff --git a/src/lib/onboard/runtime-provider/mxc.ts b/src/lib/onboard/runtime-provider/mxc.ts index 5350e0b92ed..6d7d89c389e 100644 --- a/src/lib/onboard/runtime-provider/mxc.ts +++ b/src/lib/onboard/runtime-provider/mxc.ts @@ -13,6 +13,7 @@ import { NATIVE_ARTIFACT_WORKLOAD_PLATFORM, parseNativeArtifactWorkloadReceiptV1, } from "../workload/native-artifact"; +import { exitOnSandboxGpuConfigErrors } from "../sandbox-gpu-preflight"; import { RUNTIME_PROVIDER_BUNDLE_CONTRACT_VERSION, type RuntimeProviderBundle, @@ -172,6 +173,8 @@ export function createMxcRuntimeProviderBundle({ providerId: MXC_PROVIDER_ID, supported: true, inspectHost: () => inspectMxcHost(hostFacts, qualifyAttachment), + validateSandboxGpu: (config, exitProcess) => + exitOnSandboxGpuConfigErrors(config, exitProcess), preflightLifecycle: () => ({ exitCode: 1, message: lifecycleReason }), }, gateway: { @@ -179,6 +182,9 @@ export function createMxcRuntimeProviderBundle({ supported: true, launcher: "openshell", inspectLegacyContainer: false, + prepareHostRuntime: () => { + throw new Error("OpenShell MXC does not launch a host-managed gateway."); + }, }, workload: { providerId: MXC_PROVIDER_ID, diff --git a/src/lib/onboard/runtime-provider/podman-lifecycle.test.ts b/src/lib/onboard/runtime-provider/podman-lifecycle.test.ts index 165e801ef61..defa604173b 100644 --- a/src/lib/onboard/runtime-provider/podman-lifecycle.test.ts +++ b/src/lib/onboard/runtime-provider/podman-lifecycle.test.ts @@ -14,6 +14,7 @@ import { PODMAN_SANDBOX_NAMESPACE_LABEL, PODMAN_SANDBOX_WORKSPACE, PODMAN_SANDBOX_WORKSPACE_LABEL, + recoverPodmanSandbox, startPodmanSandbox, stopPodmanSandbox, } from "./podman-lifecycle"; @@ -77,6 +78,9 @@ function harness(initial: HarnessState) { case "unpause": setState({ running: true, status: "running" }); return { status: 0, stdout: CONTAINER_ID, stderr: "" }; + case "restart": + setState({ running: true, status: "running" }); + return { status: 0, stdout: CONTAINER_ID, stderr: "" }; case "stop": setState({ running: false, status: "exited" }); return { status: 0, stdout: CONTAINER_ID, stderr: "" }; @@ -103,6 +107,18 @@ function harness(initial: HarnessState) { } describe("Podman basic CPU lifecycle", () => { + it("restarts the exact running container during gateway recovery", () => { + const runtime = harness({ running: true, status: "running" }); + + expect(recoverPodmanSandbox(runtime.input, runtime.engine)).toEqual({ exitCode: 0 }); + expect(runtime.capture.mock.calls.map(([args]) => args)).toContainEqual([ + "restart", + "--time", + "30", + CONTAINER_ID, + ]); + }); + it("stops and restarts the exact managed container", () => { const stopped = harness({ running: true, status: "running" }); const beforeStop = vi.fn(); diff --git a/src/lib/onboard/runtime-provider/podman-lifecycle.ts b/src/lib/onboard/runtime-provider/podman-lifecycle.ts index 55283815394..e3e3b77ec52 100644 --- a/src/lib/onboard/runtime-provider/podman-lifecycle.ts +++ b/src/lib/onboard/runtime-provider/podman-lifecycle.ts @@ -24,7 +24,7 @@ export const PODMAN_SANDBOX_WORKSPACE = "default"; export const PODMAN_SANDBOX_CONTAINER_PREFIX = `openshell-${PODMAN_SANDBOX_WORKSPACE}--`; const PROBE_TIMEOUT_MS = 5000; -const MUTATION_TIMEOUT_MS = 40_000; +export const PODMAN_LIFECYCLE_MUTATION_TIMEOUT_MS = 75_000; const STOP_GRACE_SECONDS = 30; const FULL_CONTAINER_ID_PATTERN = /^[0-9a-f]{64}$/u; const AT_REST_STATES = new Set(["configured", "created", "dead", "exited", "stopped"]); @@ -33,8 +33,9 @@ const CONTROL_CHARACTERS = /[\u0000-\u001f\u007f-\u009f]/u; type JsonRecord = Record; -interface PodmanManagedContainer { +export interface PodmanManagedContainer { readonly containerId: string; + readonly inspect: Readonly; readonly labels: Readonly>; readonly name: string; readonly paused: boolean; @@ -178,6 +179,7 @@ function parsePodmanManagedContainer( } return { containerId, + inspect: entry, labels: containerLabels, name, running: state.Running, @@ -201,9 +203,12 @@ function commandFailure(operation: string, result: ContainerEngineCommandResult) ); } -function requireLifecycleEngine(engine: ContainerEngine): void { - if (engine.operation !== "sandbox-lifecycle" || engine.engineId !== "podman") { - throw new Error("Podman lifecycle requires an operation-scoped Podman engine."); +function requireObservationEngine(engine: ContainerEngine): void { + if ( + engine.engineId !== "podman" || + (engine.operation !== "sandbox-lifecycle" && engine.operation !== "gateway-inspection") + ) { + throw new Error("Podman runtime observation requires an operation-scoped Podman engine."); } } @@ -225,11 +230,11 @@ function inspectExactContainer( return parsePodmanManagedContainer(inspected.stdout, expected); } -function resolveManagedContainer( +export function observePodmanManagedContainer( engine: ContainerEngine, sandboxName: string, -): PodmanManagedContainer { - requireLifecycleEngine(engine); +): PodmanManagedContainer | null { + requireObservationEngine(engine); if (!isValidName(sandboxName)) { throw new Error("Podman lifecycle requires a valid sandbox name."); } @@ -255,9 +260,7 @@ function resolveManagedContainer( .map((line) => line.trim()) .filter(Boolean); if (rows.length === 0) { - throw new Error( - `No Podman container found for sandbox '${sandboxName}'. Run '${cliName()} ${sandboxName} rebuild' if its workload was removed.`, - ); + return null; } if (rows.length !== 1) { throw new Error( @@ -268,6 +271,17 @@ function resolveManagedContainer( return inspectExactContainer(engine, { sandboxName, containerId }); } +function resolveManagedContainer( + engine: ContainerEngine, + sandboxName: string, +): PodmanManagedContainer { + const container = observePodmanManagedContainer(engine, sandboxName); + if (container) return container; + throw new Error( + `No Podman container found for sandbox '${sandboxName}'. Run '${cliName()} ${sandboxName} rebuild' if its workload was removed.`, + ); +} + function resultForFailure(error: unknown): RuntimeProviderLifecycleResult { return { exitCode: 1, @@ -277,18 +291,57 @@ function resultForFailure(error: unknown): RuntimeProviderLifecycleResult { function mutateContainer( engine: ContainerEngine, - operation: "start" | "stop" | "unpause", + operation: "restart" | "start" | "stop" | "unpause", container: PodmanManagedContainer, ): void { const args = [ operation, - ...(operation === "stop" ? ["--time", String(STOP_GRACE_SECONDS)] : []), + ...(operation === "stop" || operation === "restart" + ? ["--time", String(STOP_GRACE_SECONDS)] + : []), container.containerId, ]; - const result = engine.capture(args, MUTATION_TIMEOUT_MS); + const result = engine.capture(args, PODMAN_LIFECYCLE_MUTATION_TIMEOUT_MS); if (result.status !== 0 || result.error) throw commandFailure(operation, result); } +/** Repair a failed gateway probe without changing the pinned Podman container identity. */ +export function recoverPodmanSandbox( + input: RuntimeProviderLifecycleInput, + engine: ContainerEngine, +): RuntimeProviderLifecycleResult { + try { + let container = resolveManagedContainer(engine, input.sandboxName); + if (container.paused) { + mutateContainer(engine, "unpause", container); + container = inspectExactContainer(engine, { + sandboxName: input.sandboxName, + containerId: container.containerId, + previous: container, + }); + } + const operation = container.running ? "restart" : "start"; + if (!container.running && !AT_REST_STATES.has(container.status)) { + throw new Error( + `Refusing Podman recovery for sandbox '${input.sandboxName}': container state '${container.status}' is not safely recoverable.`, + ); + } + mutateContainer(engine, operation, container); + const verified = inspectExactContainer(engine, { + sandboxName: input.sandboxName, + containerId: container.containerId, + previous: container, + }); + if (!verified.running || verified.paused) { + throw new Error(`Podman ${operation} did not recover the exact managed container.`); + } + input.log(` Container '${container.name}' ${operation === "restart" ? "restarted" : "started"}.`); + return { exitCode: 0 }; + } catch (error) { + return resultForFailure(error); + } +} + export function startPodmanSandbox( input: RuntimeProviderLifecycleInput, engine: ContainerEngine, diff --git a/src/lib/onboard/runtime-provider/podman-privileged-sandbox-control.ts b/src/lib/onboard/runtime-provider/podman-privileged-sandbox-control.ts new file mode 100644 index 00000000000..98355b8717f --- /dev/null +++ b/src/lib/onboard/runtime-provider/podman-privileged-sandbox-control.ts @@ -0,0 +1,158 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { PodmanContainerEngine } from "../../adapters/podman"; +import type { + RuntimeProviderPrivilegedSandboxCommandInput, + RuntimeProviderPrivilegedSandboxCommandResult, + RuntimeProviderPrivilegedSandboxControl, + RuntimeProviderPrivilegedSandboxTarget, +} from "./contract"; +import { observePodmanManagedContainer } from "./podman-lifecycle"; +import { + DirectSandboxFallbackUnavailableError, + PinnedSandboxResourceIdentityChangedError, +} from "./privileged-sandbox-control-errors"; +import { + clearStoppedSandboxStateWithEngine, + sandboxStateResourceFromMounts, + type StoppedSandboxStateObservation, +} from "./stopped-sandbox-state-cleanup"; + +const SANITIZED_PRIVILEGED_ENV = [ + "BASH_ENV=", + "ENV=", + "GCONV_PATH=", + "GLIBC_TUNABLES=", + "LD_AUDIT=", + "LD_LIBRARY_PATH=", + "LD_PRELOAD=", + "LOCPATH=", + "NODE_OPTIONS=", + "PERL5OPT=", + "PYTHONHOME=", + "PYTHONINSPECT=", + "PYTHONNOUSERSITE=1", + "PYTHONPATH=", + "PYTHONSTARTUP=", + "PYTHONUSERBASE=", + "RUBYOPT=", +] as const; + +function resolveTarget( + engine: PodmanContainerEngine, + input: Pick< + RuntimeProviderPrivilegedSandboxCommandInput, + "registeredSandboxNames" | "sandbox" | "sandboxName" + >, +): RuntimeProviderPrivilegedSandboxTarget { + if (input.sandbox.name !== input.sandboxName) { + throw new Error("Podman privileged control requires the registered sandbox identity."); + } + const container = observePodmanManagedContainer(engine, input.sandboxName); + if (!container || !container.running || container.paused) { + throw new DirectSandboxFallbackUnavailableError( + `No running Podman runtime resource found for sandbox '${input.sandboxName}'.`, + ); + } + return Object.freeze({ providerId: "podman", resourceHandle: container.containerId }); +} + +function execute( + engine: PodmanContainerEngine, + input: RuntimeProviderPrivilegedSandboxCommandInput, +): RuntimeProviderPrivilegedSandboxCommandResult { + const target = resolveTarget(engine, input); + if ( + input.expectedResourceHandle !== undefined && + input.expectedResourceHandle !== target.resourceHandle + ) { + throw new PinnedSandboxResourceIdentityChangedError(input.sandboxName); + } + const environment = input.sanitizeEnvironment + ? SANITIZED_PRIVILEGED_ENV.flatMap((value) => ["--env", value]) + : []; + const result = engine.capture( + [ + "container", + "exec", + ...(input.input ? ["--interactive"] : []), + ...environment, + "--user", + "root", + target.resourceHandle, + ...input.command, + ], + input.timeoutMs, + input.input, + ); + return Object.freeze({ + status: result.status, + signal: null, + stdout: Buffer.from(result.stdout, "utf8"), + stderr: Buffer.from(result.stderr, "utf8"), + ...(result.error ? { error: result.error } : {}), + }); +} + +function observeStoppedTarget( + engine: PodmanContainerEngine, + input: Parameters< + NonNullable + >[0], +): StoppedSandboxStateObservation { + let container: ReturnType; + try { + container = observePodmanManagedContainer(engine, input.sandboxName); + } catch { + return { failure: "runtime-discovery-failed" }; + } + if (!container) return { failure: "no-eligible-stopped-runtime" }; + const stateResource = sandboxStateResourceFromMounts(container.inspect.Mounts, input.paths); + return stateResource + ? { + target: { + resourceHandle: container.containerId, + running: container.running, + stateResource, + }, + } + : { failure: "state-resource-unavailable" }; +} + +export function createPodmanPrivilegedSandboxControl( + engine: PodmanContainerEngine, + cleanupEngine?: PodmanContainerEngine, +): RuntimeProviderPrivilegedSandboxControl { + if (engine.operation !== "sandbox-lifecycle" || engine.engineId !== "podman") { + throw new Error("Podman privileged control requires its sandbox-lifecycle engine."); + } + if ( + cleanupEngine && + (cleanupEngine.operation !== "workload-cleanup" || cleanupEngine.engineId !== "podman") + ) { + throw new Error("Podman stopped-state cleanup requires its workload-cleanup engine."); + } + return Object.freeze({ + resolveTarget: ( + input: Pick< + RuntimeProviderPrivilegedSandboxCommandInput, + "registeredSandboxNames" | "sandbox" | "sandboxName" + >, + ) => resolveTarget(engine, input), + execute: (input: RuntimeProviderPrivilegedSandboxCommandInput) => execute(engine, input), + ...(cleanupEngine + ? { + clearStoppedStateRoots: ( + input: Parameters< + NonNullable + >[0], + ) => + clearStoppedSandboxStateWithEngine(input.sandboxName, input.paths, { + capture: (args, timeoutMs = 30_000) => cleanupEngine.capture(args, timeoutMs), + observe: () => observeStoppedTarget(engine, input), + }), + } + : {}), + }); +} diff --git a/src/lib/onboard/runtime-provider/podman-runtime-surfaces.test.ts b/src/lib/onboard/runtime-provider/podman-runtime-surfaces.test.ts new file mode 100644 index 00000000000..32080c6b034 --- /dev/null +++ b/src/lib/onboard/runtime-provider/podman-runtime-surfaces.test.ts @@ -0,0 +1,293 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import type { PodmanBoundContainerEngine } from "../../adapters/podman"; +import { createCurrentPodmanRuntimeProviderBundle } from "./podman"; +import { + PODMAN_MANAGED_LABEL, + PODMAN_SANDBOX_CONTAINER_PREFIX, + PODMAN_SANDBOX_ID_LABEL, + PODMAN_SANDBOX_NAME_LABEL, + PODMAN_SANDBOX_NAMESPACE, + PODMAN_SANDBOX_NAMESPACE_LABEL, + PODMAN_SANDBOX_WORKSPACE, + PODMAN_SANDBOX_WORKSPACE_LABEL, +} from "./podman-lifecycle"; +import { + capturePodmanDestroyIdentity, + capturePodmanDestroyIdentityByName, + createCurrentPodmanOperationEngine, + createPodmanRuntimeProviderSnapshotSurface, + NATIVE_PODMAN_SANDBOX_HOST_ADDRESS, + prepareNativePodmanGatewayHostRuntime, + resolveNativePodmanSocketPath, +} from "./podman-runtime-surfaces"; + +const SANDBOX_NAME = "alpha"; +const SANDBOX_ID = "sandbox-alpha"; +const CONTAINER_ID = "a".repeat(64); + +function runtimeEngine(labels: () => Readonly>): PodmanBoundContainerEngine { + return { + operation: "gateway-inspection", + engineId: "podman", + displayName: "Podman", + authorityId: "podman:test-runtime", + endpointAuthorityId: "podman-path-sha256:test", + assertAuthority: vi.fn(), + capture: vi.fn((args: readonly string[]) => { + const handlers: Readonly< + Record { status: number; stdout: string; stderr: string }> + > = { + ps: () => ({ status: 0, stdout: `${CONTAINER_ID}\n`, stderr: "" }), + "container inspect": () => ({ + status: 0, + stdout: JSON.stringify([ + { + Id: CONTAINER_ID, + Name: `${PODMAN_SANDBOX_CONTAINER_PREFIX}${SANDBOX_NAME}-${SANDBOX_ID}`, + Config: { Labels: labels() }, + State: { + Running: true, + Paused: false, + Status: "running", + StartedAt: "2026-08-22T00:00:00Z", + }, + HostConfig: {}, + Annotations: {}, + }, + ]), + stderr: "", + }), + }; + const command = args[0] === "ps" ? "ps" : args.slice(0, 2).join(" "); + return ( + handlers[command] ?? + (() => ({ + status: 125, + stdout: "", + stderr: "unexpected command", + })) + )(); + }), + captureHost: vi.fn(() => ({ status: 0, stdout: "", stderr: "" })), + }; +} + +function sandbox() { + return { name: SANDBOX_NAME, agent: "openclaw" as const, openshellDriver: "podman" }; +} + +function exactLabels(): Readonly> { + return { + [PODMAN_MANAGED_LABEL]: "true", + [PODMAN_SANDBOX_ID_LABEL]: SANDBOX_ID, + [PODMAN_SANDBOX_NAME_LABEL]: SANDBOX_NAME, + [PODMAN_SANDBOX_NAMESPACE_LABEL]: PODMAN_SANDBOX_NAMESPACE, + [PODMAN_SANDBOX_WORKSPACE_LABEL]: PODMAN_SANDBOX_WORKSPACE, + }; +} + +describe("current Podman runtime provider", () => { + it("binds destroy continuity to the full Podman ownership identity", () => { + let labels = exactLabels(); + const engine = runtimeEngine(() => labels); + const first = capturePodmanDestroyIdentity( + { sandbox: sandbox(), sandboxName: SANDBOX_NAME }, + engine, + ); + labels = { ...labels, "test.identity-drift": "true" }; + const changed = capturePodmanDestroyIdentity( + { sandbox: sandbox(), sandboxName: SANDBOX_NAME }, + engine, + ); + + expect(first.resourceHandle).toBe(CONTAINER_ID); + expect(first.ownershipSha256).toMatch(/^[a-f0-9]{64}$/u); + expect(changed.resourceHandle).toBe(first.resourceHandle); + expect(changed.ownershipSha256).not.toBe(first.ownershipSha256); + expect(capturePodmanDestroyIdentityByName(SANDBOX_NAME, engine)).toEqual(changed); + }); + + it("rejects snapshot observation outside the exact OpenShell workspace", () => { + const labels = { + ...exactLabels(), + [PODMAN_SANDBOX_WORKSPACE_LABEL]: "another-workspace", + }; + const surface = createPodmanRuntimeProviderSnapshotSurface(runtimeEngine(() => labels)); + expect(surface.supported).toBe(true); + const supported = surface as Extract; + + expect(() => supported.preflight("backup", sandbox())).toThrow( + `${PODMAN_SANDBOX_WORKSPACE_LABEL}=${PODMAN_SANDBOX_WORKSPACE}`, + ); + }); + + it("does not inherit the portable Docker compatibility socket", () => { + expect( + resolveNativePodmanSocketPath({ + DOCKER_HOST: "unix:///tmp/portable-docker-compat.sock", + XDG_RUNTIME_DIR: "/run/user/1000", + }), + ).toBe("/run/user/1000/podman/podman.sock"); + }); + + it("loads the current Docker selection without a Podman host", async () => { + const current = await import("./current"); + expect( + current.resolveCurrentRuntimeProviderBundle("linux", "x64", undefined, { + HOME: "/nonexistent/nemoclaw-podman-home", + PATH: "/nonexistent/nemoclaw-podman-bin", + OPENSHELL_PODMAN_SOCKET: "/nonexistent/run/podman/podman.sock", + }).identity.id, + ).toBe("docker"); + }); + + it("registers without probing an absent Podman executable or socket", () => { + const bundle = createCurrentPodmanRuntimeProviderBundle({ + HOME: "/nonexistent/nemoclaw-podman-home", + PATH: "/nonexistent/nemoclaw-podman-bin", + OPENSHELL_PODMAN_SOCKET: "/nonexistent/run/podman/podman.sock", + }); + + expect(bundle.identity.id).toBe("podman"); + expect(bundle.bootstrap.supported).toBe(true); + expect(bundle.snapshot.supported).toBe(true); + expect(bundle.recovery.supported).toBe(true); + expect(bundle.cleanup.supported).toBe(true); + expect(bundle.containerEngine).toMatchObject({ + supported: true, + identities: expect.arrayContaining([ + expect.objectContaining({ operation: "host-doctor", engineId: "podman" }), + expect.objectContaining({ operation: "gateway-inspection", engineId: "podman" }), + expect.objectContaining({ operation: "host-local-inference", engineId: "podman" }), + expect.objectContaining({ operation: "sandbox-lifecycle", engineId: "podman" }), + expect.objectContaining({ operation: "state-mutation", engineId: "podman" }), + expect.objectContaining({ operation: "workload-cleanup", engineId: "podman" }), + ]), + }); + }); + + it("projects managed workspace preparation through the lazy production engine", () => { + const engine = createCurrentPodmanOperationEngine("managed-bootstrap", { + HOME: "/nonexistent/nemoclaw-podman-home", + PATH: "/nonexistent/nemoclaw-podman-bin", + OPENSHELL_PODMAN_SOCKET: "/nonexistent/run/podman/podman.sock", + }); + + expect(engine.prepareManagedWorkspaceRoot).toBeTypeOf("function"); + expect(engine.prepareManagedVolumeRoot).toBeTypeOf("function"); + }); + + it("projects native gateway authority independently from the portable profile", () => { + expect( + prepareNativePodmanGatewayHostRuntime({ + environment: { + OPENSHELL_PODMAN_SOCKET: "/run/user/1000/podman/podman.sock", + NEMOCLAW_EXPERIMENTAL_PROFILE: "portable", + }, + platform: "linux", + }), + ).toEqual({ + providerId: "podman", + openShellDriver: "podman", + bindAddress: "0.0.0.0", + grpcHost: NATIVE_PODMAN_SANDBOX_HOST_ADDRESS, + sshGatewayHost: "127.0.0.1", + portCheckHost: "0.0.0.0", + socketPath: "/run/user/1000/podman/podman.sock", + requiredServerIpSans: [NATIVE_PODMAN_SANDBOX_HOST_ADDRESS], + sandboxHostAddress: NATIVE_PODMAN_SANDBOX_HOST_ADDRESS, + usesHostGatewayRoute: false, + resourceOwnership: { label: "openshell.managed", value: "true" }, + gatewayConfig: { + sandboxNamespace: "omitted", + hostGatewayIp: NATIVE_PODMAN_SANDBOX_HOST_ADDRESS, + includeSupervisorBin: false, + processOwnership: "runtime-marker", + }, + network: { + sandboxSourceCidrs: expect.any(Function), + inspect: expect.any(Function), + usesHostGatewayRoute: expect.any(Function), + run: expect.any(Function), + ensureProbeImageCached: expect.any(Function), + }, + }); + }); + + it("establishes the native gateway address before projecting host runtime authority", () => { + const order: string[] = []; + const ip = vi + .fn() + .mockImplementationOnce(() => { + order.push("inspect-absent"); + return { status: 0, stdout: "", stderr: "" }; + }) + .mockImplementationOnce(() => { + order.push("inspect-configured"); + return { status: 0, stdout: "1: lo inet 169.254.2.2/32 scope global lo\n", stderr: "" }; + }); + const sudo = vi.fn(() => { + order.push("assign"); + return { status: 0, stdout: "", stderr: "" }; + }); + + const runtime = prepareNativePodmanGatewayHostRuntime( + { + environment: { OPENSHELL_PODMAN_SOCKET: "/run/user/1000/podman/podman.sock" }, + platform: "linux", + }, + runtimeEngine(exactLabels), + { ip, sudo }, + ); + + expect(runtime.grpcHost).toBe(NATIVE_PODMAN_SANDBOX_HOST_ADDRESS); + expect(order).toEqual(["inspect-absent", "assign", "inspect-configured"]); + expect(sudo).toHaveBeenCalledWith( + ["--", "ip", "address", "replace", "169.254.2.2/32", "dev", "lo"], + expect.any(Object), + ); + }); + + it("executes native gateway networking through the injected inspection authority", () => { + const capture = vi.fn((args: readonly string[]) => ({ + status: 0, + stdout: + args[0] === "network" && args[1] === "inspect" + ? JSON.stringify([{ subnets: [{ subnet: "10.89.0.0/24", gateway: "10.89.0.1" }] }]) + : "[]", + stderr: "", + })); + const gatewayInspection = { + operation: "gateway-inspection", + engineId: "podman", + displayName: "Podman", + authorityId: "podman:test-gateway-inspection", + endpointAuthorityId: "podman-path-sha256:test", + capture, + captureHost: vi.fn(() => ({ status: 0, stdout: "", stderr: "" })), + assertAuthority: vi.fn(), + } as const satisfies PodmanBoundContainerEngine; + const runtime = prepareNativePodmanGatewayHostRuntime( + { + environment: { OPENSHELL_PODMAN_SOCKET: "/run/user/1000/podman/podman.sock" }, + platform: "linux", + }, + gatewayInspection, + ); + + expect(runtime.network.run(["network", "inspect", "openshell"], 7_500)).toMatchObject({ + status: 0, + stdout: expect.stringContaining("10.89.0.0/24"), + }); + expect(capture).toHaveBeenCalledWith(["network", "inspect", "openshell"], 7_500); + expect(runtime.network.sandboxSourceCidrs()).toEqual([ + "10.89.0.0/24", + `${NATIVE_PODMAN_SANDBOX_HOST_ADDRESS}/32`, + ]); + expect(capture).toHaveBeenCalledWith(["network", "inspect", "openshell-docker"], 30_000); + }); +}); diff --git a/src/lib/onboard/runtime-provider/podman-runtime-surfaces.ts b/src/lib/onboard/runtime-provider/podman-runtime-surfaces.ts new file mode 100644 index 00000000000..29dd794e93b --- /dev/null +++ b/src/lib/onboard/runtime-provider/podman-runtime-surfaces.ts @@ -0,0 +1,761 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash, randomUUID } from "node:crypto"; +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { + capturePodmanSocketAuthority, + createPodmanContainerEngine, + type PodmanBoundContainerEngine, + type PodmanContainerEngineOptions, +} from "../../adapters/podman"; +import { + DEFAULT_GATEWAY_BIND_ADDRESS, + WILDCARD_GATEWAY_BIND_ADDRESS, +} from "../../core/gateway-address"; +import { resolveDockerDriverNetworkName } from "../experimental/docker-network-authority"; +import type { SandboxEntry } from "../../state/registry/types"; +import { + type HostLocalInferenceRouteAuthority, + type HostLocalInferenceRouteAuthorityStore, +} from "./host-local-inference"; +import { + type RuntimeProviderCleanupInput, + type RuntimeProviderDestroyIdentityReceipt, + type RuntimeProviderGatewayHostRuntime, + type RuntimeProviderGatewayHostRuntimeInput, + type RuntimeProviderRuntimeReceipt, + RUNTIME_PROVIDER_SNAPSHOT_CONTRACT_VERSION, + type RuntimeProviderSnapshotLifecycleState, + type RuntimeProviderSnapshotSurface, + type RuntimeProviderWorkloadCleanupPlan, + type RuntimeProviderWorkloadCleanupResult, +} from "./contract"; +import { observePodmanManagedContainer, type PodmanManagedContainer } from "./podman-lifecycle"; +import { resolvePodmanStateRoot } from "./podman-state-root"; + +type SupportedSnapshotSurface = Extract< + RuntimeProviderSnapshotSurface, + { readonly supported: true } +>; + +export const NATIVE_PODMAN_SANDBOX_HOST_ADDRESS = "169.254.2.2"; +export const NATIVE_PODMAN_RESOURCE_LABEL = "openshell.managed"; +export const NATIVE_PODMAN_RESOURCE_LABEL_VALUE = "true"; + +type NativePodmanHostCommandResult = { + readonly status: number | null; + readonly stdout?: string | Buffer | null; + readonly stderr?: string | Buffer | null; + readonly error?: Error; +}; + +export interface NativePodmanGatewayHostPreparationDeps { + readonly ip?: ( + args: readonly string[], + environment: NodeJS.ProcessEnv, + ) => NativePodmanHostCommandResult; + readonly sudo?: ( + args: readonly string[], + environment: NodeJS.ProcessEnv, + ) => NativePodmanHostCommandResult; +} + +function requireNativePodmanHostCommand( + result: NativePodmanHostCommandResult, + action: string, +): void { + if (result.status === 0 && !result.error) return; + const detail = String( + result.stderr ?? result.stdout ?? result.error?.message ?? "unknown failure", + ) + .replace(/\s+/gu, " ") + .trim() + .slice(-500); + throw new Error(`${action} failed${detail ? `: ${detail}` : "."}`); +} + +function nativePodmanHostAddressState(output: string): "absent" | "configured" | "conflicting" { + const assignments = output + .split(/\r?\n/u) + .map((line) => line.match(/^\d+:\s+(\S+)\s+inet\s+169\.254\.2\.2\/(\d+)\b/u)) + .filter((match): match is RegExpMatchArray => match !== null); + if (assignments.length === 0) return "absent"; + return assignments.length === 1 && assignments[0]?.[1] === "lo" && assignments[0]?.[2] === "32" + ? "configured" + : "conflicting"; +} + +export function ensureNativePodmanGatewayHostAddress( + environment: NodeJS.ProcessEnv, + deps: NativePodmanGatewayHostPreparationDeps = {}, +): void { + const ip = + deps.ip ?? + ((args, env) => + spawnSync("ip", [...args], { + encoding: "utf8", + env, + stdio: ["ignore", "pipe", "pipe"], + timeout: 15_000, + })); + const sudo = + deps.sudo ?? + ((args, env) => + spawnSync("sudo", [...args], { + encoding: "utf8", + env, + stdio: ["ignore", "pipe", "pipe"], + timeout: 30_000, + })); + const inspect = () => { + const result = ip(["-o", "-4", "address", "show"], environment); + requireNativePodmanHostCommand(result, "Inspecting the native Podman gateway address"); + return nativePodmanHostAddressState(String(result.stdout ?? "")); + }; + const initial = inspect(); + if (initial === "configured") return; + if (initial === "conflicting") { + throw new Error( + `Native Podman gateway address ${NATIVE_PODMAN_SANDBOX_HOST_ADDRESS} has a conflicting host assignment.`, + ); + } + requireNativePodmanHostCommand( + sudo( + ["--", "ip", "address", "replace", `${NATIVE_PODMAN_SANDBOX_HOST_ADDRESS}/32`, "dev", "lo"], + environment, + ), + "Configuring the native Podman gateway address", + ); + if (inspect() !== "configured") { + throw new Error( + `Native Podman gateway address ${NATIVE_PODMAN_SANDBOX_HOST_ADDRESS}/32 was not established on loopback.`, + ); + } +} + +type PodmanMount = Readonly>; +const PODMAN_CONTAINER_ID = /^[a-f0-9]{64}$/u; +const PODMAN_IMAGE_REFERENCE = /^[A-Za-z0-9][A-Za-z0-9._:/@-]{0,511}$/u; +const PODMAN_STORAGE_PROBE_TIMEOUT_MS = 5_000; + +function mountRecord(value: unknown): PodmanMount | null { + return typeof value === "object" && value !== null && !Array.isArray(value) + ? (value as PodmanMount) + : null; +} + +function isMaterializedPodmanImageBind( + value: PodmanMount | null, + storageGraphRoot: string, + containerId: string, +): boolean { + if (value?.Type !== "bind" || value.RW !== true) return false; + const source = String(value.Source ?? ""); + if (!path.isAbsolute(source) || path.normalize(source) !== source) return false; + const relative = path.relative(storageGraphRoot, source); + const segments = relative.split(path.sep); + return ( + relative.length > 0 && + !path.isAbsolute(relative) && + segments.length === 6 && + segments[0] === "overlay-containers" && + segments[1] === containerId && + segments[2] === "userdata" && + segments[3] === "overlay" && + Boolean(segments[4]) && + segments[5] === "merge" + ); +} + +/** Read the exact graphroot from the already authority-bound Podman engine. */ +export function resolvePodmanStorageGraphRoot(engine: PodmanBoundContainerEngine): string { + const result = engine.capture( + ["info", "--format", "{{.Store.GraphRoot}}"], + PODMAN_STORAGE_PROBE_TIMEOUT_MS, + ); + if (result.status !== 0 || result.error) { + throw new Error("Native Podman storage graphroot is unavailable."); + } + return normalizedAbsolutePath(result.stdout.trim(), "storage graphroot"); +} + +/** + * Collapse one read-only image mount and its Podman storage bind into the + * provider-owned image identity. Other duplicate destinations remain visible. + */ +export function normalizePodmanLogicalMounts( + mounts: readonly unknown[], + storageGraphRoot: string, + containerId: string, +): readonly unknown[] { + const normalizedGraphRoot = normalizedAbsolutePath(storageGraphRoot, "storage graphroot"); + if (!PODMAN_CONTAINER_ID.test(containerId)) { + throw new Error("Native Podman container identity must be one full content ID."); + } + const groups = new Map(); + mounts.forEach((value, index) => { + const destination = mountRecord(value)?.Destination; + const group = typeof destination === "string" && destination.length > 0 ? destination : index; + const observed = groups.get(group) ?? []; + observed.push(value); + groups.set(group, observed); + }); + + const normalized: unknown[] = []; + for (const observed of groups.values()) { + if (observed.length === 1) { + normalized.push(observed[0]); + continue; + } + const records = observed.map(mountRecord); + const imageMounts = records.filter((candidate) => candidate?.Type === "image"); + const materializedBinds = records.filter((candidate) => candidate?.Type === "bind"); + if ( + imageMounts.length === 1 && + materializedBinds.length === 1 && + observed.length === 2 && + imageMounts[0]?.RW === false && + PODMAN_IMAGE_REFERENCE.test(String(imageMounts[0]?.Source ?? "")) && + isMaterializedPodmanImageBind(materializedBinds[0] ?? null, normalizedGraphRoot, containerId) + ) { + normalized.push(imageMounts[0] as PodmanMount); + } else { + normalized.push(...observed); + } + } + return Object.freeze(normalized); +} + +const FULL_ID = /^[a-f0-9]{64}$/u; +const ROUTE_STORE_DIRECTORY = "runtime-provider-podman"; +const ROUTE_STORE_FILE = "host-local-inference-route.json"; + +type JsonRecord = Record; + +class PodmanRuntimeSurfaceError extends Error { + constructor(message: string) { + super(`Runtime snapshot provider failed: ${message}`); + this.name = "RuntimeProviderSnapshotError"; + } +} + +function record(value: unknown, label: string): JsonRecord { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new PodmanRuntimeSurfaceError(`Podman ${label} must be an object`); + } + return value as JsonRecord; +} + +function normalizedAbsolutePath(value: string, label: string): string { + const candidate = value.trim(); + if ( + !path.isAbsolute(candidate) || + path.normalize(candidate) !== candidate || + /[\0\r\n]/u.test(candidate) + ) { + throw new Error(`Native Podman ${label} must be one normalized absolute path.`); + } + return candidate; +} + +export function resolveNativePodmanSocketPath( + environment: NodeJS.ProcessEnv = process.env, + explicitSocketPath?: string, +): string { + if (explicitSocketPath) return normalizedAbsolutePath(explicitSocketPath, "socket"); + const configured = environment.OPENSHELL_PODMAN_SOCKET?.trim(); + if (configured) return normalizedAbsolutePath(configured, "OPENSHELL_PODMAN_SOCKET"); + const uid = typeof process.getuid === "function" ? process.getuid() : os.userInfo().uid; + const runtimeDirectory = environment.XDG_RUNTIME_DIR?.trim() || `/run/user/${String(uid)}`; + return normalizedAbsolutePath(path.join(runtimeDirectory, "podman", "podman.sock"), "socket"); +} + +export function prepareNativePodmanGatewayHostRuntime( + input: RuntimeProviderGatewayHostRuntimeInput, + boundEngine?: PodmanBoundContainerEngine, + hostPreparation?: NativePodmanGatewayHostPreparationDeps, +): RuntimeProviderGatewayHostRuntime { + if (input.platform !== "linux") { + throw new Error("Native Podman gateway runtime is supported only on Linux."); + } + const engine = + boundEngine ?? createCurrentPodmanOperationEngine("gateway-inspection", input.environment); + if (engine.operation !== "gateway-inspection") { + throw new Error("Native Podman gateway runtime requires its gateway-inspection engine."); + } + if (hostPreparation) ensureNativePodmanGatewayHostAddress(input.environment, hostPreparation); + const run = (args: readonly string[], timeoutMs: number) => { + const result = engine.capture(args, timeoutMs); + const error = result.error as NodeJS.ErrnoException | undefined; + return { + status: result.status, + stderr: result.stderr, + stdout: result.stdout, + error: error?.message, + errorCode: error?.code ?? null, + timedOut: error?.code === "ETIMEDOUT", + }; + }; + const inspectNetwork = (networkName: string) => { + const result = run(["network", "inspect", networkName], 30_000); + if (result.status !== 0) return undefined; + try { + const parsed = JSON.parse(String(result.stdout ?? "")) as unknown; + const record = Array.isArray(parsed) ? parsed[0] : parsed; + if (!record || typeof record !== "object" || Array.isArray(record)) return undefined; + const subnets = (record as { subnets?: unknown }).subnets; + if (!Array.isArray(subnets)) return undefined; + for (const subnet of subnets) { + if (!subnet || typeof subnet !== "object" || Array.isArray(subnet)) continue; + const values = subnet as { subnet?: unknown; gateway?: unknown }; + const subnetValue = typeof values.subnet === "string" ? values.subnet : undefined; + const gatewayIp = typeof values.gateway === "string" ? values.gateway : undefined; + if (gatewayIp && !gatewayIp.includes(":")) { + return { ...(subnetValue ? { subnet: subnetValue } : {}), gatewayIp }; + } + } + return undefined; + } catch { + return undefined; + } + }; + return Object.freeze({ + providerId: "podman", + openShellDriver: "podman", + bindAddress: WILDCARD_GATEWAY_BIND_ADDRESS, + grpcHost: NATIVE_PODMAN_SANDBOX_HOST_ADDRESS, + sshGatewayHost: DEFAULT_GATEWAY_BIND_ADDRESS, + portCheckHost: WILDCARD_GATEWAY_BIND_ADDRESS, + socketPath: resolveNativePodmanSocketPath(input.environment, input.socketPath), + requiredServerIpSans: Object.freeze([NATIVE_PODMAN_SANDBOX_HOST_ADDRESS]), + sandboxHostAddress: NATIVE_PODMAN_SANDBOX_HOST_ADDRESS, + usesHostGatewayRoute: false, + resourceOwnership: Object.freeze({ + label: NATIVE_PODMAN_RESOURCE_LABEL, + value: NATIVE_PODMAN_RESOURCE_LABEL_VALUE, + }), + gatewayConfig: Object.freeze({ + sandboxNamespace: "omitted" as const, + hostGatewayIp: NATIVE_PODMAN_SANDBOX_HOST_ADDRESS, + includeSupervisorBin: false, + processOwnership: "runtime-marker" as const, + }), + network: Object.freeze({ + sandboxSourceCidrs: () => { + const network = inspectNetwork(resolveDockerDriverNetworkName(input.environment)); + return [ + ...(network?.subnet ? [network.subnet] : []), + `${NATIVE_PODMAN_SANDBOX_HOST_ADDRESS}/32`, + ]; + }, + inspect: inspectNetwork, + usesHostGatewayRoute: () => false, + run, + ensureProbeImageCached: (image: string) => { + const inspect = run(["image", "inspect", image], 10_000); + if (inspect.status === 0) return { ok: true as const, alreadyCached: true }; + if (inspect.status === null || inspect.errorCode) { + return { + ok: false as const, + reason: "inspect_unavailable" as const, + details: inspect.error ?? String(inspect.stderr ?? "").trim(), + }; + } + const pull = run(["pull", image], 120_000); + if (pull.status === 0) return { ok: true as const, alreadyCached: false }; + return { + ok: false as const, + reason: pull.timedOut ? ("pull_timeout" as const) : ("pull_failed" as const), + details: pull.error ?? String(pull.stderr ?? "").trim(), + }; + }, + }), + }); +} + +export type CurrentPodmanOperation = PodmanContainerEngineOptions["operation"]; + +export function createCurrentPodmanOperationEngine( + operation: CurrentPodmanOperation, + environment: NodeJS.ProcessEnv = process.env, +): PodmanBoundContainerEngine { + const socketPath = resolveNativePodmanSocketPath(environment); + const endpointAuthorityId = `podman-path-sha256:${createHash("sha256").update(socketPath, "utf8").digest("hex")}`; + let bound: PodmanBoundContainerEngine | null = null; + const resolve = (): PodmanBoundContainerEngine => { + if (bound) return bound; + const socketAuthority = capturePodmanSocketAuthority(socketPath); + bound = createPodmanContainerEngine({ + operation, + socketAuthority, + executableSearchEnv: environment, + commandEnvironment: Object.freeze({ + HOME: environment.HOME ?? os.homedir(), + PATH: environment.PATH ?? "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + ...(environment.XDG_RUNTIME_DIR ? { XDG_RUNTIME_DIR: environment.XDG_RUNTIME_DIR } : {}), + ...(operation === "managed-bootstrap" && environment.CONTAINERS_CONF + ? { CONTAINERS_CONF: environment.CONTAINERS_CONF } + : {}), + ...(operation === "managed-bootstrap" && environment.CONTAINERS_STORAGE_CONF + ? { CONTAINERS_STORAGE_CONF: environment.CONTAINERS_STORAGE_CONF } + : {}), + }), + }); + return bound; + }; + return Object.freeze({ + operation, + engineId: "podman", + displayName: "Podman", + endpointAuthorityId, + get authorityId() { + return resolve().authorityId; + }, + capture: ( + args: Parameters[0], + timeout?: number, + input?: Buffer, + ) => resolve().capture(args, timeout, input), + captureHost: ( + args: Parameters[0], + timeout?: number, + ) => resolve().captureHost(args, timeout), + assertAuthority: () => resolve().assertAuthority(), + ...(operation === "managed-bootstrap" + ? { + prepareManagedWorkspaceRoot: ( + input: Parameters< + NonNullable + >[0], + ) => { + const prepare = resolve().prepareManagedWorkspaceRoot; + if (!prepare) { + throw new Error( + "Podman managed-bootstrap engine did not expose workspace-root preparation.", + ); + } + return prepare(input); + }, + prepareManagedVolumeRoot: ( + input: Parameters< + NonNullable + >[0], + ) => { + const prepare = resolve().prepareManagedVolumeRoot; + if (!prepare) { + throw new Error( + "Podman managed-bootstrap engine did not expose volume-root preparation.", + ); + } + return prepare(input); + }, + } + : {}), + }); +} + +function capture(engine: PodmanBoundContainerEngine, args: readonly string[], label: string) { + const result = engine.capture(args, 30_000); + if (result.status !== 0 || result.error) { + throw new PodmanRuntimeSurfaceError( + `${label} failed: ${(result.stderr || result.stdout || result.error?.message || "unknown failure").replace(/\s+/gu, " ").trim().slice(-500)}`, + ); + } + return result; +} + +function lifecycle(inspect: JsonRecord): { + readonly state: RuntimeProviderSnapshotLifecycleState; + readonly generation: string; +} { + const state = record(inspect.State, "State"); + const running = state.Running; + const paused = state.Paused; + const status = String(state.Status ?? "") + .trim() + .toLowerCase(); + let normalized: RuntimeProviderSnapshotLifecycleState; + if (running === true && paused === true) normalized = "paused"; + else if (running === true && paused !== true) normalized = "running"; + else if (["configured", "created", "dead", "exited", "stopped"].includes(status)) + normalized = "stopped"; + else + throw new PodmanRuntimeSurfaceError( + `Podman lifecycle '${status || "unknown"}' cannot be represented`, + ); + return { + state: normalized, + generation: createHash("sha256") + .update( + JSON.stringify({ + id: inspect.Id, + status, + paused: paused === true, + startedAt: state.StartedAt ?? "", + finishedAt: state.FinishedAt ?? "", + restartCount: state.RestartCount ?? 0, + }), + "utf8", + ) + .digest("hex"), + }; +} + +function acceleration(inspect: JsonRecord): RuntimeProviderRuntimeReceipt["acceleration"] { + const hostConfig = record(inspect.HostConfig ?? {}, "HostConfig"); + const selectors: string[] = []; + for (const value of Array.isArray(hostConfig.Devices) ? hostConfig.Devices : []) { + const device = record(value, "HostConfig.Devices entry"); + const hostPath = String(device.PathOnHost ?? "").trim(); + const containerPath = String(device.PathInContainer ?? "").trim(); + if (/^\/dev\/(?:nvidia|dri|nvhost|nvmap|tegra)/u.test(hostPath)) { + selectors.push(`podman-device-path:${hostPath}=>${containerPath}`); + } + } + const annotations = record(inspect.Annotations ?? {}, "Annotations"); + for (const [key, value] of Object.entries(annotations)) { + if (/cdi|nvidia|gpu/iu.test(key) && typeof value === "string" && value.trim()) { + selectors.push(`podman-annotation:${key}=${value.trim()}`); + } + } + const devices = [...new Set(selectors)].sort(); + return devices.length > 0 ? { kind: "gpu", vendor: "nvidia", devices } : { kind: "none" }; +} + +function observePodmanRuntime( + sandbox: SandboxEntry, + providerId: string, + engine: PodmanBoundContainerEngine, +) { + if (sandbox.openshellDriver !== providerId) { + throw new PodmanRuntimeSurfaceError( + `sandbox '${sandbox.name}' belongs to another runtime provider`, + ); + } + let container: PodmanManagedContainer | null; + try { + container = observePodmanManagedContainer(engine, sandbox.name); + } catch (error) { + throw new PodmanRuntimeSurfaceError(error instanceof Error ? error.message : String(error)); + } + if (!container) { + throw new PodmanRuntimeSurfaceError( + `sandbox '${sandbox.name}' exact Podman runtime identity could not be inspected`, + ); + } + const inspect = container.inspect; + const id = container.containerId; + const state = lifecycle(inspect); + return Object.freeze({ + lifecycleState: state.state, + lifecycleGeneration: state.generation, + runtime: Object.freeze({ + schemaVersion: 1 as const, + providerId, + runtime: Object.freeze({ kind: "podman-container", handle: id }), + acceleration: acceleration(inspect), + }), + }); +} + +function stableLabels(labels: Readonly>): Readonly> { + return Object.freeze( + Object.fromEntries(Object.entries(labels).sort(([left], [right]) => left.localeCompare(right))), + ); +} + +export function capturePodmanDestroyIdentity( + input: RuntimeProviderCleanupInput, + engine: PodmanBoundContainerEngine, +): RuntimeProviderDestroyIdentityReceipt { + if (input.sandbox.openshellDriver !== "podman") { + throw new Error(`Sandbox '${input.sandboxName}' belongs to another runtime provider.`); + } + return capturePodmanDestroyIdentityByName(input.sandboxName, engine); +} + +export function capturePodmanDestroyIdentityByName( + sandboxName: string, + engine: PodmanBoundContainerEngine, +): RuntimeProviderDestroyIdentityReceipt { + const container = observePodmanManagedContainer(engine, sandboxName); + if (!container) { + return Object.freeze({ + schemaVersion: 1 as const, + providerId: "podman", + resourceHandle: null, + ownershipSha256: null, + }); + } + const ownershipSha256 = createHash("sha256") + .update( + JSON.stringify({ + containerId: container.containerId, + labels: stableLabels(container.labels), + name: container.name, + sandboxId: container.sandboxId, + sandboxNamespace: container.sandboxNamespace, + }), + "utf8", + ) + .digest("hex"); + return Object.freeze({ + schemaVersion: 1 as const, + providerId: "podman", + resourceHandle: container.containerId, + ownershipSha256, + }); +} + +function restoreManagedProfile( + sandbox: SandboxEntry, + authority: { readonly agent: string; readonly profileFingerprint: string }, + runtime: RuntimeProviderRuntimeReceipt, + engine: PodmanBoundContainerEngine, +): string { + if (runtime.runtime.kind !== "podman-container" || !FULL_ID.test(runtime.runtime.handle)) { + throw new PodmanRuntimeSurfaceError( + "Podman managed profile restore runtime identity is invalid", + ); + } + const result = capture( + engine, + [ + "container", + "exec", + "--user", + "root", + runtime.runtime.handle, + "/usr/bin/env", + "-i", + "HOME=/root", + "LANG=C.UTF-8", + "LC_ALL=C.UTF-8", + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "/usr/local/bin/node", + "/usr/local/lib/nemoclaw/managed-startup-image-runtime.cjs", + "--verify-completion", + "--agent", + authority.agent, + "--profile-fingerprint", + authority.profileFingerprint, + ], + "managed profile restore verification", + ); + return createHash("sha256") + .update(sandbox.name, "utf8") + .update("\0", "utf8") + .update(authority.agent, "utf8") + .update("\0", "utf8") + .update(authority.profileFingerprint, "utf8") + .update("\0", "utf8") + .update(result.stdout, "utf8") + .digest("hex"); +} + +export function createPodmanRuntimeProviderSnapshotSurface( + engine: PodmanBoundContainerEngine, +): RuntimeProviderSnapshotSurface { + if (engine.operation !== "gateway-inspection" || engine.engineId !== "podman") { + throw new Error("Podman snapshot requires a gateway-inspection engine."); + } + const surface = (): SupportedSnapshotSurface => { + const { createRuntimeProviderSnapshotSurface } = + require("./snapshot") as typeof import("./snapshot"); + return createRuntimeProviderSnapshotSurface("podman", { + observe: (sandbox, providerId) => observePodmanRuntime(sandbox, providerId, engine), + restoreManagedProfile: (sandbox, authority, runtime) => + restoreManagedProfile(sandbox, authority, runtime, engine), + }) as SupportedSnapshotSurface; + }; + return Object.freeze({ + providerId: "podman", + supported: true, + contractVersion: RUNTIME_PROVIDER_SNAPSHOT_CONTRACT_VERSION, + capabilities: Object.freeze({ backup: true, restore: true, managedProfileRestore: true }), + preflight: (...args: Parameters) => + surface().preflight(...args), + capture: (...args: Parameters) => + surface().capture(...args), + validateRestore: (...args: Parameters) => + surface().validateRestore(...args), + restore: (...args: Parameters) => + surface().restore(...args), + }); +} + +export function planOwnedPodmanWorkloadCleanup( + input: RuntimeProviderCleanupInput, +): RuntimeProviderWorkloadCleanupPlan { + const workload = input.sandbox.workload; + if (!workload || workload.kind === "native-artifact") { + return { action: "retain", reason: "no-owned-image" }; + } + if (workload.kind === "managed-image") { + return { action: "retain", reason: "shared-image" }; + } + if (workload.reference === null) return { action: "retain", reason: "no-owned-image" }; + return { action: "block", reason: "authority-unproven" }; +} + +export function removeOwnedPodmanWorkload( + input: RuntimeProviderCleanupInput, + engine: PodmanBoundContainerEngine, +): RuntimeProviderWorkloadCleanupResult { + const plan = planOwnedPodmanWorkloadCleanup(input); + if (plan.action === "retain") return { status: "skipped", reason: plan.reason }; + if (plan.action === "block") return { status: "skipped", reason: "authority-unproven" }; + const result = engine.capture(["image", "rm", plan.reference], 60_000); + return { + status: result.status === 0 && !result.error ? "removed" : "failed", + engineDisplayName: plan.engineDisplayName, + reference: plan.reference, + }; +} + +function routeAuthorityPath(stateRoot: string): string { + return path.join(stateRoot, ROUTE_STORE_DIRECTORY, ROUTE_STORE_FILE); +} + +export function createFilePodmanRouteAuthorityStore( + stateRoot = resolvePodmanStateRoot(), +): HostLocalInferenceRouteAuthorityStore { + const target = routeAuthorityPath(stateRoot); + return Object.freeze({ + load: () => { + try { + return JSON.parse(fs.readFileSync(target, "utf8")) as HostLocalInferenceRouteAuthority; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw error; + } + }, + record: (authority: HostLocalInferenceRouteAuthority) => { + fs.mkdirSync(path.dirname(target), { recursive: true, mode: 0o700 }); + const serialized = `${JSON.stringify(authority)}\n`; + const existing = (() => { + try { + return fs.readFileSync(target, "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw error; + } + })(); + if (existing !== null) { + if (existing !== serialized) + throw new Error("Podman route authority conflicts with its durable record."); + return authority; + } + const temporary = `${target}.${randomUUID()}.tmp`; + fs.writeFileSync(temporary, serialized, { mode: 0o600, flag: "wx" }); + fs.renameSync(temporary, target); + fs.chmodSync(target, 0o600); + return authority; + }, + }); +} diff --git a/src/lib/onboard/runtime-provider/podman-state-mutation.test.ts b/src/lib/onboard/runtime-provider/podman-state-mutation.test.ts index 42b4401b4bc..02233ebddd7 100644 --- a/src/lib/onboard/runtime-provider/podman-state-mutation.test.ts +++ b/src/lib/onboard/runtime-provider/podman-state-mutation.test.ts @@ -16,6 +16,7 @@ import { type HermesRuntimeStateMutationConfigTarget, } from "../../shields/hermes-runtime-state-mutation"; import { createPodmanRuntimeProviderBundle } from "./podman"; +import { createPodmanStateMutationSurface } from "./podman-state-mutation"; import { createRuntimeProviderBundleRegistry } from "./registry"; function companionEngine( @@ -55,6 +56,36 @@ function hermesConfigTarget(): HermesRuntimeStateMutationConfigTarget { afterEach(() => cleanupDockerStateMutationRoots()); describe("Podman runtime-provider state mutation", () => { + it("treats a materialized image bind as one logical Podman mount", () => { + const runtime = harness({ podmanMaterializedImageMount: true }); + const surface = createPodmanStateMutationSurface({ + engine: runtime.authority.engine as PodmanBoundContainerEngine, + resolveStateDir: () => runtime.root, + }); + + expect(surface.acquire({ ...runtime.context, plan: plan() })).toMatchObject({ + providerId: "podman", + phase: "fenced", + }); + expect( + runtime.capture.mock.calls.some(([, args]) => + (args as readonly string[]).includes("label=openshell.managed=true"), + ), + ).toBe(true); + }); + + it("rejects unrelated Podman mounts with the same destination", () => { + const runtime = harness({ podmanAmbiguousMounts: true }); + const surface = createPodmanStateMutationSurface({ + engine: runtime.authority.engine as PodmanBoundContainerEngine, + resolveStateDir: () => runtime.root, + }); + + expect(() => surface.acquire({ ...runtime.context, plan: plan() })).toThrow( + "Podman container has ambiguous mount destinations", + ); + }); + it("holds one exact Podman fence through rollback, activation, and durable release", () => { const runtime = harness(); const fence = runtime.owner.acquire({ ...runtime.context, plan: plan() }); @@ -79,6 +110,9 @@ describe("Podman runtime-provider state mutation", () => { runtime.owner.release(runtime.context, fence, proof, "e".repeat(64)); expect(runtime.lifecycleStore.listUnfinished()).toEqual([]); + expect(runtime.transportBrokerActive()).toBe(false); + expect(runtime.transportCopySourceModes.length).toBeGreaterThan(0); + expect(runtime.transportCopySourceModes.every((mode) => mode === 0o644)).toBe(true); expect(runtime.helperActions).toEqual([ "acquire", "rollback", @@ -86,13 +120,32 @@ describe("Podman runtime-provider state mutation", () => { "activate", "release", ]); + const inspectCommands = runtime.capture.mock.calls + .map(([, args]) => args as readonly string[]) + .filter((args) => args.includes("inspect")); + expect(inspectCommands.length).toBeGreaterThan(0); + expect( + inspectCommands.every((args) => args.some((value) => value.includes("{{json .ID}}"))), + ).toBe(true); + expect( + inspectCommands.every((args) => args.every((value) => !value.includes("{{json .Id}}"))), + ).toBe(true); + expect( + runtime.capture.mock.calls + .filter(([, args]) => !(args as readonly string[]).includes("--nemoclaw-broker")) + .every(([, args]) => + (args as readonly string[]) + .slice(0, 2) + .every((value, index) => + index === 0 + ? value === "--url" + : value === "unix:///run/user/1000/podman/podman.sock", + ), + ), + ).toBe(true); expect( - runtime.capture.mock.calls.every(([, args]) => - (args as readonly string[]) - .slice(0, 2) - .every((value, index) => - index === 0 ? value === "--url" : value === "unix:///run/user/1000/podman/podman.sock", - ), + runtime.capture.mock.calls.some(([, args]) => + (args as readonly string[]).includes("--detach"), ), ).toBe(true); }); diff --git a/src/lib/onboard/runtime-provider/podman-state-mutation.ts b/src/lib/onboard/runtime-provider/podman-state-mutation.ts index 60f6709cdc0..de5c3ec1b0f 100644 --- a/src/lib/onboard/runtime-provider/podman-state-mutation.ts +++ b/src/lib/onboard/runtime-provider/podman-state-mutation.ts @@ -7,6 +7,12 @@ import { createContainerStateMutationSurface, type ContainerStateMutationSurfaceOptions, } from "./container-state-mutation"; +import { + NATIVE_PODMAN_RESOURCE_LABEL, + NATIVE_PODMAN_RESOURCE_LABEL_VALUE, + normalizePodmanLogicalMounts, + resolvePodmanStorageGraphRoot, +} from "./podman-runtime-surfaces"; export interface PodmanStateMutationSurfaceOptions { readonly engine: PodmanBoundContainerEngine; @@ -25,6 +31,16 @@ export function createPodmanStateMutationSurface( providerId: "podman", providerDisplayName: "Podman", engineOperation: "state-mutation", + runtimeIdInspectField: "ID", + privatePidMode: "private", + managedLabelKey: NATIVE_PODMAN_RESOURCE_LABEL, + managedLabelValue: NATIVE_PODMAN_RESOURCE_LABEL_VALUE, + normalizeInspectionMounts: (mounts, runtimeId) => + normalizePodmanLogicalMounts( + mounts, + resolvePodmanStorageGraphRoot(options.engine), + runtimeId, + ), createAuthority: () => ({ assertAuthority: options.engine.assertAuthority, engine: options.engine, diff --git a/src/lib/onboard/runtime-provider/podman-state-root.ts b/src/lib/onboard/runtime-provider/podman-state-root.ts new file mode 100644 index 00000000000..62587c3af2e --- /dev/null +++ b/src/lib/onboard/runtime-provider/podman-state-root.ts @@ -0,0 +1,9 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { resolveNemoclawStateDir } from "../../state/paths"; + +/** Provider-owned state root shared by the registered Podman surfaces. */ +export function resolvePodmanStateRoot(homeDir?: string): string { + return resolveNemoclawStateDir(homeDir); +} diff --git a/src/lib/onboard/runtime-provider/podman.test.ts b/src/lib/onboard/runtime-provider/podman.test.ts index 9710ea2b099..57a852bb555 100644 --- a/src/lib/onboard/runtime-provider/podman.test.ts +++ b/src/lib/onboard/runtime-provider/podman.test.ts @@ -5,6 +5,7 @@ import { describe, expect, it, vi } from "vitest"; import { createPodmanHostLocalInferenceTestHarness } from "../../../../test/helpers/podman-host-local-inference-test-harness"; import { startSandbox } from "../../actions/sandbox/start"; import { stopSandbox } from "../../actions/sandbox/stop"; +import type { ContainerEngineCommandResult } from "../../adapters/container-engine"; import { createPodmanContainerEngine, type PodmanBoundContainerEngine, @@ -13,7 +14,7 @@ import { type PodmanExecutableStat, type PodmanSocketAuthority, } from "../../adapters/podman"; -import type { SandboxEntry } from "../../state/registry/types"; +import type { SandboxEntry, SandboxWorkloadReceipt } from "../../state/registry/types"; import { CURRENT_RUNTIME_PROVIDER_BUNDLES } from "./current"; import { createPodmanRuntimeProviderBundle } from "./podman"; import { @@ -30,6 +31,7 @@ import { createRuntimeProviderBundleRegistry, requireRuntimeProviderHostLocalInferenceOperation, } from "./registry"; +import { clearStoppedSandboxStateWithEngine } from "./stopped-sandbox-state-cleanup"; const AGENTS = ["openclaw", "hermes", "langchain-deepagents-code"] as const; const CONTAINER_ID = "a".repeat(64); @@ -160,6 +162,41 @@ function lifecycleEngine(sandboxName: string, authorityId = AUTHORITY_ID): Podma let running = false; const sandboxId = `id-${sandboxName}`; const containerName = `${PODMAN_SANDBOX_CONTAINER_PREFIX}${sandboxName}-${sandboxId}`; + const containerOperations: Readonly ContainerEngineCommandResult>> = { + exec: () => ({ status: 0, stdout: "uid=0\n", stderr: "" }), + inspect: () => ({ + status: 0, + stdout: JSON.stringify([ + { + Id: CONTAINER_ID, + Name: containerName, + Config: { + Labels: { + [PODMAN_MANAGED_LABEL]: "true", + [PODMAN_SANDBOX_ID_LABEL]: sandboxId, + [PODMAN_SANDBOX_NAME_LABEL]: sandboxName, + [PODMAN_SANDBOX_NAMESPACE_LABEL]: PODMAN_SANDBOX_NAMESPACE, + [PODMAN_SANDBOX_WORKSPACE_LABEL]: PODMAN_SANDBOX_WORKSPACE, + }, + }, + Mounts: [ + { + Type: "volume", + Name: `nemoclaw-${sandboxName}-state`, + Destination: "/sandbox", + RW: true, + }, + ], + State: { + Running: running, + Paused: false, + Status: running ? "running" : "exited", + }, + }, + ]), + stderr: "", + }), + }; return { operation: "sandbox-lifecycle", engineId: "podman", @@ -176,30 +213,10 @@ function lifecycleEngine(sandboxName: string, authorityId = AUTHORITY_ID): Podma stderr: "", }; case "container": - return { - status: 0, - stdout: JSON.stringify([ - { - Id: CONTAINER_ID, - Name: containerName, - Config: { - Labels: { - [PODMAN_MANAGED_LABEL]: "true", - [PODMAN_SANDBOX_ID_LABEL]: sandboxId, - [PODMAN_SANDBOX_NAME_LABEL]: sandboxName, - [PODMAN_SANDBOX_NAMESPACE_LABEL]: PODMAN_SANDBOX_NAMESPACE, - [PODMAN_SANDBOX_WORKSPACE_LABEL]: PODMAN_SANDBOX_WORKSPACE, - }, - }, - State: { - Running: running, - Paused: false, - Status: running ? "running" : "exited", - }, - }, - ]), - stderr: "", - }; + return ( + containerOperations[String(args[1])] ?? + (() => ({ status: 125, stdout: "", stderr: "unexpected container operation" })) + )(); case "start": running = true; return { status: 0, stdout: CONTAINER_ID, stderr: "" }; @@ -230,7 +247,7 @@ function providerHarness(agent: (typeof AGENTS)[number]) { return { entry, lifecycle, providers, sandboxName }; } -describe("dormant Podman runtime provider", () => { +describe("managed Podman runtime provider", () => { it.each(AGENTS)( "runs basic CPU start and stop for %s through an injected bundle", async (agent) => { @@ -292,9 +309,149 @@ describe("dormant Podman runtime provider", () => { ).toBe(true); }); - it("stays outside the production-selectable registry", () => { - expect(Object.keys(CURRENT_RUNTIME_PROVIDER_BUNDLES)).toEqual(["docker", "kubernetes"]); - expect(CURRENT_RUNTIME_PROVIDER_BUNDLES).not.toHaveProperty("podman"); + it("executes privileged control through the lifecycle-bound Podman engine", () => { + const runtime = providerHarness("openclaw"); + const lifecycle = runtime.providers.podman?.lifecycle; + expect(lifecycle).toMatchObject({ supported: true }); + const supportedLifecycle = lifecycle as Extract< + NonNullable, + { readonly supported: true } + >; + + supportedLifecycle.start({ + environment: {}, + log: vi.fn(), + sandbox: runtime.entry, + sandboxName: runtime.sandboxName, + }); + const target = supportedLifecycle.privilegedSandboxControl.resolveTarget({ + registeredSandboxNames: [runtime.sandboxName], + sandbox: runtime.entry, + sandboxName: runtime.sandboxName, + }); + const result = supportedLifecycle.privilegedSandboxControl.execute({ + registeredSandboxNames: [runtime.sandboxName], + sandbox: runtime.entry, + sandboxName: runtime.sandboxName, + command: ["/usr/bin/id", "-u"], + expectedResourceHandle: target.resourceHandle, + sanitizeEnvironment: false, + timeoutMs: 9000, + }); + + expect(target).toEqual({ providerId: "podman", resourceHandle: CONTAINER_ID }); + expect(result).toMatchObject({ status: 0, signal: null }); + expect(result.stdout.toString("utf8")).toBe("uid=0\n"); + expect(runtime.lifecycle.capture).toHaveBeenLastCalledWith( + ["container", "exec", "--user", "root", CONTAINER_ID, "/usr/bin/id", "-u"], + 9000, + undefined, + ); + expect( + JSON.stringify((runtime.lifecycle.capture as ReturnType).mock.calls), + ).not.toContain("docker"); + }); + + it("routes stopped state cleanup through the Podman workload-cleanup engine", () => { + const sandboxName = "podman-cleanup"; + const lifecycle = lifecycleEngine(sandboxName); + const cleanupCapture = vi.fn((args: readonly string[]) => ({ + status: args[0] === "image" ? 1 : 125, + stdout: "", + stderr: "expected unavailable cleanup image", + })); + const cleanup: PodmanBoundContainerEngine = { + operation: "workload-cleanup", + engineId: "podman", + displayName: "Podman", + authorityId: AUTHORITY_ID, + endpointAuthorityId: AUTHORITY_ID, + capture: cleanupCapture, + captureHost: vi.fn(), + assertAuthority: vi.fn(), + }; + const bundle = createPodmanRuntimeProviderBundle({ + engines: { + hostDoctor: hostDoctorEngine(), + sandboxLifecycle: lifecycle, + workloadCleanup: cleanup, + }, + preflight: { platform: "linux", architecture: "x64" }, + }); + const entry: SandboxEntry = { + agent: "openclaw", + name: sandboxName, + openshellDriver: "podman", + }; + const control = (bundle.lifecycle as Extract) + .privilegedSandboxControl; + + expect( + control.clearStoppedStateRoots?.({ + registeredSandboxNames: [sandboxName], + sandbox: entry, + sandboxName, + paths: ["/sandbox/.openclaw/openclaw-weixin"], + }), + ).toEqual({ cleared: false, failure: "cleanup-helper-image-unavailable" }); + expect(cleanupCapture).toHaveBeenCalledExactlyOnceWith( + ["image", "inspect", "--format", "{{.Id}}", expect.stringContaining("node:22-trixie-slim")], + 30_000, + ); + }); + + it.each([CONTAINER_ID, `sha256:${CONTAINER_ID}`])( + "accepts the cleanup image ID format returned by the container engine (%s)", + (imageId) => { + const stateResource = { + type: "volume" as const, + source: "openclaw-state", + target: "/sandbox/.openclaw", + }; + const observe = vi.fn(() => ({ + target: { resourceHandle: CONTAINER_ID, running: false, stateResource }, + })); + const capture = vi.fn((args: readonly string[]) => { + switch (args[0]) { + case "image": + return { status: 0, stdout: `${imageId}\n`, stderr: "" }; + case "inspect": + return { status: 1, stdout: "", stderr: "No such container" }; + case "create": + return { status: 0, stdout: `${CONTAINER_ID}\n`, stderr: "" }; + case "start": + case "rm": + return { status: 0, stdout: "", stderr: "" }; + default: + return { status: 125, stdout: "", stderr: `unexpected command: ${args.join(" ")}` }; + } + }); + + expect( + clearStoppedSandboxStateWithEngine( + "podman-cleanup", + ["/sandbox/.openclaw/openclaw-weixin"], + { capture, observe }, + ), + ).toEqual({ cleared: true }); + expect(observe).toHaveBeenCalledTimes(3); + expect(capture.mock.calls[0]?.[0]).toEqual([ + "image", + "inspect", + "--format", + "{{.Id}}", + expect.stringContaining("node:22-trixie-slim"), + ]); + }, + ); + + it("is available through the production-selectable registry", () => { + expect(Object.keys(CURRENT_RUNTIME_PROVIDER_BUNDLES)).toEqual([ + "docker", + "kubernetes", + "podman", + ]); + expect(CURRENT_RUNTIME_PROVIDER_BUNDLES.podman?.identity.id).toBe("podman"); }); it("declares read-only host mounts unsupported until Podman qualification lands", () => { @@ -306,6 +463,44 @@ describe("dormant Podman runtime provider", () => { }); }); + it("accepts only exact supported managed-image receipts", () => { + const runtime = providerHarness("openclaw"); + const receipt: SandboxWorkloadReceipt = { + schemaVersion: 1, + kind: "managed-image", + reference: `ghcr.io/nvidia/nemoclaw/openclaw-sandbox@sha256:${"a".repeat(64)}`, + platform: "linux/amd64", + release: "v0.0.113", + sourceRevision: "b".repeat(40), + sourceCohort: "ghrun-1-1", + startupProfileContractVersion: 1, + capabilityContractVersion: 1, + encodedProfile: "e30", + startupProfileSha256: "c".repeat(64), + credentialProxyReplayRequired: true, + shared: true, + }; + + expect(runtime.providers.podman?.workload.profile).toMatchObject({ + support: { + exactDigestReferences: true, + platforms: ["linux/amd64", "linux/arm64"], + }, + hostArchitectures: ["amd64", "arm64"], + managedImageSelectionPolicy: "require-managed", + legacyDockerfileBuilds: false, + }); + expect(runtime.providers.podman?.workload.acceptsReceipt(receipt)).toBe(true); + expect( + runtime.providers.podman?.workload.acceptsReceipt({ + schemaVersion: 1, + kind: "legacy-dockerfile", + reference: null, + shared: false, + }), + ).toBe(false); + }); + it("fails host-local inference before probing either Podman operation scope", () => { const hostDoctor = hostDoctorEngine(); const sandboxLifecycle = lifecycleEngine("unsupported"); @@ -368,7 +563,7 @@ describe("dormant Podman runtime provider", () => { env: inference.env, }), ).toThrow("service 'llama-cpp' is not enabled"); - expect(CURRENT_RUNTIME_PROVIDER_BUNDLES).not.toHaveProperty("podman"); + expect(CURRENT_RUNTIME_PROVIDER_BUNDLES.podman?.identity.id).toBe("podman"); }); it("composes real operation engines on one socket without dropping executable authority", () => { @@ -417,7 +612,7 @@ describe("dormant Podman runtime provider", () => { ]), }, }); - expect(CURRENT_RUNTIME_PROVIDER_BUNDLES).not.toHaveProperty("podman"); + expect(CURRENT_RUNTIME_PROVIDER_BUNDLES.podman?.identity.id).toBe("podman"); }); it("rejects real operation engines when one socket endpoint drifts", () => { diff --git a/src/lib/onboard/runtime-provider/podman.ts b/src/lib/onboard/runtime-provider/podman.ts index 7d13fa40609..c22ccb78c64 100644 --- a/src/lib/onboard/runtime-provider/podman.ts +++ b/src/lib/onboard/runtime-provider/podman.ts @@ -2,18 +2,28 @@ // SPDX-License-Identifier: Apache-2.0 import type { PodmanBoundContainerEngine, PodmanContainerEngine } from "../../adapters/podman"; +import { validatePodmanSandboxGpuPreflight } from "../sandbox-gpu-preflight"; import { + MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION, + MANAGED_IMAGE_PLATFORMS, + MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION, RUNTIME_PROVIDER_BUNDLE_CONTRACT_VERSION, type RuntimeProviderBundle, + type RuntimeProviderManagedImageBootstrapSurface, + type RuntimeProviderCleanupInput, type RuntimeProviderLifecycleInput, type RuntimeProviderLifecycleResult, + type RuntimeProviderMutationOperation, type RuntimeProviderWorkloadProfile, } from "./contract"; import type { HostLocalInferenceOperation, HostLocalInferenceRouteAuthorityStore, } from "./host-local-inference"; -import type { PersistedEngineAuthorityStore } from "./persisted-engine-authority"; +import { + createFilePersistedEngineAuthorityStore, + type PersistedEngineAuthorityStore, +} from "./persisted-engine-authority"; import { createPodmanHostLocalInferenceOperation, type PodmanExternalInferenceNetworkAuthority, @@ -26,7 +36,13 @@ import type { PodmanInferenceAuthorityReceipt, PodmanInferenceQualificationOptions, } from "./podman-preflight"; -import { startPodmanSandbox, stopPodmanSandbox } from "./podman-lifecycle"; +import { + PODMAN_LIFECYCLE_MUTATION_TIMEOUT_MS, + recoverPodmanSandbox, + startPodmanSandbox, + stopPodmanSandbox, +} from "./podman-lifecycle"; +import { createPodmanPrivilegedSandboxControl } from "./podman-privileged-sandbox-control"; import { inspectPodmanHost, type PodmanHostPreflightOptions, @@ -34,12 +50,28 @@ import { } from "./podman-preflight"; import { createPodmanStateMutationSurface } from "./podman-state-mutation"; import type { PodmanStateMutationSurfaceOptions } from "./podman-state-mutation"; +import { + createCurrentPodmanOperationEngine, + capturePodmanDestroyIdentity, + capturePodmanDestroyIdentityByName, + createFilePodmanRouteAuthorityStore, + createPodmanRuntimeProviderSnapshotSurface, + type NativePodmanGatewayHostPreparationDeps, + planOwnedPodmanWorkloadCleanup, + prepareNativePodmanGatewayHostRuntime, + removeOwnedPodmanWorkload, + resolveNativePodmanSocketPath, +} from "./podman-runtime-surfaces"; +import { resolvePodmanStateRoot } from "./podman-state-root"; export interface PodmanRuntimeProviderEngines { readonly hostDoctor: PodmanContainerEngine; + readonly gatewayInspection?: PodmanBoundContainerEngine; readonly hostLocalInference?: PodmanContainerEngine; + readonly managedBootstrap?: PodmanBoundContainerEngine; readonly sandboxLifecycle: PodmanContainerEngine; readonly stateMutation?: PodmanBoundContainerEngine; + readonly workloadCleanup?: PodmanBoundContainerEngine; } export interface PodmanHostLocalInferenceOptions { @@ -60,18 +92,38 @@ export interface PodmanHostLocalInferenceOptions { export interface PodmanRuntimeProviderOptions { readonly engines: PodmanRuntimeProviderEngines; + readonly environment?: NodeJS.ProcessEnv; + readonly gatewaySocketPath?: string; + readonly gatewayHostPreparation?: NativePodmanGatewayHostPreparationDeps; readonly hostLocalInference?: PodmanHostLocalInferenceOptions; readonly preflight?: PodmanHostPreflightOptions; readonly stateMutation?: Omit; } -const DORMANT_WORKLOAD_PROFILE = { - support: null, - hostArchitectures: [], +const QUALIFIED_MANAGED_WORKLOAD_PROFILE = { + support: { + exactDigestReferences: true, + platforms: MANAGED_IMAGE_PLATFORMS, + startupProfileContractVersions: [MANAGED_IMAGE_STARTUP_PROFILE_CONTRACT_VERSION], + capabilityContractVersions: [MANAGED_IMAGE_CAPABILITY_CONTRACT_VERSION], + }, + hostArchitectures: ["amd64", "arm64"], managedImageSelectionPolicy: "require-managed", legacyDockerfileBuilds: false, } as const satisfies RuntimeProviderWorkloadProfile; +function acceptsManagedWorkloadReceipt( + receipt: RuntimeProviderCleanupInput["sandbox"]["workload"], +): boolean { + if (receipt?.kind !== "managed-image" || receipt.platform === undefined) return false; + const support = QUALIFIED_MANAGED_WORKLOAD_PROFILE.support; + return ( + support.platforms.includes(receipt.platform) && + support.capabilityContractVersions.includes(receipt.capabilityContractVersion) && + support.startupProfileContractVersions.includes(receipt.startupProfileContractVersion) + ); +} + export const PODMAN_READ_ONLY_HOST_MOUNT_UNSUPPORTED_REASON = "Read-only host mounts are not qualified for the Podman runtime provider."; @@ -79,9 +131,40 @@ function unsupported(providerId: string, reason: string) { return { providerId, supported: false as const, reason }; } +function createLazyPodmanManagedBootstrapSurface( + engine: PodmanBoundContainerEngine, +): RuntimeProviderManagedImageBootstrapSurface { + const surface = (): RuntimeProviderManagedImageBootstrapSurface => { + const { createPodmanManagedBootstrapSurface } = + require("../managed-bootstrap/podman-runtime") as typeof import("../managed-bootstrap/podman-runtime"); + return createPodmanManagedBootstrapSurface(engine); + }; + return Object.freeze({ + providerId: "podman", + supported: true, + bootstrapKind: "managed-image", + createAuthorityStore: ( + input: Parameters[0], + ) => surface().createAuthorityStore(input), + createLifecycle: ( + input: Parameters[0], + ) => surface().createLifecycle(input), + createOnboardRouting: ( + input: Parameters[0], + ) => surface().createOnboardRouting(input), + }); +} + function requireEngine( engine: PodmanContainerEngine, - operation: "host-doctor" | "host-local-inference" | "sandbox-lifecycle" | "state-mutation", + operation: + | "host-doctor" + | "gateway-inspection" + | "host-local-inference" + | "managed-bootstrap" + | "sandbox-lifecycle" + | "state-mutation" + | "workload-cleanup", ): void { if (engine.engineId !== "podman" || engine.operation !== operation) { throw new Error(`Podman provider requires a '${operation}' Podman engine.`); @@ -104,25 +187,31 @@ function preflightLifecycle( } } -/** - * Construct an inert Podman provider candidate from explicitly scoped engine - * dependencies. This factory is intentionally absent from the production - * provider registry until managed startup, recovery, GPU, local inference, - * installer, and protected E2E qualification land in later slices. - */ +/** Construct the Podman provider from explicitly scoped operation authorities. */ export function createPodmanRuntimeProviderBundle( options: PodmanRuntimeProviderOptions, ): RuntimeProviderBundle { const providerId = "podman"; const { hostDoctor, + gatewayInspection, hostLocalInference: inferenceEngine, + managedBootstrap, sandboxLifecycle, stateMutation: stateMutationEngine, + workloadCleanup, } = options.engines; const inferenceOptions = options.hostLocalInference; const publishedRecoveryOperation = inferenceOptions?.hermesPortablePublishedRecoveryOperation; const stateMutationOptions = options.stateMutation; + const containerEngineOperations = new Map([ + ["host-doctor", hostDoctor], + ...(gatewayInspection ? ([["gateway-inspection", gatewayInspection]] as const) : []), + ...(inferenceEngine ? ([["host-local-inference", inferenceEngine]] as const) : []), + ["sandbox-lifecycle", sandboxLifecycle], + ...(stateMutationEngine ? ([["state-mutation", stateMutationEngine]] as const) : []), + ...(workloadCleanup ? ([["workload-cleanup", workloadCleanup]] as const) : []), + ] as const); requireEngine(hostDoctor, "host-doctor"); requireEngine(sandboxLifecycle, "sandbox-lifecycle"); const providerEndpointAuthority = hostDoctor.endpointAuthorityId; @@ -159,7 +248,19 @@ export function createPodmanRuntimeProviderBundle( if (stateMutationEngine === undefined && stateMutationOptions !== undefined) { throw new Error("Podman provider requires its state-mutation engine with its options."); } + for (const [engine, operation] of [ + [gatewayInspection, "gateway-inspection"], + [managedBootstrap, "managed-bootstrap"], + [workloadCleanup, "workload-cleanup"], + ] as const) { + if (!engine) continue; + requireEngine(engine, operation); + if (engine.endpointAuthorityId !== providerEndpointAuthority) { + throw new Error("Podman provider engines must bind the same endpoint authority."); + } + } const preflight = options.preflight ?? {}; + const environment = Object.freeze({ ...(options.environment ?? process.env) }); const deferred = "This operation is intentionally deferred to a later Podman slice."; return { @@ -175,7 +276,7 @@ export function createPodmanRuntimeProviderBundle( hostLocalInference: inferenceEngine !== undefined, directLifecycle: true, legacyGatewayContainerInspection: false, - workloadImageCleanup: false, + workloadImageCleanup: workloadCleanup !== undefined, readOnlyHostMounts: { supported: false, reason: PODMAN_READ_ONLY_HOST_MOUNT_UNSUPPORTED_REASON, @@ -185,6 +286,8 @@ export function createPodmanRuntimeProviderBundle( providerId, supported: true, inspectHost: () => inspectPodmanHost(hostDoctor, preflight), + validateSandboxGpu: (config, exitProcess) => + validatePodmanSandboxGpuPreflight(config, {}, exitProcess), preflightLifecycle: (_action, input) => preflightLifecycle(input, hostDoctor, preflight), }, gateway: { @@ -192,12 +295,31 @@ export function createPodmanRuntimeProviderBundle( supported: true, launcher: "nemoclaw", inspectLegacyContainer: false, + prepareHostRuntime: (input) => { + if ( + options.gatewaySocketPath !== undefined && + input.socketPath !== undefined && + resolveNativePodmanSocketPath(input.environment, input.socketPath) !== + options.gatewaySocketPath + ) { + throw new Error("Native Podman gateway socket differs from its bundle authority."); + } + return prepareNativePodmanGatewayHostRuntime( + { + ...input, + socketPath: options.gatewaySocketPath ?? input.socketPath, + }, + gatewayInspection, + options.gatewayHostPreparation, + ); + }, }, workload: { providerId, supported: true, - profile: DORMANT_WORKLOAD_PROFILE, - acceptsReceipt: () => false, + profile: QUALIFIED_MANAGED_WORKLOAD_PROFILE, + managedStateMountDriverId: "podman", + acceptsReceipt: acceptsManagedWorkloadReceipt, }, hostLocalInference: inferenceEngine !== undefined && inferenceOptions !== undefined @@ -251,6 +373,11 @@ export function createPodmanRuntimeProviderBundle( providerId, supported: true, channelStopTransport: "openshell", + containerMutationTimeoutMs: PODMAN_LIFECYCLE_MUTATION_TIMEOUT_MS, + privilegedSandboxControl: createPodmanPrivilegedSandboxControl( + sandboxLifecycle, + workloadCleanup, + ), start: (input) => startPodmanSandbox(input, sandboxLifecycle), verifyStarted: (input, verifyGateway) => verifyGateway(input.sandboxName), stop: (input, hooks) => stopPodmanSandbox(input, hooks, sandboxLifecycle), @@ -270,10 +397,46 @@ export function createPodmanRuntimeProviderBundle( engine: stateMutationEngine, ...(stateMutationOptions ?? {}), }), - bootstrap: unsupported(providerId, deferred), - snapshot: unsupported(providerId, deferred), - recovery: unsupported(providerId, deferred), - cleanup: unsupported(providerId, deferred), + bootstrap: + managedBootstrap === undefined + ? unsupported(providerId, deferred) + : createLazyPodmanManagedBootstrapSurface(managedBootstrap), + snapshot: + gatewayInspection === undefined + ? unsupported(providerId, deferred) + : createPodmanRuntimeProviderSnapshotSurface(gatewayInspection), + recovery: { + providerId, + supported: true, + recover: (sandbox) => + recoverPodmanSandbox( + { + environment, + log: () => undefined, + sandbox, + sandboxName: sandbox.name, + }, + sandboxLifecycle, + ), + }, + cleanup: + workloadCleanup === undefined + ? unsupported(providerId, deferred) + : { + providerId, + supported: true, + ...(gatewayInspection + ? { + captureDestroyIdentity: (input: RuntimeProviderCleanupInput) => + capturePodmanDestroyIdentity(input, gatewayInspection), + captureDestroyIdentityByName: (sandboxName: string) => + capturePodmanDestroyIdentityByName(sandboxName, gatewayInspection), + } + : {}), + prepareDestroy: (_input, operations) => operations.detachProviders(), + planOwnedWorkloadCleanup: planOwnedPodmanWorkloadCleanup, + removeOwnedWorkload: (input) => removeOwnedPodmanWorkload(input, workloadCleanup), + }, containerEngine: { providerId, supported: true, @@ -283,6 +446,15 @@ export function createPodmanRuntimeProviderBundle( engineId: hostDoctor.engineId, displayName: hostDoctor.displayName, }, + ...(gatewayInspection + ? [ + { + operation: "gateway-inspection" as const, + engineId: gatewayInspection.engineId, + displayName: gatewayInspection.displayName, + }, + ] + : []), ...(inferenceEngine ? [ { @@ -306,7 +478,94 @@ export function createPodmanRuntimeProviderBundle( }, ] : []), + ...(workloadCleanup + ? [ + { + operation: "workload-cleanup" as const, + engineId: workloadCleanup.engineId, + displayName: workloadCleanup.displayName, + }, + ] + : []), ], + capture: (operation, args, timeoutMs) => { + const engine = containerEngineOperations.get(operation); + if (!engine) { + throw new Error(`Podman provider does not register the '${operation}' engine operation.`); + } + return engine.capture(args, timeoutMs); + }, }, }; } + +function redactPodmanFailure(environment: NodeJS.ProcessEnv, value: string): string { + let redacted = value; + for (const key of ["NGC_API_KEY", "NIM_NGC_API_KEY"] as const) { + const secret = environment[key]; + if (secret) redacted = redacted.replaceAll(secret, "[REDACTED]"); + } + return redacted; +} + +function createLazyPersistedEngineAuthorityStore(stateRoot: string): PersistedEngineAuthorityStore { + let store: PersistedEngineAuthorityStore | null = null; + const resolve = () => (store ??= createFilePersistedEngineAuthorityStore(stateRoot)); + return Object.freeze({ + load: (operation: Parameters[0]) => + resolve().load(operation), + record: (authority: Parameters[0]) => + resolve().record(authority), + }); +} + +/** Production Podman bundle selected only at the managed registration boundary. */ +export function createCurrentPodmanRuntimeProviderBundle( + environment: NodeJS.ProcessEnv = process.env, +): RuntimeProviderBundle { + const stateRoot = resolvePodmanStateRoot(environment.HOME); + const engines = { + hostDoctor: createCurrentPodmanOperationEngine("host-doctor", environment), + gatewayInspection: createCurrentPodmanOperationEngine("gateway-inspection", environment), + hostLocalInference: createCurrentPodmanOperationEngine("host-local-inference", environment), + managedBootstrap: createCurrentPodmanOperationEngine("managed-bootstrap", environment), + sandboxLifecycle: createCurrentPodmanOperationEngine("sandbox-lifecycle", environment), + stateMutation: createCurrentPodmanOperationEngine("state-mutation", environment), + workloadCleanup: createCurrentPodmanOperationEngine("workload-cleanup", environment), + } as const; + const bundle = createPodmanRuntimeProviderBundle({ + engines, + environment, + gatewaySocketPath: resolveNativePodmanSocketPath(environment), + gatewayHostPreparation: {}, + hostLocalInference: { + authorityStore: createLazyPersistedEngineAuthorityStore(stateRoot), + routeAuthorityStore: createFilePodmanRouteAuthorityStore(stateRoot), + onFailureEvidence: (evidence) => { + console.error( + redactPodmanFailure(environment, `Podman ${evidence.phase}: ${evidence.message}`), + ); + }, + redactSensitive: (value) => redactPodmanFailure(environment, value), + }, + stateMutation: {}, + }); + return Object.freeze({ + ...bundle, + mutationAuthority: Object.freeze({ + providerId: "podman", + supported: true, + operations: Object.freeze([ + "registration", + "start", + "stop", + "inference-set", + "rebuild", + "clone", + "provider-cleanup", + "destroy", + "workload-cleanup", + ] satisfies readonly RuntimeProviderMutationOperation[]), + }), + }); +} diff --git a/src/lib/onboard/runtime-provider/privileged-sandbox-control-errors.ts b/src/lib/onboard/runtime-provider/privileged-sandbox-control-errors.ts new file mode 100644 index 00000000000..5e0ee7bf7c2 --- /dev/null +++ b/src/lib/onboard/runtime-provider/privileged-sandbox-control-errors.ts @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export class DirectSandboxFallbackUnavailableError extends Error { + constructor(message: string, options?: ErrorOptions) { + super(message, options); + this.name = "DirectSandboxFallbackUnavailableError"; + } +} + +export class PinnedSandboxResourceIdentityChangedError extends Error { + constructor(sandboxName: string) { + super( + `OpenShell container identity changed for sandbox '${sandboxName}'; ` + + "refusing privileged execution against a different container.", + ); + this.name = "PinnedSandboxResourceIdentityChangedError"; + } +} diff --git a/src/lib/onboard/runtime-provider/registry.ts b/src/lib/onboard/runtime-provider/registry.ts index 26dedf3456c..07f09cccb8f 100644 --- a/src/lib/onboard/runtime-provider/registry.ts +++ b/src/lib/onboard/runtime-provider/registry.ts @@ -375,6 +375,7 @@ function validateCapabilitiesSurface(surface: Record): void { function validatePreflightDoctorSurface(surface: Record): void { requireSupported("preflightDoctor", surface); requireFunction(surface, "inspectHost", "preflightDoctor"); + requireFunction(surface, "validateSandboxGpu", "preflightDoctor"); requireFunction(surface, "preflightLifecycle", "preflightDoctor"); } @@ -423,6 +424,22 @@ function validateLifecycleSurface(providerId: string, surface: Record 5 * 60_000) + ) { + throw new RuntimeProviderRegistrationError( + `lifecycle for '${providerId}' has an invalid container mutation timeout`, + ); + } + const control = requireOwnRecord(surface, "privilegedSandboxControl"); + requireFunction(control, "resolveTarget", "lifecycle.privilegedSandboxControl"); + requireFunction(control, "execute", "lifecycle.privilegedSandboxControl"); + if (control.buildLegacyDockerArgv !== undefined) { + requireFunction(control, "buildLegacyDockerArgv", "lifecycle.privilegedSandboxControl"); + } } } @@ -517,6 +534,12 @@ function validateRecoverySurface(surface: Record): void { function validateCleanupSurface(surface: Record): void { if (surface.supported === true) { + if (surface.captureDestroyIdentity !== undefined) { + requireFunction(surface, "captureDestroyIdentity", "cleanup"); + } + if (surface.captureDestroyIdentityByName !== undefined) { + requireFunction(surface, "captureDestroyIdentityByName", "cleanup"); + } requireFunction(surface, "prepareDestroy", "cleanup"); requireFunction(surface, "planOwnedWorkloadCleanup", "cleanup"); requireFunction(surface, "removeOwnedWorkload", "cleanup"); @@ -528,6 +551,7 @@ function validateContainerEngineSurface( surface: Record, ): void { if (surface.supported === true) { + requireFunction(surface, "capture", "containerEngine"); const identities = surface.identities; if (!Array.isArray(identities)) { throw new RuntimeProviderRegistrationError( @@ -795,6 +819,20 @@ export function runtimeProviderContainerEngineIdentity( return identity ? { engineId: identity.engineId, displayName: identity.displayName } : null; } +/** + * Report whether the selected provider registered the operation-scoped engine + * authority required by generic orchestration. Provider identities stay + * opaque: adding a provider changes registration, not the caller's branches. + */ +export function runtimeProviderSupportsContainerEngineOperation( + driverName: string | null | undefined, + providers: RuntimeProviderBundleRegistry, + operation: RuntimeProviderContainerEngineOperation, +): boolean { + const bundle = resolveRuntimeProviderBundle(driverName, providers); + return bundle !== null && runtimeProviderContainerEngineIdentity(bundle, operation) !== null; +} + export function requireRuntimeProviderHostLocalInferenceOperation( bundle: RuntimeProviderBundle, service: HostLocalInferenceService, diff --git a/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts b/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts index d279d5faa06..01430ecf42f 100644 --- a/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts +++ b/src/lib/onboard/runtime-provider/runtime-provider-contract.test.ts @@ -122,8 +122,12 @@ function expectSupportedSurface( } describe("RuntimeProviderBundle registry contract", () => { - it("keeps the production selectable set limited to complete Docker and Kubernetes bundles", () => { - expect(Object.keys(CURRENT_RUNTIME_PROVIDER_BUNDLES)).toEqual(["docker", "kubernetes"]); + it("registers every production-selectable provider as one complete bundle", () => { + expect(Object.keys(CURRENT_RUNTIME_PROVIDER_BUNDLES)).toEqual([ + "docker", + "kubernetes", + "podman", + ]); Object.entries(CURRENT_RUNTIME_PROVIDER_BUNDLES).forEach(([providerId, bundle]) => { expect(bundle.identity.id).toBe(providerId); expect( @@ -146,13 +150,14 @@ describe("RuntimeProviderBundle registry contract", () => { ] as const ).every((surface) => Object.is(bundle[surface].providerId, providerId)), ).toBe(true); - expect(bundle.bootstrap).toMatchObject({ supported: providerId === "docker" }); + const managedLocalProvider = providerId === "docker" || providerId === "podman"; + expect(bundle.bootstrap).toMatchObject({ supported: managedLocalProvider }); expect(bundle.stateMutation).toMatchObject({ - supported: providerId === "docker", - ...(providerId === "docker" ? { contractVersion: 2 } : {}), + supported: managedLocalProvider, + ...(managedLocalProvider ? { contractVersion: 2 } : {}), }); expect(bundle.snapshot).toMatchObject( - providerId === "docker" + managedLocalProvider ? { supported: true, capabilities: { @@ -163,7 +168,7 @@ describe("RuntimeProviderBundle registry contract", () => { } : { supported: false }, ); - expect(bundle.recovery).toMatchObject({ supported: false }); + expect(bundle.recovery).toMatchObject({ supported: providerId === "podman" }); }); expect(CURRENT_RUNTIME_PROVIDER_BUNDLES.docker?.capabilities.hostLocalInference).toBe(true); expect(CURRENT_RUNTIME_PROVIDER_BUNDLES.docker?.hostLocalInference).toMatchObject({ @@ -570,6 +575,23 @@ describe("RuntimeProviderBundle registry contract", () => { ).toThrow(/lifecycle\.verifyStarted must be a function/u); }); + it("rejects an invalid provider-owned container mutation timeout", () => { + const bundle = mxcBundle(); + expectSupportedSurface(bundle.lifecycle); + + expect(() => + createRuntimeProviderBundleRegistry([ + [ + "mxc", + replaceSurface(bundle, "lifecycle", { + ...bundle.lifecycle, + containerMutationTimeoutMs: 0, + }), + ], + ]), + ).toThrow(/invalid container mutation timeout/u); + }); + it("rejects cleanup without a side-effect-free ownership plan", () => { const bundle = mxcBundle(); expectSupportedSurface(bundle.cleanup); @@ -611,6 +633,12 @@ describe("RuntimeProviderBundle registry contract", () => { it("rejects capability/surface drift and duplicate operation-scoped engine identities", () => { const bundle = mxcBundle(); + const { capture: _capture, ...containerEngineWithoutCapture } = bundle.containerEngine; + expect(() => + createRuntimeProviderBundleRegistry([ + ["mxc", replaceSurface(bundle, "containerEngine", containerEngineWithoutCapture)], + ]), + ).toThrow(/containerEngine.*capture/u); expect(() => createRuntimeProviderBundleRegistry([ [ diff --git a/src/lib/onboard/runtime-provider/selection.ts b/src/lib/onboard/runtime-provider/selection.ts new file mode 100644 index 00000000000..05fbc2e0641 --- /dev/null +++ b/src/lib/onboard/runtime-provider/selection.ts @@ -0,0 +1,54 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { + RuntimeProviderBundle, + RuntimeProviderBundleRegistry, + RuntimeProviderContainerEngineOperation, +} from "./contract"; +import { CURRENT_RUNTIME_PROVIDER_BUNDLES, resolveCurrentRuntimeProviderBundle } from "./current"; +import { + requireRuntimeProviderBundleForSandbox, + resolveRuntimeProviderBundle, + runtimeProviderSupportsContainerEngineOperation, +} from "./registry"; + +export { requireRuntimeProviderBundleForSandbox }; + +/** Resolve persisted provider metadata through the qualification-backed registry. */ +export function resolveRegisteredRuntimeProvider( + providerId: string | null | undefined, +): RuntimeProviderBundle | null { + return resolveRuntimeProviderBundle(providerId, CURRENT_RUNTIME_PROVIDER_BUNDLES); +} + +/** Resolve configured provider intent once at the runtime selection boundary. */ +export function resolveConfiguredRuntimeProvider( + platform: NodeJS.Platform = process.platform, + arch: NodeJS.Architecture = process.arch, + environment: NodeJS.ProcessEnv = process.env, +): RuntimeProviderBundle { + return resolveCurrentRuntimeProviderBundle( + platform, + arch, + CURRENT_RUNTIME_PROVIDER_BUNDLES, + environment, + ); +} + +/** Current registry for dependency injection at provider-neutral orchestration seams. */ +export function currentRuntimeProviderBundles(): RuntimeProviderBundleRegistry { + return CURRENT_RUNTIME_PROVIDER_BUNDLES; +} + +/** Capability query for persisted provider metadata. */ +export function registeredRuntimeProviderSupportsContainerEngineOperation( + providerId: string | null | undefined, + operation: RuntimeProviderContainerEngineOperation, +): boolean { + return runtimeProviderSupportsContainerEngineOperation( + providerId, + CURRENT_RUNTIME_PROVIDER_BUNDLES, + operation, + ); +} diff --git a/src/lib/onboard/runtime-provider/stopped-sandbox-state-cleanup.ts b/src/lib/onboard/runtime-provider/stopped-sandbox-state-cleanup.ts new file mode 100644 index 00000000000..667cfe8cc75 --- /dev/null +++ b/src/lib/onboard/runtime-provider/stopped-sandbox-state-cleanup.ts @@ -0,0 +1,327 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; +import path from "node:path"; + +import type { ContainerEngineCommandResult } from "../../adapters/container-engine"; +import type { + RuntimeProviderStoppedSandboxStateCleanupFailure, + RuntimeProviderStoppedSandboxStateCleanupResult, +} from "./contract"; + +const FULL_CONTAINER_ID_RE = /^[a-f0-9]{64}$/u; +const VOLUME_NAME_RE = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,254}$/u; +const STATE_PATH_RE = /^\/sandbox\/\.(?:openclaw|hermes)\/[A-Za-z0-9_-]+$/u; +const CLEANUP_IMAGE = + "node:22-trixie-slim@sha256:db8a96a63e5264607ada2d206758876ebbed6a12be2ada7517793cbfb0c2a29c"; +const CLEANUP_LABEL = "com.nvidia.nemoclaw.channel-cleanup"; +const CLEANUP_OWNER_LABEL = `${CLEANUP_LABEL}.owner`; +const CLEANUP_VOLUME_LABEL = `${CLEANUP_LABEL}.volume`; +const NEUTRAL_ENV = [ + "--env", + "LD_AUDIT=", + "--env", + "LD_LIBRARY_PATH=", + "--env", + "LD_PRELOAD=", + "--env", + "BASH_ENV=", + "--env", + "ENV=", +] as const; + +export interface StoppedSandboxStateTarget { + readonly resourceHandle: string; + readonly running: boolean; + readonly stateResource: { + readonly type: "bind" | "volume"; + readonly source: string; + readonly target: string; + }; +} + +export type StoppedSandboxStateObservation = + | { readonly target: StoppedSandboxStateTarget } + | { readonly failure: RuntimeProviderStoppedSandboxStateCleanupFailure }; + +export interface StoppedSandboxStateCleanupEngine { + capture(args: readonly string[], timeoutMs?: number): ContainerEngineCommandResult; + observe(): StoppedSandboxStateObservation; +} + +export function buildStoppedSandboxChannelCleanupScript(root?: string): string { + return String.raw` +"use strict"; +const fs = require("node:fs"); +const path = require("node:path"); +const targets = JSON.parse(process.argv[1]); +const root = ${root === undefined ? "process.argv[2]" : JSON.stringify(root)}; +function lstat(candidate) { + try { return fs.lstatSync(candidate); } + catch (error) { if (error && error.code === "ENOENT") return null; throw error; } +} +const rootMetadata = lstat(root); +if (!rootMetadata || rootMetadata.isSymbolicLink() || !rootMetadata.isDirectory()) process.exit(40); +for (const target of targets) { + if (typeof target !== "string" || !target.startsWith(root + "/")) process.exit(41); + const relative = path.posix.relative(root, target); + const segments = relative.split("/"); + if (!relative || relative.startsWith("../") || segments.some((part) => !part || part === "." || part === "..")) process.exit(42); + let parent = root; + let absent = false; + for (const segment of segments.slice(0, -1)) { + parent = path.posix.join(parent, segment); + const metadata = lstat(parent); + if (!metadata) { absent = true; break; } + if (metadata.isSymbolicLink() || !metadata.isDirectory()) process.exit(43); + } + if (absent) continue; + const metadata = lstat(target); + if (!metadata) continue; + if (metadata.isSymbolicLink() || !metadata.isDirectory()) process.exit(44); + fs.rmSync(target, { force: false, maxRetries: 0, recursive: true }); + if (lstat(target)) process.exit(45); +} +`; +} + +const CLEANUP_SCRIPT = buildStoppedSandboxChannelCleanupScript(); + +function failure( + code: RuntimeProviderStoppedSandboxStateCleanupFailure, + cleanupHelperName?: string, +): RuntimeProviderStoppedSandboxStateCleanupResult { + return cleanupHelperName + ? { cleared: false, failure: code, cleanupHelperName } + : { cleared: false, failure: code }; +} + +export function validateStoppedSandboxStatePaths(paths: readonly string[]): boolean { + return ( + paths.length > 0 && + paths.length <= 4 && + new Set(paths).size === paths.length && + paths.every((statePath) => STATE_PATH_RE.test(statePath)) + ); +} + +export function sandboxStateResourceFromMounts( + value: unknown, + paths: readonly string[], +): StoppedSandboxStateTarget["stateResource"] | null { + if (!Array.isArray(value) || !validateStoppedSandboxStatePaths(paths)) return null; + const mounts = value + .filter((entry): entry is Record => { + if (typeof entry !== "object" || entry === null) return false; + const target = (entry as Record).Destination; + return ( + (entry as Record).RW === true && + typeof target === "string" && + path.posix.isAbsolute(target) && + path.posix.normalize(target) === target && + (target === "/sandbox" || /^\/sandbox\/\.(?:openclaw|hermes)$/u.test(target)) && + paths.every((statePath) => statePath.startsWith(`${target}/`)) + ); + }) + .sort((left, right) => String(right.Destination).length - String(left.Destination).length); + const mount = mounts[0]; + if (!mount || mounts[1]?.Destination === mount.Destination) return null; + const target = String(mount.Destination); + if ( + mount.Type === "volume" && + typeof mount.Name === "string" && + VOLUME_NAME_RE.test(mount.Name) + ) { + return { type: "volume", source: mount.Name, target }; + } + if ( + mount.Type === "bind" && + typeof mount.Source === "string" && + path.isAbsolute(mount.Source) && + path.normalize(mount.Source) === mount.Source && + mount.Source !== path.parse(mount.Source).root && + mount.Source.length <= 4096 && + !/[\u0000-\u001f\u007f]/u.test(mount.Source) + ) { + return { type: "bind", source: mount.Source, target }; + } + return null; +} + +function sameStateResource( + left: StoppedSandboxStateTarget["stateResource"], + right: StoppedSandboxStateTarget["stateResource"], +): boolean { + return left.type === right.type && left.source === right.source && left.target === right.target; +} + +function stateResourceMount(resource: StoppedSandboxStateTarget["stateResource"]): string { + return resource.type === "volume" + ? `type=volume,src=${resource.source},dst=${resource.target},volume-nocopy` + : `type=bind,src=${resource.source},dst=${resource.target}`; +} + +function identity(value: string): string { + return createHash("sha256").update(value).digest("hex"); +} + +function helperName(sandboxName: string): string { + return `nemoclaw-channel-cleanup-${identity(sandboxName).slice(0, 24)}`; +} + +function resultText(result: ContainerEngineCommandResult): string { + return `${result.stderr} ${result.stdout} ${result.error?.message ?? ""}`; +} + +function reportsMissing(result: ContainerEngineCommandResult): boolean { + return result.status !== 0 && /No such (?:container|object)/iu.test(resultText(result)); +} + +type HelperInspection = + | { readonly state: "absent" } + | { readonly state: "invalid" } + | { readonly state: "owned"; readonly id: string }; + +function inspectHelper( + engine: StoppedSandboxStateCleanupEngine, + name: string, + owner: string, + volume: string, +): HelperInspection { + const result = engine.capture([ + "inspect", + "--format", + `{{.Id}}\t{{.Config.Image}}\t{{index .Config.Labels "${CLEANUP_LABEL}"}}\t{{index .Config.Labels "${CLEANUP_OWNER_LABEL}"}}\t{{index .Config.Labels "${CLEANUP_VOLUME_LABEL}"}}`, + name, + ]); + if (reportsMissing(result)) return { state: "absent" }; + if (result.status !== 0 || result.error) return { state: "invalid" }; + const [id, image, marker, actualOwner, actualVolume, ...unexpected] = result.stdout + .trim() + .split("\t"); + return unexpected.length === 0 && + FULL_CONTAINER_ID_RE.test(id ?? "") && + image === CLEANUP_IMAGE && + marker === "1" && + actualOwner === owner && + actualVolume === volume + ? { state: "owned", id: id! } + : { state: "invalid" }; +} + +function removeHelper(engine: StoppedSandboxStateCleanupEngine, id: string): boolean { + const removed = engine.capture(["rm", "-f", id]); + if (removed.status !== 0 || removed.error) return false; + return reportsMissing(engine.capture(["inspect", id])); +} + +function reconcileHelper( + engine: StoppedSandboxStateCleanupEngine, + name: string, + owner: string, + volume: string, +): boolean { + const helper = inspectHelper(engine, name, owner, volume); + return helper.state === "absent" || (helper.state === "owned" && removeHelper(engine, helper.id)); +} + +function classifyStartFailure(result: ContainerEngineCommandResult | null) { + if (result?.status === 45) return "cleanup-deletion-unconfirmed" as const; + if (result && result.status >= 40 && result.status <= 44) { + return "cleanup-state-tree-unsafe" as const; + } + return "cleanup-helper-failed" as const; +} + +export function clearStoppedSandboxStateWithEngine( + sandboxName: string, + paths: readonly string[], + engine: StoppedSandboxStateCleanupEngine, +): RuntimeProviderStoppedSandboxStateCleanupResult { + if (!validateStoppedSandboxStatePaths(paths)) return failure("state-paths-invalid"); + const observed = engine.observe(); + if ("failure" in observed) return failure(observed.failure); + const target = observed.target; + if (target.running) return failure("runtime-not-stopped"); + const image = engine.capture(["image", "inspect", "--format", "{{.Id}}", CLEANUP_IMAGE]); + if ( + image.status !== 0 || + image.error || + !/^(?:sha256:)?[a-f0-9]{64}$/u.test(image.stdout.trim()) + ) { + return failure("cleanup-helper-image-unavailable"); + } + const name = helperName(sandboxName); + const owner = identity(sandboxName); + const stateResourceIdentity = identity(JSON.stringify(target.stateResource)); + const existing = inspectHelper(engine, name, owner, stateResourceIdentity); + if (existing.state === "invalid") return failure("cleanup-helper-ownership-invalid", name); + if (existing.state === "owned" && !removeHelper(engine, existing.id)) { + return failure("cleanup-helper-reconciliation-failed", name); + } + const revalidated = engine.observe(); + if ( + "failure" in revalidated || + revalidated.target.resourceHandle !== target.resourceHandle || + revalidated.target.running || + !sameStateResource(revalidated.target.stateResource, target.stateResource) + ) { + return failure("runtime-revalidation-failed"); + } + const created = engine.capture([ + "create", + "--name", + name, + "--pull", + "never", + "--network", + "none", + "--read-only", + "--user", + "0:0", + "--security-opt", + "no-new-privileges", + "--cap-drop", + "ALL", + "--cap-add", + "DAC_OVERRIDE", + "--pids-limit", + "64", + ...NEUTRAL_ENV, + "--label", + `${CLEANUP_LABEL}=1`, + "--label", + `${CLEANUP_OWNER_LABEL}=${owner}`, + "--label", + `${CLEANUP_VOLUME_LABEL}=${stateResourceIdentity}`, + "--mount", + stateResourceMount(target.stateResource), + "--entrypoint", + "/usr/local/bin/node", + CLEANUP_IMAGE, + "-e", + CLEANUP_SCRIPT, + JSON.stringify(paths), + target.stateResource.target, + ]); + const helperId = created.stdout.trim(); + if (created.status !== 0 || created.error || !FULL_CONTAINER_ID_RE.test(helperId)) { + return reconcileHelper(engine, name, owner, stateResourceIdentity) + ? failure("cleanup-helper-failed") + : failure("cleanup-helper-reconciliation-failed", name); + } + const cleared = engine.capture(["start", "--attach", helperId]); + if (!removeHelper(engine, helperId)) return failure("cleanup-helper-reconciliation-failed", name); + if (cleared.status !== 0 || cleared.error) return failure(classifyStartFailure(cleared)); + const confirmed = engine.observe(); + if ( + "failure" in confirmed || + confirmed.target.resourceHandle !== target.resourceHandle || + !sameStateResource(confirmed.target.stateResource, target.stateResource) || + confirmed.target.running + ) { + return failure("runtime-revalidation-failed"); + } + return { cleared: true }; +} diff --git a/src/lib/onboard/sandbox-create-intent-types.ts b/src/lib/onboard/sandbox-create-intent-types.ts index f216165619f..6487649c68d 100644 --- a/src/lib/onboard/sandbox-create-intent-types.ts +++ b/src/lib/onboard/sandbox-create-intent-types.ts @@ -2,10 +2,10 @@ // SPDX-License-Identifier: Apache-2.0 import type { SandboxHostMount } from "../state/registry/types"; +import type { MessagingChannelConfig } from "../messaging-channel-config"; import type { DockerGpuRoutePlan } from "./docker-gpu-route"; import type { InitialSandboxPolicy } from "./initial-policy"; -import type { ManagedHermesStateVolumeMount } from "./managed-workload/hermes-state-volume"; -import type { MessagingChannelConfig } from "../messaging-channel-config"; +import type { ManagedStateVolumeMount } from "./managed-workload/managed-state-volumes"; import type { MessagingTokenDef } from "./messaging-prep"; import type { MessagingChannel } from "./messaging-state"; import type { SandboxGpuCreateConfig } from "./sandbox-gpu-create"; @@ -93,10 +93,12 @@ export type ResolveSandboxCreateIntentInput = { export type MaterializeSandboxCreatePlanInput = { intent: SandboxCreateIntent; fromRef: string; + managedStateMounts?: readonly ManagedStateVolumeMount[]; + /** Opaque provider-owned OpenShell driver-config key for the managed state mount. */ + managedStateMountDriverId?: string | null; policylessCreate?: boolean; /** Keep provider mutations and attachments behind the exact post-create identity gate. */ deferSandboxEffectsUntilIdentityVerification?: boolean; - managedStateMount?: ManagedHermesStateVolumeMount | null; messagingTokenDefs: MessagingTokenDef[]; /** Non-secret config captured in the messaging plan that owns exact policy endpoints. */ messagingConfig?: MessagingChannelConfig | null; diff --git a/src/lib/onboard/sandbox-create-plan-materialization.ts b/src/lib/onboard/sandbox-create-plan-materialization.ts index ded2466efe0..61bdc4e17f0 100644 --- a/src/lib/onboard/sandbox-create-plan-materialization.ts +++ b/src/lib/onboard/sandbox-create-plan-materialization.ts @@ -28,7 +28,8 @@ const DCODE_MCP_SNAPSHOT_TMPFS_MOUNT = { function buildSandboxDriverConfig( intent: SandboxCreateIntent, - managedStateMount: MaterializeSandboxCreatePlanInput["managedStateMount"], + managedStateMounts: MaterializeSandboxCreatePlanInput["managedStateMounts"], + managedStateMountDriverId: MaterializeSandboxCreatePlanInput["managedStateMountDriverId"], ): string | null { const cdiDevice = normalizeSandboxGpuDeviceForCdi(intent.sandboxGpuDevice); if (cdiDevice && (!intent.policy.options.directGpu || !intent.gpuCreateArgs.includes("--gpu"))) { @@ -37,37 +38,56 @@ function buildSandboxDriverConfig( const dockerMounts: Array> = (intent.hostMounts ?? []).map( ({ source, target }) => ({ type: "bind", source, target, read_only: true }), ); - if (managedStateMount) { - const conflictingHostMount = intent.hostMounts?.find(({ target }) => - containerPathsOverlap(target, managedStateMount.target), - ); - if (conflictingHostMount) { - throw new Error( - `Host mount target '${conflictingHostMount.target}' conflicts with the managed Hermes state root '${managedStateMount.target}'.`, + const podmanMounts: Array> = []; + const mountsByDriver = new Map>>([ + ["docker", dockerMounts], + ["podman", podmanMounts], + ]); + if ((managedStateMounts?.length ?? 0) > 0) { + if (!managedStateMountDriverId) { + throw new Error("Managed state mounts are missing their provider-owned driver config."); + } + const providerMounts = mountsByDriver.get(managedStateMountDriverId) ?? []; + for (const managedStateMount of managedStateMounts ?? []) { + const conflictingHostMount = intent.hostMounts?.find(({ target }) => + containerPathsOverlap(target, managedStateMount.target), ); + if (conflictingHostMount) { + throw new Error( + `Host mount target '${conflictingHostMount.target}' conflicts with the managed state root '${managedStateMount.target}'.`, + ); + } + if ( + providerMounts.some(({ target }) => + typeof target === "string" && containerPathsOverlap(target, managedStateMount.target), + ) + ) { + throw new Error(`Managed state root '${managedStateMount.target}' overlaps another root.`); + } + providerMounts.push({ ...managedStateMount }); } - dockerMounts.unshift({ ...managedStateMount }); + mountsByDriver.set(managedStateMountDriverId, providerMounts); } - const podmanMounts: Array> = []; if (intent.policy.options.agentName === "langchain-deepagents-code") { dockerMounts.unshift(DCODE_MCP_SNAPSHOT_TMPFS_MOUNT); podmanMounts.push(DCODE_MCP_SNAPSHOT_TMPFS_MOUNT); } - if (dockerMounts.length === 0 && !cdiDevice) return null; - return JSON.stringify({ - docker: { - ...(cdiDevice ? { cdi_devices: [cdiDevice] } : {}), - ...(dockerMounts.length > 0 ? { mounts: dockerMounts } : {}), - }, - ...(podmanMounts.length > 0 || cdiDevice - ? { - podman: { - ...(cdiDevice ? { cdi_devices: [cdiDevice] } : {}), - ...(podmanMounts.length > 0 ? { mounts: podmanMounts } : {}), - }, - } - : {}), - }); + const driverConfig = Object.fromEntries( + [...mountsByDriver].flatMap(([driverId, mounts]) => + mounts.length > 0 || cdiDevice + ? [ + [ + driverId, + { + ...(cdiDevice ? { cdi_devices: [cdiDevice] } : {}), + ...(mounts.length > 0 ? { mounts } : {}), + }, + ], + ] + : [], + ), + ); + return Object.keys(driverConfig).length > 0 ? JSON.stringify(driverConfig) : null; } export type SandboxCreatePlan = { @@ -282,9 +302,10 @@ function assertDeferredProviderPlanSupported( export function materializeSandboxCreatePlan({ intent, fromRef, + managedStateMounts, + managedStateMountDriverId, policylessCreate = false, deferSandboxEffectsUntilIdentityVerification = false, - managedStateMount, messagingTokenDefs, messagingConfig, runProviderPreDeleteCleanup, @@ -294,7 +315,11 @@ export function materializeSandboxCreatePlan({ prepareInitialSandboxCreatePolicy = getInitialSandboxCreatePolicy, }: MaterializeSandboxCreatePlanInput): SandboxCreatePlan { const enabledMessagingTokenDefs = validateSandboxCreateIntentBindings(intent, messagingTokenDefs); - const driverConfig = buildSandboxDriverConfig(intent, managedStateMount); + const driverConfig = buildSandboxDriverConfig( + intent, + managedStateMounts, + managedStateMountDriverId, + ); const { initialSandboxPolicy, compatibilityPolicyPath } = prepareSandboxCreatePolicy( intent, prepareInitialSandboxCreatePolicy, @@ -431,7 +456,7 @@ export function materializeHermesPortableCreatePlan(input: { policyTier: intent.policy.options.policyTier, }, ); - const driverConfig = buildSandboxDriverConfig(intent, null); + const driverConfig = buildSandboxDriverConfig(intent, undefined, null); const createArgs = [ "--from", fromRef, diff --git a/src/lib/onboard/sandbox-create-plan.test.ts b/src/lib/onboard/sandbox-create-plan.test.ts index a642e9d4cbf..113f44cc18f 100644 --- a/src/lib/onboard/sandbox-create-plan.test.ts +++ b/src/lib/onboard/sandbox-create-plan.test.ts @@ -972,12 +972,15 @@ describe("resolveSandboxCreateIntent", () => { const plan = materializeSandboxCreatePlan({ intent, fromRef: `ghcr.io/nvidia/nemoclaw/hermes@sha256:${"a".repeat(64)}`, - managedStateMount: { - type: "volume", - source: "nemoclaw-hermes-state-v1-hermes-box", - target: "/sandbox/.hermes", - read_only: false, - }, + managedStateMounts: [ + { + type: "volume", + source: "nemoclaw-hermes-state-v1-hermes-box", + target: "/sandbox/.hermes", + read_only: false, + }, + ], + managedStateMountDriverId: "docker", messagingTokenDefs: [], prepareInitialSandboxCreatePolicy: vi.fn(() => ({ policyPath: "/tmp/policy.yaml", @@ -1003,6 +1006,52 @@ describe("resolveSandboxCreateIntent", () => { }); }); + it("projects the managed Hermes state volume through the selected provider driver", () => { + const intent = resolveSandboxCreateIntent({ + basePolicyPath: "/repo/policy.yaml", + sandboxName: "hermes-box", + channels: [], + enabledChannels: [], + disabledChannelNames: new Set(), + messagingProviderRequests: [], + primaryMessagingCredentialEnvKeys: [], + reusableMessagingChannels: [], + reusableMessagingProviders: [], + hermesToolGateways: [], + sandboxGpuConfig, + gpuCreateArgs: [], + gpuRoutePlan: "native-only", + sandboxGpuLogMessage: null, + agentName: "hermes", + policyTier: null, + }); + const mount = { + type: "volume" as const, + source: "nemoclaw-hermes-state-v1-hermes-box", + target: "/sandbox/.hermes" as const, + read_only: false as const, + }; + const plan = materializeSandboxCreatePlan({ + intent, + fromRef: `ghcr.io/nvidia/nemoclaw/hermes@sha256:${"a".repeat(64)}`, + managedStateMounts: [mount], + managedStateMountDriverId: "opaque-native-driver", + messagingTokenDefs: [], + prepareInitialSandboxCreatePolicy: vi.fn(() => ({ + policyPath: "/tmp/policy.yaml", + appliedPresets: [], + })), + runProviderPreDeleteCleanup: vi.fn(), + upsertMessagingProviders: vi.fn(() => []), + getHermesToolGatewayProviderName: vi.fn(), + }); + const configIndex = plan.createArgs.indexOf("--driver-config-json"); + + expect(JSON.parse(plan.createArgs[configIndex + 1]!)).toEqual({ + "opaque-native-driver": { mounts: [mount] }, + }); + }); + it("rejects host mounts that overlap the managed Hermes state root", () => { const intent = resolveSandboxCreateIntent({ basePolicyPath: "/repo/policy.yaml", @@ -1027,12 +1076,15 @@ describe("resolveSandboxCreateIntent", () => { materializeSandboxCreatePlan({ intent, fromRef: `ghcr.io/nvidia/nemoclaw/hermes@sha256:${"a".repeat(64)}`, - managedStateMount: { - type: "volume", - source: "nemoclaw-hermes-state-v1-hermes-box", - target: "/sandbox/.hermes", - read_only: false, - }, + managedStateMounts: [ + { + type: "volume", + source: "nemoclaw-hermes-state-v1-hermes-box", + target: "/sandbox/.hermes", + read_only: false, + }, + ], + managedStateMountDriverId: "docker", messagingTokenDefs: [], prepareInitialSandboxCreatePolicy: vi.fn(() => ({ policyPath: "/tmp/policy.yaml", @@ -1042,7 +1094,7 @@ describe("resolveSandboxCreateIntent", () => { upsertMessagingProviders: vi.fn(() => []), getHermesToolGatewayProviderName: vi.fn(), }), - ).toThrow(/conflicts with the managed Hermes state root/u); + ).toThrow(/conflicts with the managed state root/u); }); it("cleans up the prepared policy when disclosure fails before provider effects (#7179)", () => { diff --git a/src/lib/onboard/sandbox-create/orchestration.ts b/src/lib/onboard/sandbox-create/orchestration.ts index ee065f93bb2..44f79dbb10d 100644 --- a/src/lib/onboard/sandbox-create/orchestration.ts +++ b/src/lib/onboard/sandbox-create/orchestration.ts @@ -42,7 +42,13 @@ import type { ManagedHermesStateVolumeCleanupResult, ManagedHermesStateVolumeContext, } from "../managed-workload/hermes-state-volume"; +import { removeManagedHermesStateVolume } from "../managed-workload/hermes-state-volume"; import type { OwnedSandboxRecreateRuntime } from "../onboard-recreate-journal"; +import { managedImageRuntimeIdentity } from "../managed-image/agents"; +import { + managedStartupStateRoots, + MANAGED_HERMES_STATE_ROOT, +} from "../managed-startup/state-roots"; import type { SandboxGpuConfig } from "../sandbox-gpu-mode"; import { cliName } from "../branding"; import type { @@ -258,6 +264,7 @@ export function selectRebuildCreatePolicy( return source; }); return materializeRebuildPolicyHandoff({ + sandboxName, livePolicyPath: policySourcePath, replacementPolicy: generatedPolicy, requiredNetworkPolicyKeys, @@ -1530,15 +1537,22 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche managedWorkloadRuntime, sandboxGpuCreateFlow.resolvePortableLifecycleMode(agent), ); - const prepareHermesStateVolumeLifecycle = ( + const prepareManagedStateVolumeLifecycle = ( workload: Awaited>, - ) => - managedWorkloadOnboard.createManagedHermesStateVolumeOnboardLifecycle({ - agentName: requestedAgentName, + ) => { + const managedStateRoots = + workload.source.kind === "managed-image" + ? managedStartupStateRoots({ + agent: workload.source.contract.agent, + sandboxName, + agentIdentity: managedImageRuntimeIdentity(workload.source.contract.agent), + }) + : []; + return managedWorkloadOnboard.createManagedStateVolumeOnboardLifecycle({ + roots: managedStateRoots, runtimeProvider: managedWorkloadRuntime.runtimeProvider, - sandboxName, - workloadKind: workload.source.kind, }); + }; const finalizeRecreatedSourceHermesVolume = ( sourceConfirmedAbsent: boolean, sourceEntry: SandboxEntry | null, @@ -1553,7 +1567,10 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche }, { normalizeRuntimeProviderIdentity: managedWorkloadOnboard.normalizeRuntimeProviderIdentity, - removeManagedHermesStateVolume: managedWorkloadOnboard.removeManagedHermesStateVolume, + removeManagedHermesStateVolume: (context) => + removeManagedHermesStateVolume(context, { + runtimeProvider: managedWorkloadRuntime.runtimeProvider ?? undefined, + }), removeSourceRegistryEntry: sandboxLifecycle.removeSandboxUnlessSessionReservation, note, warn: (message) => console.warn(message), @@ -1673,7 +1690,7 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche }); let pendingStateRestoreBackupPath: string | null = null, preparedSandboxWorkload!: Awaited>, - hermesStateVolumeLifecycle!: ReturnType; + managedStateVolumeLifecycle!: ReturnType; if (!liveExists && existingEntry) ({ runtime: recreateRuntime, backupPath: pendingStateRestoreBackupPath } = recreateProtection.selectJournalBoundPreUpgradeBackup({ @@ -1963,7 +1980,7 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche pendingStateRestore = result.backup; } - hermesStateVolumeLifecycle = prepareHermesStateVolumeLifecycle(preparedSandboxWorkload); + managedStateVolumeLifecycle = prepareManagedStateVolumeLifecycle(preparedSandboxWorkload); note(` Deleting and recreating sandbox '${sandboxName}'...`); revalidateSandboxIdentity(true, `recreating sandbox '${sandboxName}'`); @@ -1998,7 +2015,13 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche ); } recreateRuntime.confirmDeleted(); - finalizeRecreatedSourceHermesVolume(true, previousEntry, hermesStateVolumeLifecycle !== null); + finalizeRecreatedSourceHermesVolume( + true, + previousEntry, + managedStateVolumeLifecycle.roots.some( + (root) => root.mountTarget === MANAGED_HERMES_STATE_ROOT, + ), + ); await hermesApiPortReservationScope.rebindAfterOwnedForwardDelete( hermesApiPortReservationInput, ); @@ -2006,17 +2029,19 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche if (resumingVerifiedCreate) { await hermesApiPortReservationScope.selectAndReserve(hermesApiPortReservationInput); preparedSandboxWorkload = await ensurePreparedSandboxWorkload(); - hermesStateVolumeLifecycle = prepareHermesStateVolumeLifecycle(preparedSandboxWorkload); + managedStateVolumeLifecycle = prepareManagedStateVolumeLifecycle(preparedSandboxWorkload); } else if (!liveExists || agentCreateInput.hermesPortableLifecycle) { if (!agentCreateInput.hermesPortableLifecycle) { await hermesApiPortReservationScope.selectAndReserve(hermesApiPortReservationInput); } preparedSandboxWorkload = await ensurePreparedSandboxWorkload(); - hermesStateVolumeLifecycle = prepareHermesStateVolumeLifecycle(preparedSandboxWorkload); + managedStateVolumeLifecycle = prepareManagedStateVolumeLifecycle(preparedSandboxWorkload); finalizeRecreatedSourceHermesVolume( !liveExists, existingEntry, - hermesStateVolumeLifecycle !== null, + managedStateVolumeLifecycle.roots.some( + (root) => root.mountTarget === MANAGED_HERMES_STATE_ROOT, + ), ); } runForNewSandboxCreate(resumingVerifiedCreate, () => { @@ -2177,12 +2202,10 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche }, dependencies: { materializeSandboxCreatePlan: (input) => - hermesStateVolumeLifecycle - ? hermesStateVolumeLifecycle.materializeSandboxCreatePlan( - input, - sandboxCreatePlanMaterialization.materializeSandboxCreatePlan, - ) - : sandboxCreatePlanMaterialization.materializeSandboxCreatePlan(input), + managedStateVolumeLifecycle.materializeSandboxCreatePlan( + input, + sandboxCreatePlanMaterialization.materializeSandboxCreatePlan, + ), prepareSandboxBuildPatchConfig: sandboxBuildPatchConfig.prepareSandboxBuildPatchConfig, }, @@ -2264,6 +2287,7 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche const managedBootstrap = managedWorkloadOnboard.resolveOnboardManagedBootstrapLaunch({ runtime: managedWorkloadRuntime, workload: preparedSandboxWorkload, + sandboxName, stateRoot: getDockerDriverGatewayStateDir(), bootstrapIdentity: managedBootstrapIdentity, request: managedStartupRootApplyRequest, @@ -2899,7 +2923,7 @@ export function createSandboxWithBaseImageResolution(runtime: SandboxCreateOrche } return runWithPostCreateRecovery( () => { - hermesStateVolumeLifecycle?.commit(); + managedStateVolumeLifecycle.commit(); if ("complete" in recreateRuntime) recreateRuntime.complete(); if (agentCreateInput.hermesPortableLifecycle) return sandboxName; return completeOrdinaryOnboardSandboxCreation( diff --git a/src/lib/onboard/sandbox-create/provider-publication.ts b/src/lib/onboard/sandbox-create/provider-publication.ts index 1489356e27e..e52b6c9fa06 100644 --- a/src/lib/onboard/sandbox-create/provider-publication.ts +++ b/src/lib/onboard/sandbox-create/provider-publication.ts @@ -9,6 +9,7 @@ import { } from "../../messaging/provider-profile"; import type { SandboxEntry } from "../../state/registry"; import { inspectGatewayCredentialFamilyProviderBinding } from "../gateway-provider-metadata"; +import { resolveRegisteredRuntimeProvider } from "../runtime-provider/selection"; import type { SandboxCreateIntent } from "../sandbox-create-intent-types"; type ProviderPreparationInput = { @@ -56,13 +57,8 @@ function inspectExpectedMessagingBinding( ): boolean { const expected = expectedBindings.get(providerName); if (!expected) return true; - const inspection = inspectGatewayCredentialFamilyProviderBinding( - expected, - (args, options) => - deps.runOpenshell( - [...args.slice(0, 2), "-g", input.gatewayName, ...args.slice(2)], - options, - ), + const inspection = inspectGatewayCredentialFamilyProviderBinding(expected, (args, options) => + deps.runOpenshell([...args.slice(0, 2), "-g", input.gatewayName, ...args.slice(2)], options), ); return inspection.kind === "exact"; } @@ -108,7 +104,13 @@ export function publishAttachedProvidersBeforeDockerSandboxCreation( input: ProviderPreparationInput, deps: ProviderPreparationDeps, ): void { - if (input.openshellDriver !== "docker") return; + const runtimeProvider = resolveRegisteredRuntimeProvider(input.openshellDriver); + if ( + !runtimeProvider || + runtimeProvider.gateway.launcher !== "nemoclaw" || + runtimeProvider.bootstrap.supported !== true + ) + return; const expectedBindings = expectedMessagingBindings(input); const providersRequiringExistenceProbe = new Set( @@ -138,7 +140,7 @@ export function publishAttachedProvidersBeforeDockerSandboxCreation( if (refreshed.status !== 0) { deps.cleanupCreateSources(); throw new Error( - `OpenShell did not publish attached provider '${attachedProvider}' before Docker sandbox creation.`, + `OpenShell did not publish attached provider '${attachedProvider}' before managed sandbox creation.`, ); } if (inspectExpectedMessagingBinding(input, deps, attachedProvider, expectedBindings)) continue; diff --git a/src/lib/onboard/sandbox-create/rebuild-policy-handoff.test.ts b/src/lib/onboard/sandbox-create/rebuild-policy-handoff.test.ts index eb56007c200..cbee0364ee2 100644 --- a/src/lib/onboard/sandbox-create/rebuild-policy-handoff.test.ts +++ b/src/lib/onboard/sandbox-create/rebuild-policy-handoff.test.ts @@ -8,7 +8,10 @@ import YAML from "yaml"; import { afterEach, describe, expect, it, vi } from "vitest"; import { loadMessagingChannelPolicyPreset } from "../../messaging"; - +import { getMessagingPolicyKeysByChannel } from "../../messaging/channels"; +import * as policies from "../../policy"; +import { getCredentialBindingProviders } from "../initial-policy"; +import { allMessagingChannelPolicyPresets } from "../messaging-policy-presets"; import { materializeRebuildPolicyHandoff, mergeReplacementPolicyAccess, @@ -364,8 +367,8 @@ network_policies: it("uses active channel preset sources without copying unrequested keys", () => { const merged = mergeReplacementPolicyAccess( - "version: 1\nnetwork_policies:\n host_edit: {}\n", - "version: 1\nnetwork_policies: {}\n", + "version: 1\nnetwork_policies:\n host_edit: {}\n googlechat_hermes: {name: googlechat_hermes}\n", + "version: 1\nnetwork_policies:\n googlechat_hermes: {name: generic_googlechat}\n", ["googlechat_hermes"], [], [ @@ -383,6 +386,21 @@ network_policies: }); }); + it("rejects conflicting active channel sources for one required policy key", () => { + expect(() => + mergeReplacementPolicyAccess( + "version: 1\nnetwork_policies: {}\n", + "version: 1\nnetwork_policies: {}\n", + ["slack"], + [], + [ + "version: 1\nnetwork_policies:\n slack: {name: slack_a}\n", + "version: 1\nnetwork_policies:\n slack: {name: slack_b}\n", + ], + ), + ).toThrow("required network policy 'slack' has conflicting replacement sources"); + }); + it("materializes one private handoff and cleans it with the generated replacement source", () => { const livePath = tempPolicy( "live.yaml", @@ -466,4 +484,204 @@ network_policies: "host-added-provider", ]); }); + + it("drops OpenShell provider-composed entries before rebuild authorization", () => { + const livePath = tempPolicy( + "live-provider-composed.yaml", + `version: 1 +network_policies: + host_edit: {name: host_edit} + _provider_disabled_channel: + endpoints: + - credential_binding: {provider: disabled-channel-provider} +`, + ); + const replacementPath = tempPolicy( + "replacement-provider-composed.yaml", + "version: 1\nnetwork_policies: {}\n", + ); + + const handoff = materializeRebuildPolicyHandoff({ + livePolicyPath: livePath, + replacementPolicy: { + policyPath: replacementPath, + appliedPresets: [], + }, + }); + + expect(YAML.parse(fs.readFileSync(handoff.policyPath, "utf8")).network_policies).toEqual({ + host_edit: { name: "host_edit" }, + }); + expect(handoff.credentialBindingProviders).toEqual([]); + expect(handoff.cleanup?.()).toBe(true); + }); + + it("removes the Teams-owned Outlook login binding when Teams is disabled", () => { + const merged = mergeReplacementPolicyAccess( + `version: 1 +network_policies: + teams: + endpoints: + - host: login.microsoftonline.com + port: 443 + credential_binding: {provider: alpha-teams-bridge} + outlook_graph: + endpoints: + - host: login.microsoftonline.com + port: 443 + credential_binding: {provider: alpha-teams-bridge} +`, + `version: 1 +network_policies: + outlook_graph: + endpoints: + - host: login.microsoftonline.com + port: 443 +`, + [], + ["teams"], + [], + "alpha", + ); + + expect(YAML.parse(merged.source).network_policies).toEqual({ + outlook_graph: { + endpoints: [{ host: "login.microsoftonline.com", port: 443 }], + }, + }); + }); + + it("restores the Teams-owned Outlook login binding when Teams is re-enabled", () => { + const merged = mergeReplacementPolicyAccess( + `version: 1 +network_policies: + outlook_graph: + endpoints: + - host: login.microsoftonline.com + port: 443 +`, + "version: 1\nnetwork_policies: {}\n", + ["teams"], + [], + [ + `version: 1 +network_policies: + teams: + endpoints: + - host: login.microsoftonline.com + port: 443 + credential_binding: {provider: alpha-teams-bridge} +`, + ], + "alpha", + ); + + expect(YAML.parse(merged.source).network_policies).toEqual({ + outlook_graph: { + endpoints: [ + { + host: "login.microsoftonline.com", + port: 443, + credential_binding: { provider: "alpha-teams-bridge" }, + }, + ], + }, + teams: { + endpoints: [ + { + host: "login.microsoftonline.com", + port: 443, + credential_binding: { provider: "alpha-teams-bridge" }, + }, + ], + }, + }); + }); + + it.each(["openclaw", "hermes"] as const)( + "preserves the complete %s messaging policy lifecycle across rebuilds", + (agent) => { + const channels = [ + "telegram", + "discord", + "wechat", + "slack", + "whatsapp", + "teams", + "googlechat", + ]; + const removedChannels = ["wechat", "teams", "googlechat"]; + const remainingChannels = ["telegram", "discord", "slack", "whatsapp"]; + const sandboxName = `lifecycle-${agent}`; + const baseSource = fs.readFileSync( + path.join(process.cwd(), "agents", agent, "policy-permissive.yaml"), + "utf8", + ); + const keysByChannel = getMessagingPolicyKeysByChannel({ agent }); + const keysFor = (selected: string[]) => + selected.flatMap((channel) => [...(keysByChannel[channel] ?? [])]); + const compose = (selected: string[]) => + policies.mergePresetNamesIntoPolicy( + baseSource, + allMessagingChannelPolicyPresets(selected), + { agent, sandboxName, credentialBoundMessagingChannels: selected }, + ).policy; + + const activeDocument = YAML.parse(compose(channels)); + activeDocument.network_policies.github.endpoints[0].host = "host-maintained.example.com"; + const activeSource = YAML.stringify(activeDocument); + expect(getCredentialBindingProviders(activeSource)).toContain( + `${sandboxName}-teams-bridge`, + ); + + const stopped = mergeReplacementPolicyAccess( + activeSource, + baseSource, + [], + keysFor(channels), + [], + sandboxName, + ).source; + expect(getCredentialBindingProviders(stopped)).toEqual([]); + + const reenabled = mergeReplacementPolicyAccess( + stopped, + compose(channels), + keysFor(channels), + [], + [], + sandboxName, + ).source; + expect(getCredentialBindingProviders(reenabled)).toContain( + `${sandboxName}-teams-bridge`, + ); + + const selectedRemoved = mergeReplacementPolicyAccess( + reenabled, + compose(remainingChannels), + keysFor(remainingChannels), + keysFor(removedChannels), + [], + sandboxName, + ).source; + expect(getCredentialBindingProviders(selectedRemoved)).not.toContain( + `${sandboxName}-teams-bridge`, + ); + + expect(YAML.parse(stopped).network_policies.github.endpoints[0].host).toBe( + "host-maintained.example.com", + ); + expect(YAML.parse(reenabled).network_policies.github.endpoints[0].host).toBe( + "host-maintained.example.com", + ); + expect(YAML.parse(selectedRemoved).network_policies.github.endpoints[0].host).toBe( + "host-maintained.example.com", + ); + const finalPolicies = YAML.parse(selectedRemoved).network_policies; + expect(Object.keys(finalPolicies)).not.toEqual( + expect.arrayContaining(keysFor(removedChannels)), + ); + expect(Object.keys(finalPolicies)).toEqual(expect.arrayContaining(keysFor(remainingChannels))); + }, + ); }); diff --git a/src/lib/onboard/sandbox-create/rebuild-policy-handoff.ts b/src/lib/onboard/sandbox-create/rebuild-policy-handoff.ts index a0926d4822f..43257ca10b1 100644 --- a/src/lib/onboard/sandbox-create/rebuild-policy-handoff.ts +++ b/src/lib/onboard/sandbox-create/rebuild-policy-handoff.ts @@ -6,7 +6,8 @@ import { isDeepStrictEqual } from "node:util"; import YAML from "yaml"; import { isReviewedMessagingChannelPolicyUpgrade } from "../../messaging/channels/policy"; -import { parseOpenShellPolicy } from "../../policy/merge"; +import * as policies from "../../policy"; +import { parseOpenShellPolicy, stripProviderComposedPolicies } from "../../policy/merge"; import { getCredentialBindingProviders, type InitialSandboxPolicy } from "../initial-policy"; import { cleanupTempDir, createExactTempFileCleanup, secureTempFile } from "../temp-files"; @@ -156,6 +157,7 @@ function mergeRequestedReplacementNetworkPolicies( policyMapping(replacement.network_policies, "replacement network_policies"), ); if (required.size > 0) { + const requiredPolicies: PolicyMapping = {}; for (const source of requiredPolicySources) { let parsed: unknown; try { @@ -172,15 +174,16 @@ function mergeRequestedReplacementNetworkPolicies( ); for (const [key, value] of Object.entries(policies)) { if (!required.has(key)) continue; - const existing = replacementPolicies[key]; + const existing = requiredPolicies[key]; if (existing !== undefined && !isDeepStrictEqual(existing, value)) { throw new Error( `Cannot prepare rebuild policy handoff: required network policy '${key}' has conflicting replacement sources.`, ); } - replacementPolicies[key] = structuredClone(value); + requiredPolicies[key] = structuredClone(value); } } + Object.assign(replacementPolicies, requiredPolicies); } const livePolicies = live.network_policies === undefined @@ -237,8 +240,25 @@ export function mergeReplacementPolicyAccess( requiredNetworkPolicyKeys: readonly string[] = [], removedNetworkPolicyKeys: readonly string[] = [], requiredNetworkPolicySources: readonly string[] = [], + sandboxName?: string, ): { readonly changed: boolean; readonly source: string } { - const live = structuredClone(parseOpenShellPolicy(livePolicySource).policy) as PolicyMapping; + const providerNormalizedLivePolicySource = stripProviderComposedPolicies(livePolicySource); + const teamsActive = requiredNetworkPolicyKeys.includes("teams") + ? true + : removedNetworkPolicyKeys.includes("teams") + ? false + : null; + const normalizedLivePolicySource = + teamsActive !== null + ? policies.reconcileTeamsOutlookLoginCredentialBinding( + providerNormalizedLivePolicySource, + sandboxName, + teamsActive, + ) + : providerNormalizedLivePolicySource; + const live = structuredClone( + parseOpenShellPolicy(normalizedLivePolicySource).policy, + ) as PolicyMapping; const replacement = parseOpenShellPolicy(replacementPolicySource).policy as PolicyMapping; const processChanged = mergeMissingReplacementProcessIdentity(live, replacement); const filesystemChanged = mergeReplacementFilesystemAccess(live, replacement); @@ -249,14 +269,19 @@ export function mergeReplacementPolicyAccess( removedNetworkPolicyKeys, requiredNetworkPolicySources, ); - const changed = processChanged || filesystemChanged || networkChanged; + const changed = + normalizedLivePolicySource !== livePolicySource || + processChanged || + filesystemChanged || + networkChanged; return changed ? { changed: true, source: YAML.stringify(live) } - : { changed: false, source: livePolicySource }; + : { changed: false, source: normalizedLivePolicySource }; } /** Materialize the single ephemeral policy input consumed by an explicit rebuild. */ export function materializeRebuildPolicyHandoff(input: { + readonly sandboxName?: string; readonly livePolicyPath: string; readonly replacementPolicy: InitialSandboxPolicy; readonly requiredNetworkPolicyKeys?: readonly string[]; @@ -274,6 +299,7 @@ export function materializeRebuildPolicyHandoff(input: { input.requiredNetworkPolicyKeys, input.removedNetworkPolicyKeys, input.requiredNetworkPolicySources, + input.sandboxName, ); if (!merged.changed) { return { diff --git a/src/lib/onboard/sandbox-dockerfile-patch-flow.test.ts b/src/lib/onboard/sandbox-dockerfile-patch-flow.test.ts index 8846fedd699..1f46aef1c2e 100644 --- a/src/lib/onboard/sandbox-dockerfile-patch-flow.test.ts +++ b/src/lib/onboard/sandbox-dockerfile-patch-flow.test.ts @@ -187,12 +187,13 @@ describe("prepareSandboxDockerfilePatch", () => { expect(enforceDockerGpuPatchPreserveNetwork).toHaveBeenCalledWith( "nvidia-prod", sandboxGpuConfig, - { + expect.objectContaining({ dockerDriverGateway: true, gatewayPort: undefined, log, + reverifyBridgeReachability: expect.any(Function), selectedRoute: "none", - }, + }), ); expect(patchStagedDockerfile).toHaveBeenCalledWith( "/tmp/Dockerfile", diff --git a/src/lib/onboard/sandbox-dockerfile-patch-flow.ts b/src/lib/onboard/sandbox-dockerfile-patch-flow.ts index 517e2ac701f..14c3386762c 100644 --- a/src/lib/onboard/sandbox-dockerfile-patch-flow.ts +++ b/src/lib/onboard/sandbox-dockerfile-patch-flow.ts @@ -99,6 +99,12 @@ function enforceDockerGpuPatchPreserveNetwork( return impl(...args); } +function reverifySandboxBridgeGatewayReachability(port?: number): Promise { + const { verifySandboxBridgeGatewayReachableOrExit } = + require("./gateway-sandbox-reachability") as typeof import("./gateway-sandbox-reachability"); + return verifySandboxBridgeGatewayReachableOrExit(true, { skip: false, port }); +} + function patchStagedDockerfile( ...args: Parameters ): ReturnType { @@ -181,6 +187,7 @@ export async function prepareSandboxDockerfilePatch({ selectedRoute: selectedGpuRoute, gatewayPort, log, + reverifyBridgeReachability: () => reverifySandboxBridgeGatewayReachability(gatewayPort), }, ); const darwinVmCompat = false; diff --git a/src/lib/onboard/sandbox-gpu-create-attempt.ts b/src/lib/onboard/sandbox-gpu-create-attempt.ts index 71279406a56..19724f614d9 100644 --- a/src/lib/onboard/sandbox-gpu-create-attempt.ts +++ b/src/lib/onboard/sandbox-gpu-create-attempt.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { hasSandboxListEntry } from "../state/gateway"; +import { getSandboxFailurePhase, hasSandboxListEntry, isSandboxReady } from "../state/gateway"; import { canFallbackToDockerGpuCompatibility, type DockerGpuRoutePlan, @@ -21,6 +21,8 @@ import { export type SandboxGpuCreateFailureStage = "create" | "readiness" | "gpu-proof"; +export { getSandboxFailurePhase, isSandboxReady }; + export type SandboxGpuCreateAttemptSuccess = { ok: true; route: SelectedDockerGpuRoute; diff --git a/src/lib/onboard/sandbox-gpu-create-flow.test.ts b/src/lib/onboard/sandbox-gpu-create-flow.test.ts index fbfe616dde5..9b46b929cfb 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.test.ts @@ -126,6 +126,7 @@ const { setupHarness, resetHarness, } = createGpuFlowTestHarness(mocks); +const READY_CHECK_ARGS = ["sandbox", "list", "-g", "nemoclaw"]; beforeEach(setupHarness); afterEach(resetHarness); @@ -254,6 +255,7 @@ describe("runSandboxGpuCreateFlow provider-owned managed create", () => { }), bootstrapIdentity: "e".repeat(64), code: "mxc-recovery-retry", + blockingScope: "sandbox", retryable: true, detail, }), @@ -343,6 +345,8 @@ describe("runSandboxGpuCreateFlow provider-owned managed create", () => { manifestDigest: `sha256:${"d".repeat(64)}`, }, agentIdentity: { uid: 1000, gid: 1000, workdir: "/sandbox" }, + workspaceRoot: { uid: 1000, gid: 1000, mode: 0o755 }, + managedStateRoots: [], intendedWorkloadArgv: launch.intendedSandboxStartupCommand, expectedSupervisorArgv: ["/mxc/supervisor"], }; @@ -404,10 +408,7 @@ describe("runSandboxGpuCreateFlow provider-owned managed create", () => { ).toEqual([2, 2]); vi.mocked(deps.runCaptureOpenshell).mockClear(); await expect(runSandboxGpuCreateFlow(input, deps)).resolves.toMatchObject({ route: "none" }); - expect(deps.runCaptureOpenshell).toHaveBeenCalledWith( - ["sandbox", "list", "-g", "nemoclaw"], - READY_CHECK_OPTIONS, - ); + expect(deps.runCaptureOpenshell).toHaveBeenCalledWith(READY_CHECK_ARGS, READY_CHECK_OPTIONS); expect(vi.mocked(console.warn).mock.calls.flat().join("\n")).toContain( "unrelated sandbox 'bravo'", ); @@ -623,10 +624,7 @@ describe("runSandboxGpuCreateFlow native failure and readiness", () => { }); const result = await runSandboxGpuCreateFlow(createInput(), deps); expect(result).toMatchObject({ route: "native" }); - expect(deps.runCaptureOpenshell).toHaveBeenCalledWith( - ["sandbox", "list", "-g", "nemoclaw"], - READY_CHECK_OPTIONS, - ); + expect(deps.runCaptureOpenshell).toHaveBeenCalledWith(READY_CHECK_ARGS, READY_CHECK_OPTIONS); }); it("defers restart-safe no-GPU recreation until the create process exits (#8720)", async () => { diff --git a/src/lib/onboard/sandbox-gpu-create-flow.ts b/src/lib/onboard/sandbox-gpu-create-flow.ts index c7a05598366..72a8b7719df 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.ts @@ -55,7 +55,10 @@ import type { RuntimeProviderManagedImageBootstrapSurface, } from "./runtime-provider/contract"; import * as sandboxGpuCreateAttempt from "./sandbox-gpu-create-attempt"; -import { createSandboxGpuCreateAttemptRunner } from "./sandbox-gpu-create-run-attempt"; +import { + createSandboxGpuCreateAttemptRunner, + verifySelectedSandboxBridgeReachability, +} from "./sandbox-gpu-create-run-attempt"; import { managedBootstrapCreateArgs } from "./sandbox-create-launch"; import type { SandboxGpuConfig } from "./sandbox-gpu-mode"; import { @@ -259,6 +262,8 @@ export interface SandboxGpuCreateFlowInput { readonly request: ManagedStartupRootApplyRequest; readonly image: ManagedBootstrapImageIdentity; readonly agentIdentity: ManagedBootstrapAgentIdentity; + readonly workspaceRoot: import("./managed-startup/state-roots").ManagedStartupWorkspaceRoot; + readonly managedStateRoots: readonly import("./managed-startup/state-roots").ManagedStartupStateRoot[]; readonly intendedWorkloadArgv: readonly string[]; readonly expectedSupervisorArgv: readonly string[]; } | null; @@ -409,6 +414,7 @@ export async function runSandboxGpuCreateFlow( installPortableDemoLifecycle: () => input.lifecycleGeneration!, } : deps, + () => verifySelectedSandboxBridgeReachability(input), ); const gpuCreateOutcome = await (input.resumeVerifiedCreate ? attemptRunner.runAttempt(input.resumeVerifiedCreate.route) @@ -515,6 +521,7 @@ export async function runSandboxGpuCreateFlow( selectedRoute: "compatibility", gatewayPort: input.gatewayPort, log: console.log, + reverifyBridgeReachability: () => verifySelectedSandboxBridgeReachability(input), }, ); } diff --git a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts index 737c5967a52..b20192f7903 100644 --- a/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-identity-gate.test.ts @@ -148,6 +148,7 @@ describe("created sandbox identity gate", () => { expect(deps.runCaptureOpenshell).toHaveBeenCalledWith(["sandbox", "list", "-g", gatewayName], { ignoreError: true, + killProcessTreeOnTimeout: true, timeout: 5_000, }); expect(deps.runOpenshell).toHaveBeenCalledWith( diff --git a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts index a136730c190..3e4c44b21c0 100644 --- a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts +++ b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts @@ -20,7 +20,6 @@ import { import { printSandboxCreateRecoveryHints } from "../build-context"; import { streamSandboxCreate, type StreamSandboxCreateResult } from "../sandbox/create-stream"; import { getReadyCheckOutputPatternsForAgent } from "../sandbox/create-stream-ready-gate"; -import { isSandboxReady } from "../state/gateway"; import type { SandboxGpuProofResult } from "../state/registry"; import { classifySandboxCreateFailure } from "../validation"; import { @@ -29,6 +28,10 @@ import { } from "./created-sandbox-failure"; import * as dockerGpuLocalInference from "./docker-gpu-local-inference"; import type { SelectedDockerGpuRoute } from "./docker-gpu-route"; +import { + isSandboxBridgeGatewayReachable, + verifySandboxBridgeGatewayReachableOrExit, +} from "./gateway-sandbox-reachability"; import { createDockerGpuSandboxCreatePatch } from "./docker-gpu-sandbox-create"; import { installPortableDemoSandboxLifecycle } from "./experimental/portable-demo-lifecycle"; import { enforceManagedBootstrapRecoveryForSandbox } from "./managed-bootstrap/adapter"; @@ -226,6 +229,7 @@ function probeExactOpenShellSandboxId( suppressOutput: true, timeout, killSignal: "SIGKILL", + killProcessTreeOnTimeout: true, }); if (result.status === 0 && !result.error) { const sandboxId = parseOpenShellSandboxId(String(result.stdout ?? "")); @@ -364,6 +368,7 @@ function checkSandboxExecutableReadiness( suppressOutput: true, timeout, killSignal: "SIGKILL", + killProcessTreeOnTimeout: true, }, ); if (result.status === 0 && !result.error) return "ready"; @@ -381,9 +386,58 @@ class ManagedBootstrapCreateStreamFailure extends Error { } } +export async function verifySelectedSandboxBridgeReachability( + input: SandboxGpuCreateFlowInput, +): Promise { + const managedBootstrap = input.managedBootstrap; + const reachabilityImpl = managedBootstrap + ? (options: Parameters[0]) => { + const gatewayRuntime = managedBootstrap.runtimeProvider.gateway.prepareHostRuntime({ + environment: input.hostEnv ?? process.env, + platform: process.platform, + }); + return isSandboxBridgeGatewayReachable({ ...options, gatewayRuntime }); + } + : undefined; + await verifySandboxBridgeGatewayReachableOrExit(true, { + skip: false, + port: input.gatewayPort, + ...(reachabilityImpl ? { reachabilityImpl } : {}), + }); +} + +async function verifyActivatedManagedCreateBeforeEffects(input: { + readonly sandboxId: string | null; + readonly createAttemptNonce: string; + readonly route: SelectedDockerGpuRoute; + readonly flow: SandboxGpuCreateFlowInput; + readonly lifecycle: ManagedBootstrapRuntimeCreateLifecycle; + readonly deferPostCreateEffects: boolean; + readonly waitForCreatedSandboxPublication: (sandboxId: string) => void; + readonly revalidatePostCreateEffect: (operation: string) => void; +}): Promise { + if (!input.sandboxId) { + throw new Error("Managed bootstrap create returned without one exact sandbox identity."); + } + input.waitForCreatedSandboxPublication(input.sandboxId); + await verifyCreatedSandboxBeforeEffects( + input.sandboxId, + input.createAttemptNonce, + input.route, + input.flow, + ); + if (input.deferPostCreateEffects) { + input.revalidatePostCreateEffect( + `activate managed sandbox network for '${input.flow.sandboxName}'`, + ); + await input.lifecycle.prepareNetwork(); + } +} + export function createSandboxGpuCreateAttemptRunner( input: SandboxGpuCreateFlowInput, deps: SandboxGpuCreateFlowDeps, + reverifyManagedBridgeReachability: () => Promise, ) { const portableLifecycle = input.portableLifecycle === true; const printCreateFailureDiagnostics = @@ -421,6 +475,15 @@ export function createSandboxGpuCreateAttemptRunner( } revalidate(operation); }; + const captureSandboxReadiness: SandboxGpuCreateFlowDeps["runCaptureOpenshell"] = ( + args, + options = {}, + ) => + deps.runCaptureOpenshell(args, { + ...options, + killProcessTreeOnTimeout: true, + timeout: SANDBOX_READY_PROBE_TIMEOUT_MS, + }); const managedRouting = input.managedBootstrap?.runtimeProvider.bootstrap.createOnboardRouting({ sandboxName: input.sandboxName, openshellArgv: deps.openshellArgv, @@ -527,11 +590,14 @@ export function createSandboxGpuCreateAttemptRunner( const managedLifecycle = managedBootstrap ? managedBootstrap.runtimeProvider.bootstrap.createLifecycle({ providerId: managedBootstrap.runtimeProvider.identity.id, + environment: input.hostEnv ?? process.env, stateRoot: managedBootstrap.stateRoot, bootstrapIdentity: attemptBootstrapIdentity ?? managedBootstrap.bootstrapIdentity, request: managedBootstrap.request, image: managedBootstrap.image, agentIdentity: managedBootstrap.agentIdentity, + workspaceRoot: managedBootstrap.workspaceRoot, + managedStateRoots: managedBootstrap.managedStateRoots, intendedWorkloadArgv: managedBootstrap.intendedWorkloadArgv, expectedSupervisorArgv: managedBootstrap.expectedSupervisorArgv, launchArgv: attemptArgv, @@ -551,6 +617,7 @@ export function createSandboxGpuCreateAttemptRunner( inferenceProvider: input.provider, gatewayUsesContainerBridge: input.dockerDriverGateway, gatewayPort: input.gatewayPort, + reverifyBridgeReachability: reverifyManagedBridgeReachability, }, dependencies: { runCaptureOpenshell: deps.runCaptureOpenshell, @@ -642,16 +709,17 @@ export function createSandboxGpuCreateAttemptRunner( readyCheck: () => { const list = deps.runCaptureOpenshell(["sandbox", "list", "-g", input.gatewayName], { ignoreError: true, + killProcessTreeOnTimeout: true, timeout: SANDBOX_READY_PROBE_TIMEOUT_MS, }); - const ready = isSandboxReady(list, input.sandboxName); + const ready = sandboxGpuCreateAttempt.isSandboxReady(list, input.sandboxName); if (!ready || !createAttemptNonce) return ready; const observation = observeCreatedOpenShellSandboxId( { sandboxName: input.sandboxName, gatewayName: input.gatewayName, createAttemptNonce, - runCaptureOpenshell: deps.runCaptureOpenshell, + runCaptureOpenshell: captureSandboxReadiness, }, SANDBOX_READY_PROBE_TIMEOUT_MS, ); @@ -713,6 +781,7 @@ export function createSandboxGpuCreateAttemptRunner( }; let createResult: Awaited> | null = null; let resumedSandboxId: string | null = null; + let managedCreatedSandboxId: string | null = null; let managedIncompleteCreateRecovered = false; let createdSandboxVerified = false; const failAfterCreatedSandboxVerification = (message: string, status: number): never => { @@ -830,15 +899,7 @@ export function createSandboxGpuCreateAttemptRunner( { cause: error }, ); } - waitForCreatedSandboxPublication(sandboxId); - await verifyCreatedSandboxBeforeEffects(sandboxId, createAttemptNonce!, route, input); - createdSandboxVerified = true; - if (deferPostCreateEffects) { - revalidatePostCreateEffect( - `activate managed sandbox network for '${input.sandboxName}'`, - ); - await managedLifecycle.prepareNetwork(); - } + managedCreatedSandboxId = sandboxId; managedIncompleteCreateRecovered = createFailure?.kind === "sandbox_create_incomplete"; return { value: result, @@ -854,6 +915,17 @@ export function createSandboxGpuCreateAttemptRunner( }; }, ); + await verifyActivatedManagedCreateBeforeEffects({ + sandboxId: managedCreatedSandboxId, + createAttemptNonce: createAttemptNonce!, + route, + flow: input, + lifecycle: managedLifecycle, + deferPostCreateEffects, + waitForCreatedSandboxPublication, + revalidatePostCreateEffect, + }); + createdSandboxVerified = true; } catch (error) { if (!(error instanceof ManagedBootstrapCreateStreamFailure)) throw error; createResult = error.result; diff --git a/src/lib/onboard/sandbox-gpu-preflight-routing.test.ts b/src/lib/onboard/sandbox-gpu-preflight-routing.test.ts index 9a1e70c3f2b..33938e5e2f2 100644 --- a/src/lib/onboard/sandbox-gpu-preflight-routing.test.ts +++ b/src/lib/onboard/sandbox-gpu-preflight-routing.test.ts @@ -13,6 +13,7 @@ import { formatSandboxGpuPassthroughNote, parseDockerRuntimeNames, sandboxGpuRemediationLines, + validatePodmanSandboxGpuPreflight, validateSandboxGpuPreflight, } from "./sandbox-gpu-preflight"; @@ -119,6 +120,40 @@ describe("sandbox GPU preflight routing", () => { ).not.toThrow(); }); + it("validates native Podman GPU support through CDI without Docker inspection", () => { + const findReadableNvidiaCdiSpecFiles = vi.fn(() => ["/etc/cdi/nvidia.yaml"]); + + expect(() => + validatePodmanSandboxGpuPreflight(sandboxGpuConfig(), { + platform: "linux", + findReadableNvidiaCdiSpecFiles, + }), + ).not.toThrow(); + expect(findReadableNvidiaCdiSpecFiles).toHaveBeenCalledWith(["/etc/cdi", "/var/run/cdi"]); + }); + + it("reports native Podman CDI admission failures without Docker remediation", () => { + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); + const exitProcess = (code: number): never => { + throw new Error(`exit:${code}`); + }; + + try { + expect(() => + validatePodmanSandboxGpuPreflight( + sandboxGpuConfig(), + { platform: "linux", findReadableNvidiaCdiSpecFiles: vi.fn(() => []) }, + exitProcess, + ), + ).toThrow("exit:1"); + const message = errorSpy.mock.calls.map((call) => call[0]).join("\n"); + expect(message).toContain("Podman CDI GPU support was not detected"); + expect(message).not.toContain("Docker"); + } finally { + errorSpy.mockRestore(); + } + }); + it("still fails when the fallback CDI spec dirs hold no NVIDIA spec (#7330)", () => { const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); const exitSpy = vi.spyOn(process, "exit").mockImplementation((( diff --git a/src/lib/onboard/sandbox-gpu-preflight.ts b/src/lib/onboard/sandbox-gpu-preflight.ts index d171f8a2291..780840ee9e2 100644 --- a/src/lib/onboard/sandbox-gpu-preflight.ts +++ b/src/lib/onboard/sandbox-gpu-preflight.ts @@ -407,3 +407,25 @@ export function validateSandboxGpuPreflight( } console.log(` ✓ Docker CDI GPU support detected (${cdiSpecFiles.join(", ")})`); } + +/** Validate native Podman GPU admission through CDI without touching Docker authority. */ +export function validatePodmanSandboxGpuPreflight( + config: SandboxGpuConfig, + deps: Pick = {}, + exitProcess: (code: number) => never = (code) => process.exit(code), +): void { + exitOnSandboxGpuConfigErrors(config, exitProcess); + if (!config.sandboxGpuEnabled || (deps.platform ?? process.platform) !== "linux") return; + const cdiSpecFiles = ( + deps.findReadableNvidiaCdiSpecFiles ?? findReadableNvidiaCdiSpecFiles + )([...DEFAULT_DOCKER_CDI_SPEC_DIRS]); + if (cdiSpecFiles.length === 0) { + console.error(""); + console.error(failLine("Podman CDI GPU support was not detected.")); + console.error(" Install/configure NVIDIA Container Toolkit CDI, then retry Podman:"); + console.error(" sudo nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml"); + console.error(" Or force CPU sandbox behavior with NEMOCLAW_SANDBOX_GPU=0."); + exitProcess(1); + } + console.log(` ✓ Podman CDI GPU support detected (${cdiSpecFiles.join(", ")})`); +} diff --git a/src/lib/onboard/sandbox-provider-cleanup.ts b/src/lib/onboard/sandbox-provider-cleanup.ts index 3d52cdd999b..0f52a220ffc 100644 --- a/src/lib/onboard/sandbox-provider-cleanup.ts +++ b/src/lib/onboard/sandbox-provider-cleanup.ts @@ -21,6 +21,15 @@ export function removeManagedHermesStateVolume( return volumeModule.removeManagedHermesStateVolume(context, deps); } +export function removeManagedAgentStateVolumes( + context: import("./managed-workload/hermes-state-volume").ManagedHermesStateVolumeContext, + deps: import("./managed-workload/hermes-state-volume").ManagedHermesStateVolumeDeps = {}, +): readonly import("./managed-workload/hermes-state-volume").ManagedAgentStateVolumeCleanupResult[] { + const volumeModule = + require("./managed-workload/hermes-state-volume") as typeof import("./managed-workload/hermes-state-volume"); + return volumeModule.removeManagedAgentStateVolumes(context, deps); +} + export type SandboxProviderRunOpenshell = ( args: string[], opts?: Record, diff --git a/src/lib/onboard/sandbox-registry-metadata.test.ts b/src/lib/onboard/sandbox-registry-metadata.test.ts index 7e1d4d19a04..2c7c92aa434 100644 --- a/src/lib/onboard/sandbox-registry-metadata.test.ts +++ b/src/lib/onboard/sandbox-registry-metadata.test.ts @@ -26,7 +26,7 @@ async function makeHelpers(driverName: string) { // resolve from TS source. Same pattern as `vm-dns-monkeypatch.test.ts`. const metadata = await import("./sandbox-registry-metadata"); return metadata.createSandboxRegistryMetadataHelpers({ - getOpenShellComputeDriverName: () => driverName, + getCurrentRuntimeProviderId: () => driverName, getInstalledOpenshellVersion: () => "0.0.42", runCaptureOpenshell: () => null, }); @@ -133,7 +133,7 @@ describe("sandbox registry metadata", () => { }); const helpers = metadata.createSandboxRegistryMetadataHelpers({ - getOpenShellComputeDriverName: () => "docker", + getCurrentRuntimeProviderId: () => "docker", getInstalledOpenshellVersion: () => "0.0.44", runCaptureOpenshell: () => "openshell 0.0.44", }); @@ -188,7 +188,7 @@ describe("sandbox registry metadata", () => { const registry = await import("../state/registry"); const authority = await import("./workload/authority"); const helpers = metadata.createSandboxRegistryMetadataHelpers({ - getOpenShellComputeDriverName: () => "docker", + getCurrentRuntimeProviderId: () => "docker", getInstalledOpenshellVersion: () => "0.0.44", runCaptureOpenshell: () => "openshell 0.0.44", }); @@ -314,7 +314,7 @@ describe("sandbox registry metadata", () => { const dashboardPorts = await import("./dashboard-port"); const gatewayRegistry = await import("../state/gateway-registry"); const helpers = metadata.createSandboxRegistryMetadataHelpers({ - getOpenShellComputeDriverName: () => "docker", + getCurrentRuntimeProviderId: () => "docker", getInstalledOpenshellVersion: () => "0.0.44", runCaptureOpenshell: () => "openshell 0.0.44", }); @@ -380,7 +380,7 @@ describe("getSandboxRuntimeRegistryFields openshellDriver", () => { const metadata = await import("./sandbox-registry-metadata"); let driverName = "docker"; const helpers = metadata.createSandboxRegistryMetadataHelpers({ - getOpenShellComputeDriverName: () => driverName, + getCurrentRuntimeProviderId: () => driverName, getInstalledOpenshellVersion: () => "0.0.42", runCaptureOpenshell: () => null, }); diff --git a/src/lib/onboard/sandbox-registry-metadata.ts b/src/lib/onboard/sandbox-registry-metadata.ts index b3442fd9f12..ffc14a4d159 100644 --- a/src/lib/onboard/sandbox-registry-metadata.ts +++ b/src/lib/onboard/sandbox-registry-metadata.ts @@ -9,7 +9,7 @@ import { getSandboxAgentRegistryFields } from "./sandbox-agent"; import type { SandboxGpuConfig } from "./sandbox-gpu-mode"; export interface SandboxRegistryMetadataDeps { - getOpenShellComputeDriverName(): string; + getCurrentRuntimeProviderId(): string; getInstalledOpenshellVersion(versionOutput?: string | null): string | null; runCaptureOpenshell(args: string[], opts?: Record): string | null; } @@ -83,9 +83,9 @@ export function createSandboxRegistryMetadataHelpers( // Only persist a proof when this run produced one; omit on reuse/update // paths so a prior proof result is preserved rather than nulled out. ...(config.sandboxGpuProof ? { sandboxGpuProof: config.sandboxGpuProof } : {}), - // Driver identity comes from the resolved compute plan, not the host - // gateway launcher; those layers may differ (#7744). - openshellDriver: deps.getOpenShellComputeDriverName(), + // Persist the selected managed provider identity. The provider may use + // compatibility compute plumbing internally without becoming Docker. + openshellDriver: deps.getCurrentRuntimeProviderId(), openshellVersion: deps.getInstalledOpenshellVersion( deps.runCaptureOpenshell(["--version"], { ignoreError: true }), ), diff --git a/src/lib/onboard/sandbox-workload-preparation.test.ts b/src/lib/onboard/sandbox-workload-preparation.test.ts index 190d177f2e8..2b8c94ef13c 100644 --- a/src/lib/onboard/sandbox-workload-preparation.test.ts +++ b/src/lib/onboard/sandbox-workload-preparation.test.ts @@ -180,6 +180,15 @@ describe("sandbox workload preparation", () => { NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG: catalogPath, }), ).toEqual({ path: catalogPath, revision: REVISION }); + expect( + liveE2eManagedImageCatalog({ + GITHUB_ACTIONS: "true", + NEMOCLAW_RUN_LIVE_E2E: "1", + NEMOCLAW_E2E_EXPECTED_SHA: "b".repeat(40), + NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG: catalogPath, + NEMOCLAW_E2E_MANAGED_IMAGE_REVISION: REVISION, + }), + ).toEqual({ path: catalogPath, revision: REVISION }); expect( liveE2eManagedImageCatalog({ GITHUB_ACTIONS: "true", @@ -223,7 +232,7 @@ describe("sandbox workload preparation", () => { } }); - it("rejects an embedded catalog without an exact candidate revision (#9464)", () => { + it("rejects an embedded catalog without an exact publication revision (#9464)", () => { const fixtureRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-live-e2e-catalog-")); const catalogPath = path.join(fixtureRoot, "catalog.json"); fs.writeFileSync(catalogPath, "{}\n", { mode: 0o600 }); @@ -234,7 +243,7 @@ describe("sandbox workload preparation", () => { NEMOCLAW_RUN_LIVE_E2E: "1", NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG: catalogPath, }), - ).toThrow("requires an exact candidate revision"); + ).toThrow("requires an exact publication revision"); expect(() => liveE2eManagedImageCatalog({ GITHUB_ACTIONS: "true", diff --git a/src/lib/onboard/workload/preparation.ts b/src/lib/onboard/workload/preparation.ts index 8995c554d97..8b5f26cc175 100644 --- a/src/lib/onboard/workload/preparation.ts +++ b/src/lib/onboard/workload/preparation.ts @@ -176,10 +176,13 @@ export function liveE2eManagedImageCatalog( { cause: error }, ); } - const revision = environment.NEMOCLAW_E2E_EXPECTED_SHA?.trim() ?? ""; + const revision = + environment.NEMOCLAW_E2E_MANAGED_IMAGE_REVISION?.trim() ?? + environment.NEMOCLAW_E2E_EXPECTED_SHA?.trim() ?? + ""; if (!/^[0-9a-f]{40}$/u.test(revision)) { throw new SandboxWorkloadPreparationError( - "the live E2E managed-image catalog requires an exact candidate revision", + "the live E2E managed-image catalog requires an exact publication revision", ); } return { path: catalogPath, revision }; diff --git a/src/lib/policy/index.ts b/src/lib/policy/index.ts index 1421e1d5825..775df355df2 100644 --- a/src/lib/policy/index.ts +++ b/src/lib/policy/index.ts @@ -1898,9 +1898,8 @@ function removePreset( const exclusionError = openClawNpmExclusionStateError(sandboxName, currentPolicy); if (exclusionError) throw new Error(exclusionError); } - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - console.error(` Refusing to remove npm policy compatibility: ${message}`); + } catch { + console.error(" Refusing to remove npm policy compatibility: validation failed."); return false; } } @@ -1924,18 +1923,16 @@ function removePreset( "teams", ); updated = reconcileTeamsOutlookLoginCredentialBinding(updated, sandboxName, teamsActive); - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - console.error(` Refusing to remove preset '${presetName}': ${message}`); + } catch { + console.error(` Refusing to remove preset '${presetName}': validation failed.`); return false; } } if (openClawNpmBaseline) { try { updated = restoreOpenClawNpmCompatibility(currentPolicy, updated, openClawNpmBaseline); - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - console.error(` Refusing to remove npm policy compatibility: ${message}`); + } catch { + console.error(" Refusing to remove npm policy compatibility: validation failed."); return false; } } @@ -2417,9 +2414,8 @@ function applyPresetContent( merged = activation.policy; npmBaselineWidened = activation.widenedBaseline; } - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - console.error(` Refusing to apply npm policy compatibility: ${message}`); + } catch { + console.error(" Refusing to apply npm policy compatibility: validation failed."); return false; } } @@ -2596,9 +2592,8 @@ function applyPresets(sandboxName: string, presetNames: string[]): boolean { merged = activation.policy; npmBaselineWidened = activation.widenedBaseline; } - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - console.error(` Refusing to apply npm policy compatibility: ${message}`); + } catch { + console.error(" Refusing to apply npm policy compatibility: validation failed."); return false; } } diff --git a/src/lib/sandbox-base-image-agent-resolution.test.ts b/src/lib/sandbox-base-image-agent-resolution.test.ts index 1557f709a62..dca8e044bac 100644 --- a/src/lib/sandbox-base-image-agent-resolution.test.ts +++ b/src/lib/sandbox-base-image-agent-resolution.test.ts @@ -123,7 +123,9 @@ describe("agent-specific sandbox base-image resolution", () => { suppressOutput: true, }); expect(validateImage).toHaveBeenCalledWith(staleRef); - expect(warn).toHaveBeenCalledWith(expect.stringContaining("deepagents-code==0.1.55")); + expect(warn).toHaveBeenCalledWith( + expect.stringContaining("lacks a required runtime capability"), + ); expect(dockerMocks.build).not.toHaveBeenCalled(); warn.mockRestore(); }); diff --git a/src/lib/sandbox-base-image.ts b/src/lib/sandbox-base-image.ts index 03bf7a96b0c..1a1702e83d3 100644 --- a/src/lib/sandbox-base-image.ts +++ b/src/lib/sandbox-base-image.ts @@ -282,11 +282,7 @@ function validatePulledCandidate( glibcVersion = check.version; if (!check.ok) { if (warn) { - console.warn( - ` Warning: ${options.label || "sandbox base image"} ${imageRef} has glibc ` + - `${glibcVersion || "unknown"}; OpenShell sandbox supervisor requires ` + - `glibc >= ${options.minGlibcVersion || OPENSHELL_SANDBOX_MIN_GLIBC}.`, - ); + console.warn(" Warning: sandbox base image does not meet the required glibc version."); } return null; } @@ -294,10 +290,7 @@ function validatePulledCandidate( if (options.validateImage && !options.validateImage(imageRef)) { if (warn) { - console.warn( - ` Warning: ${options.label || "sandbox base image"} ${imageRef} lacks ` + - `${options.validationDescription || "a required runtime capability"}.`, - ); + console.warn(" Warning: sandbox base image lacks a required runtime capability."); } return null; } @@ -434,19 +427,12 @@ function resolveLocalCandidate( ? imageMeetsMinimumGlibc(imageRef, options.minGlibcVersion || OPENSHELL_SANDBOX_MIN_GLIBC) : { ok: true, version: null }; if (!check.ok) { - console.error( - ` Local ${label} ${imageRef} has glibc ` + - `${check.version || "unknown"}; expected >= ` + - `${options.minGlibcVersion || OPENSHELL_SANDBOX_MIN_GLIBC}.`, - ); + console.error(" Local sandbox base image does not meet the required glibc version."); return null; } if (options.validateImage && !options.validateImage(imageRef)) { - console.error( - ` Local ${label} ${imageRef} lacks ` + - `${options.validationDescription || "a required runtime capability"}.`, - ); + console.error(" Local sandbox base image lacks a required runtime capability."); return null; } diff --git a/src/lib/sandbox/config.ts b/src/lib/sandbox/config.ts index c8f894ad545..a800de0a1d3 100644 --- a/src/lib/sandbox/config.ts +++ b/src/lib/sandbox/config.ts @@ -32,7 +32,6 @@ const { isIP } = require("node:net"); const { isErrnoException }: typeof import("../core/errno") = require("../core/errno"); const { validateName } = require("../runner"); const { shellQuote } = require("../core/shell-quote"); -const { dockerExecFileSync, dockerSpawnSync } = require("../adapters/docker/exec"); const credentialFilter: typeof import("../security/credential-filter") = require("../security/credential-filter"); const { stripCredentials, isConfigObject, isConfigValue, isCredentialField } = credentialFilter; const { appendAuditEntry } = require("../shields/audit"); @@ -52,8 +51,9 @@ const { isPrivateIp, }: typeof import("../private-networks") = require("../private-networks"); const { - privilegedSandboxExecArgv, - resolveDirectSandboxContainer, + capturePrivilegedSandboxCommand, + executePrivilegedSandboxCommand, + resolvePrivilegedSandboxTarget, withPrivilegedSandboxExecutionLease, }: typeof import("./privileged-exec") = require("./privileged-exec"); const { @@ -202,11 +202,11 @@ function privilegedSandboxExec( sandboxName, "sandbox config privileged execution", () => - dockerExecFileSync(privilegedSandboxExecArgv(sandboxName, cmd, hasInput, true), { - input: opts.input, - stdio: hasInput ? ["pipe", "pipe", "pipe"] : ["ignore", "pipe", "pipe"], + capturePrivilegedSandboxCommand(sandboxName, cmd, { + ...(hasInput ? { input: opts.input } : {}), + sanitizeEnvironment: true, timeout: opts.timeout ?? 30000, - }), + }).toString("utf8"), ); } @@ -215,24 +215,20 @@ function openClawConfigGuardExec(sandboxName: string, expectedContainerId?: stri run: (cmd: string[], input?: string) => { try { return withPrivilegedSandboxExecutionLease(sandboxName, "OpenClaw config guard", () => { - const argv = privilegedSandboxExecArgv( - sandboxName, - cmd, - input !== undefined, - true, - expectedContainerId, - ); - const result = dockerSpawnSync(argv, { - encoding: "utf-8", - input, + const result = executePrivilegedSandboxCommand(sandboxName, cmd, { + ...(input === undefined ? {} : { input }), + sanitizeEnvironment: true, + ...(expectedContainerId === undefined + ? {} + : { expectedResourceHandle: expectedContainerId }), timeout: OPENCLAW_CONFIG_GUARD_TIMEOUT_MS, - maxBuffer: 2 * 1024 * 1024, + maxOutputBytes: 2 * 1024 * 1024, }); return { status: result.status, signal: result.signal, - stdout: String(result.stdout ?? ""), - stderr: String(result.stderr ?? ""), + stdout: result.stdout.toString("utf8"), + stderr: result.stderr.toString("utf8"), ...(result.error ? { error: result.error.message } : {}), }; }); @@ -1364,7 +1360,7 @@ async function configSet(sandboxName: string, opts: ConfigSetOpts = {}): Promise setDotpath(config, opts.key, safeValue); const content = composeSandboxConfigBody(config, target); try { - const containerId = resolveDirectSandboxContainer(sandboxName, null); + const containerId = resolvePrivilegedSandboxTarget(sandboxName).resourceHandle; const privileged = openClawConfigGuardExec(sandboxName, containerId); const issues = validateOpenClawConfigCandidate(privileged, content); if (issues.length > 0) configFail(issues.map((issue) => ` ${issue}`)); @@ -1492,7 +1488,6 @@ export { formatConfigValueForLogs, parseConfig, parseConfigGetArgs, - privilegedSandboxExecArgv, readSandboxConfig, readStdin, recomputeSandboxConfigHash, diff --git a/src/lib/sandbox/privileged-exec.test.ts b/src/lib/sandbox/privileged-exec.test.ts index 314d4623edc..64ce1929aa0 100644 --- a/src/lib/sandbox/privileged-exec.test.ts +++ b/src/lib/sandbox/privileged-exec.test.ts @@ -9,24 +9,32 @@ import os from "node:os"; import path from "node:path"; import { describe, expect, it, vi } from "vitest"; import { createPersistedLifecycleStoreOrThrow } from "../../../test/helpers/privileged-exec-test-helpers"; +import { + dockerContainerNameMatchesSandbox as containerNameMatchesSandbox, + selectDockerPrivilegedSandboxTarget as selectDirectSandboxContainer, +} from "../onboard/runtime-provider/docker-privileged-sandbox-identity"; // The shared source hook preserves the writable CommonJS cache used by these mocks. const require = createRequire(import.meta.url); const requireCache: Record = require.cache as any; const helperPath = require.resolve("./privileged-exec"); +const currentRuntimeProvidersPath = require.resolve("../onboard/runtime-provider/current"); +const runtimeProviderRegistryPath = require.resolve("../onboard/runtime-provider/registry"); +const runtimeProviderSelectionPath = require.resolve("../onboard/runtime-provider/selection"); +const dockerControlPath = + require.resolve("../onboard/runtime-provider/docker-privileged-sandbox-control"); +const dockerOperationAuthorityPath = + require.resolve("../onboard/runtime-provider/docker-operation-authority"); const dockerRunPath = require.resolve("../adapters/docker/run"); const portableLifecyclePath = require.resolve("../onboard/experimental/portable-demo-lifecycle"); const registryPath = require.resolve("../state/registry"); const lifecycleGenerationPath = require.resolve("../state/registry/lifecycle-generation"); +const lifecycleGenerationCasPath = require.resolve("../state/registry/lifecycle-generation-cas"); const persistedLifecyclePath = require.resolve("../onboard/runtime-provider/persisted-engine-lifecycle"); const statePathsPath = require.resolve("../state/paths"); const transitionLockPath = require.resolve("../shields/transition-lock"); -const { - buildStoppedDockerSandboxChannelCleanupScript, - containerNameMatchesSandbox, - selectDirectSandboxContainer, -} = require(helperPath); +const { buildStoppedDockerSandboxChannelCleanupScript } = require(helperPath); const PINNED_CLEANUP_IMAGE = "node:22-trixie-slim@sha256:db8a96a63e5264607ada2d206758876ebbed6a12be2ada7517793cbfb0c2a29c"; const EXPECTED_WECHAT_STATE_PATHS = [ @@ -76,15 +84,47 @@ function withPrivilegedExecMocks( run: (helper: typeof import("./privileged-exec")) => T, ): T { const priorHelper = require.cache[helperPath]; + const priorCurrentRuntimeProviders = require.cache[currentRuntimeProvidersPath]; + const priorRuntimeProviderRegistry = require.cache[runtimeProviderRegistryPath]; + const priorRuntimeProviderSelection = require.cache[runtimeProviderSelectionPath]; + const priorDockerControl = require.cache[dockerControlPath]; + const priorDockerOperationAuthority = require.cache[dockerOperationAuthorityPath]; const priorDockerRun = require.cache[dockerRunPath]; const priorPortableLifecycle = require.cache[portableLifecyclePath]; const priorRegistry = require.cache[registryPath]; const priorLifecycleGeneration = require.cache[lifecycleGenerationPath]; + const priorLifecycleGenerationCas = require.cache[lifecycleGenerationCasPath]; const priorPersistedLifecycle = require.cache[persistedLifecyclePath]; const priorStatePaths = require.cache[statePathsPath]; const priorTransitionLock = require.cache[transitionLockPath]; delete require.cache[helperPath]; + delete require.cache[dockerControlPath]; + delete require.cache[runtimeProviderSelectionPath]; + requireCache[dockerOperationAuthorityPath] = { + id: dockerOperationAuthorityPath, + filename: dockerOperationAuthorityPath, + loaded: true, + exports: { + createDockerOperationAuthority: () => ({ + engine: { + capture: (args: readonly string[], timeout = 30_000) => + args[0] === "ps" + ? { + status: 0, + stdout: deps.dockerCapture(args, { timeout }), + stderr: "", + } + : (deps.dockerRun?.(args, { timeout }) ?? { + status: 0, + stdout: "", + stderr: "", + error: null, + }), + }, + }), + }, + } as any; requireCache[dockerRunPath] = { id: dockerRunPath, filename: dockerRunPath, @@ -122,6 +162,15 @@ function withPrivilegedExecMocks( deps.compareAndSetLegacySandboxLifecycleGeneration ?? (() => false), }, } as any; + requireCache[lifecycleGenerationCasPath] = { + id: lifecycleGenerationCasPath, + filename: lifecycleGenerationCasPath, + loaded: true, + exports: { + compareAndSetSandboxLifecycleGeneration: + deps.compareAndSetLegacySandboxLifecycleGeneration ?? (() => false), + }, + } as any; requireCache[persistedLifecyclePath] = { id: persistedLifecyclePath, filename: persistedLifecyclePath, @@ -152,15 +201,52 @@ function withPrivilegedExecMocks( ((_sandboxName: string, _operation: string, fn: () => T): T => fn()), }, } as any; + const dockerControl = require(dockerControlPath).createDockerPrivilegedSandboxControl(); + requireCache[currentRuntimeProvidersPath] = { + id: currentRuntimeProvidersPath, + filename: currentRuntimeProvidersPath, + loaded: true, + exports: { CURRENT_RUNTIME_PROVIDER_BUNDLES: {} }, + } as any; + const requireRuntimeProviderBundleForSandbox = (sandbox: { openshellDriver?: string | null }) => { + const providerId = + !sandbox.openshellDriver || sandbox.openshellDriver === "vm" + ? "docker" + : sandbox.openshellDriver; + return providerId === "docker" + ? { + identity: { id: "docker" }, + lifecycle: { supported: true, privilegedSandboxControl: dockerControl }, + } + : { identity: { id: providerId }, lifecycle: { supported: false } }; + }; + requireCache[runtimeProviderRegistryPath] = { + id: runtimeProviderRegistryPath, + filename: runtimeProviderRegistryPath, + loaded: true, + exports: { requireRuntimeProviderBundleForSandbox }, + } as any; + requireCache[runtimeProviderSelectionPath] = { + id: runtimeProviderSelectionPath, + filename: runtimeProviderSelectionPath, + loaded: true, + exports: { requireRuntimeProviderBundleForSandbox }, + } as any; try { return run(require(helperPath)); } finally { restoreRequireCacheEntry(helperPath, priorHelper); + restoreRequireCacheEntry(currentRuntimeProvidersPath, priorCurrentRuntimeProviders); + restoreRequireCacheEntry(runtimeProviderRegistryPath, priorRuntimeProviderRegistry); + restoreRequireCacheEntry(runtimeProviderSelectionPath, priorRuntimeProviderSelection); + restoreRequireCacheEntry(dockerControlPath, priorDockerControl); + restoreRequireCacheEntry(dockerOperationAuthorityPath, priorDockerOperationAuthority); restoreRequireCacheEntry(dockerRunPath, priorDockerRun); restoreRequireCacheEntry(portableLifecyclePath, priorPortableLifecycle); restoreRequireCacheEntry(registryPath, priorRegistry); restoreRequireCacheEntry(lifecycleGenerationPath, priorLifecycleGeneration); + restoreRequireCacheEntry(lifecycleGenerationCasPath, priorLifecycleGenerationCas); restoreRequireCacheEntry(persistedLifecyclePath, priorPersistedLifecycle); restoreRequireCacheEntry(statePathsPath, priorStatePaths); restoreRequireCacheEntry(transitionLockPath, priorTransitionLock); @@ -196,7 +282,13 @@ describe("privileged sandbox exec routing", () => { const mounts = JSON.stringify([ { Type: "volume", - Name: "nemoclaw-alpha-state", + Name: "nemoclaw-openclaw-state-v1-alpha", + Destination: "/sandbox/.openclaw", + RW: true, + }, + { + Type: "bind", + Source: "/var/lib/openshell/sandboxes/alpha", Destination: "/sandbox", RW: true, }, @@ -226,6 +318,12 @@ describe("privileged sandbox exec routing", () => { stderr: "Error: No such object: cleanup-helper", error: null, }, + { + status: 0, + stdout: `${containerId}\tfalse\t${mounts}\n`, + stderr: "", + error: null, + }, { status: 0, stdout: `${helperId}\n`, @@ -258,15 +356,15 @@ describe("privileged sandbox exec routing", () => { getSandbox: () => ({ name: "alpha", openshellDriver: "docker" }), listSandboxes: () => ({ sandboxes: [{ name: "alpha" }], defaultSandbox: "alpha" }), }, - ({ clearStoppedDockerSandboxChannelState }) => { - expect(clearStoppedDockerSandboxChannelState("alpha", EXPECTED_WECHAT_STATE_PATHS)).toEqual( - { cleared: true }, - ); + ({ clearStoppedSandboxStateRoots }) => { + expect(clearStoppedSandboxStateRoots("alpha", EXPECTED_WECHAT_STATE_PATHS)).toEqual({ + cleared: true, + }); }, ); - const helperArgv = runDocker.mock.calls[3]?.[0]; - expect(runDocker).toHaveBeenCalledTimes(8); + const helperArgv = runDocker.mock.calls[4]?.[0]; + expect(runDocker).toHaveBeenCalledTimes(9); expect(helperArgv).toEqual( expect.arrayContaining([ "create", @@ -278,7 +376,7 @@ describe("privileged sandbox exec routing", () => { "--cap-add", "DAC_OVERRIDE", "--mount", - "type=volume,src=nemoclaw-alpha-state,dst=/sandbox,volume-nocopy", + "type=volume,src=nemoclaw-openclaw-state-v1-alpha,dst=/sandbox/.openclaw,volume-nocopy", PINNED_CLEANUP_IMAGE, ]), ); @@ -287,9 +385,10 @@ describe("privileged sandbox exec routing", () => { expect(helperArgv?.join("\0")).not.toContain("/sandbox/project"); expect(helperArgv).not.toContain("/bin/sh"); expect(helperArgv?.join("\0")).not.toContain("rm -rf"); - expect(helperArgv?.at(-1)).toBe(JSON.stringify(EXPECTED_WECHAT_STATE_PATHS)); - expect(runDocker.mock.calls[4]?.[0]).toEqual(["start", "--attach", helperId]); - expect(runDocker.mock.calls[5]?.[0]).toEqual(["rm", "-f", helperId]); + expect(helperArgv?.at(-2)).toBe(JSON.stringify(EXPECTED_WECHAT_STATE_PATHS)); + expect(helperArgv?.at(-1)).toBe("/sandbox/.openclaw"); + expect(runDocker.mock.calls[5]?.[0]).toEqual(["start", "--attach", helperId]); + expect(runDocker.mock.calls[6]?.[0]).toEqual(["rm", "-f", helperId]); }); it("deletes only the exact stopped-channel directories", () => { @@ -352,16 +451,17 @@ describe("privileged sandbox exec routing", () => { getSandbox: () => ({ name: "alpha", openshellDriver: "docker" }), listSandboxes: () => ({ sandboxes: [{ name: "alpha" }], defaultSandbox: "alpha" }), }, - ({ clearStoppedDockerSandboxChannelState }) => { - expect( - clearStoppedDockerSandboxChannelState("alpha", ["/sandbox/.openclaw/../project"]), - ).toEqual({ cleared: false, failure: "state-paths-invalid" }); + ({ clearStoppedSandboxStateRoots }) => { + expect(clearStoppedSandboxStateRoots("alpha", ["/sandbox/.openclaw/../project"])).toEqual({ + cleared: false, + failure: "state-paths-invalid", + }); }, ); expect(captureDocker).not.toHaveBeenCalled(); }); - it("refuses stopped cleanup for a non-Docker sandbox before Docker discovery", () => { + it("retains legacy VM cleanup through the registered Docker provider", () => { const captureDocker = vi.fn(() => ""); const runDocker = vi.fn((_args: readonly string[]) => { return { status: 0, stdout: "", stderr: "", error: null } as const; @@ -374,17 +474,15 @@ describe("privileged sandbox exec routing", () => { getSandbox: () => ({ name: "alpha", openshellDriver: "vm" }), listSandboxes: () => ({ sandboxes: [{ name: "alpha" }], defaultSandbox: "alpha" }), }, - ({ clearStoppedDockerSandboxChannelState }) => { - expect(clearStoppedDockerSandboxChannelState("alpha", EXPECTED_WECHAT_STATE_PATHS)).toEqual( - { - cleared: false, - failure: "driver-not-docker", - }, - ); + ({ clearStoppedSandboxStateRoots }) => { + expect(clearStoppedSandboxStateRoots("alpha", EXPECTED_WECHAT_STATE_PATHS)).toEqual({ + cleared: false, + failure: "no-eligible-stopped-runtime", + }); }, ); - expect(captureDocker).not.toHaveBeenCalled(); + expect(captureDocker).toHaveBeenCalledOnce(); expect(runDocker).not.toHaveBeenCalled(); }); @@ -406,13 +504,11 @@ describe("privileged sandbox exec routing", () => { getSandbox: () => ({ name: "alpha", openshellDriver: "docker" }), listSandboxes: () => ({ sandboxes: [{ name: "alpha" }], defaultSandbox: "alpha" }), }, - ({ clearStoppedDockerSandboxChannelState }) => { - expect(clearStoppedDockerSandboxChannelState("alpha", EXPECTED_WECHAT_STATE_PATHS)).toEqual( - { - cleared: false, - failure: "sandbox-volume-unavailable", - }, - ); + ({ clearStoppedSandboxStateRoots }) => { + expect(clearStoppedSandboxStateRoots("alpha", EXPECTED_WECHAT_STATE_PATHS)).toEqual({ + cleared: false, + failure: "state-resource-unavailable", + }); }, ); @@ -428,13 +524,11 @@ describe("privileged sandbox exec routing", () => { getSandbox: () => ({ name: "alpha", openshellDriver: "docker" }), listSandboxes: () => ({ sandboxes: [{ name: "alpha" }], defaultSandbox: "alpha" }), }, - ({ clearStoppedDockerSandboxChannelState }) => { - expect(clearStoppedDockerSandboxChannelState("alpha", EXPECTED_WECHAT_STATE_PATHS)).toEqual( - { - cleared: false, - failure: "docker-discovery-failed", - }, - ); + ({ clearStoppedSandboxStateRoots }) => { + expect(clearStoppedSandboxStateRoots("alpha", EXPECTED_WECHAT_STATE_PATHS)).toEqual({ + cleared: false, + failure: "runtime-discovery-failed", + }); }, ); }); @@ -446,13 +540,11 @@ describe("privileged sandbox exec routing", () => { getSandbox: () => ({ name: "alpha", openshellDriver: "docker" }), listSandboxes: () => ({ sandboxes: [{ name: "alpha" }], defaultSandbox: "alpha" }), }, - ({ clearStoppedDockerSandboxChannelState }) => { - expect(clearStoppedDockerSandboxChannelState("alpha", EXPECTED_WECHAT_STATE_PATHS)).toEqual( - { - cleared: false, - failure: "no-eligible-stopped-container", - }, - ); + ({ clearStoppedSandboxStateRoots }) => { + expect(clearStoppedSandboxStateRoots("alpha", EXPECTED_WECHAT_STATE_PATHS)).toEqual({ + cleared: false, + failure: "no-eligible-stopped-runtime", + }); }, ); }); @@ -464,13 +556,11 @@ describe("privileged sandbox exec routing", () => { getSandbox: () => ({ name: "alpha", openshellDriver: "docker" }), listSandboxes: () => ({ sandboxes: [{ name: "alpha" }], defaultSandbox: "alpha" }), }, - ({ clearStoppedDockerSandboxChannelState }) => { - expect(clearStoppedDockerSandboxChannelState("alpha", EXPECTED_WECHAT_STATE_PATHS)).toEqual( - { - cleared: false, - failure: "container-ownership-invalid", - }, - ); + ({ clearStoppedSandboxStateRoots }) => { + expect(clearStoppedSandboxStateRoots("alpha", EXPECTED_WECHAT_STATE_PATHS)).toEqual({ + cleared: false, + failure: "runtime-ownership-invalid", + }); }, ); }); @@ -496,13 +586,11 @@ describe("privileged sandbox exec routing", () => { getSandbox: () => ({ name: "alpha", openshellDriver: "docker" }), listSandboxes: () => ({ sandboxes: [{ name: "alpha" }], defaultSandbox: "alpha" }), }, - ({ clearStoppedDockerSandboxChannelState }) => { - expect(clearStoppedDockerSandboxChannelState("alpha", EXPECTED_WECHAT_STATE_PATHS)).toEqual( - { - cleared: false, - failure: "cleanup-helper-image-unavailable", - }, - ); + ({ clearStoppedSandboxStateRoots }) => { + expect(clearStoppedSandboxStateRoots("alpha", EXPECTED_WECHAT_STATE_PATHS)).toEqual({ + cleared: false, + failure: "cleanup-helper-image-unavailable", + }); }, ); }); @@ -512,7 +600,9 @@ describe("privileged sandbox exec routing", () => { const helperId = "d".repeat(64); const sandboxVolume = "nemoclaw-alpha-state"; const ownerIdentity = createHash("sha256").update("alpha").digest("hex"); - const volumeIdentity = createHash("sha256").update(sandboxVolume).digest("hex"); + const volumeIdentity = createHash("sha256") + .update(JSON.stringify({ type: "volume", source: sandboxVolume, target: "/sandbox" })) + .digest("hex"); const helperName = `nemoclaw-channel-cleanup-${ownerIdentity.slice(0, 24)}`; const mounts = JSON.stringify([ { Type: "volume", Name: sandboxVolume, Destination: "/sandbox", RW: true }, @@ -581,10 +671,11 @@ describe("privileged sandbox exec routing", () => { getSandbox: () => ({ name: "alpha", openshellDriver: "docker" }), listSandboxes: () => ({ sandboxes: [{ name: "alpha" }], defaultSandbox: "alpha" }), }, - ({ clearStoppedDockerSandboxChannelState }) => { - expect(clearStoppedDockerSandboxChannelState("alpha", EXPECTED_WECHAT_STATE_PATHS)).toEqual( - { cleared: false, failure: "cleanup-helper-failed" }, - ); + ({ clearStoppedSandboxStateRoots }) => { + expect(clearStoppedSandboxStateRoots("alpha", EXPECTED_WECHAT_STATE_PATHS)).toEqual({ + cleared: false, + failure: "cleanup-helper-failed", + }); }, ); @@ -626,6 +717,12 @@ describe("privileged sandbox exec routing", () => { stderr: "Error: No such object: cleanup-helper", error: null, }, + { + status: 0, + stdout: `${containerId}\tfalse\t${mounts}\n`, + stderr: "", + error: null, + }, { status: 0, stdout: `${helperId}\n`, stderr: "", error: null }, { status: startStatus, stdout: "", stderr: "private helper detail", error: null }, { status: 0, stdout: helperId, stderr: "", error: null }, @@ -647,16 +744,17 @@ describe("privileged sandbox exec routing", () => { getSandbox: () => ({ name: "alpha", openshellDriver: "docker" }), listSandboxes: () => ({ sandboxes: [{ name: "alpha" }], defaultSandbox: "alpha" }), }, - ({ clearStoppedDockerSandboxChannelState }) => { - expect( - clearStoppedDockerSandboxChannelState("alpha", EXPECTED_WECHAT_STATE_PATHS), - ).toEqual({ cleared: false, failure: expectedFailure }); + ({ clearStoppedSandboxStateRoots }) => { + expect(clearStoppedSandboxStateRoots("alpha", EXPECTED_WECHAT_STATE_PATHS)).toEqual({ + cleared: false, + failure: expectedFailure, + }); }, ); - expect(runDocker.mock.calls[4]?.[0]).toEqual(["start", "--attach", helperId]); - expect(runDocker.mock.calls[5]?.[0]).toEqual(["rm", "-f", helperId]); - expect(runDocker).toHaveBeenCalledTimes(7); + expect(runDocker.mock.calls[5]?.[0]).toEqual(["start", "--attach", helperId]); + expect(runDocker.mock.calls[6]?.[0]).toEqual(["rm", "-f", helperId]); + expect(runDocker).toHaveBeenCalledTimes(8); }, ); @@ -957,7 +1055,7 @@ describe("privileged sandbox exec routing", () => { }, ({ privilegedSandboxExecArgv }) => { expect(() => privilegedSandboxExecArgv("alpha", ["id"])).toThrow( - "refusing local Docker discovery for a non-direct driver", + "Runtime provider 'kubernetes' does not support privileged sandbox control.", ); }, ); @@ -1118,7 +1216,7 @@ describe("privileged sandbox exec routing", () => { }, ({ privilegedSandboxExecArgv }) => { expect(() => privilegedSandboxExecArgv("alpha", ["id"])).toThrow( - /driver: kubernetes.*refusing local Docker discovery/i, + "Runtime provider 'kubernetes' does not support privileged sandbox control.", ); }, ); diff --git a/src/lib/sandbox/privileged-exec.ts b/src/lib/sandbox/privileged-exec.ts index 9a15b4d349f..a7f1fde21ce 100644 --- a/src/lib/sandbox/privileged-exec.ts +++ b/src/lib/sandbox/privileged-exec.ts @@ -1,652 +1,81 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { createHash } from "node:crypto"; import fs from "node:fs"; import path from "node:path"; -import { dockerCapture, dockerRun } from "../adapters/docker/run"; -import { resolveSandboxContainerOwner } from "../domain/sandbox/container-owner"; -import { resolvePortableDemoPrivilegedExecTarget } from "../onboard/experimental/portable-demo-lifecycle"; +import type { + RuntimeProviderPrivilegedSandboxCommandResult, + RuntimeProviderPrivilegedSandboxControl, + RuntimeProviderPrivilegedSandboxTarget, + RuntimeProviderStoppedSandboxStateCleanupResult, +} from "../onboard/runtime-provider/contract"; +import { CURRENT_RUNTIME_PROVIDER_BUNDLES } from "../onboard/runtime-provider/current"; +import { + DirectSandboxFallbackUnavailableError, + PinnedSandboxResourceIdentityChangedError, +} from "../onboard/runtime-provider/privileged-sandbox-control-errors"; import { createFilePersistedEngineLifecycleStore, hasActivePersistedEngineStateMutationTarget, PERSISTED_ENGINE_LIFECYCLE_DIRECTORY, } from "../onboard/runtime-provider/persisted-engine-lifecycle"; +import { requireRuntimeProviderBundleForSandbox } from "../onboard/runtime-provider/selection"; +import { + buildStoppedSandboxChannelCleanupScript, + validateStoppedSandboxStatePaths, +} from "../onboard/runtime-provider/stopped-sandbox-state-cleanup"; import { resolveShieldsStateDir, withShieldsTransitionLock } from "../shields/transition-lock"; import * as registry from "../state/registry"; -import { compareAndSetLegacySandboxLifecycleGeneration } from "../state/registry/lifecycle-generation"; - -const OPENSHELL_MANAGED_BY_LABEL = "openshell.ai/managed-by"; -const OPENSHELL_MANAGED_BY_VALUE = "openshell"; -const OPENSHELL_SANDBOX_NAME_LABEL = "openshell.ai/sandbox-name"; type SandboxEntry = import("../state/registry").SandboxEntry; -type LabeledSandboxContainer = { - id: string; - name: string; -}; - -export type StoppedDockerSandboxChannelStateCleanupFailure = - | "sandbox-registry-unavailable" - | "driver-not-docker" - | "state-paths-invalid" - | "docker-discovery-failed" - | "no-eligible-stopped-container" - | "container-ownership-invalid" - | "container-inspection-failed" - | "container-not-stopped" - | "sandbox-volume-unavailable" - | "cleanup-helper-image-unavailable" - | "cleanup-helper-ownership-invalid" - | "cleanup-helper-reconciliation-failed" - | "cleanup-state-tree-unsafe" - | "cleanup-deletion-unconfirmed" - | "cleanup-helper-failed" - | "container-revalidation-failed" - | "lifecycle-authority-unavailable"; - -export type StoppedDockerSandboxChannelStateCleanupResult = - | { readonly cleared: true } - | { - readonly cleared: false; - readonly failure: StoppedDockerSandboxChannelStateCleanupFailure; - readonly cleanupHelperName?: string; - }; - -const DIRECT_SANDBOX_DISCOVERY_TIMEOUT_MS = 5000; -const FULL_CONTAINER_ID_RE = /^[a-f0-9]{64}$/u; -const DOCKER_VOLUME_NAME_RE = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,254}$/u; -const STOPPED_CHANNEL_STATE_PATH_RE = /^\/sandbox\/\.(?:openclaw|hermes)\/[A-Za-z0-9_-]+$/u; -const STOPPED_CHANNEL_CLEANUP_IMAGE = - "node:22-trixie-slim@sha256:db8a96a63e5264607ada2d206758876ebbed6a12be2ada7517793cbfb0c2a29c"; -const STOPPED_CHANNEL_CLEANUP_LABEL = "com.nvidia.nemoclaw.channel-cleanup"; -const STOPPED_CHANNEL_CLEANUP_OWNER_LABEL = `${STOPPED_CHANNEL_CLEANUP_LABEL}.owner`; -const STOPPED_CHANNEL_CLEANUP_VOLUME_LABEL = `${STOPPED_CHANNEL_CLEANUP_LABEL}.volume`; -export function buildStoppedDockerSandboxChannelCleanupScript(root = "/sandbox"): string { - return String.raw` -"use strict"; -const fs = require("node:fs"); -const path = require("node:path"); -const root = ${JSON.stringify(root)}; -const targets = JSON.parse(process.argv[1]); -function lstat(candidate) { - try { return fs.lstatSync(candidate); } - catch (error) { if (error && error.code === "ENOENT") return null; throw error; } -} -const rootMetadata = lstat(root); -if (!rootMetadata || rootMetadata.isSymbolicLink() || !rootMetadata.isDirectory()) process.exit(40); -for (const target of targets) { - if (typeof target !== "string" || !target.startsWith(root + "/.")) process.exit(41); - const relative = path.posix.relative(root, target); - const segments = relative.split("/"); - if (!relative || relative.startsWith("../") || segments.some((part) => !part || part === "." || part === "..")) process.exit(42); - let parent = root; - let absent = false; - for (const segment of segments.slice(0, -1)) { - parent = path.posix.join(parent, segment); - const metadata = lstat(parent); - if (!metadata) { absent = true; break; } - if (metadata.isSymbolicLink() || !metadata.isDirectory()) process.exit(43); - } - if (absent) continue; - const metadata = lstat(target); - if (!metadata) continue; - if (metadata.isSymbolicLink() || !metadata.isDirectory()) process.exit(44); - fs.rmSync(target, { force: false, maxRetries: 0, recursive: true }); - if (lstat(target)) process.exit(45); -} -`; -} -const STOPPED_CHANNEL_CLEANUP_SCRIPT = buildStoppedDockerSandboxChannelCleanupScript(); -const OFFLINE_DOCKER_OPERATION_OPTIONS = { - encoding: "utf-8", - ignoreError: true, - suppressOutput: true, - timeout: 30_000, -} as const; -const SANITIZED_PRIVILEGED_ENV = [ - "BASH_ENV=", - "ENV=", - "GCONV_PATH=", - "GLIBC_TUNABLES=", - "LD_AUDIT=", - "LD_LIBRARY_PATH=", - "LD_PRELOAD=", - "LOCPATH=", - "NODE_OPTIONS=", - "PERL5OPT=", - "PYTHONHOME=", - "PYTHONINSPECT=", - "PYTHONNOUSERSITE=1", - "PYTHONPATH=", - "PYTHONSTARTUP=", - "PYTHONUSERBASE=", - "RUBYOPT=", -] as const; -const NEUTRALIZED_OFFLINE_HELPER_ENV = [ - "--env", - "LD_AUDIT=", - "--env", - "LD_LIBRARY_PATH=", - "--env", - "LD_PRELOAD=", - "--env", - "BASH_ENV=", - "--env", - "ENV=", -] as const; - -class DirectSandboxFallbackUnavailableError extends Error { - constructor(message: string, options?: ErrorOptions) { - super(message, options); - this.name = "DirectSandboxFallbackUnavailableError"; - } -} - -class PinnedSandboxContainerIdentityChangedError extends Error { - constructor(sandboxName: string) { - super( - `OpenShell container identity changed for sandbox '${sandboxName}'; ` + - "refusing privileged execution against a different container.", - ); - this.name = "PinnedSandboxContainerIdentityChangedError"; - } +export interface PrivilegedSandboxCommandOptions { + readonly input?: string | Buffer; + readonly sanitizeEnvironment?: boolean; + readonly expectedResourceHandle?: string; + readonly timeout?: number; + readonly maxOutputBytes?: number; } -function normalizeDriver(driver: unknown): string | null { - return typeof driver === "string" && driver.trim() ? driver.trim().toLowerCase() : null; -} - -function readSandboxEntry(sandboxName: string): SandboxEntry | null { - return registry.getSandbox?.(sandboxName) ?? null; -} - -function registeredSandboxNames(sandboxName: string): string[] { - const names = new Set([sandboxName]); +const DEFAULT_PRIVILEGED_SANDBOX_COMMAND_TIMEOUT_MS = 15_000; - if (registry.listSandboxes) { - const listed = registry.listSandboxes?.(); - if (Array.isArray(listed?.sandboxes)) { - for (const entry of listed.sandboxes) { - if (typeof entry.name === "string" && entry.name) names.add(entry.name); - } - } - } else { - const loaded = registry.load?.(); - const sandboxes = loaded?.sandboxes; - if (sandboxes && typeof sandboxes === "object") { - for (const [key, entry] of Object.entries(sandboxes)) { - if (key) names.add(key); - if (typeof entry?.name === "string" && entry.name) names.add(entry.name); - } - } - } - - return Array.from(names).sort((a, b) => b.length - a.length || a.localeCompare(b)); -} - -function containerNameMatchesSandbox(containerName: string, sandboxName: string): boolean { - return resolveSandboxContainerOwner(containerName, sandboxName, [sandboxName]) === containerName; -} - -function owningRegisteredSandboxName( - containerName: string, - registeredNames: readonly string[], -): string | null { - return registeredNames.find((name) => containerNameMatchesSandbox(containerName, name)) ?? null; -} - -function parseLabeledSandboxContainers(output: string): LabeledSandboxContainer[] { - return output - .split(/\r?\n/) - .map((line) => line.trim()) - .filter(Boolean) - .map((line) => { - const [id, name, ...unexpected] = line.split("\t"); - if (!id || !name || unexpected.length > 0 || /\s/.test(id)) { - throw new Error("Docker returned malformed OpenShell sandbox container metadata."); - } - return { id, name }; - }); -} - -function selectDirectSandboxContainer( - sandboxName: string, - labeledContainerRows: string, - registeredNames: readonly string[] = [sandboxName], -): string | null { - const names = Array.from(new Set([...registeredNames, sandboxName])).sort( - (a, b) => b.length - a.length || a.localeCompare(b), +function readSandboxEntry(sandboxName: string): SandboxEntry { + const entry = registry.getSandbox?.(sandboxName) ?? null; + if (entry) return entry; + throw new Error( + `No NemoClaw registry entry found for '${sandboxName}'; ` + + "refusing privileged exec without a registered sandbox owner.", ); - const candidates = parseLabeledSandboxContainers(labeledContainerRows); - if ( - candidates.some( - ({ name }) => - !containerNameMatchesSandbox(name, sandboxName) || - owningRegisteredSandboxName(name, names) !== sandboxName, - ) - ) { - throw new Error( - `OpenShell container labels and names disagree for sandbox '${sandboxName}'; ` + - "refusing lifecycle execution.", - ); - } - if (candidates.length > 1) { - throw new Error( - `Multiple running OpenShell containers are labeled for sandbox '${sandboxName}'; ` + - "refusing ambiguous lifecycle execution.", - ); - } - return candidates[0]?.id ?? null; } -function expectedDirectContainerPattern(sandboxName: string): string { - return ( - `openshell-${sandboxName}, openshell-${sandboxName}-*, or ` + - `openshell-default--${sandboxName}-*` +function privilegedSandboxControl(sandboxName: string): { + readonly sandbox: SandboxEntry; + readonly control: RuntimeProviderPrivilegedSandboxControl; +} { + const sandbox = readSandboxEntry(sandboxName); + const provider = requireRuntimeProviderBundleForSandbox( + sandbox, + CURRENT_RUNTIME_PROVIDER_BUNDLES, ); -} - -function findDirectSandboxContainer(sandboxName: string): string | null { - const names = registeredSandboxNames(sandboxName); - let output: string; - try { - output = dockerCapture( - [ - "ps", - "--no-trunc", - "--filter", - `label=${OPENSHELL_MANAGED_BY_LABEL}=${OPENSHELL_MANAGED_BY_VALUE}`, - "--filter", - `label=${OPENSHELL_SANDBOX_NAME_LABEL}=${sandboxName}`, - "--format", - "{{.ID}}\t{{.Names}}", - ], - { timeout: DIRECT_SANDBOX_DISCOVERY_TIMEOUT_MS }, - ); - } catch (error) { - const detail = error instanceof Error ? error.message : String(error); - throw new DirectSandboxFallbackUnavailableError( - `Direct sandbox container discovery failed for '${sandboxName}': ${detail}`, - { cause: error }, - ); - } - return selectDirectSandboxContainer(sandboxName, output, names); -} - -/** Select one label-owned container across all states and reject GPU rollback siblings. */ -function findStoppedDirectSandboxContainer(sandboxName: string): string | null { - const names = registeredSandboxNames(sandboxName); - let output: string; - try { - output = dockerCapture( - [ - "ps", - "-a", - "--no-trunc", - "--filter", - `label=${OPENSHELL_MANAGED_BY_LABEL}=${OPENSHELL_MANAGED_BY_VALUE}`, - "--filter", - `label=${OPENSHELL_SANDBOX_NAME_LABEL}=${sandboxName}`, - "--format", - "{{.ID}}\t{{.Names}}", - ], - { timeout: DIRECT_SANDBOX_DISCOVERY_TIMEOUT_MS }, - ); - } catch (error) { - throw new DirectSandboxFallbackUnavailableError( - `Stopped Docker sandbox discovery failed for '${sandboxName}'.`, - { cause: error }, - ); - } - const candidates = parseLabeledSandboxContainers(output); - const selected = selectDirectSandboxContainer(sandboxName, output, names); - if (/-nemoclaw-gpu-backup-\d+$/u.test(candidates[0]?.name ?? "")) return null; - return selected; -} - -type InspectedStoppedContainer = { - readonly id: string; - readonly running: boolean; - readonly sandboxVolumeName: string; -}; - -type StoppedContainerInspection = - | { readonly inspected: InspectedStoppedContainer } - | { readonly failure: StoppedDockerSandboxChannelStateCleanupFailure }; - -/** Read immutable lifecycle and shared-state mount data for one container ID. */ -function inspectStoppedContainer(containerId: string): StoppedContainerInspection { - let result: ReturnType; - try { - result = dockerRun( - ["inspect", "--format", "{{.Id}}\t{{.State.Running}}\t{{json .Mounts}}", containerId], - OFFLINE_DOCKER_OPERATION_OPTIONS, - ); - } catch { - return { failure: "container-inspection-failed" }; - } - if (result.status !== 0 || typeof result.stdout !== "string") { - return { failure: "container-inspection-failed" }; - } - const [id, running, mountsJson, ...unexpected] = result.stdout.trim().split("\t"); - if ( - unexpected.length > 0 || - !id || - !FULL_CONTAINER_ID_RE.test(id) || - !mountsJson || - (running !== "true" && running !== "false") - ) { - return { failure: "container-ownership-invalid" }; - } - let mounts: unknown; - try { - mounts = JSON.parse(mountsJson); - } catch { - return { failure: "sandbox-volume-unavailable" }; - } - if (!Array.isArray(mounts)) return { failure: "sandbox-volume-unavailable" }; - const sandboxMounts = mounts.filter( - (mount) => - typeof mount === "object" && - mount !== null && - (mount as Record).Destination === "/sandbox", - ) as Array>; - const sandboxMount = sandboxMounts.length === 1 ? sandboxMounts[0] : undefined; - const sandboxVolumeName = - sandboxMount?.Type === "volume" && - sandboxMount.RW === true && - typeof sandboxMount.Name === "string" && - DOCKER_VOLUME_NAME_RE.test(sandboxMount.Name) - ? sandboxMount.Name - : null; - return sandboxVolumeName - ? { inspected: { id, running: running === "true", sandboxVolumeName } } - : { failure: "sandbox-volume-unavailable" }; -} - -function stoppedDockerCleanupFailure( - failure: StoppedDockerSandboxChannelStateCleanupFailure, - cleanupHelperName?: string, -): StoppedDockerSandboxChannelStateCleanupResult { - return cleanupHelperName - ? { cleared: false, failure, cleanupHelperName } - : { cleared: false, failure }; -} - -function stoppedDockerCleanupPaths(paths: readonly string[]): readonly string[] | null { - if ( - paths.length === 0 || - paths.length > 4 || - new Set(paths).size !== paths.length || - paths.some((statePath) => !STOPPED_CHANNEL_STATE_PATH_RE.test(statePath)) - ) { - return null; - } - return [...paths]; -} - -function cleanupIdentity(value: string): string { - return createHash("sha256").update(value).digest("hex"); -} - -function cleanupHelperName(sandboxName: string): string { - return `nemoclaw-channel-cleanup-${cleanupIdentity(sandboxName).slice(0, 24)}`; -} - -function dockerResultText(result: ReturnType): string { - return `${String(result.stderr ?? "")} ${String(result.stdout ?? "")} ${String(result.error?.message ?? "")}`; -} - -function dockerReportsMissingContainer(result: ReturnType): boolean { - return result.status !== 0 && /No such (?:container|object)/iu.test(dockerResultText(result)); -} - -type CleanupHelperInspection = - | { readonly state: "absent" } - | { readonly state: "invalid" } - | { readonly state: "owned"; readonly id: string }; - -function inspectCleanupHelper( - helperName: string, - ownerIdentity: string, - volumeIdentity: string, -): CleanupHelperInspection { - let result: ReturnType; - try { - result = dockerRun( - [ - "inspect", - "--format", - `{{.Id}}\t{{.Config.Image}}\t{{index .Config.Labels "${STOPPED_CHANNEL_CLEANUP_LABEL}"}}\t{{index .Config.Labels "${STOPPED_CHANNEL_CLEANUP_OWNER_LABEL}"}}\t{{index .Config.Labels "${STOPPED_CHANNEL_CLEANUP_VOLUME_LABEL}"}}`, - helperName, - ], - OFFLINE_DOCKER_OPERATION_OPTIONS, - ); - } catch { - return { state: "invalid" }; - } - if (dockerReportsMissingContainer(result)) return { state: "absent" }; - if (result.status !== 0 || typeof result.stdout !== "string") return { state: "invalid" }; - const [id, image, marker, owner, volume, ...unexpected] = result.stdout.trim().split("\t"); - return unexpected.length === 0 && - !!id && - FULL_CONTAINER_ID_RE.test(id) && - image === STOPPED_CHANNEL_CLEANUP_IMAGE && - marker === "1" && - owner === ownerIdentity && - volume === volumeIdentity - ? { state: "owned", id } - : { state: "invalid" }; -} - -function removeAndConfirmCleanupHelper(containerId: string): boolean { - let removed: ReturnType; - let confirmation: ReturnType; - try { - removed = dockerRun(["rm", "-f", containerId], OFFLINE_DOCKER_OPERATION_OPTIONS); - if (removed.status !== 0) return false; - confirmation = dockerRun(["inspect", containerId], OFFLINE_DOCKER_OPERATION_OPTIONS); - } catch { - return false; - } - return dockerReportsMissingContainer(confirmation); -} - -function pinnedCleanupImageIsAvailable(): boolean { - try { - const result = dockerRun( - ["image", "inspect", "--format", "{{.Id}}", STOPPED_CHANNEL_CLEANUP_IMAGE], - OFFLINE_DOCKER_OPERATION_OPTIONS, + if (provider.lifecycle.supported !== true) { + throw new Error( + `Runtime provider '${provider.identity.id}' does not support privileged sandbox control.`, ); - return result.status === 0 && /^sha256:[a-f0-9]{64}\s*$/u.test(String(result.stdout ?? "")); - } catch { - return false; } + return { sandbox, control: provider.lifecycle.privilegedSandboxControl }; } -function reconcileCleanupHelperAfterCreate( - helperName: string, - ownerIdentity: string, - volumeIdentity: string, -): boolean { - const helper = inspectCleanupHelper(helperName, ownerIdentity, volumeIdentity); - return ( - helper.state === "absent" || - (helper.state === "owned" && removeAndConfirmCleanupHelper(helper.id)) - ); -} - -function classifyCleanupHelperFailure( - result: ReturnType | null, -): StoppedDockerSandboxChannelStateCleanupFailure { - if (result?.status === 45) return "cleanup-deletion-unconfirmed"; - if (typeof result?.status === "number" && result.status >= 40 && result.status <= 44) { - return "cleanup-state-tree-unsafe"; - } - return "cleanup-helper-failed"; -} - -/** Clear validated channel state without starting a failed Docker sandbox. */ -function clearStoppedDockerSandboxChannelState( - sandboxName: string, - paths: readonly string[], -): StoppedDockerSandboxChannelStateCleanupResult { - const cleanupPaths = stoppedDockerCleanupPaths(paths); - if (!cleanupPaths) return stoppedDockerCleanupFailure("state-paths-invalid"); - const entry = readSandboxEntry(sandboxName); - if (!entry) return stoppedDockerCleanupFailure("sandbox-registry-unavailable"); - if (normalizeDriver(entry?.openshellDriver) !== "docker") { - return stoppedDockerCleanupFailure("driver-not-docker"); - } - - try { - return withPrivilegedSandboxExecutionLease(sandboxName, "offline channel state cleanup", () => { - let containerId: string | null; - try { - containerId = findStoppedDirectSandboxContainer(sandboxName); - } catch (error) { - return stoppedDockerCleanupFailure( - isDirectSandboxFallbackUnavailableError(error) - ? "docker-discovery-failed" - : "container-ownership-invalid", - ); - } - if (!containerId) return stoppedDockerCleanupFailure("no-eligible-stopped-container"); - const inspection = inspectStoppedContainer(containerId); - if ("failure" in inspection) return stoppedDockerCleanupFailure(inspection.failure); - const { inspected } = inspection; - if (inspected.id !== containerId) { - return stoppedDockerCleanupFailure("container-ownership-invalid"); - } - if (inspected.running) return stoppedDockerCleanupFailure("container-not-stopped"); - if (!pinnedCleanupImageIsAvailable()) { - return stoppedDockerCleanupFailure("cleanup-helper-image-unavailable"); - } - const helperName = cleanupHelperName(sandboxName); - const ownerIdentity = cleanupIdentity(sandboxName); - const volumeIdentity = cleanupIdentity(inspected.sandboxVolumeName); - const existingHelper = inspectCleanupHelper(helperName, ownerIdentity, volumeIdentity); - if (existingHelper.state === "invalid") { - return stoppedDockerCleanupFailure("cleanup-helper-ownership-invalid", helperName); - } - if (existingHelper.state === "owned" && !removeAndConfirmCleanupHelper(existingHelper.id)) { - return stoppedDockerCleanupFailure("cleanup-helper-reconciliation-failed", helperName); - } - let created: ReturnType; - try { - created = dockerRun( - [ - "create", - "--name", - helperName, - "--pull", - "never", - "--network", - "none", - "--read-only", - "--user", - "0:0", - "--security-opt", - "no-new-privileges", - "--cap-drop", - "ALL", - "--cap-add", - "DAC_OVERRIDE", - "--pids-limit", - "64", - ...NEUTRALIZED_OFFLINE_HELPER_ENV, - "--label", - `${STOPPED_CHANNEL_CLEANUP_LABEL}=1`, - "--label", - `${STOPPED_CHANNEL_CLEANUP_OWNER_LABEL}=${ownerIdentity}`, - "--label", - `${STOPPED_CHANNEL_CLEANUP_VOLUME_LABEL}=${volumeIdentity}`, - "--mount", - `type=volume,src=${inspected.sandboxVolumeName},dst=/sandbox,volume-nocopy`, - "--entrypoint", - "/usr/local/bin/node", - STOPPED_CHANNEL_CLEANUP_IMAGE, - "-e", - STOPPED_CHANNEL_CLEANUP_SCRIPT, - JSON.stringify(cleanupPaths), - ], - OFFLINE_DOCKER_OPERATION_OPTIONS, - ); - } catch { - return reconcileCleanupHelperAfterCreate(helperName, ownerIdentity, volumeIdentity) - ? stoppedDockerCleanupFailure("cleanup-helper-failed") - : stoppedDockerCleanupFailure("cleanup-helper-reconciliation-failed", helperName); - } - const helperId = String(created.stdout ?? "").trim(); - if (created.status !== 0 || !FULL_CONTAINER_ID_RE.test(helperId)) { - return reconcileCleanupHelperAfterCreate(helperName, ownerIdentity, volumeIdentity) - ? stoppedDockerCleanupFailure("cleanup-helper-failed") - : stoppedDockerCleanupFailure("cleanup-helper-reconciliation-failed", helperName); - } - let cleared: ReturnType | null = null; - try { - cleared = dockerRun(["start", "--attach", helperId], OFFLINE_DOCKER_OPERATION_OPTIONS); - } catch { - cleared = null; - } - if (!removeAndConfirmCleanupHelper(helperId)) { - return stoppedDockerCleanupFailure("cleanup-helper-reconciliation-failed", helperName); - } - if (!cleared || cleared.status !== 0 || cleared.error) { - return stoppedDockerCleanupFailure(classifyCleanupHelperFailure(cleared)); - } - const confirmation = inspectStoppedContainer(containerId); - if ("failure" in confirmation) { - return stoppedDockerCleanupFailure("container-revalidation-failed"); - } - const { inspected: confirmed } = confirmation; - return confirmed.id === inspected.id && - confirmed.sandboxVolumeName === inspected.sandboxVolumeName && - !confirmed.running - ? { cleared: true } - : stoppedDockerCleanupFailure("container-revalidation-failed"); - }); - } catch { - return stoppedDockerCleanupFailure("lifecycle-authority-unavailable"); +function registeredSandboxNames(sandboxName: string): readonly string[] { + const names = new Set([sandboxName]); + const listed = registry.listSandboxes?.(); + if (Array.isArray(listed?.sandboxes)) { + for (const entry of listed.sandboxes) { + if (typeof entry.name === "string" && entry.name) names.add(entry.name); + } } -} - -function missingDirectContainerError(sandboxName: string, driver: string | null): Error { - const driverLabel = driver ?? "unspecified"; - return new DirectSandboxFallbackUnavailableError( - `No running direct OpenShell sandbox container found for '${sandboxName}' ` + - `(driver: ${driverLabel}). Expected one OpenShell-managed container labeled ` + - `'${OPENSHELL_SANDBOX_NAME_LABEL}=${sandboxName}' and named ` + - `${expectedDirectContainerPattern(sandboxName)}. Is the sandbox running?`, - ); -} - -function isDirectSandboxFallbackUnavailableError( - error: unknown, -): error is DirectSandboxFallbackUnavailableError { - return error instanceof DirectSandboxFallbackUnavailableError; -} - -function isPinnedSandboxContainerIdentityChangedError( - error: unknown, -): error is PinnedSandboxContainerIdentityChangedError { - return error instanceof PinnedSandboxContainerIdentityChangedError; -} - -function missingRegistryEntryError(sandboxName: string): Error { - return new Error( - `No NemoClaw registry entry found for '${sandboxName}'; ` + - "refusing privileged exec without a registered sandbox owner.", - ); -} - -function unsupportedDirectDriverError(sandboxName: string, driver: string): Error { - return new Error( - `Privileged direct-container control is unavailable for sandbox '${sandboxName}' ` + - `(driver: ${driver}); refusing local Docker discovery for a non-direct driver.`, + return Array.from(names).sort( + (left, right) => right.length - left.length || left.localeCompare(right), ); } @@ -667,15 +96,8 @@ function assertNoActiveStateMutationTarget(sandboxName: string): void { } } -/** - * Serialize one ordinary direct-container execution against provider fence - * acquisition. The callback must include both argv resolution and the complete - * synchronous Docker subprocess lifetime. Taking the lock before checking the - * durable target claim closes the check/acquire/exec race: an older exec drains - * before the provider can publish its fence, while a later exec observes the - * claim and is rejected before it can spawn. - */ -function withPrivilegedSandboxExecutionLease( +/** Serialize one ordinary privileged operation against provider fence acquisition. */ +export function withPrivilegedSandboxExecutionLease( sandboxName: string, operation: string, fn: () => T, @@ -690,86 +112,132 @@ function withPrivilegedSandboxExecutionLease( ); } -function resolveDirectSandboxContainer(sandboxName: string, driver: string | null): string { - const selected = findDirectSandboxContainer(sandboxName); - if (selected) return selected; - throw missingDirectContainerError(sandboxName, driver); +export function resolvePrivilegedSandboxTarget( + sandboxName: string, +): RuntimeProviderPrivilegedSandboxTarget { + assertNoActiveStateMutationTarget(sandboxName); + const { sandbox, control } = privilegedSandboxControl(sandboxName); + return control.resolveTarget({ + registeredSandboxNames: registeredSandboxNames(sandboxName), + sandbox, + sandboxName, + }); } -function privilegedSandboxExecArgv( +/** Retained name for Docker compatibility code that only needs an opaque runtime handle. */ +export function resolveDirectSandboxContainer(sandboxName: string, _driver: string | null): string { + return resolvePrivilegedSandboxTarget(sandboxName).resourceHandle; +} + +export function executePrivilegedSandboxCommand( + sandboxName: string, + command: readonly string[], + options: PrivilegedSandboxCommandOptions = {}, +): RuntimeProviderPrivilegedSandboxCommandResult { + assertNoActiveStateMutationTarget(sandboxName); + const { sandbox, control } = privilegedSandboxControl(sandboxName); + const input = + options.input === undefined + ? undefined + : Buffer.isBuffer(options.input) + ? Buffer.from(options.input) + : Buffer.from(options.input, "utf8"); + return control.execute({ + registeredSandboxNames: registeredSandboxNames(sandboxName), + sandbox, + sandboxName, + command, + sanitizeEnvironment: options.sanitizeEnvironment === true, + timeoutMs: options.timeout ?? DEFAULT_PRIVILEGED_SANDBOX_COMMAND_TIMEOUT_MS, + ...(input ? { input } : {}), + ...(options.expectedResourceHandle !== undefined + ? { expectedResourceHandle: options.expectedResourceHandle } + : {}), + ...(options.maxOutputBytes ? { maxOutputBytes: options.maxOutputBytes } : {}), + }); +} + +/** Retained Docker CLI compatibility for portable and Docker-specific probes. */ +export function privilegedSandboxExecArgv( sandboxName: string, - cmd: string[], + command: string[], stdin = false, sanitizeEnvironment = false, expectedContainerId?: string, ): string[] { - const entry = readSandboxEntry(sandboxName); - if (!entry) throw missingRegistryEntryError(sandboxName); - const driver = normalizeDriver(entry.openshellDriver); - if (driver !== null && driver !== "docker" && driver !== "vm") { - throw unsupportedDirectDriverError(sandboxName, driver); - } assertNoActiveStateMutationTarget(sandboxName); - const portableTarget = - driver === "docker" - ? resolvePortableDemoPrivilegedExecTarget(sandboxName, { - ...(entry.lifecycleGeneration ? { registryGeneration: entry.lifecycleGeneration } : {}), - backfillRegistryGeneration: (generation) => - compareAndSetLegacySandboxLifecycleGeneration(entry, generation), - }) - : null; - if (portableTarget) { - if (expectedContainerId !== undefined && portableTarget.containerId !== expectedContainerId) { - throw new PinnedSandboxContainerIdentityChangedError(sandboxName); - } - const sanitizedEnvArgs = sanitizeEnvironment - ? SANITIZED_PRIVILEGED_ENV.flatMap((value) => ["--env", value]) - : []; - portableTarget.assertRuntimeAuthority(); - return [ - "--host", - portableTarget.dockerHost, - "exec", - ...(stdin ? ["-i"] : []), - ...sanitizedEnvArgs, - "--user", - "0", - portableTarget.containerId, - ...cmd, - ]; + const { sandbox, control } = privilegedSandboxControl(sandboxName); + if (!control.buildLegacyDockerArgv) { + throw new Error( + "The selected runtime provider does not expose the retained Docker CLI compatibility path.", + ); } - // Docker/direct-container is the only supported privileged mutation path. - // Try it even when older registry entries do not record a driver, then fail - // clearly if no matching sandbox container is running. - const container = findDirectSandboxContainer(sandboxName); - if (container) { - if (expectedContainerId !== undefined && container !== expectedContainerId) { - throw new PinnedSandboxContainerIdentityChangedError(sandboxName); - } - const sanitizedEnvArgs = sanitizeEnvironment - ? SANITIZED_PRIVILEGED_ENV.flatMap((value) => ["--env", value]) - : []; - return [ - "exec", - ...(stdin ? ["-i"] : []), - ...sanitizedEnvArgs, - "--user", - "root", - container, - ...cmd, - ]; + return control.buildLegacyDockerArgv({ + registeredSandboxNames: registeredSandboxNames(sandboxName), + sandbox, + sandboxName, + command, + sanitizeEnvironment, + ...(stdin ? { input: Buffer.alloc(0) } : {}), + ...(expectedContainerId !== undefined ? { expectedResourceHandle: expectedContainerId } : {}), + }); +} + +export function capturePrivilegedSandboxCommand( + sandboxName: string, + command: readonly string[], + options: PrivilegedSandboxCommandOptions = {}, +): Buffer { + const result = executePrivilegedSandboxCommand(sandboxName, command, options); + if (result.status !== 0 || result.signal !== null || result.error) { + const detail = result.stderr.toString("utf8").replace(/\s+/gu, " ").trim().slice(-500); + const reason = + result.error?.message ?? + (result.signal ? `signal ${result.signal}` : `exit ${String(result.status)}`); + throw new Error(`Privileged sandbox command failed (${reason})${detail ? `: ${detail}` : ""}`); } + return result.stdout; +} - throw missingDirectContainerError(sandboxName, driver); +export function clearStoppedSandboxStateRoots( + sandboxName: string, + paths: readonly string[], +): RuntimeProviderStoppedSandboxStateCleanupResult { + if (!validateStoppedSandboxStatePaths(paths)) { + return { cleared: false, failure: "state-paths-invalid" }; + } + const sandbox = registry.getSandbox?.(sandboxName) ?? null; + if (!sandbox) return { cleared: false, failure: "sandbox-registry-unavailable" }; + try { + return withPrivilegedSandboxExecutionLease(sandboxName, "offline channel state cleanup", () => { + const { control } = privilegedSandboxControl(sandboxName); + if (!control.clearStoppedStateRoots) + return { cleared: false, failure: "provider-cleanup-unavailable" }; + return control.clearStoppedStateRoots({ + registeredSandboxNames: registeredSandboxNames(sandboxName), + sandbox, + sandboxName, + paths, + }); + }); + } catch { + return { cleared: false, failure: "lifecycle-authority-unavailable" }; + } } export { - clearStoppedDockerSandboxChannelState, - containerNameMatchesSandbox, - isDirectSandboxFallbackUnavailableError, - isPinnedSandboxContainerIdentityChangedError, - privilegedSandboxExecArgv, - resolveDirectSandboxContainer, - selectDirectSandboxContainer, - withPrivilegedSandboxExecutionLease, + buildStoppedSandboxChannelCleanupScript, + buildStoppedSandboxChannelCleanupScript as buildStoppedDockerSandboxChannelCleanupScript, }; + +export function isDirectSandboxFallbackUnavailableError( + error: unknown, +): error is DirectSandboxFallbackUnavailableError { + return error instanceof DirectSandboxFallbackUnavailableError; +} + +export function isPinnedSandboxContainerIdentityChangedError( + error: unknown, +): error is PinnedSandboxResourceIdentityChangedError { + return error instanceof PinnedSandboxResourceIdentityChangedError; +} diff --git a/src/lib/shields/hermes-runtime-state-mutation.test.ts b/src/lib/shields/hermes-runtime-state-mutation.test.ts index 7d567bf03d0..a26945cc9ee 100644 --- a/src/lib/shields/hermes-runtime-state-mutation.test.ts +++ b/src/lib/shields/hermes-runtime-state-mutation.test.ts @@ -195,6 +195,18 @@ describe("Hermes runtime-provider state mutation consumer", () => { testTimeout(30_000), ); + it("selects the registered native provider state-mutation surface without a provider-name branch", () => { + expect( + supportsHermesRuntimeProviderStateMutation( + { ...sandbox, openshellDriver: "podman" }, + { + content: HERMES_RUNTIME_STATE_MUTATION_CAPABILITY, + metadata: HERMES_RUNTIME_STATE_MUTATION_CAPABILITY_METADATA, + }, + ), + ).toBe(true); + }); + it("selects only an exact current managed Hermes Docker image capability", () => { const capability = { content: HERMES_RUNTIME_STATE_MUTATION_CAPABILITY, diff --git a/src/lib/shields/hermes-runtime-state-mutation.ts b/src/lib/shields/hermes-runtime-state-mutation.ts index 9a227419c7b..a8c0d040500 100644 --- a/src/lib/shields/hermes-runtime-state-mutation.ts +++ b/src/lib/shields/hermes-runtime-state-mutation.ts @@ -334,10 +334,25 @@ function releaseTarget( } } +/** Select the provider protocol only when the registered bundle owns it. */ +export function hasHermesRuntimeProviderStateMutationAuthority( + sandbox: SandboxEntry | null, + providers: RuntimeProviderBundleRegistry = currentRuntimeProviderBundles(), +): boolean { + if (sandbox?.agent !== "hermes" || sandbox.workload?.kind !== "managed-image") return false; + const provider = requireRuntimeProviderBundleForSandbox(sandbox, providers); + return ( + provider.lifecycle.supported === true && + provider.stateMutation.supported === true && + provider.stateMutation.providerId === provider.identity.id + ); +} + /** - * Select the new protocol only for a current managed Hermes Docker image. The - * fixed helper validates the same capability again after the durable runtime - * claim is acquired, so this compatibility probe never grants authority. + * Select the new protocol only for a current managed Hermes image whose + * provider owns lifecycle and state-mutation authority. The fixed helper + * validates the same capability again after the durable runtime claim is + * acquired, so this compatibility probe never grants authority. */ export function supportsHermesRuntimeProviderStateMutation( sandbox: SandboxEntry | null, @@ -345,17 +360,15 @@ export function supportsHermesRuntimeProviderStateMutation( providers: RuntimeProviderBundleRegistry = currentRuntimeProviderBundles(), ): boolean { if ( - sandbox?.agent !== "hermes" || - sandbox.openshellDriver?.trim().toLowerCase() !== "docker" || - sandbox.workload?.kind !== "managed-image" || + !hasHermesRuntimeProviderStateMutationAuthority(sandbox, providers) || + !sandbox || !sandbox.lifecycleGeneration || capability?.content !== HERMES_RUNTIME_STATE_MUTATION_CAPABILITY || capability.metadata !== HERMES_RUNTIME_STATE_MUTATION_CAPABILITY_METADATA ) { return false; } - const provider = requireRuntimeProviderBundleForSandbox(sandbox, providers); - return provider.identity.id === "docker" && provider.stateMutation.supported === true; + return true; } /** Execute one complete Hermes protection transition under its provider fence. */ diff --git a/src/lib/shields/index.ts b/src/lib/shields/index.ts index 230b0c00001..9e80575fb6f 100644 --- a/src/lib/shields/index.ts +++ b/src/lib/shields/index.ts @@ -30,12 +30,9 @@ const YAML: typeof import("yaml") = require("yaml"); const { CLI_NAME }: typeof import("../cli/branding") = require("../cli/branding"); const { isObjectRecord }: typeof import("../core/json-types") = require("../core/json-types"); const { - dockerExecFileSync, - dockerSpawnSync, -}: typeof import("../adapters/docker/exec") = require("../adapters/docker/exec"); -const { + capturePrivilegedSandboxCommand, + executePrivilegedSandboxCommand, isDirectSandboxFallbackUnavailableError, - privilegedSandboxExecArgv, withPrivilegedSandboxExecutionLease, }: typeof import("../sandbox/privileged-exec") = require("../sandbox/privileged-exec"); const { @@ -121,6 +118,7 @@ const { restoreStateDirLockPosture, restoreStateDirStartupAccess, stateLockPlanCompatibilityIssues, + verifyLockedStateDirPosture, verifyStateDirMutablePosture, }: typeof import("./state-dir-lock") = require("./state-dir-lock"); const { @@ -140,6 +138,7 @@ const { HERMES_RUNTIME_STATE_MUTATION_CAPABILITY_PATH, hermesRuntimeProviderPhaseBlocksMutation, hasActiveHermesRuntimeProviderStateMutation, + hasHermesRuntimeProviderStateMutationAuthority, runHermesRuntimeProviderStateMutation, supportsHermesRuntimeProviderStateMutation, }: typeof import("./hermes-runtime-state-mutation") = require("./hermes-runtime-state-mutation"); @@ -875,8 +874,8 @@ function openClawRollbackIssue(prefix: string, error: unknown): OpenClawRollback function privilegedSandboxExec(sandboxName: string, cmd: string[], timeout = 15000): void { withPrivilegedSandboxExecutionLease(sandboxName, "shields privileged execution", () => { - dockerExecFileSync(privilegedSandboxExecArgv(sandboxName, cmd, false, true), { - stdio: ["ignore", "pipe", "pipe"], + capturePrivilegedSandboxCommand(sandboxName, cmd, { + sanitizeEnvironment: true, timeout, }); }); @@ -884,10 +883,12 @@ function privilegedSandboxExec(sandboxName: string, cmd: string[], timeout = 150 function privilegedSandboxExecCapture(sandboxName: string, cmd: string[], timeout = 15000): string { return withPrivilegedSandboxExecutionLease(sandboxName, "shields privileged capture", () => - dockerExecFileSync(privilegedSandboxExecArgv(sandboxName, cmd, false, true), { - stdio: ["ignore", "pipe", "pipe"], + capturePrivilegedSandboxCommand(sandboxName, cmd, { + sanitizeEnvironment: true, timeout, - }).trim(), + }) + .toString("utf8") + .trim(), ); } @@ -1107,18 +1108,11 @@ function inspectHermesShieldsProtocol( if (hasActiveRuntimeProviderStateMutation(sandboxName)) { return "provider-state-mutation-v2"; } - if ( - sandbox.openshellDriver?.trim().toLowerCase() === "docker" && - sandbox.workload?.kind === "managed-image" && - !sandbox.lifecycleGeneration - ) { - throw new Error("Managed Hermes Docker registry authority has no lifecycle generation"); + const providerStateMutation = hasHermesRuntimeProviderStateMutationAuthority(sandbox); + if (providerStateMutation && !sandbox.lifecycleGeneration) { + throw new Error("Managed Hermes runtime-provider authority has no lifecycle generation"); } - if ( - sandbox.openshellDriver?.trim().toLowerCase() === "docker" && - sandbox.workload?.kind === "managed-image" && - sandbox.lifecycleGeneration - ) { + if (providerStateMutation && sandbox.lifecycleGeneration) { const capabilityPresence = privilegedSandboxExecCapture(sandboxName, [ HERMES_PYTHON, "-I", @@ -1345,12 +1339,12 @@ function requireHermesRuntimeProviderSandbox(sandboxName: string) { if ( !sandbox || sandbox.agent !== "hermes" || - sandbox.openshellDriver?.trim().toLowerCase() !== "docker" || sandbox.workload?.kind !== "managed-image" || - !sandbox.lifecycleGeneration + !sandbox.lifecycleGeneration || + !hasHermesRuntimeProviderStateMutationAuthority(sandbox) ) { throw new Error( - "Hermes runtime-provider state mutation lost its exact managed Docker registry authority", + "Hermes runtime-provider state mutation lost its exact managed provider registry authority", ); } return sandbox; @@ -1452,11 +1446,14 @@ function runHermesProviderProtectionTransition( // then asynchronously republishes the sandbox lifecycle phase; callers must // not issue route or mutation commands during that Provisioning interval. waitForHermesRuntimeProviderReleaseReady(sandboxName); - // The fenced gateway can prove local health while OpenShell PID 1 is held, - // but Hermes performs network MCP discovery before exposing that health. - // Restart once after release so configured managed bridges are discovered - // with the exact supervisor/network control path live. - restartHermesManagedMcpAfterProviderRelease(sandboxName, sandbox); + // The locked activation has already proven the exact gateway under the + // final immutable config. A second ordinary restart can only attempt a + // configuration reconciliation that lockdown forbids. Mutable activation + // still restarts once after release so configured managed bridges discover + // through the live supervisor/network control path. + if (targetPosture === "mutable") { + restartHermesManagedMcpAfterProviderRelease(sandboxName, sandbox); + } } function verifyHermesProviderMutablePosture(sandboxName: string, target: AgentConfigTarget): void { @@ -1467,6 +1464,7 @@ function verifyHermesProviderMutablePosture(sandboxName: string, target: AgentCo requireStateLockPlan(target), target.stateLockPlanInImage, [target.configPath, ...(target.sensitiveFiles || [])], + ["gateway"], ), ]; if (issues.length > 0) throw new Error(`Config not unlocked: ${issues.join(", ")}`); @@ -2473,20 +2471,17 @@ function stateDirLockExec(sandboxName: string) { return { run: (cmd: string[], input?: string) => withPrivilegedSandboxExecutionLease(sandboxName, "state directory guard", () => { - const result = dockerSpawnSync( - privilegedSandboxExecArgv(sandboxName, cmd, input !== undefined, true), - { - encoding: "utf-8", - input, - timeout: STATE_DIR_GUARD_TIMEOUT_MS, - maxBuffer: 16 * 1024 * 1024, - }, - ); + const result = executePrivilegedSandboxCommand(sandboxName, cmd, { + ...(input === undefined ? {} : { input }), + sanitizeEnvironment: true, + timeout: STATE_DIR_GUARD_TIMEOUT_MS, + maxOutputBytes: 16 * 1024 * 1024, + }); return { status: result.status, signal: result.signal, - stdout: String(result.stdout ?? ""), - stderr: String(result.stderr ?? ""), + stdout: result.stdout.toString("utf8"), + stderr: result.stderr.toString("utf8"), ...(result.error ? { error: result.error.message } : {}), }; }), @@ -2500,20 +2495,17 @@ function openClawConfigGuardExec(sandboxName: string) { const timeout = cmd.includes("unlock-failed-startup") ? OPENCLAW_CONFIG_GUARD_RECOVERY_TIMEOUT_MS : OPENCLAW_CONFIG_GUARD_TIMEOUT_MS; - const result = dockerSpawnSync( - privilegedSandboxExecArgv(sandboxName, cmd, input !== undefined, true), - { - encoding: "utf-8", - input, - timeout, - maxBuffer: 2 * 1024 * 1024, - }, - ); + const result = executePrivilegedSandboxCommand(sandboxName, cmd, { + ...(input === undefined ? {} : { input }), + sanitizeEnvironment: true, + timeout, + maxOutputBytes: 2 * 1024 * 1024, + }); return { status: result.status, signal: result.signal, - stdout: String(result.stdout ?? ""), - stderr: String(result.stderr ?? ""), + stdout: result.stdout.toString("utf8"), + stderr: result.stderr.toString("utf8"), ...(result.error ? { error: result.error.message } : {}), }; }), @@ -6231,7 +6223,13 @@ function shieldsStatusWithoutHostLock( target.agentName === "hermes" && inspectHermesShieldsProtocol(sandboxName, target) === "provider-state-mutation-v2" ) { - runHermesProviderProtectionTransition(sandboxName, target, "locked", "locked"); + driftIssues.push( + ...verifyLockedStateDirPosture( + stateDirLockExec(sandboxName), + target.configDir, + requireStateLockPlan(target), + ).map((issue) => `state lock posture: ${issue}`), + ); } try { planIssues = deps.verifyStateLockPlan diff --git a/src/lib/shields/legacy-hermes-compat.test.ts b/src/lib/shields/legacy-hermes-compat.test.ts index 47a76dcd12a..bf33f201250 100644 --- a/src/lib/shields/legacy-hermes-compat.test.ts +++ b/src/lib/shields/legacy-hermes-compat.test.ts @@ -13,7 +13,6 @@ import { createHermesShieldsProviderConsumerHarness, createRetainedUnlockSimulation, createTimerAuthorizationSender, - createTransitionFailureForPosture, hermesProviderConsumerSandbox as sandbox, hermesProviderConsumerTarget as target, writeBoundForwardPolicy, @@ -187,7 +186,7 @@ describe("legacy Hermes shields compatibility", () => { let spies: MockInstance[]; let runSpy: MockInstance; let dockerExecSpy: MockInstance; - let privilegedExecArgvSpy: MockInstance; + let privilegedCaptureSpy: MockInstance; let applyStateDirLockModeSpy: MockInstance; let inferenceConvergenceSpy: MockInstance; let auditSpy: MockInstance; @@ -224,9 +223,11 @@ describe("legacy Hermes shields compatibility", () => { runSpy = vi.spyOn(runner, "run").mockReturnValue({ status: 0 }); dockerExecSpy = vi.spyOn(dockerExec, "dockerExecFileSync"); applyStateDirLockModeSpy = vi.spyOn(stateDirLock, "applyStateDirLockMode").mockReturnValue([]); - privilegedExecArgvSpy = vi - .spyOn(privilegedExec, "privilegedSandboxExecArgv") - .mockImplementation((_sandboxName: unknown, cmd: unknown) => cmd as string[]); + privilegedCaptureSpy = vi + .spyOn(privilegedExec, "capturePrivilegedSandboxCommand") + .mockImplementation((_sandboxName: unknown, cmd: unknown) => + Buffer.from(dockerExec.dockerExecFileSync(cmd as string[])), + ); inferenceConvergenceSpy = vi .spyOn(relockReconfirm, "waitForHermesInferenceRouteConvergence") .mockReturnValue({ @@ -261,7 +262,7 @@ describe("legacy Hermes shields compatibility", () => { lifecycleGeneration: "legacy-generation", workload: { kind: "managed-image" }, })), - privilegedExecArgvSpy, + privilegedCaptureSpy, dockerExecSpy, applyStateDirLockModeSpy, vi.spyOn(stateDirLock, "preflightStateDirLock").mockReturnValue([]), @@ -558,7 +559,6 @@ describe("legacy Hermes shields compatibility", () => { expect(() => shields.unlockAgentConfig("current-hermes", hermesTarget(), true, true), ).not.toThrow(); - const guardCommands = dockerExecSpy.mock.calls .map(commandFromCall) .filter((cmd) => cmd.includes(HERMES_GUARD)); @@ -573,13 +573,15 @@ describe("legacy Hermes shields compatibility", () => { ); }), ).toBe(true); - expect(privilegedExecArgvSpy).toHaveBeenCalled(); - expect(privilegedExecArgvSpy.mock.calls.every((call) => call[3] === true)).toBe(true); + expect(privilegedCaptureSpy).toHaveBeenCalled(); + expect( + privilegedCaptureSpy.mock.calls.every( + (call) => (call[2] as { sanitizeEnvironment?: boolean }).sanitizeEnvironment === true, + ), + ).toBe(true); }); - it("pins one capability decision across policy and config mutation", () => { installExecResponses(CURRENT_GUARD_HELP); - expect(() => shields.shieldsDown("current-hermes", { throwOnError: true, @@ -795,11 +797,12 @@ describe("legacy Hermes shields compatibility", () => { }); it("migrates the current managed Hermes lock leaf and preserves the host seal result", () => { + registrySpy.mockReturnValue({ ...sandbox, mcp: { bridges: { fake: {} } } }); const result = shields.lockAgentConfig(sandbox.name, target, false, false); expect(transitionSpy).toHaveBeenCalledWith( expect.objectContaining({ - sandbox, + sandbox: expect.objectContaining(sandbox), sandboxName: sandbox.name, configTarget: target, target: "locked", @@ -811,7 +814,7 @@ describe("legacy Hermes shields compatibility", () => { "/sandbox/.hermes/.env": "c".repeat(64), "/sandbox/.hermes/.config-hash": "c".repeat(64), }); - expect(commands.some((command) => command.includes("begin-shields-transition"))).toBe(false); + expect(commands.some((command) => command.includes("nemoclaw-gateway-control"))).toBe(false); }); it("treats an already-locked provider lock as recovery plus live verification", () => { @@ -858,7 +861,8 @@ describe("legacy Hermes shields compatibility", () => { shieldsDownTimeout: 300, shieldsDownReason: "crash retry", shieldsDownPolicy: "permissive", - shieldsPolicySnapshotPath: snapshotPath, shieldsPolicySnapshot: snapshotPolicy, + shieldsPolicySnapshotPath: snapshotPath, + shieldsPolicySnapshot: snapshotPolicy, }), ); fs.writeFileSync( @@ -886,7 +890,8 @@ describe("legacy Hermes shields compatibility", () => { ownerStartIdentity: "dead-provider-owner", processToken, sandboxName: sandbox.name, - snapshotPath, snapshotPolicy, + snapshotPath, + snapshotPolicy, forwardPolicy, }), ); @@ -994,7 +999,8 @@ describe("legacy Hermes shields compatibility", () => { shieldsDownTimeout: 300, shieldsDownReason: "post-release crash", shieldsDownPolicy: "permissive", - shieldsPolicySnapshotPath: snapshotPath, shieldsPolicySnapshot: snapshotPolicy, + shieldsPolicySnapshotPath: snapshotPath, + shieldsPolicySnapshot: snapshotPolicy, }), ); fs.writeFileSync( @@ -1022,7 +1028,8 @@ describe("legacy Hermes shields compatibility", () => { ownerStartIdentity: "dead-post-release-owner", processToken, sandboxName: sandbox.name, - snapshotPath, snapshotPolicy, + snapshotPath, + snapshotPolicy, forwardPolicy, }), ); @@ -1078,7 +1085,8 @@ describe("legacy Hermes shields compatibility", () => { shieldsDownTimeout: 300, shieldsDownReason: "invalid forward policy", shieldsDownPolicy: "permissive", - shieldsPolicySnapshotPath: snapshotPath, shieldsPolicySnapshot: snapshotPolicy, + shieldsPolicySnapshotPath: snapshotPath, + shieldsPolicySnapshot: snapshotPolicy, }), ); const timerPath = path.join(stateDir, `shields-timer-${sandbox.name}.json`); @@ -1107,7 +1115,8 @@ describe("legacy Hermes shields compatibility", () => { ownerStartIdentity: "dead-forward-owner", processToken, sandboxName: sandbox.name, - snapshotPath, snapshotPolicy, + snapshotPath, + snapshotPolicy, forwardPolicy, }), ); @@ -1171,63 +1180,6 @@ describe("legacy Hermes shields compatibility", () => { expect(commands.some((command) => command.includes("--help"))).toBe(false); }); - it("does not report clean UP when provider verification finds nested skills or pairing drift", () => { - const statePaths = requireSource("../state/paths.js") as typeof import("../state/paths"); - const stateDir = statePaths.resolveNemoclawStateDir(); - fs.mkdirSync(stateDir, { recursive: true, mode: 0o700 }); - fs.writeFileSync( - path.join(stateDir, `shields-${sandbox.name}.json`), - JSON.stringify({ - shieldsDown: false, - chattrApplied: true, - fileHashes: { [target.configPath]: "c".repeat(64) }, - updatedAt: new Date().toISOString(), - }), - ); - lifecycleGateSpy.mockReturnValue(false); - transitionSpy.mockImplementation( - createTransitionFailureForPosture( - "locked", - "recursive state lock plan drift under skills/pairing", - ), - ); - const exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number) => { - throw new Error(`process exit ${String(code)}`); - }) as never); - spies.push(exitSpy); - - expect(() => shields.shieldsStatus(sandbox.name)).toThrow("process exit 2"); - - const errors = vi.mocked(console.error).mock.calls.flat().map(String).join("\n"); - const logs = vi.mocked(console.log).mock.calls.flat().map(String).join("\n"); - expect(errors).toContain("recursive state lock plan drift under skills/pairing"); - expect(errors).toContain("UP (DRIFTED"); - expect(logs).not.toContain("UP (lockdown active)"); - expect(transitionSpy).toHaveBeenCalledWith( - expect.objectContaining({ target: "locked", rollback: "locked" }), - ); - - transitionSpy.mockClear(); - vi.mocked(console.log).mockClear(); - expect(() => shields.shieldsUp(sandbox.name, { throwOnError: true })).toThrow( - "recursive state lock plan drift under skills/pairing", - ); - expect(vi.mocked(console.log).mock.calls.flat().map(String).join("\n")).not.toContain( - "already locked", - ); - expect(transitionSpy).toHaveBeenCalledWith( - expect.objectContaining({ target: "locked", rollback: "locked" }), - ); - - transitionSpy.mockClear(); - expect(() => shields.lockAgentConfig(sandbox.name, target, true, false)).toThrow( - "recursive state lock plan drift under skills/pairing", - ); - expect(transitionSpy).toHaveBeenCalledWith( - expect.objectContaining({ target: "locked", rollback: "locked" }), - ); - }); - it("does not report clean mutable-default when recursive skills or pairing posture drifts (#9485)", () => { lifecycleGateSpy.mockReturnValue(false); verifyStateDirMutablePostureSpy.mockReturnValue([ @@ -1264,6 +1216,7 @@ describe("legacy Hermes shields compatibility", () => { }), true, [target.configPath, ...(target.sensitiveFiles || [])], + ["gateway"], ); expect(vi.mocked(console.log).mock.calls.flat().map(String).join("\n")).toContain( "NOT CONFIGURED (default mutable state)", @@ -1291,7 +1244,8 @@ describe("legacy Hermes shields compatibility", () => { shieldsDownTimeout: 300, shieldsDownReason: "timed mutable status", shieldsDownPolicy: "permissive", - shieldsPolicySnapshotPath: snapshotPath, shieldsPolicySnapshot: snapshotPolicy, + shieldsPolicySnapshotPath: snapshotPath, + shieldsPolicySnapshot: snapshotPolicy, updatedAt: new Date().toISOString(), }), ); @@ -1320,7 +1274,8 @@ describe("legacy Hermes shields compatibility", () => { ownerStartIdentity: "timed-status-owner", processToken, sandboxName: sandbox.name, - snapshotPath, snapshotPolicy, + snapshotPath, + snapshotPolicy, forwardPolicy, }), ); @@ -1491,7 +1446,7 @@ describe("legacy Hermes shields compatibility", () => { registrySpy.mockReturnValue({ ...sandbox, lifecycleGeneration: undefined }); expect(() => shields.lockAgentConfig(sandbox.name, target, false, false)).toThrow( - /registry authority has no lifecycle generation/u, + /authority has no lifecycle generation/u, ); expect(commands.some((command) => command.includes("--help"))).toBe(false); }); diff --git a/src/lib/shields/mutable-config-repair.test.ts b/src/lib/shields/mutable-config-repair.test.ts index 723f47b2d4b..ca9fab2a0c6 100644 --- a/src/lib/shields/mutable-config-repair.test.ts +++ b/src/lib/shields/mutable-config-repair.test.ts @@ -13,24 +13,21 @@ const NORMALIZER = "/usr/local/lib/nemoclaw/normalize_mutable_config_perms.py"; const NORMALIZER_WATCHDOG = ["/usr/bin/timeout", "--signal=TERM", "--kill-after=5s", "15s"]; const requireSource = createRequire(import.meta.url); -type DockerExecModule = typeof import("../adapters/docker/exec"); type MutableConfigRepairModule = typeof import("./mutable-config-repair"); type PrivilegedExecModule = typeof import("../sandbox/privileged-exec"); -let dockerExec: DockerExecModule; let normalizeMutableOpenClawConfig: MutableConfigRepairModule["normalizeMutableOpenClawConfig"]; let privilegedExec: PrivilegedExecModule; -function mockPrivilegedArgv() { +function mockPrivilegedLease() { return vi - .spyOn(privilegedExec, "privilegedSandboxExecArgv") - .mockImplementation((_sandboxName, cmd) => ["privileged", ...cmd]); + .spyOn(privilegedExec, "withPrivilegedSandboxExecutionLease") + .mockImplementation((_sandboxName: string, _operation: string, fn: () => T): T => fn()); } describe("mutable OpenClaw config repair", () => { beforeEach(() => { delete require.cache[requireSource.resolve("./mutable-config-repair.js")]; - dockerExec = requireSource("../adapters/docker/exec.js"); privilegedExec = requireSource("../sandbox/privileged-exec.js"); ({ normalizeMutableOpenClawConfig } = requireSource("./mutable-config-repair.js")); }); @@ -41,18 +38,18 @@ describe("mutable OpenClaw config repair", () => { }); it("sanitizes identity probes and watchdogs the privileged normalizer", () => { - const privilegedArgv = mockPrivilegedArgv(); - const dockerExecFileSync = vi - .spyOn(dockerExec, "dockerExecFileSync") - .mockReturnValueOnce("1000\n") - .mockReturnValueOnce("1001\n") - .mockReturnValue(""); + const lease = mockPrivilegedLease(); + const capture = vi + .spyOn(privilegedExec, "capturePrivilegedSandboxCommand") + .mockReturnValueOnce(Buffer.from("1000\n")) + .mockReturnValueOnce(Buffer.from("1001\n")) + .mockReturnValue(Buffer.alloc(0)); normalizeMutableOpenClawConfig("alpha", "/sandbox/.openclaw"); - expect(privilegedArgv.mock.calls).toEqual([ - ["alpha", ["/usr/bin/id", "-u", "sandbox"], false, true], - ["alpha", ["/usr/bin/id", "-g", "sandbox"], false, true], + expect(capture.mock.calls).toEqual([ + ["alpha", ["/usr/bin/id", "-u", "sandbox"], { sanitizeEnvironment: true, timeout: 15000 }], + ["alpha", ["/usr/bin/id", "-g", "sandbox"], { sanitizeEnvironment: true, timeout: 15000 }], [ "alpha", [ @@ -64,82 +61,59 @@ describe("mutable OpenClaw config repair", () => { "1000", "1001", ], - false, - true, + { sanitizeEnvironment: true, timeout: 25000 }, ], ]); - expect(dockerExecFileSync).toHaveBeenCalledTimes(3); - expect(dockerExecFileSync.mock.calls.map(([argv]) => argv)).toEqual([ - ["privileged", "/usr/bin/id", "-u", "sandbox"], - ["privileged", "/usr/bin/id", "-g", "sandbox"], - [ - "privileged", - ...NORMALIZER_WATCHDOG, - "/usr/bin/python3", - "-I", - NORMALIZER, - "/sandbox/.openclaw", - "1000", - "1001", - ], - ]); - expect(dockerExecFileSync.mock.calls.map(([, options]) => options)).toEqual([ - { stdio: ["ignore", "pipe", "pipe"], timeout: 15000 }, - { stdio: ["ignore", "pipe", "pipe"], timeout: 15000 }, - { stdio: ["ignore", "pipe", "pipe"], timeout: 25000 }, + expect(lease.mock.calls.map(([sandboxName, operation]) => [sandboxName, operation])).toEqual([ + ["alpha", "mutable config identity lookup"], + ["alpha", "mutable config identity lookup"], + ["alpha", "mutable config permission repair"], ]); }); it("rejects an invalid sandbox UID before the GID or normalizer runs", () => { - const privilegedArgv = mockPrivilegedArgv(); - const dockerExecFileSync = vi.spyOn(dockerExec, "dockerExecFileSync").mockReturnValue("0\n"); + mockPrivilegedLease(); + const capture = vi + .spyOn(privilegedExec, "capturePrivilegedSandboxCommand") + .mockReturnValue(Buffer.from("0\n")); expect(() => normalizeMutableOpenClawConfig("alpha", "/sandbox/.openclaw")).toThrow( "sandbox identity lookup returned an invalid UID", ); - expect(privilegedArgv).toHaveBeenCalledOnce(); - expect(privilegedArgv).toHaveBeenCalledWith( - "alpha", - ["/usr/bin/id", "-u", "sandbox"], - false, - true, - ); - expect(dockerExecFileSync).toHaveBeenCalledOnce(); + expect(capture).toHaveBeenCalledOnce(); + expect(capture).toHaveBeenCalledWith("alpha", ["/usr/bin/id", "-u", "sandbox"], { + sanitizeEnvironment: true, + timeout: 15000, + }); }); it("rejects an invalid sandbox GID before the normalizer runs", () => { - const privilegedArgv = mockPrivilegedArgv(); - const dockerExecFileSync = vi - .spyOn(dockerExec, "dockerExecFileSync") - .mockReturnValueOnce("1000\n") - .mockReturnValueOnce("not-a-gid\n"); + mockPrivilegedLease(); + const capture = vi + .spyOn(privilegedExec, "capturePrivilegedSandboxCommand") + .mockReturnValueOnce(Buffer.from("1000\n")) + .mockReturnValueOnce(Buffer.from("not-a-gid\n")); expect(() => normalizeMutableOpenClawConfig("alpha", "/sandbox/.openclaw")).toThrow( "sandbox identity lookup returned an invalid GID", ); - expect(privilegedArgv).toHaveBeenCalledTimes(2); - expect(privilegedArgv).not.toHaveBeenCalledWith( - "alpha", - expect.arrayContaining([NORMALIZER]), - false, - true, - ); - expect(dockerExecFileSync).toHaveBeenCalledTimes(2); + expect(capture).toHaveBeenCalledTimes(2); + expect(capture.mock.calls.flatMap(([, command]) => command)).not.toContain(NORMALIZER); }); it("propagates a trusted normalizer execution failure", () => { - const privilegedArgv = mockPrivilegedArgv(); - const failure = new Error("docker exec failed"); - const dockerExecFileSync = vi - .spyOn(dockerExec, "dockerExecFileSync") - .mockReturnValueOnce("1000\n") - .mockReturnValueOnce("1001\n") + mockPrivilegedLease(); + const failure = new Error("provider exec failed"); + const capture = vi + .spyOn(privilegedExec, "capturePrivilegedSandboxCommand") + .mockReturnValueOnce(Buffer.from("1000\n")) + .mockReturnValueOnce(Buffer.from("1001\n")) .mockImplementationOnce(() => { throw failure; }); expect(() => normalizeMutableOpenClawConfig("alpha", "/sandbox/.openclaw")).toThrow(failure); - expect(privilegedArgv).toHaveBeenLastCalledWith( + expect(capture).toHaveBeenLastCalledWith( "alpha", [ ...NORMALIZER_WATCHDOG, @@ -150,10 +124,9 @@ describe("mutable OpenClaw config repair", () => { "1000", "1001", ], - false, - true, + { sanitizeEnvironment: true, timeout: 25000 }, ); - expect(dockerExecFileSync).toHaveBeenCalledTimes(3); + expect(capture).toHaveBeenCalledTimes(3); }); }); diff --git a/src/lib/shields/mutable-config-repair.ts b/src/lib/shields/mutable-config-repair.ts index ca7530aee2f..cc7ba4e4214 100644 --- a/src/lib/shields/mutable-config-repair.ts +++ b/src/lib/shields/mutable-config-repair.ts @@ -1,7 +1,6 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -const dockerExec: typeof import("../adapters/docker/exec") = require("../adapters/docker/exec"); const privilegedExecModule: typeof import("../sandbox/privileged-exec") = require("../sandbox/privileged-exec"); const MUTABLE_CONFIG_NORMALIZER = "/usr/local/lib/nemoclaw/normalize_mutable_config_perms.py"; @@ -18,13 +17,10 @@ function runPrivileged(sandboxName: string, cmd: string[], timeout = 15000): voi sandboxName, "mutable config permission repair", () => { - dockerExec.dockerExecFileSync( - privilegedExecModule.privilegedSandboxExecArgv(sandboxName, cmd, false, true), - { - stdio: ["ignore", "pipe", "pipe"], - timeout, - }, - ); + privilegedExecModule.capturePrivilegedSandboxCommand(sandboxName, cmd, { + sanitizeEnvironment: true, + timeout, + }); }, ); } @@ -34,14 +30,12 @@ function privilegedExecCapture(sandboxName: string, cmd: string[], timeout = 150 sandboxName, "mutable config identity lookup", () => - dockerExec - .dockerExecFileSync( - privilegedExecModule.privilegedSandboxExecArgv(sandboxName, cmd, false, true), - { - stdio: ["ignore", "pipe", "pipe"], - timeout, - }, - ) + privilegedExecModule + .capturePrivilegedSandboxCommand(sandboxName, cmd, { + sanitizeEnvironment: true, + timeout, + }) + .toString("utf8") .trim(), ); } diff --git a/src/lib/shields/openclaw-transition.test.ts b/src/lib/shields/openclaw-transition.test.ts index 9e912d68d5c..f9624bc361d 100644 --- a/src/lib/shields/openclaw-transition.test.ts +++ b/src/lib/shields/openclaw-transition.test.ts @@ -133,7 +133,7 @@ describe("OpenClaw shields top-config transaction", () => { let homeDir: string; let shields: ShieldsModule; let spies: MockInstance[]; - let privilegedExecSpy: MockInstance; + let privilegedCaptureSpy: MockInstance; let dockerExecSpy: MockInstance; let guardSpy: MockInstance; let applyStateSpy: MockInstance; @@ -190,9 +190,11 @@ describe("OpenClaw shields top-config transaction", () => { events.push(`state:restore:${locked ? "locked" : "mutable"}`); return []; }); - privilegedExecSpy = vi - .spyOn(privilegedExec, "privilegedSandboxExecArgv") - .mockImplementation((_sandboxName: unknown, cmd: unknown) => cmd as string[]); + privilegedCaptureSpy = vi + .spyOn(privilegedExec, "capturePrivilegedSandboxCommand") + .mockImplementation((_sandboxName: unknown, cmd: unknown) => + Buffer.from(dockerExec.dockerExecFileSync(cmd as string[])), + ); compatibilitySpy = vi .spyOn(stateDirLock, "stateLockPlanCompatibilityIssues") .mockReturnValue([]); @@ -201,7 +203,7 @@ describe("OpenClaw shields top-config transaction", () => { vi.spyOn(runner, "run").mockReturnValue({ status: 0 }), vi.spyOn(runner, "runCapture").mockReturnValue(""), vi.spyOn(agentConfig, "resolveAgentConfig").mockImplementation(() => openClawTarget()), - privilegedExecSpy, + privilegedCaptureSpy, dockerExecSpy, compatibilitySpy, vi.spyOn(stateDirLock, "preflightStateDirLock").mockReturnValue([]), @@ -436,7 +438,7 @@ describe("OpenClaw shields top-config transaction", () => { }); it("reports a failed mutable top-config transition without falling back to recursive unlock", () => { - privilegedExecSpy.mockImplementationOnce(() => { + privilegedCaptureSpy.mockImplementationOnce(() => { throw new Error("top-config permission repair failed"); }); diff --git a/src/lib/shields/policy-transition.test.ts b/src/lib/shields/policy-transition.test.ts index 762cd6089f7..4a5b342e6a9 100644 --- a/src/lib/shields/policy-transition.test.ts +++ b/src/lib/shields/policy-transition.test.ts @@ -11,6 +11,12 @@ import { createShieldsFlowHarness, type ShieldsFlowHarnessOptions, } from "../../../test/helpers/shields-flow-harness"; +import { + createHermesShieldsProviderConsumerHarness, + createTransitionFailureForPosture, + hermesProviderConsumerSandbox as hermesSandbox, + hermesProviderConsumerTarget as hermesTarget, +} from "../../../test/helpers/hermes-shields-provider-consumer-harness"; const requireSource = createRequire(import.meta.url); const SHIELDS_MODULE = "./index.js"; @@ -97,9 +103,7 @@ describe("shields policy transition", () => { }, stateLockPlanInImage: false, }); - vi.spyOn(privilegedExec, "privilegedSandboxExecArgv").mockImplementation( - (_sandboxName: unknown, cmd: unknown) => cmd as string[], - ); + vi.spyOn(privilegedExec, "capturePrivilegedSandboxCommand").mockReturnValue(Buffer.alloc(0)); vi.spyOn(dockerExec, "dockerExecFileSync").mockReturnValue(""); mockLivePolicy("openclaw"); vi.spyOn(console, "error").mockImplementation(() => undefined); @@ -146,6 +150,86 @@ describe("shields policy transition", () => { }); }); +describe("Hermes provider locked status", () => { + let harness: ReturnType; + let shields: typeof import("./index"); + let spies: MockInstance[]; + let transitionSpy: MockInstance; + let verifyLockedStateDirPostureSpy: MockInstance; + + beforeEach(() => { + harness = createHermesShieldsProviderConsumerHarness(requireSource); + ({ shields, spies, transitionSpy, verifyLockedStateDirPostureSpy } = harness); + }); + + afterEach(() => harness.cleanup()); + + it("does not report clean UP when provider verification finds nested skills or pairing drift", () => { + const statePaths = requireSource("../state/paths.js") as typeof import("../state/paths"); + const stateDir = statePaths.resolveNemoclawStateDir(); + fs.mkdirSync(stateDir, { recursive: true, mode: 0o700 }); + fs.writeFileSync( + path.join(stateDir, `shields-${hermesSandbox.name}.json`), + JSON.stringify({ + shieldsDown: false, + chattrApplied: true, + fileHashes: { [hermesTarget.configPath]: "c".repeat(64) }, + updatedAt: new Date().toISOString(), + }), + ); + verifyLockedStateDirPostureSpy.mockReturnValue([ + "recursive state lock plan drift under skills/pairing", + ]); + const exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number) => { + throw new Error(`process exit ${String(code)}`); + }) as never); + spies.push(exitSpy); + + expect(() => shields.shieldsStatus(hermesSandbox.name)).toThrow("process exit 2"); + + const errors = vi.mocked(console.error).mock.calls.flat().map(String).join("\n"); + const logs = vi.mocked(console.log).mock.calls.flat().map(String).join("\n"); + expect(errors).toContain("recursive state lock plan drift under skills/pairing"); + expect(errors).toContain("UP (DRIFTED"); + expect(logs).not.toContain("UP (lockdown active)"); + expect(transitionSpy).not.toHaveBeenCalled(); + expect(verifyLockedStateDirPostureSpy).toHaveBeenCalledWith( + expect.anything(), + hermesTarget.configDir, + expect.objectContaining({ + readOnlyRoots: expect.arrayContaining(["skills"]), + confidentialRoots: expect.arrayContaining(["pairing"]), + }), + ); + + transitionSpy.mockClear(); + transitionSpy.mockImplementation( + createTransitionFailureForPosture( + "locked", + "recursive state lock plan drift under skills/pairing", + ), + ); + vi.mocked(console.log).mockClear(); + expect(() => shields.shieldsUp(hermesSandbox.name, { throwOnError: true })).toThrow( + "recursive state lock plan drift under skills/pairing", + ); + expect(vi.mocked(console.log).mock.calls.flat().map(String).join("\n")).not.toContain( + "already locked", + ); + expect(transitionSpy).toHaveBeenCalledWith( + expect.objectContaining({ target: "locked", rollback: "locked" }), + ); + + transitionSpy.mockClear(); + expect(() => shields.lockAgentConfig(hermesSandbox.name, hermesTarget, true, false)).toThrow( + "recursive state lock plan drift under skills/pairing", + ); + expect(transitionSpy).toHaveBeenCalledWith( + expect.objectContaining({ target: "locked", rollback: "locked" }), + ); + }); +}); + describe("shields down policy rejection", () => { let tmpDir: string; @@ -589,39 +673,40 @@ describe("shields config lock without a shipped config hash", () => { resolveAgentConfigSpy = vi .spyOn(agentConfig, "resolveAgentConfig") .mockImplementation(() => target()); - vi.spyOn(privilegedExec, "privilegedSandboxExecArgv").mockImplementation( - (_sandboxName: unknown, cmd: unknown) => cmd as string[], + vi.spyOn(privilegedExec, "capturePrivilegedSandboxCommand").mockImplementation( + (_sandboxName: unknown, cmd: unknown) => Buffer.from(runSandboxCommand(cmd as string[])), ); vi.spyOn(dockerExec, "dockerExecFileSync").mockImplementation((cmd: unknown) => runSandboxCommand(cmd as string[]), ); - vi.spyOn(dockerExec, "dockerSpawnSync").mockImplementation((rawCommand: unknown) => { - const command = Array.isArray(rawCommand) ? rawCommand.map(String) : []; - const action = (["preflight", "lock", "unlock"] as const).find((candidate) => - command.includes(candidate), - ); - const handler = - stateDirGuardCommandHandlers.get(String(action ?? command[0])) ?? - (() => unsupportedCommand(command)); - handler(); - - return { - status: 0, - signal: null, - stdout: - action === undefined - ? "" - : `${JSON.stringify({ - type: "result", - action, - status: "ok", - issueCount: 0, - })}\n`, - stderr: "", - pid: 0, - output: [], - } as never; - }); + vi.spyOn(privilegedExec, "executePrivilegedSandboxCommand").mockImplementation( + (_sandboxName: unknown, rawCommand: unknown) => { + const command = Array.isArray(rawCommand) ? rawCommand.map(String) : []; + const action = (["preflight", "lock", "unlock"] as const).find((candidate) => + command.includes(candidate), + ); + const handler = + stateDirGuardCommandHandlers.get(String(action ?? command[0])) ?? + (() => unsupportedCommand(command)); + handler(); + + return { + status: 0, + signal: null, + stdout: Buffer.from( + action === undefined + ? "" + : `${JSON.stringify({ + type: "result", + action, + status: "ok", + issueCount: 0, + })}\n`, + ), + stderr: Buffer.alloc(0), + } as never; + }, + ); vi.spyOn(stateDirLock, "preflightStateDirLock").mockReturnValue([]); applyStateDirLockModeSpy = vi.spyOn(stateDirLock, "applyStateDirLockMode").mockReturnValue([]); restoreStateDirLockPostureSpy = vi.spyOn(stateDirLock, "restoreStateDirLockPosture"); diff --git a/src/lib/shields/state-dir-lock.test.ts b/src/lib/shields/state-dir-lock.test.ts index e5c33a96ba7..dde8d0b90ba 100644 --- a/src/lib/shields/state-dir-lock.test.ts +++ b/src/lib/shields/state-dir-lock.test.ts @@ -12,6 +12,7 @@ import { restoreStateDirLockPosture, restoreStateDirStartupAccess, stateLockPlanCompatibilityIssues, + verifyLockedStateDirPosture, verifyStateDirMutablePosture, } from "./state-dir-lock"; @@ -224,6 +225,29 @@ describe("recursive state-dir lock host wiring", () => { "--plan-json", JSON.stringify(PLAN), ]); + expect(calls[0]?.input).toContain('"verify-lock",'); + }); + + it("uses the read-only host guard for recursive locked-posture verification", () => { + const { calls, privileged } = createExec(); + + expect(verifyLockedStateDirPosture(privileged, "/sandbox/.hermes", PLAN)).toEqual([]); + expect(calls).toHaveLength(1); + expect(calls[0]?.cmd).toEqual([ + "timeout", + "--signal=TERM", + "--kill-after=5s", + "12m", + "python3", + "-I", + "-", + "verify-lock", + "--config-dir", + "/sandbox/.hermes", + "--plan-json", + JSON.stringify(PLAN), + ]); + expect(calls[0]?.input).toContain("Descriptor-safe recursive state-directory"); }); it("passes mutable top-level files to the recursive read-only posture verifier (#9485)", () => { @@ -233,7 +257,7 @@ describe("recursive state-dir lock host wiring", () => { verifyStateDirMutablePosture(privileged, "/sandbox/.hermes", PLAN, true, [ "/sandbox/.hermes/config.yaml", "/sandbox/.hermes/.credentials.json", - ]), + ], ["gateway"]), ).toEqual([]); expect(calls).toHaveLength(3); expect(calls[2]?.cmd).toEqual([ @@ -253,6 +277,8 @@ describe("recursive state-dir lock host wiring", () => { "/sandbox/.hermes/config.yaml", "--mutable-top-level-file", "/sandbox/.hermes/.credentials.json", + "--mutable-service-user", + "gateway", ]); }); diff --git a/src/lib/shields/state-dir-lock.ts b/src/lib/shields/state-dir-lock.ts index 9d64ebffeb6..185e10a60c7 100644 --- a/src/lib/shields/state-dir-lock.ts +++ b/src/lib/shields/state-dir-lock.ts @@ -35,7 +35,14 @@ const PLAN_ARRAY_FIELDS = [ ] as const; const PLAN_FIELDS = new Set(["$comment", "version", ...PLAN_ARRAY_FIELDS]); -type GuardAction = "preflight" | "lock" | "unlock" | "verify-mutable" | "startup"; +type GuardAction = + | "preflight" + | "lock" + | "unlock" + | "verify-lock" + | "verify-unlock" + | "verify-mutable" + | "startup"; type GuardIssue = { type: "issue"; @@ -192,6 +199,8 @@ function parseGuardOutput(action: GuardAction, result: PrivilegedExecResult): st (record.action === "preflight" || record.action === "lock" || record.action === "unlock" || + record.action === "verify-lock" || + record.action === "verify-unlock" || record.action === "verify-mutable" || record.action === "startup") && (record.status === "ok" || record.status === "failed") && @@ -317,6 +326,7 @@ function runHostStateDirGuard( configDir: string, plan: AgentStateLockPlan, mutableTopLevelFiles: string[] = [], + mutableServiceUsers: string[] = [], ): string[] { let input: string; try { @@ -336,6 +346,7 @@ function runHostStateDirGuard( "--plan-json", JSON.stringify(plan), ...mutableTopLevelFiles.flatMap((file) => ["--mutable-top-level-file", file]), + ...mutableServiceUsers.flatMap((user) => ["--mutable-service-user", user]), ]; return parseGuardOutput(action, privileged.run(command, input)); } @@ -352,6 +363,15 @@ export function preflightStateDirLock( return runStateDirGuard(privileged, "preflight", configDir, plan, stateLockPlanInImage); } +/** Read and verify the complete recursive locked posture without changing filesystem state. */ +export function verifyLockedStateDirPosture( + privileged: PrivilegedExec, + configDir: string, + plan: AgentStateLockPlan, +): string[] { + return runHostStateDirGuard(privileged, "verify-lock", configDir, plan); +} + // Existing images predate this read-only action, so inject the current trusted // host helper while verifying the complete recursive mutable posture. export function verifyStateDirMutablePosture( @@ -360,6 +380,7 @@ export function verifyStateDirMutablePosture( plan: AgentStateLockPlan, stateLockPlanInImage: boolean, mutableTopLevelFiles: string[] = [], + mutableServiceUsers: string[] = [], ): string[] { const compatibilityIssues = stateLockPlanCompatibilityIssues( privileged, @@ -373,6 +394,7 @@ export function verifyStateDirMutablePosture( configDir, plan, mutableTopLevelFiles, + mutableServiceUsers, ); } diff --git a/src/lib/state/mcp-lifecycle-lock-acquisition.ts b/src/lib/state/mcp-lifecycle-lock-acquisition.ts index 1e17e70b562..cecbe88def9 100644 --- a/src/lib/state/mcp-lifecycle-lock-acquisition.ts +++ b/src/lib/state/mcp-lifecycle-lock-acquisition.ts @@ -41,8 +41,8 @@ import { safelyReleaseMcpLifecycleLockSync, writeMcpLifecycleLockCandidateAndLink, writeMcpLifecycleLockCandidateAndLinkSync, + resolveNemoclawStateDir, } from "./mcp-lifecycle-lock-storage"; -import { resolveNemoclawStateDir } from "./paths"; const DEFAULT_POLL_INTERVAL_MS = 100; const DEFAULT_TIMEOUT_MS = 30 * 60_000; diff --git a/src/lib/state/mcp-lifecycle-lock-storage.ts b/src/lib/state/mcp-lifecycle-lock-storage.ts index 43c0599e381..61f3fac4975 100644 --- a/src/lib/state/mcp-lifecycle-lock-storage.ts +++ b/src/lib/state/mcp-lifecycle-lock-storage.ts @@ -13,6 +13,8 @@ import { } from "./mcp-lifecycle-lock-identity"; import { resolveNemoclawStateDir } from "./paths"; +export { resolveNemoclawStateDir } from "./paths"; + export const MCP_LIFECYCLE_LOCK_DIRNAME = "mcp-lifecycle-locks"; function lockFileStem(sandboxName: string): string { diff --git a/src/lib/state/portable-uninstall-retirement.test.ts b/src/lib/state/portable-uninstall-retirement.test.ts index d3c5e0323c4..b7c8fd7cc0a 100644 --- a/src/lib/state/portable-uninstall-retirement.test.ts +++ b/src/lib/state/portable-uninstall-retirement.test.ts @@ -354,7 +354,9 @@ describe("portable uninstall retirement state", () => { get(current, property) { const value = Reflect.get(current, property, current) as unknown; return property === "uid" - ? current.uid + 1n + ? typeof current.uid === "bigint" + ? current.uid + 1n + : current.uid + 1 : typeof value === "function" ? value.bind(current) : value; diff --git a/src/lib/state/registry.ts b/src/lib/state/registry.ts index a1350270853..2857d29089b 100644 --- a/src/lib/state/registry.ts +++ b/src/lib/state/registry.ts @@ -105,6 +105,7 @@ export type { SandboxWorkloadReceipt, } from "./registry/types"; export type { McpBridgeEntry, SandboxMcpState } from "./registry-mcp"; +export { normalizeSandboxMcpState }; export { getConfiguredMessagingChannelsFromEntry, getDisabledMessagingChannelsFromEntry, diff --git a/src/lib/state/registry/lifecycle-generation-cas.ts b/src/lib/state/registry/lifecycle-generation-cas.ts new file mode 100644 index 00000000000..d8068eb9983 --- /dev/null +++ b/src/lib/state/registry/lifecycle-generation-cas.ts @@ -0,0 +1,30 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { isDeepStrictEqual } from "node:util"; +import { withLock } from "./lock"; +import { load, save } from "./persistence"; +import type { SandboxEntry } from "./types"; + +/** Claim a lifecycle generation after the caller establishes provider authority. */ +export function compareAndSetSandboxLifecycleGeneration( + expected: SandboxEntry, + lifecycleGeneration: string, +): boolean { + if ( + expected.lifecycleGeneration !== undefined || + lifecycleGeneration.length === 0 || + lifecycleGeneration.length > 256 || + /[\u0000-\u001f\u007f-\u009f]/u.test(lifecycleGeneration) + ) { + return false; + } + return withLock(() => { + const data = load(); + const current = data.sandboxes[expected.name]; + if (!current || !isDeepStrictEqual(current, expected)) return false; + current.lifecycleGeneration = lifecycleGeneration; + save(data); + return true; + }); +} diff --git a/src/lib/state/registry/lifecycle-generation.ts b/src/lib/state/registry/lifecycle-generation.ts index f975afbefb1..9753f0940d3 100644 --- a/src/lib/state/registry/lifecycle-generation.ts +++ b/src/lib/state/registry/lifecycle-generation.ts @@ -1,31 +1,39 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { isDeepStrictEqual } from "node:util"; -import { withLock } from "./lock"; -import { load, save } from "./persistence"; +import { resolveRegisteredRuntimeProvider } from "../../onboard/runtime-provider/selection"; +import { compareAndSetSandboxLifecycleGeneration } from "./lifecycle-generation-cas"; import type { SandboxEntry } from "./types"; +export function usesLegacyRuntimeLifecycleCompatibility(entry: SandboxEntry): boolean { + const driverName = entry.openshellDriver?.trim().toLowerCase(); + if (!driverName) return false; + const provider = resolveRegisteredRuntimeProvider(driverName); + if (!provider || provider.identity.id !== driverName || provider.lifecycle.supported !== true) { + return false; + } + try { + return ( + provider.gateway.prepareHostRuntime({ + environment: process.env, + platform: process.platform, + }).socketPath === null + ); + } catch { + return false; + } +} + /** Claim a lifecycle generation for one unchanged legacy Docker registry row. */ export function compareAndSetLegacySandboxLifecycleGeneration( expected: SandboxEntry, lifecycleGeneration: string, ): boolean { if ( - expected.openshellDriver !== "docker" || - expected.lifecycleGeneration !== undefined || - lifecycleGeneration.length === 0 || - lifecycleGeneration.length > 256 || - /[\u0000-\u001f\u007f-\u009f]/u.test(lifecycleGeneration) + !usesLegacyRuntimeLifecycleCompatibility(expected) || + expected.lifecycleGeneration !== undefined ) { return false; } - return withLock(() => { - const data = load(); - const current = data.sandboxes[expected.name]; - if (!current || !isDeepStrictEqual(current, expected)) return false; - current.lifecycleGeneration = lifecycleGeneration; - save(data); - return true; - }); + return compareAndSetSandboxLifecycleGeneration(expected, lifecycleGeneration); } diff --git a/src/lib/tunnel/agent-forward-stop.ts b/src/lib/tunnel/agent-forward-stop.ts index 0b7f063ea6d..8f3527d38cd 100644 --- a/src/lib/tunnel/agent-forward-stop.ts +++ b/src/lib/tunnel/agent-forward-stop.ts @@ -107,16 +107,16 @@ function makeRunCaptureOpenshell(openshell: string): ForwardListRunner { }; } -export function stopAgentForwardPortsForStop( +function stopAndConfirmAgentForwardPorts( sandboxName: string | undefined, deps: StopAgentForwardPortsDeps = {}, -): void { - if (!sandboxName) return; +): boolean { + if (!sandboxName) return false; const warn = deps.warn ?? (() => {}); if (!SAFE_SANDBOX_NAME_RE.test(sandboxName) || sandboxName.includes("..")) { warn(`Invalid sandbox name: ${JSON.stringify(sandboxName)} - skipping host forward cleanup.`); - return; + return false; } const info = deps.info ?? (() => {}); @@ -130,13 +130,13 @@ export function stopAgentForwardPortsForStop( `${error instanceof Error ? error.message : String(error)}. ` + "Skipping agent host port forward cleanup.", ); - return; + return false; } if (!sandbox) { warn( `Could not resolve sandbox '${sandboxName}' - cannot safely stop agent host port forwards.`, ); - return; + return false; } const getRegisteredAgent = deps.getRegisteredAgent ?? agentRuntime.getRegisteredAgent; @@ -146,9 +146,9 @@ export function stopAgentForwardPortsForStop( `Could not resolve registered agent '${sandbox.agent}' for sandbox '${sandboxName}'; ` + "skipping agent host port forward cleanup.", ); - return; + return false; } - if (!agent) return; + if (!agent) return true; const displayName = deps.getAgentDisplayName ? deps.getAgentDisplayName(agent) @@ -164,16 +164,16 @@ export function stopAgentForwardPortsForStop( `${(error as Error).message ?? String(error)}. ` + `Skipping ${displayName} host port forward cleanup.`, ); - return; + return false; } const ports = getAgentForwardPorts(agent, sandbox.dashboardPort); - if (ports.length === 0) return; + if (ports.length === 0) return true; const openshell = (deps.resolveOpenshell ?? resolveOpenshell)(); if (!openshell) { warn(`openshell not found - cannot stop ${displayName} host port forwards.`); - return; + return false; } const runOpenshell = deps.runOpenshell ?? makeRunOpenshell(openshell); @@ -184,6 +184,7 @@ export function stopAgentForwardPortsForStop( runCaptureOpenshell([...args, "--gateway", gatewayName], opts); const confirmPortReleased = deps.confirmPortReleased ?? confirmForwardPortReleased; + let released = true; for (const port of ports) { const result = bestEffortForwardStopForSandbox( scopedRunOpenshell, @@ -195,6 +196,7 @@ export function stopAgentForwardPortsForStop( warn( `Keeping ${displayName} host port forward ${String(port)}; it belongs to another sandbox.`, ); + released = false; continue; } if (result === "list-failed") { @@ -203,6 +205,7 @@ export function stopAgentForwardPortsForStop( port, )} cleanup.`, ); + released = false; continue; } @@ -212,10 +215,26 @@ export function stopAgentForwardPortsForStop( `within ${String(FORWARD_RELEASE_TIMEOUT_MS / 1000)} seconds; ` + "the listener may still be running.", ); + released = false; } else if (result === "stopped") { info( `Stopped ${displayName} host port forward ${String(port)} for sandbox '${sandboxName}'.`, ); } } + return released; +} + +export function stopAgentForwardPortsForStop( + sandboxName: string | undefined, + deps: StopAgentForwardPortsDeps = {}, +): void { + stopAndConfirmAgentForwardPorts(sandboxName, deps); +} + +export function settleAgentForwardPortsForRebuild( + sandboxName: string | undefined, + deps: StopAgentForwardPortsDeps = {}, +): boolean { + return stopAndConfirmAgentForwardPorts(sandboxName, deps); } diff --git a/test/agents/hermes/hermes-config-transaction-wiring.test.ts b/test/agents/hermes/hermes-config-transaction-wiring.test.ts index c43b37f9e36..bf527291ac5 100644 --- a/test/agents/hermes/hermes-config-transaction-wiring.test.ts +++ b/test/agents/hermes/hermes-config-transaction-wiring.test.ts @@ -48,18 +48,13 @@ installMock(source("adapters", "openshell", "client.js"), { runOpenshellCommand: () => ({ status: 0 }), }); installMock(source("sandbox", "privileged-exec.js"), { - privilegedSandboxExecArgv: (sandboxName, command, stdin, sanitizeEnvironment) => { - capturedPrivilegedExec = { command, sanitizeEnvironment }; - return ["docker", "exec", ...(stdin ? ["-i"] : []), sandboxName, ...command]; + capturePrivilegedSandboxCommand: (sandboxName, command, options) => { + capturedPrivilegedExec = { command, sanitizeEnvironment: options.sanitizeEnvironment }; + captured = { argv: command, options }; + return Buffer.from("updated=1\n"); }, withPrivilegedSandboxExecutionLease: (_sandboxName, _operation, callback) => callback(), }); -installMock(source("adapters", "docker", "exec.js"), { - dockerExecFileSync: (argv, options) => { - captured = { argv, options }; - return "updated=1\n"; - }, -}); const config = require(source("sandbox", "config.js")); const target = config.resolveAgentConfig("alpha"); @@ -79,13 +74,12 @@ process.stdout.write(JSON.stringify({ ...captured, privilegedExec: capturedPrivi expect(result.status, result.stderr).toBe(0); const captured = JSON.parse(result.stdout) as { argv: string[]; - options: { input: string; timeout: number; stdio: string[] }; + options: { input: string; timeout: number; sanitizeEnvironment: boolean }; privilegedExec: { command: string[]; sanitizeEnvironment: boolean }; }; const digestFlag = captured.argv.indexOf("--expected-config-sha256"); expect(captured.argv).toEqual( expect.arrayContaining([ - "-i", "timeout", "--signal=TERM", "--kill-after=5s", @@ -106,7 +100,7 @@ process.stdout.write(JSON.stringify({ ...captured, privilegedExec: capturedPrivi expect(captured.argv[digestFlag + 1]).toBe(expectedDigest); expect(captured.options.input).toContain("default: trusted-model-v2"); expect(captured.options.timeout).toBe(150000); - expect(captured.options.stdio).toEqual(["pipe", "pipe", "pipe"]); + expect(captured.options.sanitizeEnvironment).toBe(true); expect(captured.privilegedExec.sanitizeEnvironment).toBe(true); expect(captured.privilegedExec.command).toEqual( expect.arrayContaining([ diff --git a/test/agents/hermes/hermes-gateway-auxiliary-retry.test.ts b/test/agents/hermes/hermes-gateway-auxiliary-retry.test.ts index d2d053750fd..bccdf1be59f 100644 --- a/test/agents/hermes/hermes-gateway-auxiliary-retry.test.ts +++ b/test/agents/hermes/hermes-gateway-auxiliary-retry.test.ts @@ -19,6 +19,38 @@ function writeFakeProcCmdline(procRoot: string, pid: number, args: string[]): vo } describe("Hermes gateway auxiliary retry", () => { + it("holds the exact failed supervisor for an authenticated state-mutation retry", () => { + const source = fs.readFileSync(START_SCRIPT, "utf-8"); + const result = runBashHarness([ + 'trace() { printf "%s\\n" "$*"; }', + 'nemoclaw_runtime_state_mutation_gate() { trace "gate:$1"; return 75; }', + 'kill() { trace "signal:$1:$2"; exit 0; }', + extractShellFunction(source, "nemoclaw_runtime_state_mutation_hold_supervisor_failure"), + "nemoclaw_runtime_state_mutation_hold_supervisor_failure", + ]); + + expect(result.status, result.stderr).toBe(0); + expect(result.stdout.trim().split("\n")).toEqual([ + "gate:admit", + expect.stringMatching(/^signal:-STOP:[1-9][0-9]*$/u), + ]); + expect(result.stderr).toContain("holding for authenticated retry"); + }); + + it("preserves ordinary supervisor failure without an active state mutation", () => { + const source = fs.readFileSync(START_SCRIPT, "utf-8"); + const result = runBashHarness([ + 'trace() { printf "%s\\n" "$*"; }', + 'nemoclaw_runtime_state_mutation_gate() { trace "gate:$1"; return 0; }', + 'kill() { trace "unexpected-signal:$*"; }', + extractShellFunction(source, "nemoclaw_runtime_state_mutation_hold_supervisor_failure"), + 'if nemoclaw_runtime_state_mutation_hold_supervisor_failure; then trace unsafe-success; else trace "failure:$?"; fi', + ]); + + expect(result.status, result.stderr).toBe(0); + expect(result.stdout.trim().split("\n")).toEqual(["gate:admit", "failure:1"]); + }); + it("retries transient auxiliary failures without churning the healthy gateway", () => { const source = fs.readFileSync(START_SCRIPT, "utf-8"); const result = runBashHarness([ diff --git a/test/agents/hermes/hermes-mcp-apply-race.test.ts b/test/agents/hermes/hermes-mcp-apply-race.test.ts index b92a8eabf76..c9e300ba4da 100644 --- a/test/agents/hermes/hermes-mcp-apply-race.test.ts +++ b/test/agents/hermes/hermes-mcp-apply-race.test.ts @@ -32,6 +32,7 @@ def load(name, path): transaction = load("apply_race_transaction", sys.argv[1]) guard = load("apply_race_guard", sys.argv[2]) +transaction.os.environ["FAKE_TOKEN"] = "openshell:resolve:env:FAKE_TOKEN" with tempfile.TemporaryDirectory(prefix="hermes-mcp-apply-race-") as root: hermes = os.path.join(root, ".hermes") @@ -190,6 +191,7 @@ def load(name, path): transaction = load("partial_apply_race_transaction", sys.argv[1]) guard = load("partial_apply_race_guard", sys.argv[2]) +transaction.os.environ["FAKE_TOKEN"] = "openshell:resolve:env:FAKE_TOKEN" with tempfile.TemporaryDirectory(prefix="hermes-mcp-partial-apply-race-") as root: hermes = os.path.join(root, ".hermes") @@ -390,6 +392,7 @@ def load(name, path): transaction = load("failed_reload_race_transaction", sys.argv[1]) guard = load("failed_reload_race_guard", sys.argv[2]) +transaction.os.environ["FAKE_TOKEN"] = "openshell:resolve:env:FAKE_TOKEN" with tempfile.TemporaryDirectory(prefix="hermes-mcp-failed-reload-race-") as root: hermes = os.path.join(root, ".hermes") diff --git a/test/agents/hermes/hermes-mcp-config-transaction.test.ts b/test/agents/hermes/hermes-mcp-config-transaction.test.ts index d467be55932..fbd8544ee51 100644 --- a/test/agents/hermes/hermes-mcp-config-transaction.test.ts +++ b/test/agents/hermes/hermes-mcp-config-transaction.test.ts @@ -22,8 +22,15 @@ const TRANSACTION = path.resolve( const GUARD = path.resolve(import.meta.dirname, "../../..", "agents/hermes/runtime-config-guard.py"); function runPython(source: string, args: string[] = []) { + const canonicalEnvironment = Object.fromEntries( + [...source.matchAll(/openshell:resolve:env:([A-Za-z_][A-Za-z0-9_]*)/gu)].map(([, name]) => [ + name!, + `openshell:resolve:env:${name!}`, + ]), + ); return spawnSync("python3", ["-c", source, TRANSACTION, GUARD, ...args], { encoding: "utf8", + env: { ...process.env, ...canonicalEnvironment }, }); } diff --git a/test/agents/hermes/hermes-mcp-integrity-state.test.ts b/test/agents/hermes/hermes-mcp-integrity-state.test.ts index fd1feed874d..423e944cb4d 100644 --- a/test/agents/hermes/hermes-mcp-integrity-state.test.ts +++ b/test/agents/hermes/hermes-mcp-integrity-state.test.ts @@ -611,10 +611,15 @@ spec.loader.exec_module(module) module.HERMES_DIR = "/tmp/.hermes" module.CONFIG_PATH = "/tmp/.hermes/config.yaml" module.os.geteuid = lambda: 1000 +module.os.environ["SAFE_MCP_TOKEN"] = "openshell:resolve:env:v12_SAFE_MCP_TOKEN" candidate = module._managed_candidate({ "url": "https://mcp.example.test/mcp", "headers": {"Authorization": "Bearer openshell:resolve:env:SAFE_MCP_TOKEN"}, }) +runtime_candidate = { + **candidate, + "headers": {"Authorization": "Bearer openshell:resolve:env:v12_SAFE_MCP_TOKEN"}, +} payload = {"present": {"safe": candidate}, "absent": []} outcomes = {} for integrity_state in ("current", "pending"): @@ -622,7 +627,7 @@ for integrity_state in ("current", "pending"): inspect_mcp_integrity_snapshot=lambda *_args: types.SimpleNamespace( state=state, config_text=yaml.safe_dump( - {"mcp_servers": {"safe": candidate}}, sort_keys=False + {"mcp_servers": {"safe": runtime_candidate}}, sort_keys=False ), ), assert_mcp_integrity_snapshot_current=lambda *_args: None, @@ -673,8 +678,12 @@ candidate = transaction._managed_candidate({ "url": "https://mcp.example.test/mcp", "headers": {"Authorization": "Bearer openshell:resolve:env:SAFE_MCP_TOKEN"}, }) +runtime_candidate = { + **candidate, + "headers": {"Authorization": "Bearer openshell:resolve:env:v12_SAFE_MCP_TOKEN"}, +} with open(config, "w", encoding="utf-8") as handle: - handle.write(yaml.safe_dump({"mcp_servers": {"safe": candidate}}, sort_keys=False)) + handle.write(yaml.safe_dump({"mcp_servers": {"safe": runtime_candidate}}, sort_keys=False)) with open(env, "w", encoding="utf-8") as handle: handle.write("SAFE=1\n") hash_text, _config_snapshot, _env_snapshot = guard._hash_text(config, env) @@ -685,6 +694,7 @@ transaction.HERMES_DIR = hermes transaction.CONFIG_PATH = config transaction.STRICT_HASH_PATH = strict transaction.os.geteuid = lambda: 0 +transaction.os.environ["SAFE_MCP_TOKEN"] = "openshell:resolve:env:v12_SAFE_MCP_TOKEN" transaction._load_guard = lambda: guard try: transaction.inspect_managed_config({"present": {"safe": candidate}, "absent": []}) @@ -694,7 +704,7 @@ guard._write_hash(compat, hash_text) original_inspect = guard.inspect_mcp_integrity_snapshot def race_after_authentication(*args): inspection = original_inspect(*args) - changed = {**candidate, "url": "https://attacker.example.test/mcp"} + changed = {**runtime_candidate, "url": "https://attacker.example.test/mcp"} with open(config, "w", encoding="utf-8") as handle: handle.write(yaml.safe_dump({"mcp_servers": {"safe": changed}}, sort_keys=False)) return inspection diff --git a/test/agents/hermes/hermes-mcp-rollback-pending.test.ts b/test/agents/hermes/hermes-mcp-rollback-pending.test.ts index b2c510f06f8..10242a0928d 100644 --- a/test/agents/hermes/hermes-mcp-rollback-pending.test.ts +++ b/test/agents/hermes/hermes-mcp-rollback-pending.test.ts @@ -32,6 +32,7 @@ def load(name, path): transaction = load("rollback_pending_transaction", sys.argv[1]) guard = load("rollback_pending_guard", sys.argv[2]) +transaction.os.environ["FAKE_TOKEN"] = "openshell:resolve:env:FAKE_TOKEN" with tempfile.TemporaryDirectory(prefix="hermes-mcp-rollback-pending-") as root: hermes = os.path.join(root, ".hermes") os.mkdir(hermes) diff --git a/test/agents/hermes/hermes-runtime-api-key.test.ts b/test/agents/hermes/hermes-runtime-api-key.test.ts index 7ad7b01ded9..9b2c8b50f9b 100644 --- a/test/agents/hermes/hermes-runtime-api-key.test.ts +++ b/test/agents/hermes/hermes-runtime-api-key.test.ts @@ -865,7 +865,7 @@ describe("agents/hermes/start.sh runtime API server key", () => { expect(run.result.status, run.result.stderr).toBe(0); expect(run.envFileContent).toContain( - "SLACK_BOT_TOKEN=xoxb-OPENSHELL-RESOLVE-ENV-SLACK_BOT_TOKEN\n", + "SLACK_BOT_TOKEN=xoxb-OPENSHELL-RESOLVE-ENV-v222_SLACK_BOT_TOKEN\n", ); expect(run.envFileContent).toContain( "SLACK_APP_TOKEN=xapp-OPENSHELL-RESOLVE-ENV-SLACK_APP_TOKEN\n", diff --git a/test/agents/hermes/hermes-start.test.ts b/test/agents/hermes/hermes-start.test.ts index 215cad79558..6aba3ffec4d 100644 --- a/test/agents/hermes/hermes-start.test.ts +++ b/test/agents/hermes/hermes-start.test.ts @@ -982,6 +982,7 @@ describe("agents/hermes/start.sh env secret boundary", () => { "TELEGRAM_BOT_TOKEN=openshell:resolve:env:TELEGRAM_BOT_TOKEN", "DISCORD_BOT_TOKEN='openshell:resolve:env:DISCORD_BOT_TOKEN'", "SLACK_BOT_TOKEN=xoxb-OPENSHELL-RESOLVE-ENV-SLACK_BOT_TOKEN", + "SLACK_BOT_TOKEN_ROTATED=xoxb-OPENSHELL-RESOLVE-ENV-v42_SLACK_BOT_TOKEN_ROTATED", 'SLACK_APP_TOKEN="xapp-OPENSHELL-RESOLVE-ENV-SLACK_APP_TOKEN"', "API_SERVER_PORT=18642", "API_SERVER_HOST=127.0.0.1", diff --git a/test/agents/openclaw/runtime/nemoclaw-start-runtime-env-alias.test.ts b/test/agents/openclaw/runtime/nemoclaw-start-runtime-env-alias.test.ts index 423f85f0e69..0a24d8e86db 100644 --- a/test/agents/openclaw/runtime/nemoclaw-start-runtime-env-alias.test.ts +++ b/test/agents/openclaw/runtime/nemoclaw-start-runtime-env-alias.test.ts @@ -110,7 +110,9 @@ describe("messaging runtime env aliases", () => { timeout: 5000, }); expect(result.status, result.stderr).toBe(0); - expect(result.stdout).toContain("SLACK_BOT_TOKEN=xoxb-OPENSHELL-RESOLVE-ENV-SLACK_BOT_TOKEN"); + expect(result.stdout).toContain( + "SLACK_BOT_TOKEN=xoxb-OPENSHELL-RESOLVE-ENV-v42_SLACK_BOT_TOKEN", + ); expect(result.stderr).toContain("[channels] normalized Slack alias"); } finally { fs.rmSync(tmpDir, { recursive: true, force: true }); diff --git a/test/agents/openclaw/runtime/nemoclaw-start-slack-runtime.test.ts b/test/agents/openclaw/runtime/nemoclaw-start-slack-runtime.test.ts index 2fd171b8d27..b28c7a00f86 100644 --- a/test/agents/openclaw/runtime/nemoclaw-start-slack-runtime.test.ts +++ b/test/agents/openclaw/runtime/nemoclaw-start-slack-runtime.test.ts @@ -8,7 +8,13 @@ import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; -const START_SCRIPT = path.join(import.meta.dirname, "..", "../../..", "scripts", "nemoclaw-start.sh"); +const START_SCRIPT = path.join( + import.meta.dirname, + "..", + "../../..", + "scripts", + "nemoclaw-start.sh", +); function messagingRuntimeSetupSection(src: string, planPath: string): string { const start = src.indexOf("# ── Messaging runtime setup from manifest metadata"); @@ -131,19 +137,19 @@ describe("Slack runtime env normalization (#4274)", () => { }); expect(run.result.status, run.result.stderr).toBe(0); - expect(run.bot).toBe("xoxb-OPENSHELL-RESOLVE-ENV-SLACK_BOT_TOKEN"); - expect(run.app).toBe("xapp-OPENSHELL-RESOLVE-ENV-SLACK_APP_TOKEN"); + expect(run.bot).toBe("xoxb-OPENSHELL-RESOLVE-ENV-v51_SLACK_BOT_TOKEN"); + expect(run.app).toBe("xapp-OPENSHELL-RESOLVE-ENV-v51_SLACK_APP_TOKEN"); }); - it("does not leak the revision suffix into the normalized env or logs", () => { + it("preserves the revision suffix in aliases without leaking it to logs", () => { const run = runNormalize({ SLACK_BOT_TOKEN: "openshell:resolve:env:v51_SLACK_BOT_TOKEN", SLACK_APP_TOKEN: "openshell:resolve:env:v51_SLACK_APP_TOKEN", }); expect(run.result.status, run.result.stderr).toBe(0); - expect(run.bot).not.toContain("v51_"); - expect(run.app).not.toContain("v51_"); + expect(run.bot).toContain("v51_"); + expect(run.app).toContain("v51_"); expect(run.result.stderr).not.toContain("v51_"); expect(run.bot).not.toContain("openshell:resolve:env:"); expect(run.app).not.toContain("openshell:resolve:env:"); diff --git a/test/agents/openclaw/runtime/nemoclaw-start-wechat-placeholder.test.ts b/test/agents/openclaw/runtime/nemoclaw-start-wechat-placeholder.test.ts index d22c3f9ec75..7b9fe6c83d8 100644 --- a/test/agents/openclaw/runtime/nemoclaw-start-wechat-placeholder.test.ts +++ b/test/agents/openclaw/runtime/nemoclaw-start-wechat-placeholder.test.ts @@ -448,6 +448,30 @@ describe("OpenClaw WeChat provider placeholder refresh (#10079)", () => { expect(run.result.stderr).not.toContain("Refusing WeChat provider placeholder refresh"); }); + it("leaves an intentionally removed managed WeChat tree absent", () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-wechat-placeholder-")); + const openclawDir = path.join(tmpDir, ".openclaw"); + const configPath = path.join(openclawDir, "openclaw.json"); + fs.mkdirSync(openclawDir); + fs.writeFileSync(configPath, `${JSON.stringify(wechatConfig(true), null, 2)}\n`); + + try { + const result = spawnSync("python3", ["-I", REFRESH_HELPER, configPath], { + encoding: "utf-8", + env: { + PATH: process.env.PATH || "", + WECHAT_BOT_TOKEN: "openshell:resolve:env:v42_WECHAT_BOT_TOKEN", + }, + timeout: 5000, + }); + + expect(result.status, result.stderr).toBe(0); + expect(fs.existsSync(path.join(openclawDir, "openclaw-weixin"))).toBe(false); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); + it.each([ [ "symlinked", diff --git a/test/agents/openclaw/runtime/nemoclaw-start.test.ts b/test/agents/openclaw/runtime/nemoclaw-start.test.ts index f0d9a8022b7..9204193499c 100644 --- a/test/agents/openclaw/runtime/nemoclaw-start.test.ts +++ b/test/agents/openclaw/runtime/nemoclaw-start.test.ts @@ -2814,13 +2814,12 @@ describe("provider placeholder refresh (#4251)", () => { expect(run.result.status, run.result.stderr).toBe(0); expect(run.result.stderr).not.toContain("slack.default"); - // The Bolt-compatible alias is never rewritten on disk; it does not match - // the canonical "openshell:resolve:env:SLACK_BOT_TOKEN" placeholder key. + // The Bolt-compatible alias follows the revision-scoped runtime placeholder. expect(run.config.channels.slack.accounts.default.botToken).toBe( - "xoxb-OPENSHELL-RESOLVE-ENV-SLACK_BOT_TOKEN", + "xoxb-OPENSHELL-RESOLVE-ENV-v42_SLACK_BOT_TOKEN", ); expect(run.config.channels.slack.accounts.default.appToken).toBe( - "xapp-OPENSHELL-RESOLVE-ENV-SLACK_APP_TOKEN", + "xapp-OPENSHELL-RESOLVE-ENV-v42_SLACK_APP_TOKEN", ); }); diff --git a/test/automation/e2e/e2e-recommendations.test.ts b/test/automation/e2e/e2e-recommendations.test.ts index 64bd0122cf9..79d1574a489 100644 --- a/test/automation/e2e/e2e-recommendations.test.ts +++ b/test/automation/e2e/e2e-recommendations.test.ts @@ -125,6 +125,7 @@ describe("E2E recommendation normalizer", () => { "tools/advisors/risk-plan.mts", "tools/e2e/credential-free-tests.mts", "tools/e2e/execution-coverage.mts", + "tools/e2e/gateway-runtime.mts", "tools/e2e/onboard-timeout-contract.mts", "tools/e2e/selector-aliases.mts", "tools/e2e/target-catalogue.mts", diff --git a/test/automation/pull-requests/pr-review-advisor-security-boundaries.test.ts b/test/automation/pull-requests/pr-review-advisor-security-boundaries.test.ts index fd59c59245d..0e403e0432c 100644 --- a/test/automation/pull-requests/pr-review-advisor-security-boundaries.test.ts +++ b/test/automation/pull-requests/pr-review-advisor-security-boundaries.test.ts @@ -18,7 +18,7 @@ describe("PR review advisor security boundaries", () => { vi.restoreAllMocks(); }); - it("removes the model credential from the tool environment after in-memory setup", async () => { + it("removes the model credential after registering the selected model in memory", async () => { const credentialEnv = "PR_REVIEW_ADVISOR_TEST_API_KEY"; vi.stubEnv(credentialEnv, "test-secret"); const configDir = fs.mkdtempSync(path.join(ROOT, ".tmp-pr-advisor-config-")); diff --git a/test/channels/channels-add-bridge-lifecycle.test.ts b/test/channels/channels-add-bridge-lifecycle.test.ts index afc62686aaa..73f573a2ab3 100644 --- a/test/channels/channels-add-bridge-lifecycle.test.ts +++ b/test/channels/channels-add-bridge-lifecycle.test.ts @@ -286,7 +286,8 @@ describe("channels add owns the bridge-provider lifecycle (#6120)", () => { expect(exitSpy).toHaveBeenCalledWith(1); expect(providerSpy).not.toHaveBeenCalled(); - expect(printedText()).toContain("GOOGLECHAT_SERVICE_ACCOUNT"); + expect(printedText()).toContain("Missing required inputs for this channel."); + expect(printedText()).not.toContain("GOOGLECHAT_SERVICE_ACCOUNT"); }); it("tears the just-created bridge provider back down when gateway registration fails", async () => { @@ -298,7 +299,7 @@ describe("channels add owns the bridge-provider lifecycle (#6120)", () => { code: 1, }); - expect(printedText()).toContain("Failed to register 'googlechat' providers"); + expect(printedText()).toContain("Failed to register channel providers with the gateway."); expect(openshellCalls()).toEqual( expect.arrayContaining([ ["sandbox", "provider", "detach", "test-sb", "test-sb-googlechat-bridge"], diff --git a/test/channels/channels-add-deepagents-rejection.test.ts b/test/channels/channels-add-deepagents-rejection.test.ts index 925ad1f503e..95d987e7cb7 100644 --- a/test/channels/channels-add-deepagents-rejection.test.ts +++ b/test/channels/channels-add-deepagents-rejection.test.ts @@ -178,20 +178,12 @@ const ctx = module.exports; ); assert.equal(payload.exitCode, 1, "expected addSandboxChannel to exit with code 1"); assert.ok( - payload.errors.some((msg) => - /Channel 'discord' does not support agent 'langchain-deepagents-code'/.test(msg), - ), - `missing unsupported channel-agent error in stderr: ${JSON.stringify(payload.errors)}`, + payload.errors.includes(" This channel does not support the configured agent."), + `missing redacted unsupported channel-agent error in stderr: ${JSON.stringify(payload.errors)}`, ); assert.ok( - payload.errors.some((msg) => /Channel-supported agents: openclaw, hermes/.test(msg)), - `missing channel-supported agents hint in stderr: ${JSON.stringify(payload.errors)}`, - ); - assert.ok( - payload.errors.some((msg) => - /Channels supported by agent 'langchain-deepagents-code': \(none\)/.test(msg), - ), - `missing agent-supported channels hint in stderr: ${JSON.stringify(payload.errors)}`, + payload.errors.every((msg) => !msg.includes("langchain-deepagents-code")), + `agent identity leaked in stderr: ${JSON.stringify(payload.errors)}`, ); assert.deepEqual(payload.policyCalls.loadPreset, [], "loadPreset must not run before the gate"); diff --git a/test/channels/channels-remove-full-teardown.test.ts b/test/channels/channels-remove-full-teardown.test.ts index 22bbc0ae3c6..1d011c85efb 100644 --- a/test/channels/channels-remove-full-teardown.test.ts +++ b/test/channels/channels-remove-full-teardown.test.ts @@ -73,7 +73,7 @@ function buildPreamble({ sshFallbackResult = null as { status: number; stdout: string; stderr: string } | null, stoppedDockerCleanupResult = { cleared: false, - failure: "sandbox-volume-unavailable", + failure: "state-resource-unavailable", } as { cleared: true } | { cleared: false; failure: string; cleanupHelperName?: string }, }: { presetNamesApplied?: string[]; @@ -195,7 +195,7 @@ policies.removePreset = (sandboxName, presetName) => { const callOrder = []; const stoppedDockerCleanupCalls = []; const policyChannelDeps = require(${j("actions/sandbox/policy-channel-dependencies.js")}); -policyChannelDeps.policyChannelDependencies.clearStoppedDockerSandboxChannelState = (sandboxName, paths) => { +policyChannelDeps.policyChannelDependencies.clearStoppedSandboxStateRoots = (sandboxName, paths) => { stoppedDockerCleanupCalls.push({ sandboxName, paths }); return ${JSON.stringify(stoppedDockerCleanupResult)}; }; @@ -364,7 +364,7 @@ const ctx = module.exports; { failure: "cleanup-helper-failed", cleanup: { cleared: false, failure: "cleanup-helper-failed" }, - guidance: "Inspect the stopped sandbox and Docker daemon.", + guidance: "Inspect the stopped sandbox and selected runtime provider.", }, { failure: "cleanup-helper-ownership-invalid", @@ -427,7 +427,7 @@ const ctx = module.exports; assert.deepEqual(payload.removedPresets, []); assert.deepEqual(payload.registryUpdates, []); assert.ok(!payload.callOrder.includes("promptAndRebuild")); - assert.ok(result.stderr.includes(`Stopped-Docker cleanup failed (${failure}).`)); + assert.ok(result.stderr.includes(`Stopped-runtime cleanup failed (${failure}).`)); assert.ok(result.stderr.includes(guidance)); }, ); diff --git a/test/e2e-runtime/entrypoint-env-wrapper.test.ts b/test/e2e-runtime/entrypoint-env-wrapper.test.ts index 21911bcaf9d..a631486fa69 100644 --- a/test/e2e-runtime/entrypoint-env-wrapper.test.ts +++ b/test/e2e-runtime/entrypoint-env-wrapper.test.ts @@ -31,6 +31,7 @@ function runNormalizer(argv: readonly string[]) { "printf 'NO_PROXY=%s\\n' \"${NO_PROXY-__UNSET__}\"", "printf 'FAST_REENTRY_INTERVAL=%s\\n' \"${NEMOCLAW_AUTO_PAIR_FAST_REENTRY_INTERVAL_SECS-__UNSET__}\"", "printf 'FAST_REENTRY_POLLS=%s\\n' \"${NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS-__UNSET__}\"", + "printf 'HERMES_API_PORT=%s\\n' \"${NEMOCLAW_HERMES_API_PORT-__UNSET__}\"", `printf 'ARG=%s\\n' "$@"`, ].join("\n"); return spawnSync("/bin/bash", ["-c", harness, "entrypoint-env-wrapper-test", HELPER, ...argv], { @@ -52,6 +53,7 @@ describe("OCI entrypoint env-wrapper normalization", () => { "NO_PROXY=localhost,127.0.0.1", "NEMOCLAW_AUTO_PAIR_FAST_REENTRY_INTERVAL_SECS=0.25", "NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS=3", + "NEMOCLAW_HERMES_API_PORT=8645", "nemoclaw-start", "/bin/sh", "-c", @@ -66,6 +68,7 @@ describe("OCI entrypoint env-wrapper normalization", () => { expect(result.stdout).toContain("NO_PROXY=localhost,127.0.0.1"); expect(result.stdout).toContain("FAST_REENTRY_INTERVAL=0.25"); expect(result.stdout).toContain("FAST_REENTRY_POLLS=3"); + expect(result.stdout).toContain("HERMES_API_PORT=8645"); expect(result.stdout).toContain("ARG=/bin/sh\nARG=-c\nARG=printf managed command\n"); }); @@ -192,7 +195,11 @@ describe("OCI entrypoint env-wrapper normalization", () => { return spawnSync("bash", [scriptPath], { encoding: "utf-8", timeout: 5000, - env: { ...baseEnv, PATH: `${fakeBin}:${process.env.PATH || ""}`, ...extraEnv }, + env: { + ...baseEnv, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + ...extraEnv, + }, }); } diff --git a/test/e2e/fixtures/availability-env.ts b/test/e2e/fixtures/availability-env.ts index 2e6f0724f6b..eddbb3cce92 100644 --- a/test/e2e/fixtures/availability-env.ts +++ b/test/e2e/fixtures/availability-env.ts @@ -4,6 +4,9 @@ import { buildChildEnv } from "./redaction.ts"; const AVAILABILITY_PROBE_EXTRA_ENV_KEYS = [ + "CONTAINERS_CONF", + "CONTAINERS_STORAGE_CONF", + "DBUS_SESSION_BUS_ADDRESS", "DOCKER_CONFIG", "DOCKER_CONTEXT", "DOCKER_HOST", @@ -18,18 +21,21 @@ const AVAILABILITY_PROBE_EXTRA_ENV_KEYS = [ "NEMOCLAW_E2E_EXPECTED_SHA", "NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG", "NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG_JSON", + "NEMOCLAW_E2E_MANAGED_IMAGE_REVISION", "NEMOCLAW_OLLAMA_PULL_TIMEOUT", "NEMOCLAW_EXPERIMENTAL_PROFILE", + "NEMOCLAW_GATEWAY_RUNTIME", "NEMOCLAW_RUN_LIVE_E2E", "NEMOCLAW_TRACE_DIR", + "OPENSHELL_PODMAN_SOCKET", ]; export function buildAvailabilityProbeEnv( base: NodeJS.ProcessEnv = process.env, ): NodeJS.ProcessEnv { - // Availability probes run outside live target phases but need the shared - // child environment and PATH policy. Add Docker discovery settings, the - // workflow-owned local-model pull budget, and the selected managed-image + // Availability probes run outside live target phases, but they need + // the same child-env and PATH policy. Add container-runtime discovery, + // the workflow-owned local-model pull budget, and the selected managed-image // cohort revision and receipt to that boundary. return buildChildEnv(base, { additionalAllowedEnv: AVAILABILITY_PROBE_EXTRA_ENV_KEYS, diff --git a/test/e2e/fixtures/clients/gateway.ts b/test/e2e/fixtures/clients/gateway.ts index 1fc49ee46de..327f6b2f2df 100644 --- a/test/e2e/fixtures/clients/gateway.ts +++ b/test/e2e/fixtures/clients/gateway.ts @@ -7,6 +7,7 @@ import { buildAvailabilityProbeEnv } from "../availability-env.ts"; import type { NemoClawInstance } from "../phases/onboarding.ts"; import { pollUntil } from "../polling.ts"; import type { ShellProbeResult, ShellProbeRunOptions } from "../shell-probe.ts"; +import { RuntimeProviderPrerequisite } from "../runtime-provider.ts"; import { assertExitZero } from "./command.ts"; import type { HostCliClient } from "./host.ts"; import type { SandboxClient } from "./sandbox.ts"; @@ -111,10 +112,20 @@ function isMissingManagedSupervisorProof(result: ShellProbeResult): boolean { export class GatewayClient { private readonly host: HostCliClient; private readonly sandbox: SandboxClient; + private readonly runtimeProvider: RuntimeProviderPrerequisite; - constructor(host: HostCliClient, sandbox: SandboxClient) { + constructor( + host: HostCliClient, + sandbox: SandboxClient, + runtimeProvider?: RuntimeProviderPrerequisite, + ) { this.host = host; this.sandbox = sandbox; + this.runtimeProvider = + runtimeProvider ?? + new RuntimeProviderPrerequisite(host, (reason) => { + throw new Error(reason); + }); } status(options: ShellProbeRunOptions = {}): Promise { @@ -151,16 +162,22 @@ export class GatewayClient { return { kind: "pid", id: pid.stdout.trim() }; } - const container = await this.host.command( - "docker", - ["ps", "-qf", `name=${DEFAULT_GATEWAY_CONTAINER}`], + const container = await this.runtimeProvider.command( + ["container", "ps", "--format", "{{.ID}}\t{{.Names}}"], { artifactName: "gateway-runtime-container-probe", env: probeEnv(), timeoutMs: 15_000, }, ); - const id = container.stdout.trim().split(/\r?\n/).find(Boolean); + const ids = container.stdout + .split(/\r?\n/u) + .map((line) => line.trim().split(/\s+/u)) + .filter(([, name]) => name === DEFAULT_GATEWAY_CONTAINER) + .map(([id]) => id) + .filter((id): id is string => Boolean(id)); + if (ids.length > 1) throw new Error("OpenShell gateway runtime identity is ambiguous."); + const [id] = ids; return id ? { kind: "container", id } : null; } diff --git a/test/e2e/fixtures/e2e-test.ts b/test/e2e/fixtures/e2e-test.ts index 63ae4b0b664..85fac01b091 100644 --- a/test/e2e/fixtures/e2e-test.ts +++ b/test/e2e/fixtures/e2e-test.ts @@ -36,10 +36,11 @@ import { type TestProgress, type TestProgressOptions, } from "./progress.ts"; +import { RuntimeProviderPrerequisite } from "./runtime-provider.ts"; import { SecretStore } from "./secrets.ts"; import { ShellProbe } from "./shell-probe.ts"; -declare module "@vitest/runner" { +declare module "vitest" { interface TaskMeta { e2eArtifactRootId?: string; e2eCleanupTimeoutMs?: number; @@ -52,6 +53,7 @@ export interface E2ETargetFixtures { cleanup: CleanupRegistry; secrets: SecretStore; docker: DockerPrerequisite; + runtimeProvider: RuntimeProviderPrerequisite; shellProbe: ShellProbe; host: HostCliClient; gateway: GatewayClient; @@ -260,14 +262,17 @@ export const test = base.extend({ host: async ({ shellProbe }, use) => { await use(new HostCliClient(shellProbe)); }, + runtimeProvider: async ({ host, skip }, use) => { + await use(new RuntimeProviderPrerequisite(host, skip)); + }, sandbox: async ({ shellProbe }, use) => { await use(new SandboxClient(shellProbe)); }, - gateway: async ({ host, sandbox }, use) => { + gateway: async ({ host, runtimeProvider, sandbox }, use) => { // GatewayClient depends on `sandbox` for in-sandbox probes // (guard-chain inspection, log tailing, gateway-PID polling). // The fixture chain is sandbox → gateway so the dependency stays acyclic. - await use(new GatewayClient(host, sandbox)); + await use(new GatewayClient(host, sandbox, runtimeProvider)); }, provider: async ({ shellProbe }, use) => { await use(new ProviderClient(shellProbe)); @@ -283,14 +288,14 @@ export const test = base.extend({ state: async ({}, use) => { await use(new StateClient()); }, - environment: async ({ artifacts, host }, use) => { - await use(new EnvironmentPhaseFixture(host, artifacts)); + environment: async ({ artifacts, host, runtimeProvider }, use) => { + await use(new EnvironmentPhaseFixture(host, artifacts, runtimeProvider)); }, onboard: async ({ artifacts, cleanup, host, secrets }, use) => { await use(new OnboardingPhaseFixture(host, secrets, cleanup, artifacts)); }, - lifecycle: async ({ cleanup, gateway, host, sandbox }, use) => { - await use(new LifecyclePhaseFixture(host, sandbox, cleanup, gateway)); + lifecycle: async ({ cleanup, gateway, host, runtimeProvider, sandbox }, use) => { + await use(new LifecyclePhaseFixture(host, sandbox, cleanup, gateway, runtimeProvider)); }, runtime: async ({ provider, sandbox }, use) => { await use(new RuntimePhaseFixture(sandbox, provider)); diff --git a/test/e2e/fixtures/gateway-providers.ts b/test/e2e/fixtures/gateway-providers.ts index f9930157710..a9cc71ad6c9 100644 --- a/test/e2e/fixtures/gateway-providers.ts +++ b/test/e2e/fixtures/gateway-providers.ts @@ -1,10 +1,16 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + import { resultText } from "./clients/command.ts"; import type { HostCliClient } from "./clients/host.ts"; import type { SandboxClient } from "./clients/sandbox.ts"; import { expect } from "./e2e-test.ts"; +import { bindHermesDiscordPolicyEndpoint } from "./hermes-discord-policy-binding.ts"; +import type { ShellProbeResult } from "./shell-probe.ts"; const PROVIDER_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]*$/u; const CREDENTIAL_ENV = /^[A-Z_][A-Z0-9_]*$/u; @@ -19,6 +25,105 @@ function assertProviderName(providerName: string): void { } } +async function runFixtureOpenShell( + host: HostCliClient, + args: string[], + options: { + readonly artifactName: string; + readonly env: NodeJS.ProcessEnv; + readonly redactionValues: readonly string[]; + }, +): Promise { + const result = await host.command(host.openshellCommandPath, args, { + artifactName: options.artifactName, + env: options.env, + redactionValues: [...options.redactionValues], + timeoutMs: 120_000, + }); + expect(result.exitCode, resultText(result)).toBe(0); + return result; +} + +/** Bind a fixture endpoint without rotating the already-attached provider credential revision. */ +export async function rebindFixtureProviderPolicyEndpoint( + host: HostCliClient, + sandboxName: string, + options: { + readonly artifactName: string; + readonly credentialEnv: string; + readonly endpoint: { + readonly host: string; + readonly port: number | string; + readonly protocol: "rest" | "websocket"; + }; + readonly env: NodeJS.ProcessEnv; + readonly providerName: string; + readonly redactionValues?: readonly string[]; + }, +): Promise { + assertProviderName(options.providerName); + if (!CREDENTIAL_ENV.test(options.credentialEnv)) { + throw new Error(`Unsafe provider credential env name: ${options.credentialEnv}`); + } + if (!options.env[options.credentialEnv]) { + throw new Error(`Missing provider credential env value: ${options.credentialEnv}`); + } + const gatewayName = options.env.OPENSHELL_GATEWAY ?? "nemoclaw"; + assertProviderName(gatewayName); + const endpointPort = Number(options.endpoint.port); + if (!Number.isInteger(endpointPort) || endpointPort < 1 || endpointPort > 65_535) { + throw new Error("Fixture provider endpoint port must be an integer between 1 and 65535."); + } + + const redactionValues = options.redactionValues ?? []; + const policy = await host.command( + host.openshellCommandPath, + ["policy", "get", "--base", sandboxName], + { + artifactName: `${options.artifactName}-policy-before-rebind`, + env: options.env, + redactionValues: [...redactionValues], + timeoutMs: 60_000, + }, + ); + expect(policy.exitCode, resultText(policy)).toBe(0); + + const temporary = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-provider-rebind-")); + const boundPolicy = path.join(temporary, "bound-policy.yaml"); + try { + fs.writeFileSync(boundPolicy, policy.stdout, { mode: 0o600 }); + const attachments = await runFixtureOpenShell( + host, + ["sandbox", "provider", "list", "-g", gatewayName, sandboxName], + { + artifactName: `${options.artifactName}-provider-attachments`, + env: options.env, + redactionValues, + }, + ); + expect(resultText(attachments).split(/\s+/u)).toContain(options.providerName); + + bindHermesDiscordPolicyEndpoint( + boundPolicy, + options.providerName, + options.endpoint.host, + endpointPort, + options.endpoint.protocol, + ); + await runFixtureOpenShell( + host, + ["policy", "set", "--policy", boundPolicy, "--wait", sandboxName], + { + artifactName: `${options.artifactName}-policy-rebound`, + env: options.env, + redactionValues, + }, + ); + } finally { + fs.rmSync(temporary, { force: true, recursive: true }); + } +} + export async function upsertGenericGatewayProvider( host: HostCliClient, providerName: string, diff --git a/test/e2e/fixtures/hermes-discord-policy-binding.ts b/test/e2e/fixtures/hermes-discord-policy-binding.ts index 36eaabe3449..731311ef237 100644 --- a/test/e2e/fixtures/hermes-discord-policy-binding.ts +++ b/test/e2e/fixtures/hermes-discord-policy-binding.ts @@ -61,6 +61,32 @@ export function bindHermesDiscordPolicyEndpoint( fs.chmodSync(policyFile, 0o600); } +export function unbindProviderPolicyEndpoints(policyFile: string, providerName: string): void { + const policy = readPolicy(policyFile); + Object.values(policy.network_policies ?? {}) + .map((entry) => + typeof entry === "object" && entry !== null && !Array.isArray(entry) + ? ((entry as { endpoints?: unknown }).endpoints ?? []) + : [], + ) + .filter((endpoints): endpoints is unknown[] => Array.isArray(endpoints)) + .flat() + .filter( + (endpoint): endpoint is Record => + typeof endpoint === "object" && endpoint !== null && !Array.isArray(endpoint), + ) + .filter( + (endpoint) => + typeof endpoint.credential_binding === "object" && + endpoint.credential_binding !== null && + !Array.isArray(endpoint.credential_binding) && + (endpoint.credential_binding as { provider?: unknown }).provider === providerName, + ) + .forEach((endpoint) => delete endpoint.credential_binding); + fs.writeFileSync(policyFile, YAML.stringify(policy)); + fs.chmodSync(policyFile, 0o600); +} + export function assertHermesDiscordPolicyEndpointBinaries( policyFile: string, host: string, @@ -113,13 +139,16 @@ function main(): void { return; } - const [policyFile, providerName, host, rawPort, protocol] = args; - if (!policyFile || !providerName || !host || !rawPort || !protocol) { + const unbind = args[0] === "--unbind-provider"; + const [policyFile, providerName, host, rawPort, protocol] = args.slice(unbind ? 1 : 0); + if (!policyFile || !providerName || (!unbind && (!host || !rawPort || !protocol))) { throw new Error( "usage: hermes-discord-policy-binding ", ); } - bindHermesDiscordPolicyEndpoint(policyFile, providerName, host, Number(rawPort), protocol); + unbind + ? unbindProviderPolicyEndpoints(policyFile, providerName) + : bindHermesDiscordPolicyEndpoint(policyFile, providerName, host!, Number(rawPort), protocol!); } if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) main(); diff --git a/test/e2e/fixtures/host-address.ts b/test/e2e/fixtures/host-address.ts index 75e8b98a08c..ccbf10fee37 100644 --- a/test/e2e/fixtures/host-address.ts +++ b/test/e2e/fixtures/host-address.ts @@ -3,12 +3,15 @@ import { isIP } from "node:net"; +import { prepareConfiguredGatewayHostRuntime } from "../../../src/lib/onboard/docker-driver-gateway-env.ts"; +import { isPortableExperimentalProfile } from "../../../src/lib/onboard/docker-driver-platform.ts"; import { buildAvailabilityProbeEnv } from "./availability-env.ts"; import { assertExitZero } from "./clients/command.ts"; import type { HostCliClient } from "./clients/host.ts"; import type { ShellProbeResult } from "./shell-probe.ts"; export type HostAddressSource = + | "runtime-provider" | "route" | "hostname" | "darwin-interface" @@ -17,7 +20,17 @@ export type HostAddressSource = export interface HostAddressResult { address: string; source: HostAddressSource; - probe: ShellProbeResult; + probe: ShellProbeResult | null; +} + +export function configuredRuntimeProviderHostAddress( + environment: NodeJS.ProcessEnv = process.env, + platform: NodeJS.Platform = process.platform, +): string | null { + if (!environment.NEMOCLAW_GATEWAY_RUNTIME || isPortableExperimentalProfile(environment)) { + return null; + } + return prepareConfiguredGatewayHostRuntime({ environment, platform }).sandboxHostAddress; } export function parseHostAddressProbe( @@ -46,7 +59,16 @@ export function parseHostAddressProbe( export async function discoverHostAddress( host: HostCliClient, artifactName = "host-address-for-sandbox", + environment: NodeJS.ProcessEnv = process.env, + platform: NodeJS.Platform = process.platform, ): Promise { + const runtimeProviderAddress = configuredRuntimeProviderHostAddress(environment, platform); + if (runtimeProviderAddress !== null) { + if (isIP(runtimeProviderAddress) !== 4) { + throw new Error(`runtime provider returned invalid sandbox host address: ${runtimeProviderAddress}`); + } + return { source: "runtime-provider", address: runtimeProviderAddress, probe: null }; + } const probe = await host.command( "bash", [ diff --git a/test/e2e/fixtures/managed-image-receipt.ts b/test/e2e/fixtures/managed-image-receipt.ts index 0604a8b11ba..0b340071abb 100644 --- a/test/e2e/fixtures/managed-image-receipt.ts +++ b/test/e2e/fixtures/managed-image-receipt.ts @@ -244,7 +244,11 @@ export function shouldAssertStockManagedImageReceipt( path.basename(args[0] ?? "") === "nemoclaw.js" && args[1] === "onboard" ? 1 : -1; } if (onboardArgumentIndex < 0) return false; - return !args - .slice(onboardArgumentIndex + 1) - .some((argument) => argument === "--from" || argument.startsWith("--from=")); + const onboardArguments = args.slice(onboardArgumentIndex + 1); + if (onboardArguments.some((argument) => argument === "--help" || argument === "-h")) { + return false; + } + return !onboardArguments.some( + (argument) => argument === "--from" || argument.startsWith("--from="), + ); } diff --git a/test/e2e/fixtures/phases/environment.ts b/test/e2e/fixtures/phases/environment.ts index e57193aeeb8..3ecbc80ce2a 100644 --- a/test/e2e/fixtures/phases/environment.ts +++ b/test/e2e/fixtures/phases/environment.ts @@ -5,24 +5,26 @@ import { buildAvailabilityProbeEnv } from "../availability-env.ts"; import type { ArtifactSink } from "../artifacts.ts"; import { artifactLabel, assertExitZero } from "../clients/command.ts"; import type { HostCliClient } from "../clients/host.ts"; +import { RuntimeProviderPrerequisite } from "../runtime-provider.ts"; import type { ShellProbeResult } from "../shell-probe.ts"; import type { TargetEnvironment } from "../../registry/types.ts"; const SUPPORTED_INSTALLS = new Set(["repo-current", "launchable"]); -const DOCKER_RUNTIME_EXPECTATIONS = { +const RUNTIME_EXPECTATIONS = { + "managed-runtime-running": "required", "docker-running": "required", "gpu-docker-cdi": "required", "docker-missing": "missing", "macos-docker-optional": "optional", } as const; -export type DockerRuntimeExpectation = - (typeof DOCKER_RUNTIME_EXPECTATIONS)[keyof typeof DOCKER_RUNTIME_EXPECTATIONS]; +export type RuntimeExpectation = (typeof RUNTIME_EXPECTATIONS)[keyof typeof RUNTIME_EXPECTATIONS]; -export interface DockerRuntimeReady { +export interface RuntimeReady { id: string; - expectation: DockerRuntimeExpectation; + expectation: RuntimeExpectation; + providerId: "docker" | "podman"; available: boolean; result?: ShellProbeResult; probeError?: string; @@ -30,16 +32,15 @@ export interface DockerRuntimeReady { export interface EnvironmentReady extends TargetEnvironment { cliPath: string; - docker: DockerRuntimeReady; + runtimeProvider: RuntimeReady; } function errorMessage(error: unknown): string { return error instanceof Error ? error.message : String(error); } -function supportedRuntime(runtime: string): DockerRuntimeExpectation { - const expectation = - DOCKER_RUNTIME_EXPECTATIONS[runtime as keyof typeof DOCKER_RUNTIME_EXPECTATIONS]; +function supportedRuntime(runtime: string): RuntimeExpectation { + const expectation = RUNTIME_EXPECTATIONS[runtime as keyof typeof RUNTIME_EXPECTATIONS]; if (!expectation) { throw new Error(`Unsupported target runtime '${runtime}'.`); } @@ -50,16 +51,19 @@ export class EnvironmentPhaseFixture { constructor( private readonly host: HostCliClient, private readonly artifacts?: ArtifactSink, + private readonly runtimeProvider = new RuntimeProviderPrerequisite(host, (reason) => { + throw new Error(reason); + }), ) {} async assertReady(environment: TargetEnvironment): Promise { try { await this.assertInstallReady(environment.install); - const docker = await this.assertRuntimeReady(environment.runtime); + const runtimeProvider = await this.assertRuntimeReady(environment.runtime); const result = { ...environment, cliPath: this.host.commandPath, - docker, + runtimeProvider, }; await this.writeResult("passed", result); return result; @@ -89,27 +93,34 @@ export class EnvironmentPhaseFixture { return this.host.expectNemoclawAvailable(); } - private async assertRuntimeReady(runtime: string): Promise { + private async assertRuntimeReady(runtime: string): Promise { const expectation = supportedRuntime(runtime); - const result = await this.probeDocker(runtime, expectation); + const result = await this.probeRuntime(runtime, expectation); if (!result.result) { return result; } if (expectation === "required") { - assertExitZero(result.result, `docker runtime ${runtime}`); + assertExitZero(result.result, `${result.providerId} runtime ${runtime}`); } // Missing-runtime targets simulate Docker failure at the phase that // needs it; this probe records host reality without blocking composition. return result; } - private async probeDocker( + private async probeRuntime( runtime: string, - expectation: DockerRuntimeExpectation, - ): Promise { + expectation: RuntimeExpectation, + ): Promise { try { - const result = await this.host.command("docker", ["info"], { + const selectedProvider = runtime === "managed-runtime-running"; + const providerId = selectedProvider ? this.runtimeProvider.id : "docker"; + const result = selectedProvider + ? await this.runtimeProvider.command(["info"], { + artifactName: `runtime-${providerId}-info-${artifactLabel(runtime)}`, + timeoutMs: 30_000, + }) + : await this.host.command("docker", ["info"], { artifactName: `runtime-docker-info-${artifactLabel(runtime)}`, env: buildAvailabilityProbeEnv(), timeoutMs: 30_000, @@ -117,6 +128,7 @@ export class EnvironmentPhaseFixture { return { id: runtime, expectation, + providerId, available: result.exitCode === 0, result, }; @@ -127,6 +139,7 @@ export class EnvironmentPhaseFixture { return { id: runtime, expectation, + providerId: runtime === "managed-runtime-running" ? this.runtimeProvider.id : "docker", available: false, probeError: errorMessage(error), }; diff --git a/test/e2e/fixtures/phases/index.ts b/test/e2e/fixtures/phases/index.ts index dcb7c319e3f..a2a18c4a38d 100644 --- a/test/e2e/fixtures/phases/index.ts +++ b/test/e2e/fixtures/phases/index.ts @@ -2,10 +2,10 @@ // SPDX-License-Identifier: Apache-2.0 export { - type DockerRuntimeExpectation, - type DockerRuntimeReady, EnvironmentPhaseFixture, type EnvironmentReady, + type RuntimeExpectation, + type RuntimeReady, } from "./environment.ts"; export { type DcodeInvalidCredentialRebuildOptions, diff --git a/test/e2e/fixtures/phases/lifecycle.ts b/test/e2e/fixtures/phases/lifecycle.ts index 9dcb98debd6..298400d9d57 100644 --- a/test/e2e/fixtures/phases/lifecycle.ts +++ b/test/e2e/fixtures/phases/lifecycle.ts @@ -8,6 +8,7 @@ import { assertExitZero, outputContainsReadySandbox } from "../clients/command.t import type { GatewayClient, HostGatewayRuntime } from "../clients/gateway.ts"; import type { HostCliClient } from "../clients/host.ts"; import type { SandboxClient } from "../clients/sandbox.ts"; +import { RuntimeProviderPrerequisite } from "../runtime-provider.ts"; import type { ShellProbeResult } from "../shell-probe.ts"; import { type DcodeInvalidCredentialRebuildOptions, @@ -29,7 +30,7 @@ export { // a real onboarded sandbox through the docker-sandbox-container-present // probe. const OPENSHELL_SANDBOX_NAME_LABEL = "openshell.ai/sandbox-name"; -const DOCKER_PROBE_TIMEOUT_MS = 15_000; +const RUNTIME_PROBE_TIMEOUT_MS = 15_000; // Recovery can take several minutes while gateway and host-forward // readiness converge, so keep the status budget generous. const STATUS_TIMEOUT_MS = 5 * 60_000; @@ -225,13 +226,25 @@ function instanceName(instance: NemoClawInstance | string): string { export class LifecyclePhaseFixture { private postRebootUserServiceStage: UserServiceStageResult | undefined; + private readonly runtimeProvider: RuntimeProviderPrerequisite; constructor( private readonly host: HostCliClient, private readonly sandbox: SandboxClient, private readonly cleanup: LifecycleCleanup, private readonly gateway?: GatewayClient, - ) {} + runtimeProvider?: RuntimeProviderPrerequisite, + ) { + this.runtimeProvider = + runtimeProvider ?? + new RuntimeProviderPrerequisite(host, (reason) => { + throw new Error(reason); + }); + } + + private requireRuntimeProvider(): RuntimeProviderPrerequisite { + return this.runtimeProvider; + } /** * Ensure OpenShell is installed and stage the OpenShell gateway user service @@ -399,48 +412,54 @@ export class LifecyclePhaseFixture { const containerNames = await this.discoverLabeledContainerNames(instance); if (containerNames.length === 0) { throw new Error( - `lifecycle.post-reboot-recovery expected at least one Docker container labeled ` + - `'${OPENSHELL_SANDBOX_NAME_LABEL}=${instance.sandboxName}', but docker ps -a returned none. ` + + `lifecycle.post-reboot-recovery expected at least one managed runtime resource labeled ` + + `'${OPENSHELL_SANDBOX_NAME_LABEL}=${instance.sandboxName}', but the selected provider returned none. ` + `Did onboarding create the sandbox?`, ); } const originalName = containerNames[0]; let bootContainerName = originalName; - const stop = await this.host.command("docker", ["stop", originalName], { - artifactName: `lifecycle-post-reboot-docker-stop-${originalName}`, + const stop = await this.requireRuntimeProvider().command(["container", "stop", originalName], { + artifactName: `lifecycle-post-reboot-runtime-stop-${originalName}`, env: buildAvailabilityProbeEnv(), - timeoutMs: DOCKER_PROBE_TIMEOUT_MS, + timeoutMs: RUNTIME_PROBE_TIMEOUT_MS, }); - assertExitZero(stop, `docker stop ${originalName}`); - steps.push({ id: `docker-stop:${originalName}`, results: [stop] }); - this.cleanup.add(`lifecycle.docker-start:${originalName}`, async () => { - await this.host.command("docker", ["start", originalName], { - artifactName: `lifecycle-cleanup-docker-start-${originalName}`, + assertExitZero(stop, `stop managed runtime resource ${originalName}`); + steps.push({ id: `runtime-stop:${originalName}`, results: [stop] }); + this.cleanup.add(`lifecycle.runtime-start:${originalName}`, async () => { + await this.requireRuntimeProvider().command(["container", "start", originalName], { + artifactName: `lifecycle-cleanup-runtime-start-${originalName}`, env: buildAvailabilityProbeEnv(), - timeoutMs: DOCKER_PROBE_TIMEOUT_MS, + timeoutMs: RUNTIME_PROBE_TIMEOUT_MS, }); }); if (mode === "rename-to-gpu-backup") { const backupName = buildBackupContainerName(originalName, Date.now()); - const rename = await this.host.command("docker", ["rename", originalName, backupName], { - artifactName: `lifecycle-post-reboot-docker-rename-${originalName}`, - env: buildAvailabilityProbeEnv(), - timeoutMs: DOCKER_PROBE_TIMEOUT_MS, - }); - assertExitZero(rename, `docker rename ${originalName} ${backupName}`); + const rename = await this.requireRuntimeProvider().command( + ["container", "rename", originalName, backupName], + { + artifactName: `lifecycle-post-reboot-runtime-rename-${originalName}`, + env: buildAvailabilityProbeEnv(), + timeoutMs: RUNTIME_PROBE_TIMEOUT_MS, + }, + ); + assertExitZero(rename, `rename managed runtime resource ${originalName} ${backupName}`); steps.push({ - id: `docker-rename:${originalName}->${backupName}`, + id: `runtime-rename:${originalName}->${backupName}`, results: [rename], }); bootContainerName = backupName; - this.cleanup.add(`lifecycle.docker-rename-back:${backupName}`, async () => { - await this.host.command("docker", ["rename", backupName, originalName], { - artifactName: `lifecycle-cleanup-docker-rename-back-${backupName}`, - env: buildAvailabilityProbeEnv(), - timeoutMs: DOCKER_PROBE_TIMEOUT_MS, - }); + this.cleanup.add(`lifecycle.runtime-rename-back:${backupName}`, async () => { + await this.requireRuntimeProvider().command( + ["container", "rename", backupName, originalName], + { + artifactName: `lifecycle-cleanup-runtime-rename-back-${backupName}`, + env: buildAvailabilityProbeEnv(), + timeoutMs: RUNTIME_PROBE_TIMEOUT_MS, + }, + ); }); } @@ -459,14 +478,17 @@ export class LifecyclePhaseFixture { // `docker stop` suppresses Docker restart-policy handling until the // daemon restarts. Start the same container here to model that boot-owned // transition without restarting the GitHub-hosted runner's Docker daemon. - const bootStart = await this.host.command("docker", ["start", bootContainerName], { - artifactName: `lifecycle-post-reboot-docker-start-${bootContainerName}`, - env: buildAvailabilityProbeEnv(), - timeoutMs: DOCKER_PROBE_TIMEOUT_MS, - }); - assertExitZero(bootStart, `docker start ${bootContainerName}`); + const bootStart = await this.requireRuntimeProvider().command( + ["container", "start", bootContainerName], + { + artifactName: `lifecycle-post-reboot-runtime-start-${bootContainerName}`, + env: buildAvailabilityProbeEnv(), + timeoutMs: RUNTIME_PROBE_TIMEOUT_MS, + }, + ); + assertExitZero(bootStart, `start managed runtime resource ${bootContainerName}`); steps.push({ - id: `docker-boot-start:${bootContainerName}`, + id: `runtime-boot-start:${bootContainerName}`, results: [bootStart], }); @@ -584,20 +606,36 @@ export class LifecyclePhaseFixture { // is explicitly anchored. The unanchored form can select a sandbox whose // name contains the gateway prefix; stopping that container remounts its // tmpfs and turns a gateway-restart probe into a sandbox-restart probe. - const containerStop = await this.host.command( - "sh", - [ - "-lc", - `cid="$(docker ps --filter 'name=^/openshell-cluster-nemoclaw$' --format '{{.ID}}' 2>/dev/null)"; ` + - `if [ -n "$cid" ]; then docker stop "$cid" >/dev/null; fi`, - ], + const runtimeProvider = this.requireRuntimeProvider(); + const gatewayResources = await runtimeProvider.command( + ["container", "ps", "--format", "{{.ID}}\t{{.Names}}"], { - artifactName: "lifecycle-gateway-container-stop", + artifactName: "lifecycle-gateway-runtime-discover", env: buildAvailabilityProbeEnv(), timeoutMs: 60_000, }, ); - assertExitZero(containerStop, "stop OpenShell gateway container"); + assertExitZero(gatewayResources, "discover OpenShell gateway runtime resource"); + const gatewayHandles = gatewayResources.stdout + .split(/\r?\n/u) + .map((line) => line.trim().split(/\s+/u)) + .filter(([, name]) => name === "openshell-cluster-nemoclaw") + .map(([handle]) => handle) + .filter((handle): handle is string => Boolean(handle)); + if (gatewayHandles.length > 1) { + throw new Error("OpenShell gateway runtime resource identity is ambiguous."); + } + if (gatewayHandles[0]) { + const containerStop = await runtimeProvider.command( + ["container", "stop", gatewayHandles[0]], + { + artifactName: "lifecycle-gateway-container-stop", + env: buildAvailabilityProbeEnv(), + timeoutMs: 60_000, + }, + ); + assertExitZero(containerStop, "stop OpenShell gateway runtime resource"); + } return runtime; } @@ -605,6 +643,13 @@ export class LifecyclePhaseFixture { previousRuntime: HostGatewayRuntime | null, options: { requireUserService?: boolean; sandboxName?: string } = {}, ): Promise { + if (options.sandboxName && options.requireUserService !== true) { + return await this.host.nemoclaw([options.sandboxName, "status"], { + artifactName: `lifecycle-gateway-recover-through-nemoclaw-status-${options.sandboxName}`, + env: buildAvailabilityProbeEnv(), + timeoutMs: 120_000, + }); + } const userServiceStart = await this.startOpenShellGatewayUserService({ requireAvailable: options.requireUserService, }); @@ -720,25 +765,25 @@ export class LifecyclePhaseFixture { } private async discoverLabeledContainerNames(instance: NemoClawInstance): Promise { - const result = await this.host.command( - "docker", + const result = await this.requireRuntimeProvider().command( [ + "container", "ps", - "-a", + "--all", "--filter", `label=${OPENSHELL_SANDBOX_NAME_LABEL}=${instance.sandboxName}`, "--format", "{{.Names}}", ], { - artifactName: `lifecycle-post-reboot-docker-discover-${instance.sandboxName}`, + artifactName: `lifecycle-post-reboot-runtime-discover-${instance.sandboxName}`, env: buildAvailabilityProbeEnv(), - timeoutMs: DOCKER_PROBE_TIMEOUT_MS, + timeoutMs: RUNTIME_PROBE_TIMEOUT_MS, }, ); if (result.exitCode !== 0) { throw new Error( - `lifecycle.post-reboot-recovery could not query Docker for label ` + + `lifecycle.post-reboot-recovery could not query the selected runtime provider for label ` + `'${OPENSHELL_SANDBOX_NAME_LABEL}=${instance.sandboxName}' (exit ${result.exitCode}).`, ); } diff --git a/test/e2e/fixtures/phases/onboarding.ts b/test/e2e/fixtures/phases/onboarding.ts index de996524d44..e4e3034a81a 100644 --- a/test/e2e/fixtures/phases/onboarding.ts +++ b/test/e2e/fixtures/phases/onboarding.ts @@ -207,8 +207,8 @@ export class OnboardingPhaseFixture { environment: EnvironmentReady, options: OnboardingOptions = {}, ): Promise { - if (!environment.docker.available) { - throw new Error("cloud-openclaw onboarding requires an available Docker runtime."); + if (!environment.runtimeProvider.available) { + throw new Error("cloud-openclaw onboarding requires an available managed runtime provider."); } const sandboxName = sandboxNameFromOptions(environment.onboarding, options); const apiKey = this.secrets.required("NVIDIA_INFERENCE_API_KEY"); @@ -250,9 +250,9 @@ export class OnboardingPhaseFixture { environment: EnvironmentReady, options: OnboardingOptions = {}, ): Promise { - if (!environment.docker.available) { + if (!environment.runtimeProvider.available) { throw new Error( - "cloud-langchain-deepagents-code onboarding requires an available Docker runtime.", + "cloud-langchain-deepagents-code onboarding requires an available managed runtime provider.", ); } const sandboxName = sandboxNameFromOptions(environment.onboarding, options); @@ -308,7 +308,7 @@ export class OnboardingPhaseFixture { environment: EnvironmentReady, options: OnboardingOptions = {}, ): Promise { - if (environment.docker.expectation !== "missing") { + if (environment.runtimeProvider.expectation !== "missing") { throw new Error( "cloud-openclaw-no-docker onboarding requires the docker-missing runtime expectation.", ); @@ -360,9 +360,9 @@ export class OnboardingPhaseFixture { environment: EnvironmentReady, options: OnboardingOptions = {}, ): Promise { - if (!environment.docker.available) { + if (!environment.runtimeProvider.available) { throw new Error( - "cloud-openclaw-policy-custom-missing-presets onboarding requires an available Docker runtime.", + "cloud-openclaw-policy-custom-missing-presets onboarding requires an available managed runtime provider.", ); } const sandboxName = sandboxNameFromOptions(environment.onboarding, options); diff --git a/test/e2e/fixtures/routed-private-relay.ts b/test/e2e/fixtures/routed-private-relay.ts new file mode 100644 index 00000000000..9fee94a6abb --- /dev/null +++ b/test/e2e/fixtures/routed-private-relay.ts @@ -0,0 +1,100 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import { randomBytes } from "node:crypto"; +import { isIP } from "node:net"; + +import { isOperatorTrustablePrivateIp } from "../../../src/lib/security/trusted-private-endpoint.ts"; +import type { HostCliClient } from "./clients/host.ts"; +import { RuntimeProviderPrerequisite } from "./runtime-provider.ts"; + +const RELAY_PORT = 8443; +const RELAY_SOURCE = String.raw` +const net = require("node:net"); +const [host, portText] = process.argv.slice(1); +const port = Number(portText); +const server = net.createServer((client) => { + const upstream = net.connect({ host, port }); + client.pipe(upstream); + upstream.pipe(client); + const close = () => { client.destroy(); upstream.destroy(); }; + client.on("error", close); + upstream.on("error", close); +}); +server.listen(${String(RELAY_PORT)}, "0.0.0.0"); +`; + +export interface RoutedPrivateRelay { + readonly address: string; + readonly port: number; + close(): Promise; +} + +export async function startRoutedPrivateRelay(options: { + host: HostCliClient; + sandboxName: string; + upstreamHost: string; + upstreamPort: number; +}): Promise { + const runtime = new RuntimeProviderPrerequisite(options.host, (reason) => { + throw new Error(reason); + }); + const sandboxHandle = await runtime.resolveSandboxResourceHandle(options.sandboxName, { + artifactName: "routed-private-relay-sandbox-resource", + timeoutMs: 30_000, + }); + const networks = await runtime.command( + ["container", "inspect", "--format", "{{json .NetworkSettings.Networks}}", sandboxHandle], + { artifactName: "routed-private-relay-sandbox-network", timeoutMs: 30_000 }, + ); + assert.equal(networks.exitCode, 0, `${networks.stdout}\n${networks.stderr}`); + const networkNames = Object.keys(JSON.parse(networks.stdout) as Record); + assert.equal(networkNames.length, 1, "sandbox must have one exact runtime network"); + const networkName = networkNames[0] as string; + const relayName = `nemoclaw-private-relay-${process.pid}-${randomBytes(4).toString("hex")}`; + const close = async (): Promise => { + await runtime.command(["container", "rm", "--force", relayName], { + artifactName: "cleanup-routed-private-relay", + timeoutMs: 60_000, + }); + }; + const start = await runtime.command( + [ + "run", + "--detach", + "--rm", + "--name", + relayName, + "--network", + networkName, + "node:22-bookworm-slim", + "node", + "-e", + RELAY_SOURCE, + options.upstreamHost, + String(options.upstreamPort), + ], + { artifactName: "start-routed-private-relay", timeoutMs: 120_000 }, + ); + assert.equal(start.exitCode, 0, `${start.stdout}\n${start.stderr}`); + const addressResult = await runtime.command( + [ + "container", + "inspect", + "--format", + "{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}", + relayName, + ], + { artifactName: "inspect-routed-private-relay-address", timeoutMs: 30_000 }, + ); + assert.equal(addressResult.exitCode, 0, `${addressResult.stdout}\n${addressResult.stderr}`); + const address = addressResult.stdout.trim(); + assert.equal(isIP(address), 4, "routed-private relay must have one IPv4 address"); + assert.equal( + isOperatorTrustablePrivateIp(address), + true, + "routed-private relay must use an operator-trustable private network", + ); + return Object.freeze({ address, port: RELAY_PORT, close }); +} diff --git a/test/e2e/fixtures/runtime-provider.ts b/test/e2e/fixtures/runtime-provider.ts new file mode 100644 index 00000000000..934744394b1 --- /dev/null +++ b/test/e2e/fixtures/runtime-provider.ts @@ -0,0 +1,199 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import path from "node:path"; + +import { buildAvailabilityProbeEnv } from "./availability-env.ts"; +import type { HostCliClient } from "./clients/host.ts"; +import type { ShellProbeResult, ShellProbeRunOptions } from "./shell-probe.ts"; + +export type RuntimeProviderSkip = (reason: string) => never; + +export type E2eRuntimeProviderId = "docker" | "podman"; + +const SANITIZED_PRIVILEGED_ENVIRONMENT = [ + "BASH_ENV=", + "ENV=", + "GCONV_PATH=", + "GLIBC_TUNABLES=", + "LD_AUDIT=", + "LD_LIBRARY_PATH=", + "LD_PRELOAD=", + "LOCPATH=", + "NODE_OPTIONS=", + "PERL5OPT=", + "PYTHONHOME=", + "PYTHONINSPECT=", + "PYTHONNOUSERSITE=1", + "PYTHONPATH=", + "PYTHONSTARTUP=", + "PYTHONUSERBASE=", + "RUBYOPT=", +] as const; + +const SANDBOX_NAME_LABEL = "openshell.ai/sandbox-name"; +const CONTAINER_ID = /^[a-f0-9]{12,64}$/u; + +interface RuntimeProviderInvocation { + readonly argsPrefix: readonly string[]; + readonly command: E2eRuntimeProviderId; + readonly displayName: "Docker" | "Podman"; + readonly id: E2eRuntimeProviderId; +} + +export interface E2eRuntimeProviderCommand { + readonly args: readonly string[]; + readonly command: E2eRuntimeProviderId; +} + +function configuredRuntimeProviderInvocation( + environment: NodeJS.ProcessEnv, +): RuntimeProviderInvocation { + const portable = environment.NEMOCLAW_EXPERIMENTAL_PROFILE === "portable"; + const configured = environment.NEMOCLAW_GATEWAY_RUNTIME?.trim() || "docker"; + if (configured !== "docker" && configured !== "podman") { + throw new Error(`unsupported E2E gateway runtime: ${configured}`); + } + + const providerId = portable ? "docker" : configured; + if (providerId === "docker") { + return { + argsPrefix: [], + command: "docker", + displayName: "Docker", + id: providerId, + }; + } + + const socketPath = environment.OPENSHELL_PODMAN_SOCKET?.trim(); + if ( + !socketPath || + !path.isAbsolute(socketPath) || + path.normalize(socketPath) !== socketPath || + /[\u0000-\u001f\u007f-\u009f]/u.test(socketPath) + ) { + throw new Error("native Podman E2E requires one absolute provider-owned socket path"); + } + return { + argsPrefix: ["--url", `unix://${socketPath}`], + command: "podman", + displayName: "Podman", + id: providerId, + }; +} + +export class RuntimeProviderPrerequisite { + readonly displayName: "Docker" | "Podman"; + readonly id: E2eRuntimeProviderId; + private readonly invocation: RuntimeProviderInvocation; + + constructor( + private readonly host: HostCliClient, + private readonly skip: RuntimeProviderSkip, + private readonly environment: NodeJS.ProcessEnv = process.env, + ) { + this.invocation = configuredRuntimeProviderInvocation(environment); + this.displayName = this.invocation.displayName; + this.id = this.invocation.id; + } + + command(args: readonly string[], options: ShellProbeRunOptions = {}): Promise { + const invocation = this.hostInvocation(args); + return this.host.command(invocation.command, [...invocation.args], { + env: buildAvailabilityProbeEnv(this.environment), + ...options, + }); + } + + hostInvocation(args: readonly string[]): E2eRuntimeProviderCommand { + return Object.freeze({ + command: this.invocation.command, + args: Object.freeze([...this.invocation.argsPrefix, ...args]), + }); + } + + async requireAvailable(options: { artifactName: string; scenarioLabel: string }): Promise { + const result = await this.command(["info"], { + artifactName: options.artifactName, + timeoutMs: 30_000, + }); + if (result.exitCode === 0) return; + + const detail = [result.stdout, result.stderr].filter(Boolean).join("\n"); + const reason = `${this.displayName} is required for ${options.scenarioLabel} live E2E: ${detail}`; + if (process.env.GITHUB_ACTIONS === "true") throw new Error(reason); + this.skip(reason); + } + + async resolveSandboxResourceHandle( + sandboxName: string, + options: ShellProbeRunOptions = {}, + ): Promise { + const result = await this.command( + [ + "container", + "ps", + "--all", + "--no-trunc", + "--filter", + `label=${SANDBOX_NAME_LABEL}=${sandboxName}`, + "--format", + "{{.ID}}", + ], + options, + ); + if (result.exitCode !== 0) { + throw new Error( + `${this.displayName} sandbox resource discovery failed for '${sandboxName}': ${[ + result.stdout, + result.stderr, + ] + .filter(Boolean) + .join("\n")}`, + ); + } + const handles = result.stdout + .split(/\r?\n/u) + .map((line) => line.trim().toLowerCase()) + .filter(Boolean); + if (handles.length !== 1 || !CONTAINER_ID.test(handles[0] ?? "")) { + throw new Error( + `${this.displayName} sandbox '${sandboxName}' resolved ${String(handles.length)} runtime resources; expected exactly one.`, + ); + } + return handles[0] as string; + } + + async execSandboxAsRoot( + sandboxName: string, + args: readonly string[], + options: ShellProbeRunOptions & { sanitizeEnvironment?: boolean } = {}, + ): Promise { + const { sanitizeEnvironment = false, ...runOptions } = options; + const resourceHandle = await this.resolveSandboxResourceHandle(sandboxName, { + ...runOptions, + artifactName: runOptions.artifactName ? `${runOptions.artifactName}-resource` : undefined, + }); + const environment = sanitizeEnvironment + ? SANITIZED_PRIVILEGED_ENVIRONMENT.flatMap((value) => ["--env", value]) + : []; + return this.command( + ["container", "exec", ...environment, "--user", "root", resourceHandle, ...args], + runOptions, + ); + } +} + +export async function ensureConfiguredRuntimeProviderAvailable(options: { + artifactName: string; + environment?: NodeJS.ProcessEnv; + host: HostCliClient; + scenarioLabel: string; + skip: RuntimeProviderSkip; +}): Promise { + const environment = options.environment ?? process.env; + await new RuntimeProviderPrerequisite(options.host, options.skip, environment).requireAvailable({ + artifactName: options.artifactName, + scenarioLabel: options.scenarioLabel, + }); +} diff --git a/test/e2e/fixtures/security-posture.ts b/test/e2e/fixtures/security-posture.ts index ae7f43ef47f..0d1b1093629 100644 --- a/test/e2e/fixtures/security-posture.ts +++ b/test/e2e/fixtures/security-posture.ts @@ -1,8 +1,14 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { privilegedSandboxExecArgv } from "../../../src/lib/sandbox/privileged-exec.ts"; -import { buildSubprocessEnv } from "../../../src/lib/subprocess-env.ts"; +import type { + RuntimeProviderPrivilegedSandboxCommandResult, + RuntimeProviderPrivilegedSandboxTarget, +} from "../../../src/lib/onboard/runtime-provider/contract.ts"; +import { + executePrivilegedSandboxCommand, + resolvePrivilegedSandboxTarget, +} from "../../../src/lib/sandbox/privileged-exec.ts"; import { buildAvailabilityProbeEnv } from "./availability-env.ts"; import type { HostCliClient } from "./clients/host.ts"; import { type SandboxClient, trustedSandboxShellScript } from "./clients/sandbox.ts"; @@ -60,12 +66,10 @@ export interface SecurityPostureExpectations { } export interface SecurityPostureDependencies { - privilegedExecArgv?: typeof privilegedSandboxExecArgv; + executePrivilegedCommand?: typeof executePrivilegedSandboxCommand; + resolvePrivilegedTarget?: typeof resolvePrivilegedSandboxTarget; } -const OPENSHELL_DEFAULT_WORKSPACE = "default"; -const OPENSHELL_SANDBOX_ID_LABEL = "openshell.ai/sandbox-id"; -const OPENSHELL_SANDBOX_WORKSPACE_LABEL = "openshell.ai/sandbox-workspace"; const OPENSHELL_SUPERVISOR_EXECUTABLE = "/opt/openshell/bin/openshell-sandbox"; const OPENSHELL_SUPERVISOR_ARGV = [ OPENSHELL_SUPERVISOR_EXECUTABLE, @@ -79,7 +83,6 @@ const NEMOCLAW_START_SUPERVISOR_PATHS = [ ] as const; const BASH_ARGV0 = ["bash", ...SYSTEM_BASH_EXECUTABLES] as const; const LIVE_PROCESS_STATES = ["D", "R", "S"] as const; -const SAFE_OPENSHELL_IDENTITY_COMPONENT = /^[a-z0-9][a-z0-9_.-]*$/u; const MAX_PROC_ENTRIES = 32_768; const MAX_CENSUS_STABILITY_ATTEMPTS = 4; const MAX_CENSUS_DIAGNOSTIC_IDENTITIES = 16; @@ -88,6 +91,22 @@ const MAX_CENSUS_DIAGNOSTIC_IDENTITIES = 16; // resulting Linux capability mask so additions and removals both require an // explicit security review. export const OPENSHELL_SUPERVISOR_CAPABILITY_MASK = "00000004a82c35fb"; +export const PODMAN_OPENSHELL_SUPERVISOR_CAPABILITY_MASK = "00000004002811cd"; + +const OPENSHELL_SUPERVISOR_CAPABILITY_MASKS = Object.freeze({ + docker: OPENSHELL_SUPERVISOR_CAPABILITY_MASK, + podman: PODMAN_OPENSHELL_SUPERVISOR_CAPABILITY_MASK, +}); + +function supervisorCapabilityMask(providerId: string): string { + const mask = OPENSHELL_SUPERVISOR_CAPABILITY_MASKS[ + providerId as keyof typeof OPENSHELL_SUPERVISOR_CAPABILITY_MASKS + ]; + if (!mask) { + throw new Error(`security-posture has no reviewed capability mask for '${providerId}'`); + } + return mask; +} export const SPLIT_PROCESS_SECURITY_PROBE = String.raw`import grp import json @@ -370,20 +389,6 @@ function probeEnv(): NodeJS.ProcessEnv { }; } -function subprocessEnvironmentIdentity(env: NodeJS.ProcessEnv): string { - return JSON.stringify( - Object.entries(env) - .filter((entry): entry is [string, string] => entry[1] !== undefined) - .sort(([left], [right]) => left.localeCompare(right)), - ); -} - -function requireStablePrivilegedDockerEnvironment(expectedIdentity: string): void { - if (subprocessEnvironmentIdentity(buildSubprocessEnv()) !== expectedIdentity) { - throw new Error("privileged Docker environment changed during security posture inspection"); - } -} - function resultText(result: Pick): string { return [result.stdout, result.stderr].filter(Boolean).join("\n"); } @@ -487,11 +492,20 @@ function requireExactSupplementaryGroups( values: string[], expected: readonly number[], label: string, + alternatives: readonly (readonly number[])[] = [], ): void { - const exact = expected.map(String).sort(); + const exact = [expected, ...alternatives].map((groupSet) => groupSet.map(String).sort()); const actual = [...values].sort(); - if (actual.length !== exact.length || actual.some((value, index) => value !== exact[index])) { - throw new Error(`${label} expected exactly ${exact.join(" ")}, got ${values.join(" ")}`); + if ( + !exact.some( + (groupSet) => + actual.length === groupSet.length && + actual.every((value, index) => value === groupSet[index]), + ) + ) { + throw new Error( + `${label} expected exactly ${exact.map((groupSet) => groupSet.join(" ")).join(" or ")}, got ${values.join(" ")}`, + ); } } @@ -509,7 +523,11 @@ function canonicalNemoclawStartSupervisorArgv(argv: string[]): boolean { ); } -function validateSupervisor(process: ProcessSecurityIdentity, sandboxGid: number): void { +function validateSupervisor( + process: ProcessSecurityIdentity, + sandboxGid: number, + expectedCapabilityMask: string, +): void { if (process.pid !== 1 || process.ppid !== 0) { throw new Error( `OpenShell supervisor expected pid=1 ppid=0, got ${process.pid}/${process.ppid}`, @@ -526,8 +544,9 @@ function validateSupervisor(process: ProcessSecurityIdentity, sandboxGid: number requireExactIds(process.status.gid, 0, "OpenShell supervisor Gid"); requireExactSupplementaryGroups( process.status.groups, - [0, sandboxGid], + [0], "OpenShell supervisor Groups", + [[0, sandboxGid]], ); for (const field of ["capInh", "capPrm", "capEff", "capBnd", "capAmb"] as const) { requireCapabilityHex(process.status[field], `OpenShell supervisor ${field}`); @@ -536,9 +555,9 @@ function validateSupervisor(process: ProcessSecurityIdentity, sandboxGid: number throw new Error(`OpenShell supervisor CapInh drifted to ${process.status.capInh}`); } for (const field of ["capPrm", "capEff", "capBnd"] as const) { - if (process.status[field] !== OPENSHELL_SUPERVISOR_CAPABILITY_MASK) { + if (process.status[field] !== expectedCapabilityMask) { throw new Error( - `OpenShell supervisor ${field} expected ${OPENSHELL_SUPERVISOR_CAPABILITY_MASK}, got ${process.status[field]}`, + `OpenShell supervisor ${field} expected ${expectedCapabilityMask}, got ${process.status[field]}`, ); } } @@ -641,7 +660,11 @@ function processIdentityArray(value: unknown, label: string): ProcessSecurityIde return value.map((entry, index) => processIdentity(entry, `${label}[${index}]`)); } -export function validateSplitProcessSecurityReport(value: unknown): SplitProcessSecurityReport { +export function validateSplitProcessSecurityReport( + value: unknown, + expectedCapabilityMask = OPENSHELL_SUPERVISOR_CAPABILITY_MASK, +): SplitProcessSecurityReport { + requireCapabilityHex(expectedCapabilityMask, "reviewed OpenShell supervisor capability mask"); const report = requiredRecord(value, "split-process security report"); if (report.version !== 2) throw new Error("split-process security report version must be 2"); const observedProcEntries = requiredInteger( @@ -668,7 +691,7 @@ export function validateSplitProcessSecurityReport(value: unknown): SplitProcess "split-process security report retained more child supervisors than observed processes", ); } - validateSupervisor(supervisor, sandboxGid); + validateSupervisor(supervisor, sandboxGid, expectedCapabilityMask); for (const process of childSupervisors) { validateNemoclawStartProcess(process, sandboxUid, sandboxGid); } @@ -701,55 +724,17 @@ export function validateSplitProcessSecurityReport(value: unknown): SplitProcess }; } -export function parseSplitProcessSecurityReport(output: string): SplitProcessSecurityReport { +export function parseSplitProcessSecurityReport( + output: string, + expectedCapabilityMask = OPENSHELL_SUPERVISOR_CAPABILITY_MASK, +): SplitProcessSecurityReport { let parsed: unknown; try { parsed = JSON.parse(output.trim()); } catch (error) { throw new Error("split-process security probe emitted invalid JSON", { cause: error }); } - return validateSplitProcessSecurityReport(parsed); -} - -export function parseOpenShellContainerId(output: string, sandboxName: string): string { - const rows = output - .split(/\r?\n/u) - .map((line) => line.trim()) - .filter(Boolean); - if (rows.length !== 1) { - throw new Error( - `expected exactly one running OpenShell Docker container for ${sandboxName}, found ${rows.length}`, - ); - } - const [id, name, sandboxId, sandboxWorkspace, ...unexpected] = rows[0]!.split("\t"); - const expectedName = `openshell-${OPENSHELL_DEFAULT_WORKSPACE}--${sandboxName}-${sandboxId}`; - if ( - !id || - !/^[0-9a-f]{64}$/u.test(id) || - !name || - !sandboxId || - !SAFE_OPENSHELL_IDENTITY_COMPONENT.test(sandboxId) || - sandboxWorkspace !== OPENSHELL_DEFAULT_WORKSPACE || - unexpected.length > 0 || - name !== expectedName - ) { - throw new Error(`unexpected OpenShell Docker container identity for ${sandboxName}`); - } - return id; -} - -export function dockerRuntimeEndpointArgs(privilegedExecArgs: readonly string[]): string[] { - if (privilegedExecArgs[0] === "exec") return []; - const dockerHost = privilegedExecArgs[1]; - if ( - privilegedExecArgs[0] !== "--host" || - !dockerHost || - /[\u0000-\u001f\u007f-\u009f]/u.test(dockerHost) || - privilegedExecArgs[2] !== "exec" - ) { - throw new Error("privileged Docker execution did not identify a supported runtime endpoint"); - } - return ["--host", dockerHost]; + return validateSplitProcessSecurityReport(parsed, expectedCapabilityMask); } export function securityPostureEnabled(): boolean { @@ -798,66 +783,45 @@ export async function assertSecurityPosture( ); requireSuccess("non-root host user", hostUser); - const privilegedExecArgv = dependencies.privilegedExecArgv ?? privilegedSandboxExecArgv; + const resolvePrivilegedTarget = + dependencies.resolvePrivilegedTarget ?? resolvePrivilegedSandboxTarget; + const executePrivilegedCommand = + dependencies.executePrivilegedCommand ?? executePrivilegedSandboxCommand; const splitProcessProbeCommand = ["/usr/bin/python3", "-I", "-c", SPLIT_PROCESS_SECURITY_PROBE]; - const privilegedDockerEnv = buildSubprocessEnv(); - const privilegedDockerEnvironmentIdentity = subprocessEnvironmentIdentity(privilegedDockerEnv); - const initialPrivilegedExecArgs = privilegedExecArgv( + const initialTarget: RuntimeProviderPrivilegedSandboxTarget = + resolvePrivilegedTarget(sandboxName); + const splitProcessProbe: RuntimeProviderPrivilegedSandboxCommandResult = executePrivilegedCommand( sandboxName, splitProcessProbeCommand, - false, - true, - ); - requireStablePrivilegedDockerEnvironment(privilegedDockerEnvironmentIdentity); - const dockerEndpointArgs = dockerRuntimeEndpointArgs(initialPrivilegedExecArgs); - - const containers = await host.command( - "docker", - [ - ...dockerEndpointArgs, - "ps", - "--no-trunc", - "--filter", - "label=openshell.ai/managed-by=openshell", - "--filter", - `label=openshell.ai/sandbox-name=${sandboxName}`, - "--format", - `{{.ID}}\t{{.Names}}\t{{.Label "${OPENSHELL_SANDBOX_ID_LABEL}"}}\t{{.Label "${OPENSHELL_SANDBOX_WORKSPACE_LABEL}"}}`, - ], { - artifactName: "security-posture-container-identity", - env: privilegedDockerEnv, - timeoutMs: 30_000, + expectedResourceHandle: initialTarget.resourceHandle, + sanitizeEnvironment: true, + timeout: 30_000, }, ); - requireSuccess("OpenShell Docker container discovery", containers); - const containerId = parseOpenShellContainerId(containers.stdout, sandboxName); - requireStablePrivilegedDockerEnvironment(privilegedDockerEnvironmentIdentity); - const finalPrivilegedExecArgs = privilegedExecArgv( - sandboxName, - splitProcessProbeCommand, - false, - true, - containerId, - ); - requireStablePrivilegedDockerEnvironment(privilegedDockerEnvironmentIdentity); - const finalDockerEndpointArgs = dockerRuntimeEndpointArgs(finalPrivilegedExecArgs); + const finalTarget = resolvePrivilegedTarget(sandboxName); if ( - finalDockerEndpointArgs.length !== dockerEndpointArgs.length || - finalDockerEndpointArgs.some((argument, index) => argument !== dockerEndpointArgs[index]) + finalTarget.providerId !== initialTarget.providerId || + finalTarget.resourceHandle !== initialTarget.resourceHandle ) { - throw new Error("container runtime endpoint changed before privileged inspection"); + throw new Error("runtime provider resource identity changed during privileged inspection"); } - const splitProcessProbe = await host.command("docker", finalPrivilegedExecArgs, { - artifactName: "security-posture-split-processes", - env: privilegedDockerEnv, - timeoutMs: 30_000, - }); - requireSuccess( - "OpenShell and nemoclaw-start child supervisor security posture", - splitProcessProbe, + if (splitProcessProbe.status !== 0 || splitProcessProbe.signal || splitProcessProbe.error) { + const detail = [ + splitProcessProbe.stdout.toString("utf8"), + splitProcessProbe.stderr.toString("utf8"), + splitProcessProbe.error?.message, + ] + .filter(Boolean) + .join("\n"); + throw new Error( + `OpenShell and nemoclaw-start child supervisor security posture failed: ${detail}`, + ); + } + const splitProcess = parseSplitProcessSecurityReport( + splitProcessProbe.stdout.toString("utf8"), + supervisorCapabilityMask(initialTarget.providerId), ); - const splitProcess = parseSplitProcessSecurityReport(splitProcessProbe.stdout); const rcFiles = await sandbox.execShell( sandboxName, diff --git a/test/e2e/fixtures/workflow-e2e-test.ts b/test/e2e/fixtures/workflow-e2e-test.ts index d10ea26c9e0..7e000ddd794 100644 --- a/test/e2e/fixtures/workflow-e2e-test.ts +++ b/test/e2e/fixtures/workflow-e2e-test.ts @@ -9,7 +9,7 @@ import { createArtifactSink } from "./artifacts.ts"; import { type ProgressPhaseOutcome, startTestProgress, type TestProgress } from "./progress.ts"; import { SecretStore } from "./secrets.ts"; -declare module "@vitest/runner" { +declare module "vitest" { interface TaskMeta { e2ePhases?: readonly string[]; } diff --git a/test/e2e/live/agent-turn-latency.test.ts b/test/e2e/live/agent-turn-latency.test.ts index 87f59e2df11..ae63ebeb357 100644 --- a/test/e2e/live/agent-turn-latency.test.ts +++ b/test/e2e/live/agent-turn-latency.test.ts @@ -56,7 +56,7 @@ runAgentTurnLatencyTest( ], }, }, - async ({ artifacts, cleanup, host, inference, progress, sandbox }) => { + async ({ artifacts, cleanup, host, inference, progress, runtimeProvider, sandbox }) => { const results: Record = { model: inference.model, maxTurnSeconds: MAX_TURN_SECONDS, @@ -106,13 +106,10 @@ runAgentTurnLatencyTest( await cleanupTurnSandbox(host, OPENCLAW_SANDBOX, "openclaw", inference, progress); }); - const docker = await host.command("docker", ["info"], { - artifactName: "docker-info", - env: buildAvailabilityProbeEnv(), - onOutput: progress.onOutput, - timeoutMs: 30_000, + await runtimeProvider.requireAvailable({ + artifactName: "runtime-info", + scenarioLabel: "agent-turn latency", }); - expect(docker.exitCode, resultText(docker)).toBe(0); const cleanBeforeRetry = () => cleanupTurnSandboxes(host, sandbox, inference, progress); await cleanupTurnSandboxes(host, sandbox, inference, progress); diff --git a/test/e2e/live/bedrock-runtime-compatible-anthropic.test.ts b/test/e2e/live/bedrock-runtime-compatible-anthropic.test.ts index 506a33524a1..664e4f2b273 100644 --- a/test/e2e/live/bedrock-runtime-compatible-anthropic.test.ts +++ b/test/e2e/live/bedrock-runtime-compatible-anthropic.test.ts @@ -1115,7 +1115,9 @@ async function assertNoBedrockLeaks(options: { expect(leaks).toEqual([]); } -test("bedrock runtime compatible Anthropic endpoint routes through managed inference.local", { +test( + "bedrock runtime compatible Anthropic endpoint routes through managed inference.local", + { timeout: TEST_TIMEOUT_MS, meta: { e2ePhases: [ @@ -1126,7 +1128,8 @@ test("bedrock runtime compatible Anthropic endpoint routes through managed infer "audit Bedrock traffic and secret isolation", ], }, -}, async ({ artifacts, cleanup, host, progress, sandbox, secrets, skip }) => { + }, + async ({ artifacts, cleanup, host, progress, runtimeProvider, sandbox, secrets, skip }) => { assertAgent(AGENT); const shard = process.env.GITHUB_ACTIONS === "true" @@ -1188,7 +1191,7 @@ test("bedrock runtime compatible Anthropic endpoint routes through managed infer sandboxName: SANDBOX_NAME, boundary: "host-bedrock-mock-source-cli-onboard-and-sandbox-exec", contracts: [ - "Docker, python3, source CLI, and OpenShell are available", + "the selected runtime, python3, source CLI, and OpenShell are available", "bedrock-runtime.us-east-1.amazonaws.com maps to the host fake endpoint", "non-interactive anthropicCompatible onboarding selects compatible-anthropic-endpoint", "OpenShell owns the hidden Bedrock adapter token while sandbox config uses inference.local", @@ -1199,19 +1202,10 @@ test("bedrock runtime compatible Anthropic endpoint routes through managed infer ], }); - const docker = await host.command("docker", ["info"], { - artifactName: "prereq-docker-info-bedrock-runtime", - env: testEnv(home), - timeoutMs: 30_000, + await runtimeProvider.requireAvailable({ + artifactName: "prereq-runtime-info-bedrock-runtime", + scenarioLabel: "Bedrock Runtime compatible Anthropic", }); - if (docker.exitCode !== 0) { - if (process.env.GITHUB_ACTIONS === "true") { - throw new Error( - `Docker is required for Bedrock Runtime compatible Anthropic E2E: ${resultText(docker)}`, - ); - } - skip("Docker is required for Bedrock Runtime compatible Anthropic E2E"); - } expectExitZero( await host.command("python3", ["--version"], { artifactName: "prereq-python-version-bedrock-runtime", @@ -1313,4 +1307,5 @@ test("bedrock runtime compatible Anthropic endpoint routes through managed infer leakScanPassed: true, }, }); -}); + }, +); diff --git a/test/e2e/live/brave-search.test.ts b/test/e2e/live/brave-search.test.ts index a39c3e3ec5a..1af9b58b7c1 100644 --- a/test/e2e/live/brave-search.test.ts +++ b/test/e2e/live/brave-search.test.ts @@ -2,7 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 import { parseOpenShellSandboxId } from "../../../src/lib/adapters/openshell/sandbox-identity.ts"; -import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import { resultText } from "../fixtures/clients/index.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; import { parseOpenClawAgentText } from "../fixtures/openclaw-agent-output.ts"; @@ -11,7 +10,6 @@ import { assertBraveConfig, assertBraveResponse, assertBraveShellCredentialBoundary, - assertDockerAvailable, cleanupBraveNemoClawSandbox, cleanupBraveState, commandEnv, @@ -24,173 +22,175 @@ import { const LIVE_TIMEOUT_MS = 35 * 60_000; -test("Brave search preset wires policy/config, performs real searches, and survives disabled-search reuse (#2687, #10404)", { - timeout: LIVE_TIMEOUT_MS, - meta: { - e2ePhases: [ - "check Brave search prerequisites", - "onboard Brave-enabled OpenClaw sandbox", - "validate Brave policy and secret isolation", - "run Brave-backed OpenClaw search", - "assert sandbox shell cannot read the real Brave key", - "query Brave API through credential resolver", - "re-onboard the existing sandbox with web search disabled", - "verify reused runtime identity and retained Brave egress", - ], +test( + "Brave search preset wires policy/config, performs real searches, and survives disabled-search reuse (#2687, #10404)", + { + timeout: LIVE_TIMEOUT_MS, + meta: { + e2ePhases: [ + "check Brave search prerequisites", + "onboard Brave-enabled OpenClaw sandbox", + "validate Brave policy and secret isolation", + "run Brave-backed OpenClaw search", + "assert sandbox shell cannot read the real Brave key", + "query Brave API through credential resolver", + "re-onboard the existing sandbox with web search disabled", + "verify reused runtime identity and retained Brave egress", + ], + }, }, -}, async ({ artifacts, cleanup, host, progress, sandbox, secrets, skip }) => { - const braveKey = secrets.required("BRAVE_API_KEY"); - const inferenceKey = secrets.required("NVIDIA_INFERENCE_API_KEY"); - const redactionValues = [braveKey, inferenceKey]; - - await artifacts.target.declare({ - id: "brave-search", - boundary: "source CLI onboard + OpenShell policy/config + in-sandbox OpenClaw/Brave API calls", - sandboxName: SANDBOX_NAME, - contracts: [ - "onboard succeeds with BRAVE_API_KEY present", - "the brave network policy preset includes api.search.brave.com", - "OpenClaw web search config is enabled and selects provider=brave", - "OpenClaw stores a BRAVE_API_KEY placeholder rather than the raw key", - "OpenClaw agent can perform a Brave-backed web search", - "BRAVE_API_KEY is absent or a placeholder in the live agent and sandbox shell environments", - "curl from inside the sandbox can query Brave using the placeholder token header", - "re-onboard reuse with web search disabled exits zero and retains the durable OpenShell sandbox identity", - "the reused OpenClaw config records web search as disabled", - "the reused Balanced-tier policy retains api.search.brave.com and can reach it", - ], - }); - - const dockerInfo = await host.command("docker", ["info"], { - artifactName: "phase-0-docker-info", - env: buildAvailabilityProbeEnv(), - timeoutMs: 30_000, - }); - assertDockerAvailable(dockerInfo, skip); - - cleanup.trackDisposable(`delete Brave search OpenShell sandbox ${SANDBOX_NAME}`, () => - sandbox.cleanupSandbox(SANDBOX_NAME, { - artifactName: "cleanup-openshell-delete-brave-search", + async ({ artifacts, cleanup, host, progress, runtimeProvider, sandbox, secrets }) => { + const braveKey = secrets.required("BRAVE_API_KEY"); + const inferenceKey = secrets.required("NVIDIA_INFERENCE_API_KEY"); + const redactionValues = [braveKey, inferenceKey]; + + await artifacts.target.declare({ + id: "brave-search", + boundary: + "source CLI onboard + OpenShell policy/config + in-sandbox OpenClaw/Brave API calls", + sandboxName: SANDBOX_NAME, + contracts: [ + "onboard succeeds with BRAVE_API_KEY present", + "the brave network policy preset includes api.search.brave.com", + "OpenClaw web search config is enabled and selects provider=brave", + "OpenClaw stores a BRAVE_API_KEY placeholder rather than the raw key", + "OpenClaw agent can perform a Brave-backed web search", + "BRAVE_API_KEY is absent or a placeholder in the live agent and sandbox shell environments", + "curl from inside the sandbox can query Brave using the placeholder token header", + "re-onboard reuse with web search disabled exits zero and retains the durable OpenShell sandbox identity", + "the reused OpenClaw config records web search as disabled", + "the reused Balanced-tier policy retains api.search.brave.com and can reach it", + ], + }); + + await runtimeProvider.requireAvailable({ + artifactName: "phase-0-runtime-info", + scenarioLabel: "Brave search", + }); + + cleanup.trackDisposable(`delete Brave search OpenShell sandbox ${SANDBOX_NAME}`, () => + sandbox.cleanupSandbox(SANDBOX_NAME, { + artifactName: "cleanup-openshell-delete-brave-search", + env: commandEnv(), + timeoutMs: 60_000, + }), + ); + cleanup.trackDisposable(`destroy Brave search sandbox ${SANDBOX_NAME}`, () => + cleanupBraveNemoClawSandbox(host), + ); + await cleanupBraveState(host, sandbox); + + progress.phase("onboard Brave-enabled OpenClaw sandbox"); + const onboard = await onboardBrave(host, braveKey, inferenceKey); + expect(onboard.exitCode, resultText(onboard)).toBe(0); + + progress.phase("validate Brave policy and secret isolation"); + const policy = await sandbox.openshell(["policy", "get", "--full", SANDBOX_NAME], { + artifactName: "phase-2-brave-policy", + env: commandEnv(), + timeoutMs: 60_000, + }); + expect(policy.exitCode, resultText(policy)).toBe(0); + expect(resultText(policy)).toContain("api.search.brave.com"); + + const config = await sandbox.exec(SANDBOX_NAME, ["cat", "/sandbox/.openclaw/openclaw.json"], { + artifactName: "phase-2-openclaw-config", env: commandEnv(), + redactionValues, timeoutMs: 60_000, - }), - ); - cleanup.trackDisposable(`destroy Brave search sandbox ${SANDBOX_NAME}`, () => - cleanupBraveNemoClawSandbox(host), - ); - await cleanupBraveState(host, sandbox); - - progress.phase("onboard Brave-enabled OpenClaw sandbox"); - const onboard = await onboardBrave(host, braveKey, inferenceKey); - expect(onboard.exitCode, resultText(onboard)).toBe(0); - - progress.phase("validate Brave policy and secret isolation"); - const policy = await sandbox.openshell(["policy", "get", "--full", SANDBOX_NAME], { - artifactName: "phase-2-brave-policy", - env: commandEnv(), - timeoutMs: 60_000, - }); - expect(policy.exitCode, resultText(policy)).toBe(0); - expect(resultText(policy)).toContain("api.search.brave.com"); - - const config = await sandbox.exec(SANDBOX_NAME, ["cat", "/sandbox/.openclaw/openclaw.json"], { - artifactName: "phase-2-openclaw-config", - env: commandEnv(), - redactionValues, - timeoutMs: 60_000, - }); - expect(config.exitCode, resultText(config)).toBe(0); - - const placeholder = assertBraveConfig(config.stdout); - - progress.phase("run Brave-backed OpenClaw search"); - const agent = await runBraveAgentWithSecretBoundaryCheck(sandbox, redactionValues); - expect(resultText(agent)).not.toMatch( - /SsrFBlockedError|Blocked hostname|ECONNREFUSED|EAI_AGAIN|gateway unavailable|network connection error/i, - ); - expect(agent.exitCode, resultText(agent)).toBe(0); - expect(parseOpenClawAgentText(agent.stdout), resultText(agent)).toMatch( - /nvidia|geforce|cuda|gpu/i, - ); - - progress.phase("assert sandbox shell cannot read the real Brave key"); - // #7425 reproduction, reframed to the real boundary. The reporter's leak came - // from the raw key being readable by the agent (a generic-typed provider - // injects it into the sandbox env), not from the model choosing to print it. - // The benign search above proves Brave still works; the checks cover the live - // agent and sandbox login-shell environment without feeding the key through - // the live LLM loop or deriving portable test material from it. - await assertBraveShellCredentialBoundary(sandbox, redactionValues); - - progress.phase("query Brave API through credential resolver"); - const curl = await sandboxShell( - sandbox, - `curl -sS --max-time 20 -G 'https://api.search.brave.com/res/v1/web/search' --data-urlencode 'q=NVIDIA' --data-urlencode 'count=1' -H 'X-Subscription-Token: ${placeholder}' -w '\nHTTP_STATUS:%{http_code}\n'`, - { artifactName: "phase-4b-direct-brave-curl", timeoutMs: 60_000, redactionValues }, - ); - assertBraveResponse(resultText(curl)); - - progress.phase("re-onboard the existing sandbox with web search disabled"); - const sandboxBeforeReuse = await sandbox.openshell(["sandbox", "get", SANDBOX_NAME], { - artifactName: "phase-5-pre-reuse-sandbox-identity", - env: commandEnv({ NEMOCLAW_RECREATE_SANDBOX: "0" }), - timeoutMs: 60_000, - }); - expect(sandboxBeforeReuse.exitCode, resultText(sandboxBeforeReuse)).toBe(0); - const sandboxIdBeforeReuse = parseOpenShellSandboxId(resultText(sandboxBeforeReuse)); - expect(sandboxIdBeforeReuse, resultText(sandboxBeforeReuse)).not.toBeNull(); - - const reuse = await reuseBraveSandboxWithWebSearchDisabled(host, inferenceKey); - expect(reuse.exitCode, resultText(reuse)).toBe(0); - - progress.phase("verify reused runtime identity and retained Brave egress"); - const sandboxAfterReuse = await sandbox.openshell(["sandbox", "get", SANDBOX_NAME], { - artifactName: "phase-6-post-reuse-sandbox-identity", - env: commandEnv({ NEMOCLAW_RECREATE_SANDBOX: "0" }), - timeoutMs: 60_000, - }); - expect(sandboxAfterReuse.exitCode, resultText(sandboxAfterReuse)).toBe(0); - expect( - parseOpenShellSandboxId(resultText(sandboxAfterReuse)), - resultText(sandboxAfterReuse), - ).toBe(sandboxIdBeforeReuse); - - const status = await host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "status"], { - artifactName: "phase-6-reused-runtime-status", - cwd: REPO_ROOT, - env: commandEnv({ NEMOCLAW_RECREATE_SANDBOX: "0" }), - timeoutMs: 60_000, - }); - expect(status.exitCode, resultText(status)).toBe(0); - - const reusedConfig = await sandbox.exec( - SANDBOX_NAME, - ["cat", "/sandbox/.openclaw/openclaw.json"], - { - artifactName: "phase-6-reused-openclaw-config", + }); + expect(config.exitCode, resultText(config)).toBe(0); + + const placeholder = assertBraveConfig(config.stdout); + + progress.phase("run Brave-backed OpenClaw search"); + const agent = await runBraveAgentWithSecretBoundaryCheck(sandbox, redactionValues); + expect(resultText(agent)).not.toMatch( + /SsrFBlockedError|Blocked hostname|ECONNREFUSED|EAI_AGAIN|gateway unavailable|network connection error/i, + ); + expect(agent.exitCode, resultText(agent)).toBe(0); + expect(parseOpenClawAgentText(agent.stdout), resultText(agent)).toMatch( + /nvidia|geforce|cuda|gpu/i, + ); + + progress.phase("assert sandbox shell cannot read the real Brave key"); + // #7425 reproduction, reframed to the real boundary. The reporter's leak came + // from the raw key being readable by the agent (a generic-typed provider + // injects it into the sandbox env), not from the model choosing to print it. + // The benign search above proves Brave still works; the checks cover the live + // agent and sandbox login-shell environment without feeding the key through + // the live LLM loop or deriving portable test material from it. + await assertBraveShellCredentialBoundary(sandbox, redactionValues); + + progress.phase("query Brave API through credential resolver"); + const curl = await sandboxShell( + sandbox, + `curl -sS --max-time 20 -G 'https://api.search.brave.com/res/v1/web/search' --data-urlencode 'q=NVIDIA' --data-urlencode 'count=1' -H 'X-Subscription-Token: ${placeholder}' -w '\nHTTP_STATUS:%{http_code}\n'`, + { artifactName: "phase-4b-direct-brave-curl", timeoutMs: 60_000, redactionValues }, + ); + assertBraveResponse(resultText(curl)); + progress.phase("re-onboard the existing sandbox with web search disabled"); + const sandboxBeforeReuse = await sandbox.openshell(["sandbox", "get", SANDBOX_NAME], { + artifactName: "phase-5-pre-reuse-sandbox-identity", env: commandEnv({ NEMOCLAW_RECREATE_SANDBOX: "0" }), timeoutMs: 60_000, - }, - ); - expect(reusedConfig.exitCode, resultText(reusedConfig)).toBe(0); - const parsedReusedConfig = JSON.parse(reusedConfig.stdout) as { - tools?: { web?: { search?: { enabled?: unknown } } }; - }; - expect(parsedReusedConfig.tools?.web?.search?.enabled, reusedConfig.stdout).toBe(false); - - const reusedPolicy = await sandbox.openshell(["policy", "get", "--full", SANDBOX_NAME], { - artifactName: "phase-6-reused-brave-policy", - env: commandEnv({ NEMOCLAW_RECREATE_SANDBOX: "0" }), - timeoutMs: 60_000, - }); - expect(reusedPolicy.exitCode, resultText(reusedPolicy)).toBe(0); - expect(resultText(reusedPolicy)).toContain("api.search.brave.com"); - - const reachable = await sandboxShell( - sandbox, - "curl -sS -o /dev/null --max-time 20 -w 'HTTP_STATUS:%{http_code}\\n' 'https://api.search.brave.com/res/v1/web/search'", - { artifactName: "phase-6-reused-brave-egress", timeoutMs: 60_000 }, - ); - expect(reachable.exitCode, resultText(reachable)).toBe(0); - expect(resultText(reachable)).toMatch(/HTTP_STATUS:(?!000)[0-9]{3}/u); -}); + }); + expect(sandboxBeforeReuse.exitCode, resultText(sandboxBeforeReuse)).toBe(0); + const sandboxIdBeforeReuse = parseOpenShellSandboxId(resultText(sandboxBeforeReuse)); + expect(sandboxIdBeforeReuse, resultText(sandboxBeforeReuse)).not.toBeNull(); + + const reuse = await reuseBraveSandboxWithWebSearchDisabled(host, inferenceKey); + expect(reuse.exitCode, resultText(reuse)).toBe(0); + + progress.phase("verify reused runtime identity and retained Brave egress"); + const sandboxAfterReuse = await sandbox.openshell(["sandbox", "get", SANDBOX_NAME], { + artifactName: "phase-6-post-reuse-sandbox-identity", + env: commandEnv({ NEMOCLAW_RECREATE_SANDBOX: "0" }), + timeoutMs: 60_000, + }); + expect(sandboxAfterReuse.exitCode, resultText(sandboxAfterReuse)).toBe(0); + expect( + parseOpenShellSandboxId(resultText(sandboxAfterReuse)), + resultText(sandboxAfterReuse), + ).toBe(sandboxIdBeforeReuse); + + const status = await host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "status"], { + artifactName: "phase-6-reused-runtime-status", + cwd: REPO_ROOT, + env: commandEnv({ NEMOCLAW_RECREATE_SANDBOX: "0" }), + timeoutMs: 60_000, + }); + expect(status.exitCode, resultText(status)).toBe(0); + + const reusedConfig = await sandbox.exec( + SANDBOX_NAME, + ["cat", "/sandbox/.openclaw/openclaw.json"], + { + artifactName: "phase-6-reused-openclaw-config", + env: commandEnv({ NEMOCLAW_RECREATE_SANDBOX: "0" }), + timeoutMs: 60_000, + }, + ); + expect(reusedConfig.exitCode, resultText(reusedConfig)).toBe(0); + const parsedReusedConfig = JSON.parse(reusedConfig.stdout) as { + tools?: { web?: { search?: { enabled?: unknown } } }; + }; + expect(parsedReusedConfig.tools?.web?.search?.enabled, reusedConfig.stdout).toBe(false); + + const reusedPolicy = await sandbox.openshell(["policy", "get", "--full", SANDBOX_NAME], { + artifactName: "phase-6-reused-brave-policy", + env: commandEnv({ NEMOCLAW_RECREATE_SANDBOX: "0" }), + timeoutMs: 60_000, + }); + expect(reusedPolicy.exitCode, resultText(reusedPolicy)).toBe(0); + expect(resultText(reusedPolicy)).toContain("api.search.brave.com"); + + const reachable = await sandboxShell( + sandbox, + "curl -sS -o /dev/null --max-time 20 -w 'HTTP_STATUS:%{http_code}\\n' 'https://api.search.brave.com/res/v1/web/search'", + { artifactName: "phase-6-reused-brave-egress", timeoutMs: 60_000 }, + ); + expect(reachable.exitCode, resultText(reachable)).toBe(0); + expect(resultText(reachable)).toMatch(/HTTP_STATUS:(?!000)[0-9]{3}/u); + }, +); diff --git a/test/e2e/live/channels-add-remove.test.ts b/test/e2e/live/channels-add-remove.test.ts index 24461ac6aca..68a428c9e1a 100644 --- a/test/e2e/live/channels-add-remove.test.ts +++ b/test/e2e/live/channels-add-remove.test.ts @@ -409,7 +409,7 @@ test( await environment.assertReady({ platform: "ubuntu-local", install: "repo-current", - runtime: "docker-running", + runtime: "managed-runtime-running", onboarding: "cloud-openclaw", }); diff --git a/test/e2e/live/channels-stop-start-helpers.ts b/test/e2e/live/channels-stop-start-helpers.ts index 112693fc520..e484b00b26f 100644 --- a/test/e2e/live/channels-stop-start-helpers.ts +++ b/test/e2e/live/channels-stop-start-helpers.ts @@ -12,7 +12,7 @@ import * as openshellRuntimeModule from "../../../src/lib/adapters/openshell/run import * as credentialProviderRegistrationModule from "../../../src/lib/onboard/credential-provider-registration.ts"; import * as messagingBridgeProviderModule from "../../../src/lib/onboard/messaging-bridge-provider.ts"; import * as legacyProvidersModule from "../../../src/lib/onboard/providers.ts"; -import { clearStoppedDockerSandboxChannelState } from "../../../src/lib/sandbox/privileged-exec.ts"; +import { clearStoppedSandboxStateRoots } from "../../../src/lib/sandbox/privileged-exec.ts"; import * as statePathsModule from "../../../src/lib/state/paths.ts"; import { assertCleanupSucceededOrAbsent, @@ -39,7 +39,7 @@ import { type AgentKind, runSecondaryCleanup as bestEffortPreclean, CLI, - dockerInfo, + requirePhase6RuntimeProvider, expectExitZero, expectSandboxReady, installSandboxOrSkipOnRateLimit, @@ -247,6 +247,9 @@ export function installGooglechatCredentialFixture( const run = dependencies.run ?? runOpenshell; const originalRegistrationUpsert = providerDependencies.upsertMessagingProviders; const originalLegacyUpsert = effectiveLegacyProviderDependencies.upsertMessagingProviders; + const delegatedUpsert = dependencies.legacyProviderDependencies + ? originalLegacyUpsert + : originalRegistrationUpsert; const fixtureUpsert: ProviderDependencies["upsertMessagingProviders"] = ( tokenDefs, @@ -267,7 +270,7 @@ export function installGooglechatCredentialFixture( const delegatedProviderNames = delegatedTokenDefs.length === 0 ? [] - : originalLegacyUpsert(delegatedTokenDefs, providerRun, options); + : delegatedUpsert(delegatedTokenDefs, providerRun, options); const baseRun = providerRun ?? run; const revalidate = () => options.revalidateSandboxIdentity?.( @@ -1068,7 +1071,7 @@ async function removeChannelsAndRebuild( expectExitZero(stop, "stop OpenClaw before WeChat cleanup"); const cleanupResult = await withLiveE2eEnvironment(env, async () => - clearStoppedDockerSandboxChannelState(SANDBOX_NAME, [ + clearStoppedSandboxStateRoots(SANDBOX_NAME, [ "/sandbox/.openclaw/wechat", "/sandbox/.openclaw/openclaw-weixin", ]), @@ -1203,6 +1206,7 @@ export async function runChannelsStopStartTarget({ cleanup, host, progress, + runtimeProvider, sandbox, secrets, skip, @@ -1252,8 +1256,7 @@ export async function runChannelsStopStartTarget({ ); await precleanProviders(host, env, redactions, `preclean-channels-stop-start-${AGENT}`); - const docker = await dockerInfo(host, env); - expect(docker.exitCode, resultText(docker)).toBe(0); + await requirePhase6RuntimeProvider(runtimeProvider, `${AGENT} channels stop/start`); progress.phase("onboard sandbox with all messaging channels"); const onboardingEnv = withoutGooglechatOnboardInputs(env); const install = await installSandboxOrSkipOnRateLimit( diff --git a/test/e2e/live/cloud-inference.test.ts b/test/e2e/live/cloud-inference.test.ts index 107fcaedfaf..62903dfa894 100644 --- a/test/e2e/live/cloud-inference.test.ts +++ b/test/e2e/live/cloud-inference.test.ts @@ -13,7 +13,6 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import type { ArtifactSink } from "../fixtures/artifacts.ts"; -import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import { assertCleanupSucceededOrAbsent } from "../fixtures/cleanup-resources.ts"; import { resultText } from "../fixtures/clients/command.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; @@ -331,7 +330,7 @@ test( ], }, }, - async ({ artifacts, cleanup, host, progress, sandbox, secrets, skip }) => { + async ({ artifacts, cleanup, host, progress, runtimeProvider, sandbox, secrets, skip }) => { const hosted = requireHostedInferenceConfig(secrets); const apiKey = hosted.apiKey; @@ -349,7 +348,7 @@ test( id: "cloud-inference", boundary: "install-sh-onboard-sandbox-inference-local-skill-filesystem", contracts: [ - "Docker is running before install/onboard", + "the selected runtime is available before install/onboard", "NVIDIA_INFERENCE_API_KEY is staged as the compatible endpoint credential", "install.sh --non-interactive creates or recreates the named OpenClaw sandbox", "nemoclaw and openshell are available on PATH after install", @@ -367,17 +366,10 @@ test( }, }); - const docker = await host.command("docker", ["info"], { - artifactName: "phase-1-docker-info-cloud-inference", - env: buildAvailabilityProbeEnv(), - timeoutMs: 30_000, + await runtimeProvider.requireAvailable({ + artifactName: "phase-1-runtime-info-cloud-inference", + scenarioLabel: "cloud inference", }); - if (docker.exitCode !== 0) { - if (process.env.GITHUB_ACTIONS === "true") { - throw new Error(`Docker is required for cloud inference E2E: ${resultText(docker)}`); - } - return skip("Docker is required for cloud inference E2E"); - } const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cloud-inference-home-")); cleanup.trackDisposable(`remove cloud inference test home ${home}`, () => @@ -460,7 +452,7 @@ test( id: "cloud-inference", status: "passed", assertions: { - dockerRunning: docker.exitCode === 0, + runtimeProviderAvailable: true, installCompleted: install.exitCode === 0, chatReturnedPong: /pong/i.test(chat.content), sandboxCredentialBoundaryValidated: true, diff --git a/test/e2e/live/cloud-onboard.test.ts b/test/e2e/live/cloud-onboard.test.ts index 5ea4a7808cf..2eb8c20f79f 100644 --- a/test/e2e/live/cloud-onboard.test.ts +++ b/test/e2e/live/cloud-onboard.test.ts @@ -111,7 +111,9 @@ function publicInstallRef(): string { return process.env.NEMOCLAW_PUBLIC_INSTALL_REF || process.env.GITHUB_SHA || "main"; } -test("cloud onboard: public installer creates healthy sandbox with security checks", { +test( + "cloud onboard: public installer creates healthy sandbox with security checks", + { timeout: LIVE_TIMEOUT_MS, meta: { e2ePhases: [ @@ -126,7 +128,16 @@ test("cloud onboard: public installer creates healthy sandbox with security chec "remove cloud sandbox", ], }, -}, async ({ artifacts, cleanup: cleanupRegistry, host, progress, sandbox, secrets, skip }) => { + }, + async ({ + artifacts, + cleanup: cleanupRegistry, + host, + progress, + runtimeProvider, + sandbox, + secrets, + }) => { const hosted = requireHostedInferenceConfig(secrets); const ref = publicInstallRef(); const installUrl = @@ -172,15 +183,10 @@ test("cloud onboard: public installer creates healthy sandbox with security chec ], }); - const docker = await host.command("docker", ["info"], { + await runtimeProvider.requireAvailable({ artifactName: "phase-0-docker-info", - env: testEnv(), - timeoutMs: 30_000, + scenarioLabel: "cloud onboarding", }); - if (docker.exitCode !== 0) { - if (process.env.GITHUB_ACTIONS === "true") throw new Error(resultText(docker)); - skip(`Docker is required: ${resultText(docker)}`); - } cleanupRegistry.trackDisposable("remove cloud-onboard sandbox", () => cleanup(host, sandbox, { home: testHome, label: "cleanup", verify: true }), @@ -235,9 +241,10 @@ test("cloud onboard: public installer creates healthy sandbox with security chec if (ref !== "main") expect(resultText(install)).toContain(`Resolved install ref: ${ref}`); progress.phase("verify migrated gateway credential"); - expect(fs.existsSync(legacyFile), "successful onboard must remove legacy credentials.json").toBe( - false, - ); + expect( + fs.existsSync(legacyFile), + "successful onboard must remove legacy credentials.json", + ).toBe(false); const providers = await host.command( "openshell", ["-g", "nemoclaw", "provider", "list", "--names"], @@ -361,4 +368,5 @@ test("cloud onboard: public installer creates healthy sandbox with security chec !providerNames.includes("OPENSHELL_GATEWAY") && !providerNames.includes("NODE_OPTIONS"), }, }); -}); + }, +); diff --git a/test/e2e/live/common-egress-agent.test.ts b/test/e2e/live/common-egress-agent.test.ts index 3866b0a0379..d51b0e4c76e 100644 --- a/test/e2e/live/common-egress-agent.test.ts +++ b/test/e2e/live/common-egress-agent.test.ts @@ -25,6 +25,7 @@ import { import { CLI_DIST_ENTRYPOINT, CLI_ENTRYPOINT, REPO_ROOT } from "../fixtures/paths.ts"; import type { SecretStore } from "../fixtures/secrets.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; +import type { RuntimeProviderPrerequisite } from "../fixtures/runtime-provider.ts"; import { assessPersonalPublicFetchToolEvidence, classifyHermesAgentAssertion, @@ -238,25 +239,18 @@ function cleanupAttempt(result: ShellProbeResult): CleanupAttempt { async function assertPrerequisites( host: HostCliClient, + runtimeProvider: RuntimeProviderPrerequisite, secrets: SecretStore, - skip: SkipFn, ): Promise { expect( fs.existsSync(CLI_DIST_ENTRYPOINT), "run `npm run build:cli` before live repo CLI targets", ).toBe(true); - const docker = await host.command("docker", ["info"], { - artifactName: "prereq-docker-info-common-egress", - env: buildAvailabilityProbeEnv(), - timeoutMs: 30_000, + await runtimeProvider.requireAvailable({ + artifactName: "prereq-runtime-info-common-egress", + scenarioLabel: "common-egress agent", }); - if (docker.exitCode !== 0) { - if (process.env.GITHUB_ACTIONS === "true") { - throw new Error(`Docker is required for common-egress agent E2E: ${text(docker)}`); - } - skip("Docker is required for common-egress agent E2E"); - } const openshell = await host.command("openshell", ["--version"], { artifactName: "prereq-openshell-version-common-egress", @@ -613,8 +607,8 @@ describe.sequential("common-egress agent live targets", () => { ], }, }, - async ({ artifacts, cleanup, host, progress, sandbox, secrets, skip }) => { - const hosted = await assertPrerequisites(host, secrets, skip); + async ({ artifacts, cleanup, host, progress, runtimeProvider, sandbox, secrets, skip }) => { + const hosted = await assertPrerequisites(host, runtimeProvider, secrets); const apiKey = hosted.apiKey; const braveApiKey = secrets.required("BRAVE_API_KEY"); await artifacts.target.declare({ @@ -755,8 +749,8 @@ After it returns, reply with only WEATHER_AGENT_OK. Do not fetch any other URL.` ], }, }, - async ({ artifacts, cleanup, host, progress, sandbox, secrets, skip }) => { - const hosted = await assertPrerequisites(host, secrets, skip); + async ({ artifacts, cleanup, host, progress, runtimeProvider, sandbox, secrets, skip }) => { + const hosted = await assertPrerequisites(host, runtimeProvider, secrets); const apiKey = hosted.apiKey; await artifacts.target.declare({ id: "common-egress-agent", @@ -814,8 +808,8 @@ After web_fetch returns, reply exactly REFERENCE_AGENT_OK if the fetched respons ], }, }, - async ({ artifacts, cleanup, host, progress, sandbox, secrets, skip }) => { - const hosted = await assertPrerequisites(host, secrets, skip); + async ({ artifacts, cleanup, host, progress, runtimeProvider, sandbox, secrets, skip }) => { + const hosted = await assertPrerequisites(host, runtimeProvider, secrets); await artifacts.target.declare({ id: "common-egress-agent", case: "hermes-open-public-reference", @@ -878,8 +872,8 @@ After web_fetch returns, reply exactly REFERENCE_AGENT_OK if the fetched respons ], }, }, - async ({ artifacts, cleanup, host, progress, sandbox, secrets, skip }) => { - const hosted = await assertPrerequisites(host, secrets, skip); + async ({ artifacts, cleanup, host, progress, runtimeProvider, sandbox, secrets, skip }) => { + const hosted = await assertPrerequisites(host, runtimeProvider, secrets); const apiKey = hosted.apiKey; await artifacts.target.declare({ id: "common-egress-agent", diff --git a/test/e2e/live/cron-preflight-inference-local.test.ts b/test/e2e/live/cron-preflight-inference-local.test.ts index 0b80b87016a..26482db0ce1 100644 --- a/test/e2e/live/cron-preflight-inference-local.test.ts +++ b/test/e2e/live/cron-preflight-inference-local.test.ts @@ -193,7 +193,9 @@ async function preCleanCronSandbox(sandbox: SandboxClient): Promise { ); } -test("cron preflight reaches managed inference.local provider without EAI_AGAIN", { +test( + "cron preflight reaches managed inference.local provider without EAI_AGAIN", + { timeout: LIVE_TIMEOUT_MS, meta: { e2ePhases: [ @@ -203,7 +205,8 @@ test("cron preflight reaches managed inference.local provider without EAI_AGAIN" "validate managed route availability", ], }, -}, async ({ artifacts, cleanup, host, progress, sandbox, secrets, skip }) => { + }, + async ({ artifacts, cleanup, host, progress, runtimeProvider, sandbox, secrets }) => { const hosted = requireHostedInferenceConfig(secrets, process.env, { model: MODEL }); const apiKey = hosted.apiKey; @@ -221,17 +224,10 @@ test("cron preflight reaches managed inference.local provider without EAI_AGAIN" ], }); - const dockerInfo = await host.command("docker", ["info"], { - artifactName: "phase-0-docker-info", - env: buildAvailabilityProbeEnv(), - timeoutMs: 30_000, + await runtimeProvider.requireAvailable({ + artifactName: "phase-0-runtime-info", + scenarioLabel: "cron preflight", }); - if (dockerInfo.exitCode !== 0) { - if (process.env.GITHUB_ACTIONS === "true") { - throw new Error(`Docker is required for cron preflight E2E: ${resultText(dockerInfo)}`); - } - skip(`Docker is required for cron preflight E2E: ${resultText(dockerInfo)}`); - } const cleanupEnv = commandEnv(); cleanup.trackDisposable(`delete OpenShell sandbox ${SANDBOX_NAME}`, () => @@ -302,4 +298,5 @@ test("cron preflight reaches managed inference.local provider without EAI_AGAIN" expect(probe.exitCode, output).toBe(0); expect(parsed?.result?.status, output).toBe("available"); expect(parsed?.baseUrl, output).toBe("https://inference.local/v1"); -}); + }, +); diff --git a/test/e2e/live/dashboard-remote-bind.test.ts b/test/e2e/live/dashboard-remote-bind.test.ts index c8651fdfdb3..0fb70935014 100644 --- a/test/e2e/live/dashboard-remote-bind.test.ts +++ b/test/e2e/live/dashboard-remote-bind.test.ts @@ -3,7 +3,6 @@ import os from "node:os"; -import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import { resultText } from "../fixtures/clients/command.ts"; import { sandboxAccessEnv, trustedSandboxShellScript } from "../fixtures/clients/sandbox.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; @@ -71,7 +70,7 @@ runDashboardRemoteBindTest( ], }, }, - async ({ artifacts, cleanup, host, progress, sandbox, secrets, skip }) => { + async ({ artifacts, cleanup, host, progress, runtimeProvider, sandbox, secrets }) => { const sandboxName = SANDBOX_NAME; const dashboardPort = process.env.NEMOCLAW_DASHBOARD_PORT || "18789"; const remoteHost = remoteHostCandidate(); @@ -93,17 +92,10 @@ runDashboardRemoteBindTest( ], }); - const docker = await host.command("docker", ["info"], { - artifactName: "dashboard-remote-bind-docker-info", - env: buildAvailabilityProbeEnv(), - timeoutMs: 30_000, + await runtimeProvider.requireAvailable({ + artifactName: "dashboard-remote-bind-runtime-info", + scenarioLabel: "dashboard remote-bind", }); - if (docker.exitCode !== 0) { - if (process.env.GITHUB_ACTIONS === "true") { - throw new Error(`Docker is required for dashboard remote-bind E2E: ${resultText(docker)}`); - } - skip("Docker is required for dashboard remote-bind E2E"); - } cleanup.trackGateway(host, "nemoclaw", { artifactName: "dashboard-remote-bind-cleanup-gateway", diff --git a/test/e2e/live/device-auth-health.test.ts b/test/e2e/live/device-auth-health.test.ts index 2b0871f31fe..4b1f1ceada3 100644 --- a/test/e2e/live/device-auth-health.test.ts +++ b/test/e2e/live/device-auth-health.test.ts @@ -14,7 +14,6 @@ import { trustedSandboxShellScript } from "../fixtures/clients/sandbox.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; import { startFakeOpenAiCompatibleServer } from "../fixtures/fake-openai-compatible.ts"; import { - assertDockerAvailable, cleanupDeviceAuthSandbox, commandEnv, DASHBOARD_PORT, @@ -35,7 +34,9 @@ function assertStatusNotOffline(output: string, context: string): void { ); } -test("device auth health probes treat 401 as live instead of offline (#2342)", { +test( + "device auth health probes treat 401 as live instead of offline (#2342)", + { timeout: LIVE_TIMEOUT_MS, meta: { e2ePhases: [ @@ -45,7 +46,8 @@ test("device auth health probes treat 401 as live instead of offline (#2342)", { "recover stopped OpenClaw gateway", ], }, -}, async ({ artifacts, cleanup, host, progress, sandbox, skip }) => { + }, + async ({ artifacts, cleanup, host, progress, runtimeProvider, sandbox }) => { const installLog = artifacts.pathFor("phase-1-install-device-auth-health.log"); // The sandbox cannot reach runner loopback, so expose the fixture through // OpenShell's host bridge while keeping readiness checks local to the runner. @@ -82,12 +84,10 @@ test("device auth health probes treat 401 as live instead of offline (#2342)", { ], }); - const dockerInfo = await host.command("docker", ["info"], { - artifactName: "phase-0-docker-info", - env: buildAvailabilityProbeEnv(), - timeoutMs: 30_000, + await runtimeProvider.requireAvailable({ + artifactName: "phase-0-runtime-info", + scenarioLabel: "device auth health", }); - assertDockerAvailable(dockerInfo, skip); const cleanupEnv = commandEnv(); cleanup.trackDisposable(`delete OpenShell sandbox ${SANDBOX_NAME}`, () => @@ -224,4 +224,5 @@ test("device auth health probes treat 401 as live instead of offline (#2342)", { expect(recoveryStatus.exitCode, resultText(recoveryStatus)).toBe(0); assertStatusNotOffline(resultText(recoveryStatus), "recovery status"); await waitForRecoveryArtifact(artifacts, sandbox); -}); + }, +); diff --git a/test/e2e/live/dns-rebinding-hosts-fixture.ts b/test/e2e/live/dns-rebinding-hosts-fixture.ts index 8f5c70c15c3..14412f3afa3 100644 --- a/test/e2e/live/dns-rebinding-hosts-fixture.ts +++ b/test/e2e/live/dns-rebinding-hosts-fixture.ts @@ -7,6 +7,7 @@ import path from "node:path"; import { shellQuote } from "../../../src/lib/core/shell-quote"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; +import { RuntimeProviderPrerequisite } from "../fixtures/runtime-provider.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; export interface DnsRebindingHostsFixture { @@ -20,6 +21,12 @@ function assertHostFixtureProbeSucceeded(result: ShellProbeResult, label: string throw new Error(`${label}\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`); } +function runtimeInvocation(host: HostCliClient) { + return new RuntimeProviderPrerequisite(host, (reason) => { + throw new Error(reason); + }).hostInvocation([]); +} + export async function setupDnsRebindingHostsFixture( host: HostCliClient, sandboxName: string, @@ -32,24 +39,29 @@ export async function setupDnsRebindingHostsFixture( hostBackupPath: path.join(tempDir, `nemoclaw-rebind-hosts-host-${suffix}`), sandboxBackupPath: path.join(tempDir, `nemoclaw-rebind-hosts-sandbox-${suffix}`), }; + const runtime = runtimeInvocation(host); const result = await host.command( "bash", [ "-lc", [ "set -euo pipefail", + 'runtime_command=("$@")', `sandbox_name=${shellQuote(sandboxName)}`, `hostname=${shellQuote(hostname)}`, `host_backup=${shellQuote(fixture.hostBackupPath)}`, `sandbox_backup=${shellQuote(fixture.sandboxBackupPath)}`, - 'container_id="$(docker ps --filter "label=openshell.ai/sandbox-name=${sandbox_name}" --format \'{{.ID}}\' | head -n 1)"', + 'container_id="$("${runtime_command[@]}" container ps --filter "label=openshell.ai/sandbox-name=${sandbox_name}" --format \'{{.ID}}\' | head -n 1)"', '[ -n "$container_id" ] || { echo "OpenShell sandbox container not found" >&2; exit 1; }', "sudo -n true", 'rm -f "$host_backup" "$sandbox_backup"', 'sudo -n cat /etc/hosts > "$host_backup"', - 'docker exec "$container_id" cat /etc/hosts > "$sandbox_backup"', + '"${runtime_command[@]}" container exec "$container_id" cat /etc/hosts > "$sandbox_backup"', 'if grep -Fq "$hostname" "$host_backup" || grep -Fq "$hostname" "$sandbox_backup"; then rm -f "$host_backup" "$sandbox_backup"; echo "DNS rebinding fixture hostname already exists in /etc/hosts" >&2; exit 1; fi', ].join("\n"), + "dns-rebinding-backup-hosts", + runtime.command, + ...runtime.args, ], { artifactName: "dns-rebinding-backup-hosts", @@ -81,27 +93,32 @@ export async function remapDnsRebindingHostname( " process.exit(addresses.length === 1 && addresses[0] === expected ? 0 : 1);", "});", ].join(" "); + const runtime = runtimeInvocation(host); const result = await host.command( "bash", [ "-lc", [ "set -euo pipefail", + 'runtime_command=("$@")', `sandbox_name=${shellQuote(sandboxName)}`, `hostname=${shellQuote(fixture.hostname)}`, `expected_ip=${shellQuote(address)}`, `host_backup=${shellQuote(fixture.hostBackupPath)}`, `sandbox_backup=${shellQuote(fixture.sandboxBackupPath)}`, '[ -s "$host_backup" ] && [ -s "$sandbox_backup" ] || { echo "DNS rebinding hosts backups are missing" >&2; exit 1; }', - 'container_id="$(docker ps --filter "label=openshell.ai/sandbox-name=${sandbox_name}" --format \'{{.ID}}\' | head -n 1)"', + 'container_id="$("${runtime_command[@]}" container ps --filter "label=openshell.ai/sandbox-name=${sandbox_name}" --format \'{{.ID}}\' | head -n 1)"', '[ -n "$container_id" ] || { echo "OpenShell sandbox container not found" >&2; exit 1; }', 'sudo -n tee /etc/hosts < "$host_backup" >/dev/null', 'printf "\\n%s %s\\n" "$expected_ip" "$hostname" | sudo -n tee -a /etc/hosts >/dev/null', - 'docker exec --user 0 -i "$container_id" sh -c \'cat > /etc/hosts\' < "$sandbox_backup"', - 'printf "\\n%s %s\\n" "$expected_ip" "$hostname" | docker exec --user 0 -i "$container_id" tee -a /etc/hosts >/dev/null', + '"${runtime_command[@]}" container exec --user 0 --interactive "$container_id" sh -c \'cat > /etc/hosts\' < "$sandbox_backup"', + 'printf "\\n%s %s\\n" "$expected_ip" "$hostname" | "${runtime_command[@]}" container exec --user 0 --interactive "$container_id" tee -a /etc/hosts >/dev/null', `node -e ${shellQuote(resolverCheck)} "$hostname" "$expected_ip"`, - 'docker exec "$container_id" grep -F "$expected_ip $hostname" /etc/hosts >/dev/null', + '"${runtime_command[@]}" container exec "$container_id" grep -F "$expected_ip $hostname" /etc/hosts >/dev/null', ].join("\n"), + "dns-rebinding-remap-hosts", + runtime.command, + ...runtime.args, ], { artifactName, @@ -117,6 +134,7 @@ export async function restoreDnsRebindingHostsFixture( sandboxName: string, fixture: DnsRebindingHostsFixture, ): Promise { + const runtime = runtimeInvocation(host); const result = await host.command( "bash", [ @@ -126,6 +144,7 @@ export async function restoreDnsRebindingHostsFixture( // here can turn a transient file/container race into an unexplained // exit 1 with empty stdout/stderr, which defeats the cleanup artifact. "set -uo pipefail", + 'runtime_command=("$@")', `sandbox_name=${shellQuote(sandboxName)}`, `host_backup=${shellQuote(fixture.hostBackupPath)}`, `sandbox_backup=${shellQuote(fixture.sandboxBackupPath)}`, @@ -145,8 +164,8 @@ export async function restoreDnsRebindingHostsFixture( 'if [ -f "$sandbox_backup" ]; then', " sandbox_restored=0", " for attempt in 1 2 3; do", - ' container_id="$(docker ps --filter "label=openshell.ai/sandbox-name=${sandbox_name}" --format \'{{.ID}}\' 2>/dev/null | head -n 1 || true)"', - ' if [ -n "$container_id" ] && docker exec --user 0 -i "$container_id" sh -c \'cat > /etc/hosts\' < "$sandbox_backup"; then sandbox_restored=1; break; fi', + ' container_id="$("${runtime_command[@]}" container ps --filter "label=openshell.ai/sandbox-name=${sandbox_name}" --format \'{{.ID}}\' 2>/dev/null | head -n 1 || true)"', + ' if [ -n "$container_id" ] && "${runtime_command[@]}" container exec --user 0 --interactive "$container_id" sh -c \'cat > /etc/hosts\' < "$sandbox_backup"; then sandbox_restored=1; break; fi', ' [ "$attempt" -eq 3 ] || sleep 1', " done", ' if [ "$sandbox_restored" -eq 1 ]; then echo "restored sandbox /etc/hosts"; else echo "::warning::could not restore ephemeral sandbox /etc/hosts; cleanup will destroy the sandbox" >&2; fi', @@ -156,6 +175,9 @@ export async function restoreDnsRebindingHostsFixture( 'echo "removed DNS rebinding hosts backups"', "exit 0", ].join("\n"), + "dns-rebinding-restore-hosts", + runtime.command, + ...runtime.args, ], { artifactName: "dns-rebinding-restore-hosts", diff --git a/test/e2e/live/double-onboard.test.ts b/test/e2e/live/double-onboard.test.ts index 959cede60bb..f59cc6c6af0 100644 --- a/test/e2e/live/double-onboard.test.ts +++ b/test/e2e/live/double-onboard.test.ts @@ -7,11 +7,13 @@ import path from "node:path"; import { parseOpenShellSandboxId } from "../../../src/lib/adapters/openshell/sandbox-identity.ts"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import { resultText } from "../fixtures/clients/command.ts"; +import type { GatewayClient } from "../fixtures/clients/gateway.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; import type { SandboxClient } from "../fixtures/clients/sandbox.ts"; import { validateSandboxName } from "../fixtures/clients/sandbox.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; import { startFakeOpenAiCompatibleServer } from "../fixtures/fake-openai-compatible.ts"; +import type { LifecyclePhaseFixture } from "../fixtures/phases/lifecycle.ts"; import { CLI_DIST_ENTRYPOINT, CLI_ENTRYPOINT } from "../fixtures/paths.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; @@ -143,6 +145,7 @@ async function runProbeOnlyConnect( async function cleanupDoubleOnboardState( host: HostCliClient, + lifecycle: LifecyclePhaseFixture, sandbox: SandboxClient, ): Promise { const names = [INSTALL_SANDBOX_NAME, SANDBOX_A, SANDBOX_B].filter(Boolean); @@ -171,7 +174,7 @@ async function cleanupDoubleOnboardState( timeoutMs: 30_000, }), ); - await stopGatewayRuntime(host, "cleanup-stop-gateway-runtime"); + await lifecycle.stopGatewayRuntime(); await ignoreCleanupError(() => sandbox.openshell(["gateway", "destroy", "-g", "nemoclaw"], { artifactName: "cleanup-openshell-gateway-destroy-nemoclaw", @@ -188,63 +191,11 @@ async function cleanupDoubleOnboardState( ); } -async function gatewayRuntimeId(host: HostCliClient, artifactName: string): Promise { - const script = String.raw` -set -euo pipefail -pid_file="$HOME/.local/state/nemoclaw/openshell-docker-gateway/openshell-gateway.pid" -if [ -f "$pid_file" ]; then - pid="$(tr -d '[:space:]' <"$pid_file" 2>/dev/null || true)" - if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then - printf 'pid:%s\n' "$pid" - exit 0 - fi -fi -cid="$(docker ps -qf "name=openshell-cluster-nemoclaw" 2>/dev/null | head -1)" -if [ -n "$cid" ]; then - printf 'container:%s\n' "$cid" - exit 0 -fi -exit 1 -`; - const result = await host.command("bash", ["-lc", script], { - artifactName, - env: commandEnv(), - timeoutMs: 30_000, - }); - const observedRuntimeId = result.stdout - .split("\n") - .map((line) => line.trim()) - .find((line) => /^(pid|container):/.test(line)); - return observedRuntimeId ?? (result.exitCode === 0 ? result.stdout.trim() : ""); -} - -async function stopGatewayRuntime(host: HostCliClient, artifactName: string): Promise { - const script = String.raw` -set +e -openshell forward stop 18789 >/dev/null 2>&1 -openshell gateway stop -g nemoclaw >/dev/null 2>&1 -pid_file="$HOME/.local/state/nemoclaw/openshell-docker-gateway/openshell-gateway.pid" -if [ -f "$pid_file" ]; then - pid="$(tr -d '[:space:]' <"$pid_file" 2>/dev/null || true)" - if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then - kill "$pid" >/dev/null 2>&1 || true - for _ in $(seq 1 10); do - kill -0 "$pid" 2>/dev/null || break - sleep 1 - done - kill -0 "$pid" 2>/dev/null && kill -9 "$pid" >/dev/null 2>&1 || true - fi -fi -cid="$(docker ps -qf "name=openshell-cluster-nemoclaw" 2>/dev/null | head -1)" -[ -n "$cid" ] && docker stop "$cid" >/dev/null 2>&1 || true -exit 0 -`; - const stop = await host.command("bash", ["-lc", script], { - artifactName, - env: commandEnv(), - timeoutMs: 60_000, - }); - expect(stop.exitCode, resultText(stop)).toBe(0); +async function gatewayRuntimeId( + gateway: GatewayClient, +): Promise { + const runtime = await gateway.resolveHostRuntime(); + return runtime?.kind === "container" ? `${runtime.kind}:${runtime.id}` : (runtime?.kind ?? ""); } function gatewayAliasEndpoint(): string { @@ -296,7 +247,11 @@ async function waitForForwardOwner( port: string, owner: string | undefined, artifactPrefix: string, -): Promise<{ owner: string | undefined; output: string; querySucceeded: boolean }> { +): Promise<{ + owner: string | undefined; + output: string; + querySucceeded: boolean; +}> { let observedOwner: string | undefined; let lastOutput = ""; let querySucceeded = false; @@ -447,29 +402,41 @@ async function prerequisiteOrSkip( skip(message); } -test("double-onboard: reuses gateway, preserves sibling sandbox, and replaces stale registry", { - timeout: TEST_TIMEOUT_MS, - meta: { - e2ePhases: [ - "validate double-onboard lifecycle prerequisites", - "onboard first sandbox", - "re-onboard same sandbox on existing gateway", - "recreate same sandbox on existing gateway", - "onboard sibling sandbox with isolated dashboard", - "stop sibling sandbox without disturbing the first forward", - "replace sandbox after stale registry refusal", - "validate gateway-stop lifecycle guidance", - "remove double-onboard resources", - ], +test( + "double-onboard: reuses gateway, preserves sibling sandbox, and replaces stale registry", + { + timeout: TEST_TIMEOUT_MS, + meta: { + e2ePhases: [ + "validate double-onboard lifecycle prerequisites", + "onboard first sandbox", + "re-onboard same sandbox on existing gateway", + "recreate same sandbox on existing gateway", + "onboard sibling sandbox with isolated dashboard", + "stop sibling sandbox without disturbing the first forward", + "replace sandbox after stale registry refusal", + "validate gateway-stop lifecycle guidance", + "remove double-onboard resources", + ], + }, }, -}, async ({ artifacts, cleanup, host, progress, sandbox, skip }) => { + async ({ artifacts, cleanup, gateway, host, lifecycle, progress, runtimeProvider, sandbox, skip }) => { expect( fs.existsSync(CLI_DIST_ENTRYPOINT), "run `npm run build:cli` before live repo CLI targets", ).toBe(true); - await prerequisiteOrSkip(host, skip, "docker", ["info"], "prereq-docker-info"); - await prerequisiteOrSkip(host, skip, "bash", ["-lc", "command -v openshell"], "prereq-openshell"); + await runtimeProvider.requireAvailable({ + artifactName: "prereq-runtime-info", + scenarioLabel: "double-onboard", + }); + await prerequisiteOrSkip( + host, + skip, + "bash", + ["-lc", "command -v openshell"], + "prereq-openshell", + ); await prerequisiteOrSkip( host, skip, @@ -501,9 +468,9 @@ test("double-onboard: reuses gateway, preserves sibling sandbox, and replaces st env: commandEnv(), timeoutMs: 60_000, }); - cleanup.trackDisposable("stop double-onboard gateway runtime", () => - stopGatewayRuntime(host, "cleanup-stop-gateway-runtime"), - ); + cleanup.trackDisposable("stop double-onboard gateway runtime", async () => { + await lifecycle.stopGatewayRuntime(); + }); cleanup.trackForward(host, 18789, { artifactName: "cleanup-openshell-forward-stop-18789", env: commandEnv(), @@ -540,7 +507,7 @@ test("double-onboard: reuses gateway, preserves sibling sandbox, and replaces st ], }); - await cleanupDoubleOnboardState(host, sandbox); + await cleanupDoubleOnboardState(host, lifecycle, sandbox); progress.phase("onboard first sandbox"); // Phase 2: first onboard. @@ -588,13 +555,13 @@ test("double-onboard: reuses gateway, preserves sibling sandbox, and replaces st progress.phase("re-onboard same sandbox on existing gateway"); // Phase 3: second onboard with the same name must reuse the healthy gateway. - const gatewayBeforeSecond = await gatewayRuntimeId(host, "phase-3-gateway-id-before"); + const gatewayBeforeSecond = await gatewayRuntimeId(gateway); await artifacts.writeJson("phase-3-registry-before-second.json", registryEntry(SANDBOX_A)); const second = await runOnboard(host, SANDBOX_A, fake.baseUrl, "phase-3-second-onboard"); await artifacts.writeJson("phase-3-registry-after-second.json", registryEntry(SANDBOX_A)); const secondText = resultText(second); expect(second.exitCode, secondText).toBe(0); - const gatewayAfterSecond = await gatewayRuntimeId(host, "phase-3-gateway-id-after"); + const gatewayAfterSecond = await gatewayRuntimeId(gateway); expect(gatewayBeforeSecond, "gateway runtime id before second onboard").not.toBe(""); expect(gatewayAfterSecond).toBe(gatewayBeforeSecond); expect(secondText).toContain("Reusing healthy NemoClaw gateway."); @@ -621,7 +588,7 @@ test("double-onboard: reuses gateway, preserves sibling sandbox, and replaces st expect(stripAnsi(listAfterSecond.stdout)).toContain(SANDBOX_A); progress.phase("recreate same sandbox on existing gateway"); - const gatewayBeforeRecreate = await gatewayRuntimeId(host, "phase-3-recreate-gateway-id-before"); + const gatewayBeforeRecreate = await gatewayRuntimeId(gateway); const recreated = await runOnboard( host, SANDBOX_A, @@ -631,9 +598,7 @@ test("double-onboard: reuses gateway, preserves sibling sandbox, and replaces st ); const recreatedText = resultText(recreated); expect(recreated.exitCode, recreatedText).toBe(0); - expect(await gatewayRuntimeId(host, "phase-3-recreate-gateway-id-after")).toBe( - gatewayBeforeRecreate, - ); + expect(await gatewayRuntimeId(gateway)).toBe(gatewayBeforeRecreate); expect(recreatedText).not.toContain("Port 8080 is not available"); expect(recreatedText).not.toContain("Port 18789 is not available"); @@ -653,22 +618,12 @@ test("double-onboard: reuses gateway, preserves sibling sandbox, and replaces st timeoutMs: 30_000, }); expect(selectAlt.exitCode, resultText(selectAlt)).toBe(0); - const selectedAlt = await host.command( - "bash", - ["-lc", "openshell status 2>&1 || true; openshell gateway info 2>&1 || true"], - { - artifactName: "phase-4-selected-alt-gateway", - env: commandEnv(), - timeoutMs: 30_000, - }, - ); - expect(gatewayNameFromOutput(resultText(selectedAlt))).toBe(ALT_GATEWAY_NAME); - const gatewayBeforeThird = await gatewayRuntimeId(host, "phase-4-gateway-id-before"); + const gatewayBeforeThird = await gatewayRuntimeId(gateway); const third = await runOnboard(host, SANDBOX_B, fake.baseUrl, "phase-4-third-onboard"); const thirdText = resultText(third); expect(third.exitCode, thirdText).toBe(0); - const gatewayAfterThird = await gatewayRuntimeId(host, "phase-4-gateway-id-after"); + const gatewayAfterThird = await gatewayRuntimeId(gateway); expect(gatewayBeforeThird, "gateway runtime id before third onboard").not.toBe(""); expect(gatewayAfterThird).toBe(gatewayBeforeThird); expect(thirdText).not.toContain("Port 8080 is not available"); @@ -887,7 +842,8 @@ test("double-onboard: reuses gateway, preserves sibling sandbox, and replaces st env: commandEnv(), timeoutMs: 30_000, }); - await stopGatewayRuntime(host, "phase-6-stop-gateway-runtime"); + await lifecycle.stopGatewayRuntime(); + await gateway.expectHostRuntimeStopped({ artifactName: "phase-6-gateway-runtime-stopped" }); const postStopStatus = await command(host, [SANDBOX_B, "status"], { artifactName: "phase-6-status-after-gateway-stop", env: commandEnv(), @@ -898,11 +854,13 @@ test("double-onboard: reuses gateway, preserves sibling sandbox, and replaces st expect(postStopText).toMatch( /Recovered NemoClaw gateway runtime|gateway is no longer configured after restart\/rebuild|gateway is still refusing connections after restart|gateway trust material rotated after restart/, ); - expect(registryHas(SANDBOX_B), "gateway-stop status removed sandbox B registry entry").toBe(true); + expect(registryHas(SANDBOX_B), "gateway-stop status removed sandbox B registry entry").toBe( + true, + ); progress.phase("remove double-onboard resources"); // Phase 7: final cleanup with explicit assertions. - await cleanupDoubleOnboardState(host, sandbox); + await cleanupDoubleOnboardState(host, lifecycle, sandbox); const sandboxAAfterCleanup = await sandbox.openshell(["sandbox", "get", SANDBOX_A], { artifactName: "phase-7-openshell-sandbox-a-after-cleanup", env: commandEnv(), @@ -948,4 +906,5 @@ test("double-onboard: reuses gateway, preserves sibling sandbox, and replaces st ), }, }); -}); + }, +); diff --git a/test/e2e/live/full-e2e.test.ts b/test/e2e/live/full-e2e.test.ts index 4d6509e5fa9..0f611d1ec11 100644 --- a/test/e2e/live/full-e2e.test.ts +++ b/test/e2e/live/full-e2e.test.ts @@ -31,6 +31,7 @@ import { } from "../fixtures/onboard-performance.ts"; import { CLI_ENTRYPOINT, REPO_ROOT } from "../fixtures/paths.ts"; import { pollUntil } from "../fixtures/polling.ts"; +import { ensureConfiguredRuntimeProviderAvailable } from "../fixtures/runtime-provider.ts"; import { assertSecurityPosture, securityPostureEnabled, @@ -425,15 +426,12 @@ test("full e2e: install, onboard, inference, cli operations, and cleanup", { ], }); - const docker = await host.command("docker", ["info"], { - artifactName: "phase-0-docker-info", - env: env(), - timeoutMs: 30_000, + await ensureConfiguredRuntimeProviderAvailable({ + artifactName: "phase-0-runtime-provider-info", + host, + scenarioLabel: FULL_E2E_TARGET_ID, + skip, }); - if (docker.exitCode !== 0) { - if (process.env.GITHUB_ACTIONS === "true") throw new Error(resultText(docker)); - skip(`Docker is required: ${resultText(docker)}`); - } cleanupRegistry.trackGateway(host, "nemoclaw", { artifactName: "cleanup-openshell-gateway-destroy", diff --git a/test/e2e/live/gpu-double-onboard.test.ts b/test/e2e/live/gpu-double-onboard.test.ts index 25896d0e65d..9e5d97d2ba3 100644 --- a/test/e2e/live/gpu-double-onboard.test.ts +++ b/test/e2e/live/gpu-double-onboard.test.ts @@ -156,11 +156,13 @@ async function expectSandboxInference42( expect(containsAnswer(response.stdout, "42"), resultText(response)).toBe(true); } -test("gpu double onboard keeps Ollama auth proxy token consistent after re-onboard", { +test( + "gpu double onboard keeps Ollama auth proxy token consistent after re-onboard", + { timeout: LIVE_TIMEOUT_MS, meta: { e2ePhases: [ - "validate GPU and Docker prerequisites", + "validate GPU and runtime prerequisites", "install Ollama runtime", "perform first Ollama onboard", "validate first proxy token and inference", @@ -169,13 +171,22 @@ test("gpu double onboard keeps Ollama auth proxy token consistent after re-onboa "remove GPU double-onboard sandbox", ], }, -}, async ({ artifacts, cleanup: cleanupRegistry, host, progress, sandbox, skip }) => { + }, + async ({ + artifacts, + cleanup: cleanupRegistry, + host, + progress, + runtimeProvider, + sandbox, + skip, + }) => { await artifacts.target.declare({ id: "gpu-double-onboard", sandboxName: SANDBOX_NAME, proxyPort: PROXY_PORT, contracts: [ - "GPU and Docker prerequisites are present", + "GPU and runtime prerequisites are present", "install.sh onboards with the Ollama provider", "the persisted Ollama auth-proxy token works after first onboard", "nemoclaw onboard --non-interactive --yes recreates the sandbox", @@ -184,15 +195,10 @@ test("gpu double onboard keeps Ollama auth proxy token consistent after re-onboa ], }); - const docker = await host.command("docker", ["info"], { - artifactName: "phase-0-docker-info", - env: env(), - timeoutMs: 30_000, + await runtimeProvider.requireAvailable({ + artifactName: "phase-0-runtime-info", + scenarioLabel: "GPU double-onboard", }); - if (docker.exitCode !== 0) { - if (process.env.GITHUB_ACTIONS === "true") throw new Error(resultText(docker)); - skip(`Docker is required: ${resultText(docker)}`); - } const smi = await host.command("nvidia-smi", [], { artifactName: "phase-0-nvidia-smi", env: env(), @@ -399,4 +405,5 @@ exit "$status"`, id: "gpu-double-onboard", status: "passed", }); -}); + }, +); diff --git a/test/e2e/live/gpu-e2e.test.ts b/test/e2e/live/gpu-e2e.test.ts index e17d42d43f3..0d514a868c5 100644 --- a/test/e2e/live/gpu-e2e.test.ts +++ b/test/e2e/live/gpu-e2e.test.ts @@ -59,7 +59,7 @@ test( ], }, }, - async ({ artifacts, cleanup, host, progress, sandbox, skip }) => { + async ({ artifacts, cleanup, host, progress, runtimeProvider, sandbox, skip }) => { await artifacts.target.declare({ id: "gpu-e2e", boundary: @@ -98,12 +98,10 @@ test( }); await cleanupGpu(host, sandbox); - const docker = await host.command("docker", ["info"], { - artifactName: "docker-info", - env: buildAvailabilityProbeEnv(), - timeoutMs: 30_000, + await runtimeProvider.requireAvailable({ + artifactName: "runtime-info", + scenarioLabel: "GPU", }); - expect(docker.exitCode, resultText(docker)).toBe(0); const nvidia = await host.command("nvidia-smi", [], { artifactName: "nvidia-smi", env: buildAvailabilityProbeEnv(), @@ -146,15 +144,15 @@ test( ); expect(installLog).not.toContain("Docker GPU mode selected"); - const sandboxContainers = await host.command( - "docker", + const sandboxContainers = await runtimeProvider.command( [ + "container", "ps", - "-a", + "--all", "--filter", `label=openshell.ai/sandbox-name=${SANDBOX_NAME}`, "--format", - "{{json .}}", + "{{.Names}}\t{{.State}}\t{{.Status}}", ], { artifactName: "gpu-native-route-sandbox-containers", @@ -167,14 +165,10 @@ test( .split(/\r?\n/) .map((line) => line.trim()) .filter(Boolean) - .map( - (line) => - JSON.parse(line) as { - Names?: string; - State?: string; - Status?: string; - }, - ); + .map((line) => { + const [Names = "", State = "", Status = ""] = line.split("\t"); + return { Names, State, Status }; + }); expect( sandboxContainerInventory, `native GPU route must retain exactly one sandbox container; got ${sandboxContainers.stdout}`, @@ -350,7 +344,7 @@ test( ], }, }, - async ({ artifacts, cleanup, host, progress, sandbox, skip }) => { + async ({ artifacts, cleanup, host, progress, runtimeProvider, sandbox, skip }) => { await artifacts.target.declare({ id: "gpu-e2e", boundary: "Hermes sandbox + GPU Ollama + initial, resumed, and continued CLI replies", @@ -383,12 +377,10 @@ test( progress.phase("prepare clean GPU Ollama runtime for Hermes"); await cleanupGpu(host, sandbox); - const docker = await host.command("docker", ["info"], { - artifactName: "docker-info-hermes-response", - env: buildAvailabilityProbeEnv(), - timeoutMs: 30_000, + await runtimeProvider.requireAvailable({ + artifactName: "runtime-info-hermes-response", + scenarioLabel: "Hermes GPU response validation", }); - expect(docker.exitCode, resultText(docker)).toBe(0); const nvidia = await host.command("nvidia-smi", [], { artifactName: "nvidia-smi-hermes-response", env: buildAvailabilityProbeEnv(), diff --git a/test/e2e/live/hermes-discord-proxy.ts b/test/e2e/live/hermes-discord-proxy.ts index 97619659442..47a56269bc6 100644 --- a/test/e2e/live/hermes-discord-proxy.ts +++ b/test/e2e/live/hermes-discord-proxy.ts @@ -4,3 +4,70 @@ export function hermesDiscordHttpProxyWebSocketUrl(host: string, port: number | string): string { return `http://${host}:${port}/gateway`; } + +export function isDiscordExternalAccessDenial(statusCode: number, body: string): boolean { + return statusCode === 403 && /^error code:\s*1010\s*$/iu.test(body.trim()); +} + +export async function verifyDiscordRestBoundary( + stdout: string, + recordUnavailable: (reason: string) => Promise, +): Promise { + const rows = stdout + .split(/\r?\n/u) + .filter((line) => line.trim().startsWith("{")) + .map( + (line) => JSON.parse(line) as { statusCode?: number; body?: string; error?: string }, + ); + const result = rows.at(-1) ?? {}; + switch (result.error ?? "") { + case "timeout": + await recordUnavailable("Discord API timed out, matching legacy skip behavior"); + return; + case "": + if (isDiscordExternalAccessDenial(result.statusCode ?? 0, result.body ?? "")) { + await recordUnavailable("Discord edge denied this runner before the API boundary (error 1010)"); + return; + } + if ([200, 401].includes(result.statusCode ?? 0)) return; + throw new Error( + `Unexpected Discord users/@me response (got ${String(result.statusCode)}): ${stdout}`, + ); + default: + throw new Error(`Discord API call failed: ${result.error}`); + } +} + +export const HERMES_DISCORD_REST_PROOF_SOURCE = String.raw` +import json +import os +import re +import socket +import urllib.error +import urllib.request + +token = os.environ.get("DISCORD_BOT_TOKEN", "") +if not re.fullmatch(r"openshell:resolve:env:v[0-9]{1,20}_DISCORD_BOT_TOKEN", token): + print(json.dumps({"error": "missing_current_revision_scoped_token"})) + raise SystemExit(0) + +request = urllib.request.Request( + "https://discord.com/api/v10/users/@me", + method="GET", + headers={"Authorization": "Bot " + token}, +) +try: + with urllib.request.urlopen(request, timeout=20) as response: + status = response.status + body = response.read().decode("utf-8", errors="replace") +except urllib.error.HTTPError as error: + status = error.code + body = error.read().decode("utf-8", errors="replace") +except (TimeoutError, socket.timeout): + print(json.dumps({"error": "timeout"})) + raise SystemExit(0) +except Exception as error: + print(json.dumps({"error": str(error)})) + raise SystemExit(0) +print(json.dumps({"statusCode": status, "body": body[:200]})) +`; diff --git a/test/e2e/live/hermes-discord.test.ts b/test/e2e/live/hermes-discord.test.ts index 40669ebd50c..c001299a453 100644 --- a/test/e2e/live/hermes-discord.test.ts +++ b/test/e2e/live/hermes-discord.test.ts @@ -6,14 +6,20 @@ import path from "node:path"; import { setTimeout as sleep } from "node:timers/promises"; import { HERMES_DISCORD_TEST_TIMEOUT_MS } from "../../../tools/e2e/hermes-timeout-contract.mts"; +import type { CleanupRegistry } from "../fixtures/cleanup.ts"; import { cleanupWhenOpenShellAvailable } from "../fixtures/cleanup-resources.ts"; import type { HostCliClient, SandboxClient } from "../fixtures/clients/index.ts"; import { sandboxAccessEnv, validateSandboxName } from "../fixtures/clients/sandbox.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; +import { rebindFixtureProviderPolicyEndpoint } from "../fixtures/gateway-providers.ts"; import { REPO_ROOT } from "../fixtures/paths.ts"; import { buildProcessTokenProbe } from "../fixtures/process-token-probe.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; -import { hermesDiscordHttpProxyWebSocketUrl } from "./hermes-discord-proxy.ts"; +import { + hermesDiscordHttpProxyWebSocketUrl, + HERMES_DISCORD_REST_PROOF_SOURCE, + verifyDiscordRestBoundary, +} from "./hermes-discord-proxy.ts"; import { assertDiscordGatewayCapture, type FakeDockerApi, @@ -21,11 +27,10 @@ import { } from "./messaging-providers-helpers.ts"; import { runSecondaryCleanup as bestEffortLifecycleCleanup, - dockerInfo, expectExitZero, phase6Env, + requirePhase6RuntimeProvider, resultText, - sandboxNode, sandboxSh, sandboxShWithArgs, shellQuote, @@ -112,6 +117,26 @@ async function precleanHermesDiscord( ); } +async function startHermesFakeDiscordGateway( + host: HostCliClient, + cleanup: CleanupRegistry, + env: NodeJS.ProcessEnv, + token: string, + redactionValues: string[], +): Promise { + return startFakeDockerApi(host, cleanup.trackDisposable.bind(cleanup), { + kind: "discord-gateway", + imageScript: "fake-discord-gateway.cjs", + containerPrefix: "nemoclaw-fake-discord-hermes", + portEnv: "FAKE_DISCORD_GATEWAY_PORT", + portFileEnv: "FAKE_DISCORD_GATEWAY_PORT_FILE", + captureFileEnv: "FAKE_DISCORD_GATEWAY_CAPTURE_FILE", + expectedEnv: { FAKE_DISCORD_GATEWAY_EXPECTED_TOKEN: token }, + env, + redactionValues, + }); +} + async function applyHermesFakeDiscordPolicy(options: { host: HostCliClient; sandboxName: string; @@ -144,7 +169,21 @@ async function applyHermesFakeDiscordPolicy(options: { ); expectExitZero(result, "apply Hermes fake Discord Gateway policy"); - const binding = await options.host.command( + const providerName = `${options.sandboxName}-discord-bridge`; + await rebindFixtureProviderPolicyEndpoint(options.host, options.sandboxName, { + artifactName: "bind-hermes-fake-discord-gateway-credential", + credentialEnv: "DISCORD_BOT_TOKEN", + endpoint: { + host: FAKE_DISCORD_HOST, + port: options.api.port, + protocol: "websocket", + }, + env: options.env, + providerName, + redactionValues: options.redactions, + }); + + const binaryAssertion = await options.host.command( "bash", [ "-lc", @@ -152,27 +191,23 @@ async function applyHermesFakeDiscordPolicy(options: { policy_file="$(mktemp)" trap 'rm -f "$policy_file"' EXIT "$1" policy get --base "$2" >"$policy_file" -node --import tsx "$6" "$policy_file" "$3" "$4" "$5" websocket -"$1" policy set --policy "$policy_file" --wait "$2" -"$1" policy get --base "$2" >"$policy_file" -node --import tsx "$6" --assert-binaries "$policy_file" "$4" "$5" websocket /opt/hermes/.venv/bin/python`, - "bind-hermes-fake-discord-policy", +node --import tsx "$5" --assert-binaries "$policy_file" "$3" "$4" websocket /opt/hermes/.venv/bin/python`, + "assert-hermes-fake-discord-policy-binaries", options.host.openshellCommandPath, options.sandboxName, - `${options.sandboxName}-discord-bridge`, FAKE_DISCORD_HOST, String(options.api.port), path.join(REPO_ROOT, "test/e2e/fixtures/hermes-discord-policy-binding.ts"), ], { - artifactName: "bind-hermes-fake-discord-gateway-credential", + artifactName: "assert-hermes-fake-discord-gateway-binaries", cwd: REPO_ROOT, env: options.env, redactionValues: options.redactions, timeoutMs: 120_000, }, ); - expectExitZero(binding, "bind Hermes fake Discord Gateway credential"); + expectExitZero(binaryAssertion, "assert Hermes fake Discord Gateway binary restriction"); } const HERMES_DISCORD_PYTHON_GATEWAY_PROOF = String.raw` @@ -361,151 +396,159 @@ async function rawTokenSurfaceProbe( }); } -test("hermes-discord: Hermes Discord schema, credential isolation, and native gateway rewrite", { - timeout: HERMES_DISCORD_TEST_TIMEOUT_MS, - meta: { - e2ePhases: [ - "prepare clean Hermes Discord runner", - "install Hermes Discord sandbox", - "validate Discord provider and Hermes health", - "validate Discord config and placeholders", - "exercise native Discord gateway rewrite", - "verify Discord token isolation and REST boundary", - "finalize Hermes Discord resources", - ], +test( + "hermes-discord: Hermes Discord schema, credential isolation, and native gateway rewrite", + { + timeout: HERMES_DISCORD_TEST_TIMEOUT_MS, + meta: { + e2ePhases: [ + "prepare clean Hermes Discord runner", + "install Hermes Discord sandbox", + "validate Discord provider and Hermes health", + "validate Discord config and placeholders", + "exercise native Discord gateway rewrite", + "verify Discord token isolation and REST boundary", + "finalize Hermes Discord resources", + ], + }, }, -}, async ({ artifacts, cleanup, host, progress, sandbox, secrets }) => { - const apiKey = secrets.required("NVIDIA_INFERENCE_API_KEY"); - const env = commandEnv(apiKey); - const redactionValues = redactions(apiKey); - - await artifacts.target.declare({ - id: "hermes-discord", - boundary: - "install.sh --non-interactive Hermes sandbox + Discord config + OpenShell provider rewrite + sandbox leak probes", - sandboxName: SANDBOX_NAME, - discordServerIds: DISCORD_SERVER_IDS, - discordAllowedIds: DISCORD_ALLOWED_IDS, - discordRequireMention: DISCORD_REQUIRE_MENTION, - }); + async ({ artifacts, cleanup, host, progress, runtimeProvider, sandbox, secrets }) => { + const apiKey = secrets.required("NVIDIA_INFERENCE_API_KEY"); + const env = commandEnv(apiKey); + const redactionValues = redactions(apiKey); + + await artifacts.target.declare({ + id: "hermes-discord", + boundary: + "install.sh --non-interactive Hermes sandbox + Discord config + OpenShell provider rewrite + sandbox leak probes", + sandboxName: SANDBOX_NAME, + discordServerIds: DISCORD_SERVER_IDS, + discordAllowedIds: DISCORD_ALLOWED_IDS, + discordRequireMention: DISCORD_REQUIRE_MENTION, + }); - const gatewayCleanupOptions = { - artifactName: "cleanup-hermes-discord-openshell-gateway-destroy", - env, - redactionValues, - timeoutMs: 120_000, - }; - cleanup.trackGateway( - { - cleanupGatewayRegistration: (name: string) => - cleanupWhenOpenShellAvailable( - host, - { - artifactName: "cleanup-hermes-discord-probe-openshell-gateway", - env, - redactionValues, - timeoutMs: 30_000, - }, - () => host.cleanupGatewayRegistration(name, gatewayCleanupOptions), - ), - }, - "nemoclaw", - gatewayCleanupOptions, - ); - trackPreinstallSandboxCleanup( - cleanup, - host, - sandbox, - SANDBOX_NAME, - env, - redactionValues, - "cleanup-hermes-discord", - ); + const gatewayCleanupOptions = { + artifactName: "cleanup-hermes-discord-openshell-gateway-destroy", + env, + redactionValues, + timeoutMs: 120_000, + }; + cleanup.trackGateway( + { + cleanupGatewayRegistration: (name: string) => + cleanupWhenOpenShellAvailable( + host, + { + artifactName: "cleanup-hermes-discord-probe-openshell-gateway", + env, + redactionValues, + timeoutMs: 30_000, + }, + () => host.cleanupGatewayRegistration(name, gatewayCleanupOptions), + ), + }, + "nemoclaw", + gatewayCleanupOptions, + ); + trackPreinstallSandboxCleanup( + cleanup, + host, + sandbox, + SANDBOX_NAME, + env, + redactionValues, + "cleanup-hermes-discord", + ); - await precleanHermesDiscord(host, SANDBOX_NAME, env, redactionValues, "preclean-hermes-discord"); + await precleanHermesDiscord( + host, + SANDBOX_NAME, + env, + redactionValues, + "preclean-hermes-discord", + ); - const docker = await dockerInfo(host, env); - expectExitZero(docker, "Docker is running"); - expect(process.env.NEMOCLAW_NON_INTERACTIVE ?? env.NEMOCLAW_NON_INTERACTIVE).toBe("1"); - expect( - process.env.NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE ?? env.NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE, - ).toBe("1"); + await requirePhase6RuntimeProvider(runtimeProvider, "Hermes Discord"); + expect(process.env.NEMOCLAW_NON_INTERACTIVE ?? env.NEMOCLAW_NON_INTERACTIVE).toBe("1"); + expect( + process.env.NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE ?? env.NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE, + ).toBe("1"); - progress.phase("install Hermes Discord sandbox"); - const install = await host.command("bash", ["install.sh", "--non-interactive"], { - artifactName: "phase-1-install-hermes-discord", - cwd: REPO_ROOT, - env, - redactionValues, - timeoutMs: 60 * 60_000, - }); - expectExitZero(install, "install.sh --non-interactive with Hermes Discord"); - - const cliProbe = await host.command( - "bash", - [ - "-lc", - 'command -v nemoclaw && command -v "$1" && "$1" --version', - "cli-probe-hermes-discord", - host.openshellCommandPath, - ], - { - artifactName: "phase-1-cli-probe", + progress.phase("install Hermes Discord sandbox"); + const install = await host.command("bash", ["install.sh", "--non-interactive"], { + artifactName: "phase-1-install-hermes-discord", + cwd: REPO_ROOT, env, redactionValues, - timeoutMs: 30_000, - }, - ); - expectExitZero(cliProbe, "nemoclaw and openshell installed"); - expect(cliProbe.stdout).toContain("nemoclaw"); + timeoutMs: 60 * 60_000, + }); + expectExitZero(install, "install.sh --non-interactive with Hermes Discord"); - progress.phase("validate Discord provider and Hermes health"); - const list = await host.command("nemoclaw", ["list"], { - artifactName: "phase-2-nemoclaw-list", - env, - redactionValues, - timeoutMs: 60_000, - }); - expectExitZero(list, "nemoclaw list"); - expect(resultText(list)).toContain(SANDBOX_NAME); + const cliProbe = await host.command( + "bash", + [ + "-lc", + 'command -v nemoclaw && command -v "$1" && "$1" --version', + "cli-probe-hermes-discord", + host.openshellCommandPath, + ], + { + artifactName: "phase-1-cli-probe", + env, + redactionValues, + timeoutMs: 30_000, + }, + ); + expectExitZero(cliProbe, "nemoclaw and openshell installed"); + expect(cliProbe.stdout).toContain("nemoclaw"); - const provider = await host.command( - host.openshellCommandPath, - ["provider", "get", `${SANDBOX_NAME}-discord-bridge`], - { - artifactName: "phase-2-discord-provider-get", + progress.phase("validate Discord provider and Hermes health"); + const list = await host.command("nemoclaw", ["list"], { + artifactName: "phase-2-nemoclaw-list", env, redactionValues, timeoutMs: 60_000, - }, - ); - expectExitZero(provider, "Discord provider exists in gateway"); - - let health: ShellProbeResult | undefined; - for (let attempt = 1; attempt <= 15; attempt += 1) { - health = await sandboxSh(sandbox, SANDBOX_NAME, `curl -sf ${shellQuote(HERMES_HEALTH_URL)}`, { - artifactName: `phase-3-hermes-health-${attempt}`, - redactionValues, - timeoutMs: 20_000, }); - switch (health.exitCode === 0 && /"ok"/i.test(resultText(health))) { - case true: - attempt = 16; - break; - default: - await sleep(4_000); + expectExitZero(list, "nemoclaw list"); + expect(resultText(list)).toContain(SANDBOX_NAME); + + const provider = await host.command( + host.openshellCommandPath, + ["provider", "get", `${SANDBOX_NAME}-discord-bridge`], + { + artifactName: "phase-2-discord-provider-get", + env, + redactionValues, + timeoutMs: 60_000, + }, + ); + expectExitZero(provider, "Discord provider exists in gateway"); + + let health: ShellProbeResult | undefined; + for (let attempt = 1; attempt <= 15; attempt += 1) { + health = await sandboxSh(sandbox, SANDBOX_NAME, `curl -sf ${shellQuote(HERMES_HEALTH_URL)}`, { + artifactName: `phase-3-hermes-health-${attempt}`, + redactionValues, + timeoutMs: 20_000, + }); + switch (health.exitCode === 0 && /"ok"/i.test(resultText(health))) { + case true: + attempt = 16; + break; + default: + await sleep(4_000); + } } - } - expect(health, "Hermes health probe did not run").toBeTruthy(); - expect(health?.exitCode, health ? resultText(health) : "missing health result").toBe(0); - expect(resultText(health!)).toMatch(/"ok"/i); - - progress.phase("validate Discord config and placeholders"); - const expectedRequireMention = DISCORD_REQUIRE_MENTION === "0" ? "false" : "true"; - const configProbe = await sandboxShWithArgs( - sandbox, - SANDBOX_NAME, - `EXPECTED_REQUIRE_MENTION=${shellQuote(expectedRequireMention)} python3 - <<'PY' + expect(health, "Hermes health probe did not run").toBeTruthy(); + expect(health?.exitCode, health ? resultText(health) : "missing health result").toBe(0); + expect(resultText(health!)).toMatch(/"ok"/i); + + progress.phase("validate Discord config and placeholders"); + const expectedRequireMention = DISCORD_REQUIRE_MENTION === "0" ? "false" : "true"; + const configProbe = await sandboxShWithArgs( + sandbox, + SANDBOX_NAME, + `EXPECTED_REQUIRE_MENTION=${shellQuote(expectedRequireMention)} python3 - <<'PY' import os import sys, yaml with open("/sandbox/.hermes/config.yaml", "r", encoding="utf-8") as f: @@ -543,16 +586,16 @@ if errors: raise SystemExit(1) print("OK") PY`, - [], - { artifactName: "phase-4-hermes-discord-config-shape", redactionValues }, - ); - expectExitZero(configProbe, "Hermes Discord config shape"); - expect(configProbe.stdout.trim()).toBe("OK"); + [], + { artifactName: "phase-4-hermes-discord-config-shape", redactionValues }, + ); + expectExitZero(configProbe, "Hermes Discord config shape"); + expect(configProbe.stdout.trim()).toBe("OK"); - const envProbe = await sandboxShWithArgs( - sandbox, - SANDBOX_NAME, - `EXPECTED_ALLOWED_USERS=${shellQuote(normalizedCsv(DISCORD_ALLOWED_IDS))} EXPECTED_GUILD_IDS=${shellQuote(normalizedCsv(DISCORD_SERVER_IDS))} python3 - <<'PY' + const envProbe = await sandboxShWithArgs( + sandbox, + SANDBOX_NAME, + `EXPECTED_ALLOWED_USERS=${shellQuote(normalizedCsv(DISCORD_ALLOWED_IDS))} EXPECTED_GUILD_IDS=${shellQuote(normalizedCsv(DISCORD_SERVER_IDS))} python3 - <<'PY' import os from pathlib import Path text = Path("/sandbox/.hermes/.env").read_text(encoding="utf-8") @@ -571,155 +614,111 @@ if errors: raise SystemExit(1) print("OK") PY`, - [], - { artifactName: "phase-4-hermes-discord-env-shape", redactionValues }, - ); - expectExitZero(envProbe, "Hermes Discord .env shape"); - expect(envProbe.stdout.trim()).toBe("OK"); + [], + { artifactName: "phase-4-hermes-discord-env-shape", redactionValues }, + ); + expectExitZero(envProbe, "Hermes Discord .env shape"); + expect(envProbe.stdout.trim()).toBe("OK"); - progress.phase("exercise native Discord gateway rewrite"); - const fakeGateway = await startFakeDockerApi( - host, - cleanup.trackDisposable.bind(cleanup), - { - kind: "discord-gateway", - imageScript: "fake-discord-gateway.cjs", - containerPrefix: "nemoclaw-fake-discord-hermes", - portEnv: "FAKE_DISCORD_GATEWAY_PORT", - captureFileEnv: "FAKE_DISCORD_GATEWAY_CAPTURE_FILE", - expectedEnv: { FAKE_DISCORD_GATEWAY_EXPECTED_TOKEN: DISCORD_TOKEN }, + progress.phase("exercise native Discord gateway rewrite"); + const fakeGateway = await startHermesFakeDiscordGateway( + host, + cleanup, env, + DISCORD_TOKEN, redactionValues, - }, - ); - await applyHermesFakeDiscordPolicy({ - host, - sandboxName: SANDBOX_NAME, - api: fakeGateway, - env, - redactions: redactionValues, - }); - - const nativeGateway = await runHermesPythonDiscordGatewayProof( - sandbox, - fakeGateway.port, - redactionValues, - ); - const gatewayCapture = await host.command( - "bash", - [ - "-lc", - 'if [ -f "$1" ]; then sed -n "1,80p" "$1"; else printf "MISSING_CAPTURE\\n"; fi', - "read-hermes-discord-gateway-capture", - fakeGateway.captureFile, - ], - { - artifactName: "hermes-discord-gateway-capture", + ); + await applyHermesFakeDiscordPolicy({ + host, + sandboxName: SANDBOX_NAME, + api: fakeGateway, env, - redactionValues, - timeoutMs: 30_000, - }, - ); - expectExitZero(gatewayCapture, "Hermes Discord Gateway capture"); - expectExitZero(nativeGateway, "Hermes Python Discord Gateway protocol proof"); - expect(resultText(nativeGateway)).toContain("UPGRADE"); - expect(resultText(nativeGateway)).toContain("HELLO"); - expect(resultText(nativeGateway)).toContain("IDENTIFY_SENT_PLACEHOLDER"); - expect(resultText(nativeGateway)).toContain("READY"); - expect(resultText(nativeGateway)).toContain("HEARTBEAT_ACK"); - expect(resultText(nativeGateway)).not.toContain("IMPORT_DISCORD_FAILED"); - assertDiscordGatewayCapture(fakeGateway.captureFile, DISCORD_TOKEN); - - progress.phase("verify Discord token isolation and REST boundary"); - await assertRawTokenAbsentFromFiles(sandbox, DISCORD_TOKEN, redactionValues); - - const envSurface = await rawTokenSurfaceProbe( - sandbox, - DISCORD_TOKEN, - "env", - "phase-5-raw-token-env-probe", - redactionValues, - ); - expectExitZero(envSurface, "sandbox environment token isolation"); - expect(envSurface.stdout.trim()).toBe("ABSENT"); - - const processSurface = await rawTokenSurfaceProbe( - sandbox, - DISCORD_TOKEN, - "process", - "phase-5-raw-token-process-probe", - redactionValues, - ); - expectExitZero(processSurface, "sandbox process token isolation"); - expect(processSurface.stdout.trim()).toBe("ABSENT"); - - const filesystemSurface = await rawTokenSurfaceProbe( - sandbox, - DISCORD_TOKEN, - "filesystem", - "phase-5-raw-token-filesystem-probe", - redactionValues, - ); - expectExitZero(filesystemSurface, "sandbox filesystem token isolation"); - expect(filesystemSurface.stdout.trim()).toBe("ABSENT"); + redactions: redactionValues, + }); - const discordApi = await sandboxNode( - sandbox, - SANDBOX_NAME, - ` -import https from "node:https"; -const token = process.env.DISCORD_BOT_TOKEN ?? ""; -if (!/^openshell:resolve:env:v[1-9][0-9]*_DISCORD_BOT_TOKEN$/.test(token)) { - console.log(JSON.stringify({ error: "invalid_token_placeholder" })); - process.exit(0); -} -const req = https.request({ - hostname: "discord.com", - path: "/api/v10/users/@me", - method: "GET", - headers: { Authorization: "Bot " + token }, -}, (res) => { - let body = ""; - res.on("data", (d) => { body += d; }); - res.on("end", () => console.log(JSON.stringify({ statusCode: res.statusCode, body: body.slice(0, 200) }))); -}); -req.on("error", (error) => console.log(JSON.stringify({ error: error.message }))); -req.setTimeout(20000, () => { req.destroy(); console.log(JSON.stringify({ error: "timeout" })); }); -req.end(); -`, - {}, - { - artifactName: "phase-6-discord-users-me", + const nativeGateway = await runHermesPythonDiscordGatewayProof( + sandbox, + fakeGateway.port, redactionValues, - timeoutMs: 30_000, - }, - ); - expectExitZero(discordApi, "Discord REST users/@me probe command"); - const discordApiRows = discordApi.stdout - .split(/\r?\n/) - .filter((line) => line.trim().startsWith("{")) - .map((line) => JSON.parse(line) as { statusCode?: number; error?: string }); - const discordApiResult = discordApiRows.at(-1) ?? {}; - switch (discordApiResult.error ?? "") { - case "timeout": - await artifacts.writeJson("phase-6-discord-users-me-skip.json", { - reason: "Discord API timed out, matching legacy skip behavior", - }); - break; - case "": - expect( - [200, 401].includes(discordApiResult.statusCode ?? 0), - `Unexpected Discord users/@me response (got ${discordApiResult.statusCode}): ${discordApi.stdout}`, - ).toBe(true); - break; - default: - throw new Error(`Discord API call failed: ${discordApiResult.error}`); - } - - const bridgeResidue = await sandboxShWithArgs( - sandbox, - SANDBOX_NAME, - String.raw`set +e + ); + const gatewayCapture = await host.command( + "bash", + [ + "-lc", + 'if [ -f "$1" ]; then sed -n "1,80p" "$1"; else printf "MISSING_CAPTURE\\n"; fi', + "read-hermes-discord-gateway-capture", + fakeGateway.captureFile, + ], + { + artifactName: "hermes-discord-gateway-capture", + env, + redactionValues, + timeoutMs: 30_000, + }, + ); + expectExitZero(gatewayCapture, "Hermes Discord Gateway capture"); + expectExitZero(nativeGateway, "Hermes Python Discord Gateway protocol proof"); + expect(resultText(nativeGateway)).toContain("UPGRADE"); + expect(resultText(nativeGateway)).toContain("HELLO"); + expect(resultText(nativeGateway)).toContain("IDENTIFY_SENT_PLACEHOLDER"); + expect(resultText(nativeGateway)).toContain("READY"); + expect(resultText(nativeGateway)).toContain("HEARTBEAT_ACK"); + expect(resultText(nativeGateway)).not.toContain("IMPORT_DISCORD_FAILED"); + assertDiscordGatewayCapture(fakeGateway.captureFile, DISCORD_TOKEN); + + progress.phase("verify Discord token isolation and REST boundary"); + await assertRawTokenAbsentFromFiles(sandbox, DISCORD_TOKEN, redactionValues); + + const envSurface = await rawTokenSurfaceProbe( + sandbox, + DISCORD_TOKEN, + "env", + "phase-5-raw-token-env-probe", + redactionValues, + ); + expectExitZero(envSurface, "sandbox environment token isolation"); + expect(envSurface.stdout.trim()).toBe("ABSENT"); + + const processSurface = await rawTokenSurfaceProbe( + sandbox, + DISCORD_TOKEN, + "process", + "phase-5-raw-token-process-probe", + redactionValues, + ); + expectExitZero(processSurface, "sandbox process token isolation"); + expect(processSurface.stdout.trim()).toBe("ABSENT"); + + const filesystemSurface = await rawTokenSurfaceProbe( + sandbox, + DISCORD_TOKEN, + "filesystem", + "phase-5-raw-token-filesystem-probe", + redactionValues, + ); + expectExitZero(filesystemSurface, "sandbox filesystem token isolation"); + expect(filesystemSurface.stdout.trim()).toBe("ABSENT"); + + const discordApi = await sandboxShWithArgs( + sandbox, + SANDBOX_NAME, + `/opt/hermes/.venv/bin/python - <<'PY'\n${HERMES_DISCORD_REST_PROOF_SOURCE}\nPY\n`, + [], + { + artifactName: "phase-6-discord-users-me", + redactionValues, + timeoutMs: 30_000, + }, + ); + expectExitZero(discordApi, "Discord REST users/@me probe command"); + await verifyDiscordRestBoundary(discordApi.stdout, (reason) => + artifacts.writeJson("phase-6-discord-users-me-skip.json", { reason }), + ); + + const bridgeResidue = await sandboxShWithArgs( + sandbox, + SANDBOX_NAME, + String.raw`set +e env_needle="$(printf "%s%s" "NEMOCLAW_DISCORD_" "FACADE_URL")" name_needle="$(printf "%s%s" "nemoclaw-discord-" "facade")" proxy_needle="$(printf "%s" "DISCORD_PROXY")" @@ -739,65 +738,66 @@ for p in /proc/[0-9]*; do case "$cmd" in *"$name_needle"*) echo PROCESS_FACADE ;; esac case "$cmd" in *"$decode_needle"*) echo PROCESS_DECODE_PROXY ;; esac done`, - [], - { artifactName: "phase-7-no-local-discord-bridge", redactionValues }, - ); - expectExitZero(bridgeResidue, "no local Discord bridge residue probe"); - expect(bridgeResidue.stdout.trim()).toBe(""); - - progress.phase("finalize Hermes Discord resources"); - await (async (): Promise => { - switch (process.env.NEMOCLAW_E2E_KEEP_SANDBOX) { - case "1": - return; - default: - } - const destroy = await host.command("nemoclaw", [SANDBOX_NAME, "destroy", "--yes"], { - artifactName: "phase-8-nemoclaw-destroy", - env, - redactionValues, - timeoutMs: 15 * 60_000, - }); - expectExitZero(destroy, "destroy Hermes Discord sandbox"); - await bestEffortLifecycleCleanup(() => - host.command(host.openshellCommandPath, ["gateway", "destroy", "-g", "nemoclaw"], { - artifactName: "phase-8-openshell-gateway-destroy", - env, - redactionValues, - timeoutMs: 120_000, - }), + [], + { artifactName: "phase-7-no-local-discord-bridge", redactionValues }, ); - const registryProbe = await host.command( - "bash", - [ - "-lc", - `registry="$HOME/.nemoclaw/sandboxes.json"; if [ -f "$registry" ] && grep -Fq ${shellQuote(`"${SANDBOX_NAME}"`)} "$registry"; then echo FOUND; exit 1; else echo ABSENT; fi`, - ], - { - artifactName: "phase-8-registry-removal-probe", - env: sandboxAccessEnv(), + expectExitZero(bridgeResidue, "no local Discord bridge residue probe"); + expect(bridgeResidue.stdout.trim()).toBe(""); + + progress.phase("finalize Hermes Discord resources"); + await (async (): Promise => { + switch (process.env.NEMOCLAW_E2E_KEEP_SANDBOX) { + case "1": + return; + default: + } + const destroy = await host.command("nemoclaw", [SANDBOX_NAME, "destroy", "--yes"], { + artifactName: "phase-8-nemoclaw-destroy", + env, redactionValues, - timeoutMs: 30_000, + timeoutMs: 15 * 60_000, + }); + expectExitZero(destroy, "destroy Hermes Discord sandbox"); + await bestEffortLifecycleCleanup(() => + host.command(host.openshellCommandPath, ["gateway", "destroy", "-g", "nemoclaw"], { + artifactName: "phase-8-openshell-gateway-destroy", + env, + redactionValues, + timeoutMs: 120_000, + }), + ); + const registryProbe = await host.command( + "bash", + [ + "-lc", + `registry="$HOME/.nemoclaw/sandboxes.json"; if [ -f "$registry" ] && grep -Fq ${shellQuote(`"${SANDBOX_NAME}"`)} "$registry"; then echo FOUND; exit 1; else echo ABSENT; fi`, + ], + { + artifactName: "phase-8-registry-removal-probe", + env: sandboxAccessEnv(), + redactionValues, + timeoutMs: 30_000, + }, + ); + expectExitZero(registryProbe, "sandbox removed from registry"); + expect(registryProbe.stdout.trim()).toBe("ABSENT"); + })(); + + await artifacts.target.complete({ + id: "hermes-discord", + assertions: { + dockerAndNonInteractivePrereqs: true, + installHermesDiscord: true, + providerRegistered: true, + hermesHealthy: true, + configSchema: true, + envPlaceholders: true, + nativePythonDiscordGatewayRewrite: true, + rawTokenAbsentFromConfigEnvProcessAndFilesystem: true, + discordRestBoundaryReachedOrExternallyUnavailable: true, + noLocalDiscordBridgeResidue: true, + cleanupVerified: process.env.NEMOCLAW_E2E_KEEP_SANDBOX !== "1", }, - ); - expectExitZero(registryProbe, "sandbox removed from registry"); - expect(registryProbe.stdout.trim()).toBe("ABSENT"); - })(); - - await artifacts.target.complete({ - id: "hermes-discord", - assertions: { - dockerAndNonInteractivePrereqs: true, - installHermesDiscord: true, - providerRegistered: true, - hermesHealthy: true, - configSchema: true, - envPlaceholders: true, - nativePythonDiscordGatewayRewrite: true, - rawTokenAbsentFromConfigEnvProcessAndFilesystem: true, - discordRestBoundaryReachedOrSkippedOnTimeout: true, - noLocalDiscordBridgeResidue: true, - cleanupVerified: process.env.NEMOCLAW_E2E_KEEP_SANDBOX !== "1", - }, - }); -}); + }); + }, +); diff --git a/test/e2e/live/hermes-e2e.test.ts b/test/e2e/live/hermes-e2e.test.ts index ed00ae661fc..673946c5985 100644 --- a/test/e2e/live/hermes-e2e.test.ts +++ b/test/e2e/live/hermes-e2e.test.ts @@ -210,7 +210,7 @@ test( timeout: HERMES_E2E_TEST_TIMEOUT_MS, meta: { e2ePhases: HERMES_E2E_PHASES }, }, - async ({ artifacts, cleanup, host, inference, progress, sandbox }) => { + async ({ artifacts, cleanup, host, inference, progress, runtimeProvider, sandbox }) => { await artifacts.target.declare({ id: "hermes-e2e", boundary: `install.sh --non-interactive --fresh + Hermes sandbox runtime + ${inference.mode} inference adapter`, @@ -295,12 +295,11 @@ test( progress.phase("prepare clean Hermes runner"); await cleanupHermes("pre-cleanup"); - const dockerInfo = await host.command("docker", ["info"], { - artifactName: "phase-1-docker-info", - env: buildAvailabilityProbeEnv(), - timeoutMs: 30_000, + // Phase 1: prerequisites. + await runtimeProvider.requireAvailable({ + artifactName: "phase-1-runtime-info", + scenarioLabel: "Hermes", }); - expect(dockerInfo.exitCode, resultText(dockerInfo)).toBe(0); await expect(inference.probeModels("phase-1-inference-models")).resolves.toMatchObject({ data: expect.arrayContaining([expect.objectContaining({ id: inference.model })]), @@ -414,7 +413,7 @@ test( await expectPackageDatabaseReadOnly({ artifactPrefix: "phase-3", env: commandEnv(), - host, + runtimeProvider, sandbox, sandboxName: SANDBOX_NAME, timeoutMs: 30_000, diff --git a/test/e2e/live/hermes-gpu-startup-proof.ts b/test/e2e/live/hermes-gpu-startup-proof.ts index e0581220de8..76f81cec72f 100644 --- a/test/e2e/live/hermes-gpu-startup-proof.ts +++ b/test/e2e/live/hermes-gpu-startup-proof.ts @@ -20,6 +20,7 @@ import { trustedSandboxShellScript, } from "../fixtures/clients/index.ts"; import { expect } from "../fixtures/e2e-test.ts"; +import type { RuntimeProviderPrerequisite } from "../fixtures/runtime-provider.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; import { buildHermesManagedStartupIntegrityScript } from "./hermes-gpu-startup-integrity.ts"; import { stripAnsi } from "./json-envelope.ts"; @@ -37,21 +38,32 @@ interface HermesGpuStartupProofOptions { gpuRoute: "compatibility-fallback" | "compatibility-only" | "native-success"; host: HostCliClient; install: Pick; + runtimeProvider: RuntimeProviderPrerequisite; sandbox: SandboxClient; sandboxName: string; status: Pick; } const IMMUTABLE_IMAGE_REFERENCE = /^[^@\s]+@sha256:[a-f0-9]{64}$/u; +const IMMUTABLE_IMAGE_CONTENT_ID = /^sha256:[a-f0-9]{64}$/u; +const BARE_IMMUTABLE_IMAGE_CONTENT_ID = /^[a-f0-9]{64}$/u; + +export function normalizeImmutableImageContentId(value: unknown): unknown { + return typeof value === "string" && BARE_IMMUTABLE_IMAGE_CONTENT_ID.test(value) + ? `sha256:${value}` + : value; +} export function assertHermesGpuStartupOutputContract( gpuRoute: HermesGpuStartupProofOptions["gpuRoute"], + runtimeProviderId: RuntimeProviderPrerequisite["id"], installText: string, ): void { - expect(installText).toContain("Starting OpenShell Docker-driver gateway..."); - expect(installText).toContain("Docker-driver gateway is healthy"); + expect(installText).toContain(`Container runtime: ${runtimeProviderId}`); + expect(installText).toMatch(/Starting OpenShell .*gateway/u); + expect(installText).toMatch(/gateway is healthy/u); expect(installText).not.toContain("Reusing healthy NemoClaw gateway."); - expect(installText).not.toContain("Reusing existing Docker-driver gateway"); + expect(installText).not.toMatch(/Reusing existing .*gateway/u); expect(installText).not.toContain("[reuse] Skipping gateway (running)"); if (gpuRoute === "compatibility-fallback") { expect(installText).toContain( @@ -102,8 +114,14 @@ export function assertHermesManagedWorkloadAuthority( export function assertHermesContainerImageAuthority( containerImage: unknown, authorityReference: string, + authorityContentId?: string, ): void { - expect(containerImage).toBe(authorityReference); + const normalizedContainerImage = normalizeImmutableImageContentId(containerImage); + expect( + normalizedContainerImage === authorityReference || + (IMMUTABLE_IMAGE_CONTENT_ID.test(authorityContentId ?? "") && + normalizedContainerImage === authorityContentId), + ).toBe(true); } export async function assertHermesGpuStartupProof({ @@ -111,12 +129,13 @@ export async function assertHermesGpuStartupProof({ gpuRoute, host, install, + runtimeProvider, sandbox, sandboxName, status, }: HermesGpuStartupProofOptions): Promise { const installText = resultText(install); - assertHermesGpuStartupOutputContract(gpuRoute, installText); + assertHermesGpuStartupOutputContract(gpuRoute, runtimeProvider.id, installText); const plainStatus = stripAnsi(resultText(status)); expect(plainStatus).toMatch(/Phase:\s*Ready/i); expect(plainStatus).toContain("Sandbox GPU: enabled"); @@ -148,9 +167,9 @@ export async function assertHermesGpuStartupProof({ has_nemoclaw_start: false, }); - const runningContainers = await host.command( - "docker", + const runningContainers = await runtimeProvider.command( [ + "container", "ps", "--filter", `label=openshell.ai/sandbox-name=${sandboxName}`, @@ -185,6 +204,22 @@ export async function assertHermesGpuStartupProof({ registryEntry.imageTag, managedAuthority, ); + const managedImageInspection = await runtimeProvider.command( + ["image", "inspect", managedImageReference], + { + artifactName: "phase-4-gpu-startup-managed-image-content-authority", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }, + ); + expect(managedImageInspection.exitCode, resultText(managedImageInspection)).toBe(0); + const managedImageInspectEntry = ( + JSON.parse(managedImageInspection.stdout) as Array<{ Id?: unknown; ID?: unknown }> + )[0]; + const managedImageContentId = normalizeImmutableImageContentId( + managedImageInspectEntry?.Id ?? managedImageInspectEntry?.ID, + ); + expect(managedImageContentId).toMatch(IMMUTABLE_IMAGE_CONTENT_ID); const guardWithoutStartupOwner = await sandbox.execShell( sandboxName, @@ -230,28 +265,60 @@ export async function assertHermesGpuStartupProof({ expect(startupConfig.exitCode, resultText(startupConfig)).toBe(0); expect(startupConfig.stdout.trim()).toBe("OK"); - const dockerCommandBoundary = await host.command( - "bash", - [ - "-lc", - String.raw`docker inspect "$1" | python3 -c 'import json, sys; config=json.load(sys.stdin)[0]["Config"]; env=dict(item.split("=", 1) for item in (config.get("Env") or []) if "=" in item); command=env.get("OPENSHELL_SANDBOX_COMMAND", ""); tokens=command.split(); print(json.dumps({"cmd": config.get("Cmd"), "entrypoint": config.get("Entrypoint"), "image": config.get("Image"), "has_openshell_sandbox_command": bool(command), "command_is_sleep_infinity": tokens == ["sleep", "infinity"], "command_ends_with_nemoclaw_start": bool(tokens) and tokens[-1] in ("nemoclaw-start", "/usr/local/bin/nemoclaw-start")}))'`, - "hermes-gpu-command-boundary", - containerId, - ], + const runtimeCommandBoundary = await runtimeProvider.command( + ["container", "inspect", containerId], { - artifactName: "phase-4-gpu-startup-docker-command-boundary", + artifactName: "phase-4-gpu-startup-runtime-command-boundary", env: buildAvailabilityProbeEnv(), timeoutMs: 30_000, }, ); - expect(dockerCommandBoundary.exitCode, resultText(dockerCommandBoundary)).toBe(0); - const commandBoundary = JSON.parse(dockerCommandBoundary.stdout); + expect(runtimeCommandBoundary.exitCode, resultText(runtimeCommandBoundary)).toBe(0); + const runtimeInspection = ( + JSON.parse(runtimeCommandBoundary.stdout) as Array<{ + Image?: unknown; + ImageName?: unknown; + Config?: { + Cmd?: unknown; + Entrypoint?: unknown; + Env?: string[]; + Image?: unknown; + }; + }> + )[0]; + const runtimeConfig = runtimeInspection?.Config ?? {}; + const runtimeEnvironment = Object.fromEntries( + (runtimeConfig.Env ?? []) + .filter((entry) => entry.includes("=")) + .map((entry) => entry.split(/=(.*)/su).slice(0, 2) as [string, string]), + ); + const intendedCommand = runtimeEnvironment.OPENSHELL_SANDBOX_COMMAND ?? ""; + const intendedTokens = intendedCommand.trim().split(/\s+/u).filter(Boolean); + const commandBoundary = { + cmd: runtimeConfig.Cmd, + entrypoint: runtimeConfig.Entrypoint, + image: runtimeInspection?.Image ?? runtimeInspection?.ImageName ?? runtimeConfig.Image, + has_openshell_sandbox_command: Boolean(intendedCommand), + command_is_sleep_infinity: + intendedTokens.length === 2 && + intendedTokens[0] === "sleep" && + intendedTokens[1] === "infinity", + command_ends_with_nemoclaw_start: + intendedTokens.length > 0 && + ["nemoclaw-start", "/usr/local/bin/nemoclaw-start"].includes(intendedTokens.at(-1) ?? ""), + }; const verifiedManagedAuthority = managedAuthority!; expect(verifiedManagedAuthority.agent).toBe("hermes"); const managedBootstrapCommand = commandBoundary.cmd; expect(Array.isArray(managedBootstrapCommand)).toBe(true); + if (!Array.isArray(managedBootstrapCommand)) { + throw new TypeError("managed bootstrap command must be an argument array"); + } const bootstrapIdentity = managedBootstrapCommand[5]; expect(typeof bootstrapIdentity).toBe("string"); + if (typeof bootstrapIdentity !== "string") { + throw new TypeError("managed bootstrap identity must be a string"); + } assertManagedBootstrapIdentity(bootstrapIdentity); const agentIdentity = managedImageRuntimeIdentity(verifiedManagedAuthority.agent); expect(commandBoundary.entrypoint).toEqual([MANAGED_BOOTSTRAP_TRAMPOLINE_EXECUTABLE]); @@ -274,13 +341,16 @@ export async function assertHermesGpuStartupProof({ ...OPENSHELL_SANDBOX_SUPERVISOR_ARGV, ]); expect(commandBoundary.has_openshell_sandbox_command).toBe(true); - assertHermesContainerImageAuthority(commandBoundary.image, managedImageReference); + assertHermesContainerImageAuthority( + commandBoundary.image, + managedImageReference, + managedImageContentId as string, + ); expect(commandBoundary.command_ends_with_nemoclaw_start).toBe(true); expect(commandBoundary.command_is_sleep_infinity).toBe(false); - const containerState = await host.command( - "docker", - ["inspect", "--format", "{{.State.Status}} {{.RestartCount}}", containerId], + const containerState = await runtimeProvider.command( + ["container", "inspect", "--format", "{{.State.Status}} {{.RestartCount}}", containerId], { artifactName: "phase-4-gpu-startup-container-state", env: buildAvailabilityProbeEnv(), @@ -290,11 +360,11 @@ export async function assertHermesGpuStartupProof({ expect(containerState.exitCode, resultText(containerState)).toBe(0); expect(containerState.stdout.trim()).toBe("running 0"); - const allContainers = await host.command( - "docker", + const allContainers = await runtimeProvider.command( [ + "container", "ps", - "-a", + "--all", "--filter", `label=openshell.ai/sandbox-name=${sandboxName}`, "--format", diff --git a/test/e2e/live/hermes-gpu-startup.test.ts b/test/e2e/live/hermes-gpu-startup.test.ts index c85aa37d6a7..404842e82e1 100644 --- a/test/e2e/live/hermes-gpu-startup.test.ts +++ b/test/e2e/live/hermes-gpu-startup.test.ts @@ -18,6 +18,7 @@ import { import { expect, test } from "../fixtures/e2e-test.ts"; import { startFakeOpenAiCompatibleServer } from "../fixtures/fake-openai-compatible.ts"; import { REPO_ROOT } from "../fixtures/paths.ts"; +import type { RuntimeProviderPrerequisite } from "../fixtures/runtime-provider.ts"; import { createHermesGpuFallbackWrapper, extractHermesGpuDiagnosticsDirectory, @@ -147,7 +148,11 @@ async function cleanupGatewayRegistrationBeforeTest( }); } -async function expectGatewayPortAvailable(host: HostCliClient, label: string): Promise { +async function expectGatewayPortAvailable( + host: HostCliClient, + runtimeProvider: RuntimeProviderPrerequisite, + label: string, +): Promise { const gatewayPort = process.env.NEMOCLAW_GATEWAY_PORT ?? "8080"; const portAvailable = await host.command( "node", @@ -167,9 +172,15 @@ async function expectGatewayPortAvailable(host: HostCliClient, label: string): P `gateway port ${gatewayPort} remains occupied after cleanup: ${resultText(portAvailable)}`, ).toBe(0); - const labeledContainers = await host.command( - "docker", - ["ps", "-aq", "--filter", `label=openshell.ai/sandbox-name=${SANDBOX_NAME}`], + const labeledContainers = await runtimeProvider.command( + [ + "container", + "ps", + "--all", + "--quiet", + "--filter", + `label=openshell.ai/sandbox-name=${SANDBOX_NAME}`, + ], { artifactName: `${label}-labeled-containers-absent`, env: buildAvailabilityProbeEnv(), @@ -179,9 +190,8 @@ async function expectGatewayPortAvailable(host: HostCliClient, label: string): P expect(labeledContainers.exitCode, resultText(labeledContainers)).toBe(0); expect(labeledContainers.stdout.trim()).toBe(""); - const namedContainers = await host.command( - "docker", - ["ps", "-a", "--filter", `name=${SANDBOX_NAME}`, "--format", "{{.Names}}"], + const namedContainers = await runtimeProvider.command( + ["container", "ps", "--all", "--filter", `name=${SANDBOX_NAME}`, "--format", "{{.Names}}"], { artifactName: `${label}-backup-containers-absent`, env: buildAvailabilityProbeEnv(), @@ -198,6 +208,7 @@ async function expectGatewayPortAvailable(host: HostCliClient, label: string): P async function cleanupHermes( host: HostCliClient, + runtimeProvider: RuntimeProviderPrerequisite, sandbox: SandboxClient, label: string, ): Promise { @@ -218,11 +229,12 @@ async function cleanupHermes( env: commandEnv(), timeoutMs: 60_000, }); - await expectGatewayPortAvailable(host, label); + await expectGatewayPortAvailable(host, runtimeProvider, label); } async function preCleanHermes( host: HostCliClient, + runtimeProvider: RuntimeProviderPrerequisite, sandbox: SandboxClient, label: string, ): Promise { @@ -243,16 +255,21 @@ async function preCleanHermes( await expectSandboxAbsent(host, label); await cleanupOwnedGatewayRuntime(host, label); await cleanupGatewayRegistrationBeforeTest(host, label); - await expectGatewayPortAvailable(host, label); + await expectGatewayPortAvailable(host, runtimeProvider, label); } async function captureFailedGpuContainer( host: HostCliClient, + runtimeProvider: RuntimeProviderPrerequisite, preRollbackDiagnosticsDir: string, ): Promise { const sandboxFilter = `label=openshell.ai/sandbox-name=${SANDBOX_NAME}`; + const runtimeInvocation = runtimeProvider.hostInvocation([]); const script = String.raw`set -u +sandbox_filter="$1" diagnostics_dir="$2" +shift 2 +runtime_command=("$@") if [ -n "$diagnostics_dir" ] && [ -d "$diagnostics_dir" ]; then printf '%s\n' "== pre-rollback diagnostics $diagnostics_dir ==" for name in summary.txt patched-container-state.json docker-inspect.json docker-network-summary.txt docker-top.txt docker-logs.txt openshell-sandbox-get.txt openshell-sandbox-list.txt openshell-logs.txt; do @@ -269,23 +286,31 @@ if [ -n "$diagnostics_dir" ] && [ -d "$diagnostics_dir" ]; then else printf '%s\n' "pre-rollback diagnostics directory unavailable: $diagnostics_dir" fi -ids="$(docker ps -aq --filter "$1")" +ids="$("\${runtime_command[@]}" container ps --all --quiet --filter "$sandbox_filter")" if [ -z "$ids" ]; then - printf '%s\n' "no Docker container found for $1" + printf '%s\n' "no runtime container found for $sandbox_filter" exit 0 fi for id in $ids; do printf '%s\n' "== container $id inspect ==" - docker inspect --format '{{json .Name}} {{json .Config.User}} {{json .Config.Entrypoint}} {{json .Config.Cmd}} {{json .State}} {{json .HostConfig.RestartPolicy}}' "$id" 2>&1 || true + "\${runtime_command[@]}" container inspect --format '{{json .Name}} {{json .Config.User}} {{json .Config.Entrypoint}} {{json .Config.Cmd}} {{json .State}} {{json .HostConfig.RestartPolicy}}' "$id" 2>&1 || true printf '%s\n' "== container $id top ==" - docker top "$id" -eo user,pid,ppid,stat,args 2>&1 || true + "\${runtime_command[@]}" container top "$id" -eo user,pid,ppid,stat,args 2>&1 || true printf '%s\n' "== container $id logs ==" - docker logs --tail 300 "$id" 2>&1 || true + "\${runtime_command[@]}" container logs --tail 300 "$id" 2>&1 || true done`; await captureDiagnosticsBestEffort(() => host.command( "bash", - ["-lc", script, "hermes-gpu-failure-diagnostics", sandboxFilter, preRollbackDiagnosticsDir], + [ + "-lc", + script, + "hermes-gpu-failure-diagnostics", + sandboxFilter, + preRollbackDiagnosticsDir, + runtimeInvocation.command, + ...runtimeInvocation.args, + ], { artifactName: "phase-2-hermes-gpu-startup-failure-diagnostics", env: buildAvailabilityProbeEnv(), @@ -310,7 +335,7 @@ test( ], }, }, - async ({ artifacts, cleanup, host, progress, sandbox }) => { + async ({ artifacts, cleanup, host, progress, runtimeProvider, sandbox }) => { await artifacts.target.declare({ id: "hermes-gpu-startup", boundary: "install.sh --non-interactive --fresh + Hermes GPU-supervised startup", @@ -320,14 +345,12 @@ test( scenario: GPU_STARTUP_SCENARIO, }); - await preCleanHermes(host, sandbox, "pre-cleanup"); + await preCleanHermes(host, runtimeProvider, sandbox, "pre-cleanup"); - const dockerInfo = await host.command("docker", ["info"], { - artifactName: "phase-1-docker-info", - env: buildAvailabilityProbeEnv(), - timeoutMs: 30_000, + await runtimeProvider.requireAvailable({ + artifactName: "phase-1-runtime-info", + scenarioLabel: "Hermes GPU startup", }); - expect(dockerInfo.exitCode, resultText(dockerInfo)).toBe(0); const hostAddress = "host.openshell.internal"; @@ -358,7 +381,7 @@ test( // resource operations after their OpenShell gateway has been removed. cleanup.trackDisposable("verify Hermes GPU gateway port is available", () => cleanupUnlessVerified(cleanTeardownVerified, () => - expectGatewayPortAvailable(host, "cleanup"), + expectGatewayPortAvailable(host, runtimeProvider, "cleanup"), ), ); cleanup.trackGateway(cleanupHost, "nemoclaw", { @@ -448,7 +471,7 @@ test( }); const gpuDiagnosticsDir = extractHermesGpuDiagnosticsDirectory(resultText(install)); await (install.exitCode !== 0 - ? captureFailedGpuContainer(host, gpuDiagnosticsDir) + ? captureFailedGpuContainer(host, runtimeProvider, gpuDiagnosticsDir) : Promise.resolve()); expect(install.exitCode, resultText(install)).toBe(0); assertStockManagedImageReceipt({ @@ -487,6 +510,7 @@ test( gpuRoute: GPU_ROUTE, host, install, + runtimeProvider, sandbox, sandboxName: SANDBOX_NAME, status, @@ -528,7 +552,7 @@ test( expect(inferencePosts.filter((request) => request.authorizationSent !== true)).toEqual([]); progress.phase("remove Hermes GPU resources"); - await cleanupHermes(host, sandbox, "phase-5-clean-teardown"); + await cleanupHermes(host, runtimeProvider, sandbox, "phase-5-clean-teardown"); cleanTeardownVerified = true; await artifacts.target.complete({ diff --git a/test/e2e/live/hermes-inference-switch.test.ts b/test/e2e/live/hermes-inference-switch.test.ts index 63f7957d8ed..703fa4c1f5f 100644 --- a/test/e2e/live/hermes-inference-switch.test.ts +++ b/test/e2e/live/hermes-inference-switch.test.ts @@ -65,7 +65,9 @@ function canonicalEndpoint(value: unknown): string | null { return typeof value === "string" ? new URL(value).toString() : null; } -test("Hermes inference set updates route/config and preserves live runtime", { +test( + "Hermes inference set updates route/config and preserves live runtime", + { timeout: TIMEOUT_MS, meta: { e2ePhases: [ @@ -78,7 +80,8 @@ test("Hermes inference set updates route/config and preserves live runtime", { "prove split provider/model credential resolution", ], }, -}, async ({ artifacts, cleanup, host, progress, sandbox, secrets }) => { + }, + async ({ artifacts, cleanup, host, progress, runtimeProvider, sandbox, secrets }) => { await artifacts.target.declare({ id: "hermes-inference-switch", boundary: @@ -110,12 +113,10 @@ test("Hermes inference set updates route/config and preserves live runtime", { }); await cleanupHermesSwitch(host, sandbox); - const docker = await host.command("docker", ["info"], { - artifactName: "docker-info", - env: buildAvailabilityProbeEnv(), - timeoutMs: 30_000, + await runtimeProvider.requireAvailable({ + artifactName: "runtime-info", + scenarioLabel: "Hermes inference switch", }); - expect(docker.exitCode, resultText(docker)).toBe(0); // OpenShell reaches this fixture from its gateway network namespace, where // the runner's loopback address is not routable. @@ -263,15 +264,11 @@ test("Hermes inference set updates route/config and preserves live runtime", { expect(dashboardModel.provider).toBe(SWITCH_PROVIDER); expect(dashboardModel.base_url).toBe(expectedBaseUrl()); expect(dashboardModel.api_mode).toBe(expectedApiMode()); - [ - "approvals", - "browser", - "session_reset", - "display", - "updates", - ].forEach((reviewedPolicySection) => { + ["approvals", "browser", "session_reset", "display", "updates"].forEach( + (reviewedPolicySection) => { expect(dashboardConfig.stdout).toMatch(new RegExp(`^${reviewedPolicySection}:`, "mu")); - }); + }, + ); const dashboardModelInfo = await sandbox.exec( SANDBOX_NAME, @@ -318,7 +315,9 @@ test("Hermes inference set updates route/config and preserves live runtime", { const publicSwitch = SWITCH_PROVIDER === PUBLIC_NVIDIA_SWITCH_PROVIDER; const durableEndpointUrl = publicSwitch ? null - : (switchEndpointUrl ?? process.env.NEMOCLAW_ENDPOINT_URL ?? DEFAULT_HOSTED_INFERENCE_BASE_URL); + : (switchEndpointUrl ?? + process.env.NEMOCLAW_ENDPOINT_URL ?? + DEFAULT_HOSTED_INFERENCE_BASE_URL); const durableCredentialEnv = publicSwitch ? null : switchEndpointUrl @@ -335,7 +334,9 @@ test("Hermes inference set updates route/config and preserves live runtime", { expect(canonicalEndpoint(state.session.endpointUrl)).toBe( canonicalEndpoint(publicSwitch ? "https://inference.local/v1" : durableEndpointUrl), ); - expect(state.session.credentialEnv).toBe(publicSwitch ? "OPENAI_API_KEY" : durableCredentialEnv); + expect(state.session.credentialEnv).toBe( + publicSwitch ? "OPENAI_API_KEY" : durableCredentialEnv, + ); expect(state.session.preferredInferenceApi).toBe(RUNTIME_SWITCH_API); expect(state.session.nimContainer).toBeNull(); @@ -472,4 +473,5 @@ test("Hermes inference set updates route/config and preserves live runtime", { expect(proxyResolutionCli.stdout).toMatch(/\bPONG\b/iu); expectAuthenticatedProxyResolutionRequests(mockBaseline, requestOffset, proxyResolutionModel); -}); + }, +); diff --git a/test/e2e/live/hermes-shields-config.test.ts b/test/e2e/live/hermes-shields-config.test.ts index f2a598d969f..daab52bb6e7 100644 --- a/test/e2e/live/hermes-shields-config.test.ts +++ b/test/e2e/live/hermes-shields-config.test.ts @@ -20,6 +20,7 @@ import { import { expect, test } from "../fixtures/e2e-test.ts"; import { startFakeOpenAiCompatibleServer } from "../fixtures/fake-openai-compatible.ts"; import { REPO_ROOT } from "../fixtures/paths.ts"; +import type { RuntimeProviderPrerequisite } from "../fixtures/runtime-provider.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; import { stripAnsi } from "./json-envelope.ts"; @@ -107,21 +108,17 @@ async function expectShieldsStatus( expect(resultText(status)).toContain(`Shields: ${expected}`); } -async function collectStartFailureDockerLogs( - host: HostCliClient, +async function collectStartFailureRuntimeLogs( + runtimeProvider: RuntimeProviderPrerequisite, artifactPrefix: string, ): Promise { - const lookup = await host.command( - "docker", + const lookup = await runtimeProvider.command( [ + "container", "ps", "--all", "--filter", - "label=openshell.ai/managed-by=openshell", - "--filter", `label=openshell.ai/sandbox-name=${SANDBOX_NAME}`, - "--filter", - "label=openshell.ai/sandbox-workspace=default", "-q", ], { @@ -135,8 +132,8 @@ async function collectStartFailureDockerLogs( const result = lookup.exitCode !== 0 || !containerId ? lookup - : await host.command("docker", ["logs", "--tail", "200", containerId], { - artifactName: `${artifactPrefix}-failure-docker-logs`, + : await runtimeProvider.command(["container", "logs", "--tail", "200", containerId], { + artifactName: `${artifactPrefix}-failure-runtime-logs`, env: commandEnv(), redactionValues: [COMPATIBLE_API_KEY], timeoutMs: 30_000, @@ -146,6 +143,7 @@ async function collectStartFailureDockerLogs( async function expectStopStartRecovery( host: HostCliClient, + runtimeProvider: RuntimeProviderPrerequisite, sandbox: SandboxClient, posture: "DOWN" | "UP", artifactPrefix: string, @@ -165,12 +163,14 @@ async function expectStopStartRecovery( timeoutMs: 5 * 60_000, }); const startFailureLogs = - start.exitCode === 0 ? "" : await collectStartFailureDockerLogs(host, artifactPrefix); + start.exitCode === 0 + ? "" + : await collectStartFailureRuntimeLogs(runtimeProvider, artifactPrefix); expect( start.exitCode, [ `start Hermes with shields ${posture.toLowerCase()}: ${resultText(start)}`, - startFailureLogs && `Docker logs:\n${startFailureLogs}`, + startFailureLogs && `Runtime logs:\n${startFailureLogs}`, ] .filter(Boolean) .join("\n"), @@ -273,7 +273,9 @@ async function completeShieldsCycle( await expectLockedPosture(sandbox, cycle); } -test("hermes-shields-config: stopped Hermes restores under both Shields postures (#6381, #8112)", { +test( + "hermes-shields-config: stopped Hermes restores under both Shields postures (#6381, #8112)", + { timeout: HERMES_SHIELDS_CONFIG_TEST_TIMEOUT_MS, meta: { e2ePhases: [ @@ -287,7 +289,8 @@ test("hermes-shields-config: stopped Hermes restores under both Shields postures "verify preserved config and ready state", ], }, -}, async ({ artifacts, cleanup: cleanupRegistry, host, progress, sandbox }) => { + }, + async ({ artifacts, cleanup: cleanupRegistry, host, progress, runtimeProvider, sandbox }) => { await artifacts.target.declare({ id: "hermes-shields-config", boundary: @@ -306,12 +309,10 @@ test("hermes-shields-config: stopped Hermes restores under both Shields postures sandboxName: SANDBOX_NAME, }); - const docker = await host.command("docker", ["info"], { - artifactName: "prereq-docker-info", - env: buildAvailabilityProbeEnv(), - timeoutMs: 30_000, + await runtimeProvider.requireAvailable({ + artifactName: "prereq-runtime-info", + scenarioLabel: "Hermes shields", }); - assertExitZero(docker, "Docker prerequisite for Hermes shields E2E"); const fake = await startFakeOpenAiCompatibleServer({ apiKey: COMPATIBLE_API_KEY, @@ -421,7 +422,13 @@ test("hermes-shields-config: stopped Hermes restores under both Shields postures await completeShieldsCycle(host, sandbox, 1); progress.phase("restart Hermes with shields up"); - await expectStopStartRecovery(host, sandbox, "UP", "cycle-1-shields-up-start-recovery"); + await expectStopStartRecovery( + host, + runtimeProvider, + sandbox, + "UP", + "cycle-1-shields-up-start-recovery", + ); await expectLockedPosture(sandbox, 1); progress.phase("unlock shields and restart Hermes"); @@ -434,7 +441,13 @@ test("hermes-shields-config: stopped Hermes restores under both Shields postures await expectImmediateInferenceRoute(sandbox, 2); await expectShieldsStatus(host, "DOWN", "cycle-2-status-down"); await expectMutablePosture(sandbox, 2); - await expectStopStartRecovery(host, sandbox, "DOWN", "cycle-2-shields-down-start-recovery"); + await expectStopStartRecovery( + host, + runtimeProvider, + sandbox, + "DOWN", + "cycle-2-shields-down-start-recovery", + ); await expectMutablePosture(sandbox, 2); progress.phase("complete second shields cycle"); @@ -475,4 +488,5 @@ test("hermes-shields-config: stopped Hermes restores under both Shields postures secondCycle: true, }, }); -}); + }, +); diff --git a/test/e2e/live/hermes-slack-e2e-helpers.ts b/test/e2e/live/hermes-slack-e2e-helpers.ts index 7d6061b37cc..eaa26b21aa9 100644 --- a/test/e2e/live/hermes-slack-e2e-helpers.ts +++ b/test/e2e/live/hermes-slack-e2e-helpers.ts @@ -20,7 +20,7 @@ import { assertHermesSlackApiProof } from "./hermes-slack-proof.ts"; import { runSecondaryCleanup as bestEffortLifecycleCleanup, CLI, - dockerInfo, + requirePhase6RuntimeProvider, expectExitZero, installSandboxOrSkipOnRateLimit, phase6Env, @@ -330,6 +330,7 @@ export async function runHermesSlackE2E({ cleanup, host, progress, + runtimeProvider, sandbox, secrets, skip, @@ -356,14 +357,7 @@ export async function runHermesSlackE2E({ providerNames: [`${SANDBOX_NAME}-slack-bridge`, `${SANDBOX_NAME}-slack-app`], }); - const docker = await dockerInfo(host, env); - if (docker.exitCode !== 0) { - if (process.env.GITHUB_ACTIONS === "true") { - throw new Error(`Docker is required for Hermes Slack E2E: ${resultText(docker)}`); - } - skip("Docker is required for Hermes Slack E2E"); - return; - } + await requirePhase6RuntimeProvider(runtimeProvider, "Hermes Slack"); await precleanHermesSlack({ host, apiKey, artifactPrefix: "preclean-hermes-slack" }); await precleanSandbox(host, SANDBOX_NAME, env, redactionValues, "preclean-hermes-slack-cli"); @@ -807,7 +801,7 @@ PY`, "bash", [ "-lc", - 'test ! -f "$HOME/.nemoclaw/sandboxes.json" || ! grep -Fq "\\\"${SANDBOX_NAME}\\\"" "$HOME/.nemoclaw/sandboxes.json"', + 'test ! -f "$HOME/.nemoclaw/sandboxes.json" || ! grep -Fq "\\\"$SANDBOX_NAME\\\"" "$HOME/.nemoclaw/sandboxes.json"', ], { artifactName: "phase-7-registry-removed", diff --git a/test/e2e/live/inference-routing-helpers.ts b/test/e2e/live/inference-routing-helpers.ts index d467b748118..23c7a4933a5 100644 --- a/test/e2e/live/inference-routing-helpers.ts +++ b/test/e2e/live/inference-routing-helpers.ts @@ -12,7 +12,8 @@ import type { SandboxClient } from "../fixtures/clients/sandbox.ts"; import { validateSandboxName } from "../fixtures/clients/sandbox.ts"; import { expect } from "../fixtures/e2e-test.ts"; import { captureIssue4462FailureDiagnostics } from "../fixtures/issue-4462-diagnostics.ts"; -import { CLI_DIST_ENTRYPOINT, CLI_ENTRYPOINT, REPO_ROOT } from "../fixtures/paths.ts"; +import { CLI_DIST_ENTRYPOINT, CLI_ENTRYPOINT } from "../fixtures/paths.ts"; +import type { RuntimeProviderPrerequisite } from "../fixtures/runtime-provider.ts"; import { type RawRunOptions, type RawRunResult, @@ -130,22 +131,19 @@ async function runOpenShell( }); } -async function requireLivePrerequisites(host: HostCliClient, skip: SkipFn): Promise { +async function requireLivePrerequisites( + host: HostCliClient, + runtimeProvider: RuntimeProviderPrerequisite, +): Promise { expect( fs.existsSync(DIST_ENTRYPOINT), "run `npm run build:cli` before live inference-routing targets", ).toBe(true); - const docker = await host.command("docker", ["info"], { + await runtimeProvider.requireAvailable({ artifactName: "prereq-docker-info-inference-routing", - env: buildAvailabilityProbeEnv(), - timeoutMs: 30_000, + scenarioLabel: "inference routing", }); - if (docker.exitCode !== 0) { - const message = `Docker is required for live inference-routing coverage: ${resultText(docker)}`; - if (process.env.GITHUB_ACTIONS === "true") throw new Error(message); - skipLive(skip, message); - } try { const openshell = await host.command("openshell", ["--version"], { diff --git a/test/e2e/live/inference-routing-provider-smoke.test.ts b/test/e2e/live/inference-routing-provider-smoke.test.ts index 4f0239ea61e..b217c6e257a 100644 --- a/test/e2e/live/inference-routing-provider-smoke.test.ts +++ b/test/e2e/live/inference-routing-provider-smoke.test.ts @@ -40,11 +40,11 @@ test("TC-INF-05 real NVIDIA key is isolated from sandbox env, process list, and "confirm placeholder credential injection", ], }, -}, async ({ artifacts, cleanup, host, progress, sandbox, secrets, skip }) => { +}, async ({ artifacts, cleanup, host, progress, runtimeProvider, sandbox, secrets, skip }) => { const apiKey = secrets.optional("NVIDIA_INFERENCE_API_KEY") ?? skipLive(skip, "NVIDIA_INFERENCE_API_KEY not set — cannot test credential isolation"); - await requireLivePrerequisites(host, skip); + await requireLivePrerequisites(host, runtimeProvider); const sandboxName = inferenceSandboxName("e2e-cred"); cleanup.add(`best-effort inference-routing credential-isolation cleanup for ${sandboxName}`, () => cleanupSandbox(host, sandbox, sandboxName), @@ -218,10 +218,10 @@ test("TC-INF-02 OpenAI provider responds through inference.local", { "request OpenAI chat through inference.local", ], }, -}, async ({ artifacts, cleanup, host, progress, sandbox, secrets, skip }) => { +}, async ({ artifacts, cleanup, host, progress, runtimeProvider, sandbox, secrets, skip }) => { requireProviderSmokeSelected("openai", skip); const apiKey = secrets.optional("OPENAI_API_KEY") ?? skipLive(skip, "OPENAI_API_KEY not set"); - await requireLivePrerequisites(host, skip); + await requireLivePrerequisites(host, runtimeProvider); const sandboxName = inferenceSandboxName("e2e-openai"); const model = process.env.NEMOCLAW_OPENAI_MODEL || "gpt-4o-mini"; cleanup.add(`best-effort inference-routing OpenAI cleanup for ${sandboxName}`, () => @@ -269,11 +269,11 @@ test("TC-INF-03 Anthropic provider responds through inference.local", { "request Anthropic messages through inference.local", ], }, -}, async ({ artifacts, cleanup, host, progress, sandbox, secrets, skip }) => { +}, async ({ artifacts, cleanup, host, progress, runtimeProvider, sandbox, secrets, skip }) => { requireProviderSmokeSelected("anthropic", skip); const apiKey = secrets.optional("ANTHROPIC_API_KEY") ?? skipLive(skip, "ANTHROPIC_API_KEY not set"); - await requireLivePrerequisites(host, skip); + await requireLivePrerequisites(host, runtimeProvider); const sandboxName = inferenceSandboxName("e2e-anth"); const model = process.env.NEMOCLAW_ANTHROPIC_MODEL || "claude-sonnet-4-6"; cleanup.add(`best-effort inference-routing Anthropic cleanup for ${sandboxName}`, () => diff --git a/test/e2e/live/inference-routing.test.ts b/test/e2e/live/inference-routing.test.ts index 0b5a82979a3..7f80405301e 100644 --- a/test/e2e/live/inference-routing.test.ts +++ b/test/e2e/live/inference-routing.test.ts @@ -58,8 +58,8 @@ test("TC-INF-06 invalid API key fails with credential classification and cleanup "confirm credential failure and no sandbox residue", ], }, -}, async ({ artifacts, cleanup, host, progress, sandbox, skip }) => { - await requireLivePrerequisites(host, skip); +}, async ({ artifacts, cleanup, host, progress, runtimeProvider, sandbox }) => { + await requireLivePrerequisites(host, runtimeProvider); const sandboxName = inferenceSandboxName("e2e-badkey"); cleanup.add(`remove inference-routing invalid-key residue for ${sandboxName}`, () => cleanupSandbox(host, sandbox, sandboxName), @@ -109,8 +109,8 @@ test("TC-INF-07 unreachable endpoint fails with transport classification and cle "confirm transport failure and no sandbox residue", ], }, -}, async ({ artifacts, cleanup, host, progress, sandbox, skip }) => { - await requireLivePrerequisites(host, skip); +}, async ({ artifacts, cleanup, host, progress, runtimeProvider, sandbox }) => { + await requireLivePrerequisites(host, runtimeProvider); const sandboxName = inferenceSandboxName("e2e-unreach"); cleanup.add(`remove inference-routing unreachable residue for ${sandboxName}`, () => cleanupSandbox(host, sandbox, sandboxName), @@ -241,7 +241,9 @@ await main(["apply"]); }, ); const raw = resultText(result); - const openshellLog = fs.existsSync(commandLogPath) ? fs.readFileSync(commandLogPath, "utf8") : ""; + const openshellLog = fs.existsSync(commandLogPath) + ? fs.readFileSync(commandLogPath, "utf8") + : ""; await artifacts.writeText("tc-inf-10-openshell-commands.jsonl", openshellLog); progress.phase("confirm rejection before OpenShell handoff"); @@ -306,7 +308,7 @@ const RUNTIME_IDENTITY_E2E_SCENARIOS = [ type RuntimeIdentityE2EContext = Pick< E2ETargetFixtures, - "artifacts" | "cleanup" | "host" | "progress" | "sandbox" + "artifacts" | "cleanup" | "host" | "progress" | "runtimeProvider" | "sandbox" > & { skip: (note?: string) => never; }; @@ -330,15 +332,13 @@ const RUNTIME_IDENTITY_E2E_OPTIONS = { } as const; async function runRuntimeIdentityE2EScenario( - _testNumber: string, - _providerLabel: string, scenario: RuntimeIdentityE2EScenario, context: RuntimeIdentityE2EContext, ): Promise { - const { artifacts, cleanup, host, progress, sandbox, skip } = context; + const { artifacts, cleanup, host, progress, runtimeProvider, sandbox } = context; const artifactPrefix = scenario.testId.toLowerCase(); progress.phase("confirm live runtime identity prerequisites"); - await requireLivePrerequisites(host, skip); + await requireLivePrerequisites(host, runtimeProvider); const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-runtime-identity-e2e-")); const workdir = path.join(root, "blueprint"); const profileDir = path.join(workdir, "provider-profiles"); @@ -346,7 +346,6 @@ async function runRuntimeIdentityE2EScenario( cleanup.add(`remove runtime identity E2E temp root ${root}`, () => { fs.rmSync(root, { recursive: true, force: true }); }); - const model = "nemoclaw-e2e-runtime-identity"; const inferenceKey = "sk-runtime-identity-TEST-NOT-A-REAL-VALUE"; const sandboxName = inferenceSandboxName(`e2e-i${scenario.testId.slice(-2)}`); @@ -676,14 +675,11 @@ async function runRuntimeIdentityE2EScenario( `Inference route 'compatible-endpoint / ${model}' is already active, reusing.`, ); for (const secret of redactionValues) expect(applyText).not.toContain(secret); - const attachedProviders = await sandbox.openshell( - ["sandbox", "provider", "list", sandboxName], - { + const attachedProviders = await sandbox.openshell(["sandbox", "provider", "list", sandboxName], { artifactName: `${artifactPrefix}-attached-providers`, env: openshellEnv, timeoutMs: 30_000, - }, - ); + }); expect(attachedProviders.exitCode, resultText(attachedProviders)).toBe(0); expect(resultText(attachedProviders)).toContain(providerName); expect(oauth.tokenRequests()).toEqual([ @@ -987,20 +983,21 @@ async function runRuntimeIdentityE2EScenario( } // OpenShell 0.0.106 does not project provider-refresh credentials into Docker sandboxes. -test.skipIf(!OPENSHELL_V0106_QUALIFICATION.supportsRuntimeIdentityRefreshProjection).for( - RUNTIME_IDENTITY_E2E_SCENARIOS, -)( +test + .skipIf(!OPENSHELL_V0106_QUALIFICATION.supportsRuntimeIdentityRefreshProjection) + .for(RUNTIME_IDENTITY_E2E_SCENARIOS)( "TC-INF-%s %sruntime identity refreshes and injects a delegated bearer through real OpenShell", RUNTIME_IDENTITY_E2E_OPTIONS, async ( - [testNumber, providerLabel, scenario], - { artifacts, cleanup, host, progress, sandbox, skip }, + [, , scenario], + { artifacts, cleanup, host, progress, runtimeProvider, sandbox, skip }, ) => { - await runRuntimeIdentityE2EScenario(testNumber, providerLabel, scenario, { + await runRuntimeIdentityE2EScenario(scenario, { artifacts, cleanup, host, progress, + runtimeProvider, sandbox, skip, }); @@ -1019,13 +1016,14 @@ test("TC-INF-09 Deep Agents Code uses a local compatible endpoint through infere "request a dcode completion through the route", ], }, -}, async ({ artifacts, cleanup, host, progress, sandbox, skip }) => { +}, async ({ artifacts, cleanup, host, progress, runtimeProvider, sandbox }) => { const model = "nemoclaw-e2e-compatible"; const apiKey = "sk-compatible-TEST-NOT-A-REAL-VALUE"; - await requireLivePrerequisites(host, skip); + await requireLivePrerequisites(host, runtimeProvider); const sandboxName = inferenceSandboxName("e2e-compat"); - cleanup.add(`best-effort inference-routing compatible-endpoint cleanup for ${sandboxName}`, () => - cleanupSandbox(host, sandbox, sandboxName), + cleanup.add( + `best-effort inference-routing compatible-endpoint cleanup for ${sandboxName}`, + () => cleanupSandbox(host, sandbox, sandboxName), ); cleanup.add(`strict inference-routing compatible-endpoint cleanup for ${sandboxName}`, () => cleanupSandbox(host, sandbox, sandboxName, { strict: true }), @@ -1162,9 +1160,9 @@ test("TC-INF-11 DNS-backed HTTPS custom endpoint routes through the local pinnin "verify private redirect rejection", ], }, -}, async ({ artifacts, cleanup, host, progress, sandbox, skip }) => { +}, async ({ artifacts, cleanup, host, progress, runtimeProvider, sandbox }) => { progress.phase("confirm live inference prerequisites"); - await requireLivePrerequisites(host, skip); + await requireLivePrerequisites(host, runtimeProvider); const model = "nemoclaw-e2e-https-pin"; const apiKey = "sk-https-pin-TEST-NOT-A-REAL-VALUE"; const sandboxName = inferenceSandboxName("e2e-https"); diff --git a/test/e2e/live/issue-2478-crash-loop-recovery.test.ts b/test/e2e/live/issue-2478-crash-loop-recovery.test.ts index 2b1aada179a..3de67f19b57 100644 --- a/test/e2e/live/issue-2478-crash-loop-recovery.test.ts +++ b/test/e2e/live/issue-2478-crash-loop-recovery.test.ts @@ -20,9 +20,9 @@ import type { CleanupRegistry } from "../fixtures/cleanup.ts"; import type { HostCliClient } from "../fixtures/clients/index.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; import type { NemoClawInstance } from "../fixtures/phases/onboarding.ts"; -import { ubuntuRepoDocker } from "../registry/matrix.ts"; +import { ubuntuRepoManagedRuntime } from "../registry/matrix.ts"; -const ENVIRONMENT = ubuntuRepoDocker("cloud-openclaw"); +const ENVIRONMENT = ubuntuRepoManagedRuntime("cloud-openclaw"); const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-2478"; const STABILITY_SECONDS = 15; const COMPATIBLE_MODEL = process.env.NEMOCLAW_COMPAT_MODEL ?? "test-model"; @@ -297,7 +297,9 @@ function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } -test("gateway recovery restores the guard chain and keeps the recovered process identity for 15 seconds (#2478)", { +test( + "gateway recovery restores the guard chain and keeps the recovered process identity for 15 seconds (#2478)", + { meta: { e2ePhases: [ "start the compatible endpoint and confirm host readiness", @@ -307,7 +309,8 @@ test("gateway recovery restores the guard chain and keeps the recovered process "verify the recovered process identity remains unchanged for 15 seconds", ], }, -}, async ({ artifacts, cleanup, environment, gateway, host, progress, runtime, sandbox }) => { + }, + async ({ artifacts, cleanup, environment, gateway, host, progress, runtime, sandbox }) => { await artifacts.target.declare({ id: "issue-2478-crash-loop-recovery", issues: ["#2478", "#2701"], @@ -346,9 +349,10 @@ test("gateway recovery restores the guard chain and keeps the recovered process timeoutMs: 60_000, }); const preRecoveryIdentity = await gateway.resolveGatewayIdentity(instance); - expect(preRecoveryIdentity, "gateway process identity changed before the recovery probe").toEqual( - initialIdentity, - ); + expect( + preRecoveryIdentity, + "gateway process identity changed before the recovery probe", + ).toEqual(initialIdentity); progress.phase("terminate one live gateway and verify production recovery"); await terminateGatewayIdentity( @@ -357,11 +361,7 @@ test("gateway recovery restores the guard chain and keeps the recovered process preRecoveryIdentity!, "functional-recovery-terminate-gateway", ); - await runProbeOnly( - host, - instance.sandboxName, - "functional-recovery-connect-probe-only", - ); + await runProbeOnly(host, instance.sandboxName, "functional-recovery-connect-probe-only"); const recoveredIdentity = await waitForGatewayIdentity(gateway, instance, 45_000); expect( recoveredIdentity, @@ -390,4 +390,5 @@ test("gateway recovery restores the guard chain and keeps the recovered process stableIdentity, stabilitySeconds: STABILITY_SECONDS, }); -}); + }, +); diff --git a/test/e2e/live/issue-4462-admin-approval-helper.ts b/test/e2e/live/issue-4462-admin-approval-helper.ts index d3b2cd2b8f3..b43229047d0 100644 --- a/test/e2e/live/issue-4462-admin-approval-helper.ts +++ b/test/e2e/live/issue-4462-admin-approval-helper.ts @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 export const ISSUE_4462_SCOPE_UPGRADE_PHASES = [ - "confirm Docker availability and clear the scope-upgrade sandbox", + "confirm configured runtime availability and clear the scope-upgrade sandbox", "install the OpenClaw sandbox", "prove onboarding settled operator.write and the first agent turn used the gateway", "trigger and approve an operator.admin request through connect", diff --git a/test/e2e/live/issue-4462-scope-upgrade-approval.test.ts b/test/e2e/live/issue-4462-scope-upgrade-approval.test.ts index d6d993b5b25..ce199b16e1d 100644 --- a/test/e2e/live/issue-4462-scope-upgrade-approval.test.ts +++ b/test/e2e/live/issue-4462-scope-upgrade-approval.test.ts @@ -103,7 +103,15 @@ test( timeout: LIVE_TIMEOUT_MS, meta: { e2ePhases: ISSUE_4462_SCOPE_UPGRADE_PHASES }, }, - async ({ artifacts, cleanup: cleanupRegistry, host, progress, sandbox, secrets, skip }) => { + async ({ + artifacts, + cleanup: cleanupRegistry, + host, + progress, + runtimeProvider, + sandbox, + secrets, + }) => { const apiKey = secrets.required("NVIDIA_INFERENCE_API_KEY"); await artifacts.target.declare({ id: "issue-4462-scope-upgrade-approval", @@ -119,15 +127,10 @@ test( ], }); - const docker = await host.command("docker", ["info"], { - artifactName: "phase-0-docker-info", - env: env(), - timeoutMs: 30_000, + await runtimeProvider.requireAvailable({ + artifactName: "phase-0-runtime-info", + scenarioLabel: "scope-upgrade approval", }); - if (docker.exitCode !== 0) { - if (process.env.GITHUB_ACTIONS === "true") throw new Error(resultText(docker)); - skip(`Docker is required: ${resultText(docker)}`); - } cleanupRegistry.trackGateway(host, "nemoclaw", { artifactName: "cleanup-openshell-gateway-destroy", diff --git a/test/e2e/live/kimi-inference-compat.test.ts b/test/e2e/live/kimi-inference-compat.test.ts index 664536b43c9..770e23ef897 100644 --- a/test/e2e/live/kimi-inference-compat.test.ts +++ b/test/e2e/live/kimi-inference-compat.test.ts @@ -1,7 +1,6 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import { resultText } from "../fixtures/clients/index.ts"; import { trustedSandboxShellScript } from "../fixtures/clients/sandbox.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; @@ -27,12 +26,14 @@ import { const TIMEOUT_MS = 40 * 60_000; -test("Kimi-compatible endpoint config enables plugin wiring and managed inference route", { +test( + "Kimi-compatible endpoint config enables plugin wiring and managed inference route", + { timeout: TIMEOUT_MS, meta: { e2ePhases: [ "select the Kimi endpoint mode and credentials", - "confirm Docker and clear the Kimi sandbox", + "confirm the selected runtime and clear the Kimi sandbox", "onboard the Kimi-compatible endpoint", "inspect the generated Kimi OpenClaw configuration", "probe the managed inference models route", @@ -40,7 +41,8 @@ test("Kimi-compatible endpoint config enables plugin wiring and managed inferenc "confirm Kimi upstream traffic", ], }, -}, async ({ artifacts, cleanup, host, progress, sandbox, secrets }) => { + }, + async ({ artifacts, cleanup, host, progress, runtimeProvider, sandbox, secrets }) => { const mode = resolveKimiInferenceMode(); const apiKey = mode === "public-nvidia" @@ -74,13 +76,11 @@ test("Kimi-compatible endpoint config enables plugin wiring and managed inferenc model: KIMI_MODEL, }); - progress.phase("confirm Docker and clear the Kimi sandbox"); - const docker = await host.command("docker", ["info"], { - artifactName: "docker-info", - env: buildAvailabilityProbeEnv(), - timeoutMs: 30_000, + progress.phase("confirm the selected runtime and clear the Kimi sandbox"); + await runtimeProvider.requireAvailable({ + artifactName: "runtime-info", + scenarioLabel: "Kimi inference compatibility", }); - expect(docker.exitCode, resultText(docker)).toBe(0); await cleanupKimi(host, sandbox); @@ -152,4 +152,5 @@ test("Kimi-compatible endpoint config enables plugin wiring and managed inferenc await assertTrajectory(sandbox, mode); progress.phase("confirm Kimi upstream traffic"); await assertKimiUpstreamTraffic({ fake, host, mode, apiKey }); -}); + }, +); diff --git a/test/e2e/live/launchable-smoke.test.ts b/test/e2e/live/launchable-smoke.test.ts index 7016ee0bc6d..6e25b785730 100644 --- a/test/e2e/live/launchable-smoke.test.ts +++ b/test/e2e/live/launchable-smoke.test.ts @@ -43,7 +43,11 @@ const ONBOARD_ATTEMPTS = 3; type ChatCompletion = { choices?: Array<{ - message?: { content?: unknown; reasoning_content?: unknown; reasoning?: unknown }; + message?: { + content?: unknown; + reasoning_content?: unknown; + reasoning?: unknown; + }; }>; }; @@ -170,7 +174,9 @@ async function expectPongFromSandboxInference( ); } -test("bootstrap install smoke: bootstrap, onboard, sandbox health, live inference, cleanup", { +test( + "bootstrap install smoke: bootstrap, onboard, sandbox health, live inference, cleanup", + { timeout: TEST_TIMEOUT_MS, meta: { e2ePhases: [ @@ -184,7 +190,8 @@ test("bootstrap install smoke: bootstrap, onboard, sandbox health, live inferenc "destroy the bootstrap sandbox and clone", ], }, -}, async ({ artifacts, cleanup, host, progress, sandbox, secrets, skip }) => { + }, + async ({ artifacts, cleanup, host, progress, runtimeProvider, sandbox, secrets, skip }) => { validateSandboxName(SANDBOX_NAME); await artifacts.target.declare({ @@ -215,12 +222,10 @@ test("bootstrap install smoke: bootstrap, onboard, sandbox health, live inferenc }); if (sudo.exitCode !== 0) skip("passwordless sudo is required for bootstrap install smoke"); - const dockerInfo = await host.command("docker", ["info"], { + await runtimeProvider.requireAvailable({ artifactName: "prereq-docker-info", - env: runEnv(), - timeoutMs: 30_000, + scenarioLabel: "bootstrap install smoke", }); - expectExitZero(dockerInfo, "Docker is running"); const network = await host.command( "bash", @@ -263,7 +268,9 @@ test("bootstrap install smoke: bootstrap, onboard, sandbox health, live inferenc expectExitZero(install, "Brev bootstrap script completed"); progress.phase("inspect installed CLI and runtime artifacts"); - const pathEnv = runEnv({ PATH: `/usr/local/bin:${process.env.PATH ?? ""}` }); + const pathEnv = runEnv({ + PATH: `/usr/local/bin:${process.env.PATH ?? ""}`, + }); const nemoclawHelp = await runBash(host, "command -v nemoclaw && nemoclaw --help >/dev/null", { artifactName: "phase-3-nemoclaw-help", @@ -294,19 +301,22 @@ test("bootstrap install smoke: bootstrap, onboard, sandbox health, live inferenc { artifactName: "phase-3-node-version", env: pathEnv, timeoutMs: 30_000 }, ); expectExitZero(nodeVersion, "node version probe"); - const node = JSON.parse(nodeVersion.stdout) as { version: string; major: number }; + const node = JSON.parse(nodeVersion.stdout) as { + version: string; + major: number; + }; await artifacts.writeJson("node-version.json", node); expect( node.major, `Node.js too old after bootstrap install: ${node.version}`, ).toBeGreaterThanOrEqual(20); - const dockerAfterInstall = await host.command("docker", ["info"], { - artifactName: "phase-3-docker-info-after-install", + const runtimeAfterInstall = await runtimeProvider.command(["info"], { + artifactName: "phase-3-runtime-info-after-install", env: pathEnv, timeoutMs: 30_000, }); - expectExitZero(dockerAfterInstall, "Docker running after install"); + expectExitZero(runtimeAfterInstall, `${runtimeProvider.displayName} running after install`); expect(fs.existsSync(BOOTSTRAP_SENTINEL), `${BOOTSTRAP_SENTINEL} missing`).toBe(true); expect(fs.existsSync(path.join(cloneDir, ".git")), `${cloneDir}/.git missing`).toBe(true); expect(fs.existsSync(path.join(cloneDir, "dist")), `${cloneDir}/dist missing`).toBe(true); @@ -378,12 +388,19 @@ test("bootstrap install smoke: bootstrap, onboard, sandbox health, live inferenc expectExitZero(inferenceConfig, "openshell inference get"); expect(inferenceConfig.stdout).toMatch(new RegExp(EXPECTED_ROUTE_PROVIDER, "i")); - const gatewayContainer = await runBash( - host, - "docker ps --format '{{.Names}}' | grep -E 'nemoclaw|openshell'", - { artifactName: "phase-5-gateway-container", env: pathEnv, timeoutMs: 30_000 }, + const gatewayContainer = await runtimeProvider.command( + ["container", "ps", "--format", "{{.Names}}"], + { + artifactName: "phase-5-gateway-runtime-resource", + env: pathEnv, + timeoutMs: 30_000, + }, ); - const gatewayContainerNames = gatewayContainer.stdout.trim(); + expectExitZero(gatewayContainer, "list gateway runtime resources"); + const gatewayContainerNames = gatewayContainer.stdout + .split(/\r?\n/u) + .filter((name) => /nemoclaw|openshell/iu.test(name)) + .join("\n"); await artifacts.writeJson("gateway-container.json", { confirmed: gatewayContainerNames.length > 0, stdout: gatewayContainer.stdout, @@ -472,4 +489,5 @@ test("bootstrap install smoke: bootstrap, onboard, sandbox health, live inferenc } await cleanupBootstrapState(host, cloneDir); -}); + }, +); diff --git a/test/e2e/live/mcp-bridge-deepagents-config.ts b/test/e2e/live/mcp-bridge-deepagents-config.ts new file mode 100644 index 00000000000..0aa21cfe6b8 --- /dev/null +++ b/test/e2e/live/mcp-bridge-deepagents-config.ts @@ -0,0 +1,40 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { assertExitZero } from "../fixtures/clients/command.ts"; +import { type SandboxClient, trustedSandboxShellScript } from "../fixtures/clients/sandbox.ts"; +import { buildRevisionScopedMcpAuthorizationPattern } from "./mcp-provider-rewrite-probe.ts"; + +export async function assertDeepAgentsMcpConfig( + sandbox: SandboxClient, + options: { + sandboxName: string; + serverName: string; + mcpUrl: string; + hostSecret: string; + }, +): Promise { + const authorizationPattern = buildRevisionScopedMcpAuthorizationPattern("FAKE_MCP_SECRET"); + const script = [ + "set -eu", + "python3 - <<'PY'", + "import json, pathlib, re", + "path = pathlib.Path('/sandbox/.deepagents/.nemoclaw-mcp.json')", + "text = path.read_text(encoding='utf-8')", + "data = json.loads(text)", + `entry = data['mcpServers'][${JSON.stringify(options.serverName)}]`, + "assert entry['type'] == 'http'", + `assert entry['url'] == ${JSON.stringify(options.mcpUrl)}`, + `assert re.fullmatch(${JSON.stringify(authorizationPattern)}, entry['headers']['Authorization'])`, + `assert ${JSON.stringify(options.hostSecret)} not in text`, + "PY", + ].join("\n"); + const result = await sandbox.execShell(options.sandboxName, trustedSandboxShellScript(script), { + artifactName: "deepagents-mcp-config-assertions", + env: buildAvailabilityProbeEnv(), + redactionValues: [options.hostSecret, Buffer.from(script, "utf8").toString("base64")], + timeoutMs: 60_000, + }); + assertExitZero(result, "Deep Agents MCP config contains placeholder and no raw host secret"); +} diff --git a/test/e2e/live/mcp-bridge-hermes-http.ts b/test/e2e/live/mcp-bridge-hermes-http.ts index 3f9b796586c..b2bda8ccc84 100644 --- a/test/e2e/live/mcp-bridge-hermes-http.ts +++ b/test/e2e/live/mcp-bridge-hermes-http.ts @@ -50,7 +50,7 @@ export function buildHermesMcpChatProbeScript(payload: string, resultToken: stri "set -e", `printf '\\n${HERMES_MCP_HTTP_STATUS_MARKER}%s\\n' "$status" >&2`, 'if [ "$curl_rc" -ne 0 ]; then exit "$curl_rc"; fi', - `case "$status" in 2??) if grep -Fq -- ${shellQuote(resultToken)} "$response_file"; then printf '${HERMES_MCP_RESULT_TOKEN_MARKER}present\\n' >&2; else printf '${HERMES_MCP_RESULT_TOKEN_MARKER}missing\\n' >&2; fi ;; *) emit_failure_body ;; esac`, + `case "$status" in 2??) if grep -Fq -- ${shellQuote(resultToken)} "$response_file"; then printf '${HERMES_MCP_RESULT_TOKEN_MARKER}present\\n' >&2; else printf '${HERMES_MCP_RESULT_TOKEN_MARKER}missing\\n' >&2; emit_failure_body; fi ;; *) emit_failure_body ;; esac`, ].join("\n"); } @@ -122,7 +122,10 @@ export function assertHermesMcpHttpResponse( ); } if (!hasResultToken(result.stderr)) { - throw new Error("Hermes real MCP tool call response did not contain the fixture result token"); + const body = sanitizedPreview(result.stdout, explicitRedactionValues); + throw new Error( + `Hermes real MCP tool call response did not contain the fixture result token; redacted response body: ${body}`, + ); } if (result.stdout !== "") { throw new Error("Hermes real MCP tool call success path emitted response contents"); diff --git a/test/e2e/live/mcp-bridge-onboard-env.ts b/test/e2e/live/mcp-bridge-onboard-env.ts index 31b0c66e685..ad39fc482cd 100644 --- a/test/e2e/live/mcp-bridge-onboard-env.ts +++ b/test/e2e/live/mcp-bridge-onboard-env.ts @@ -18,6 +18,7 @@ const MCP_BRIDGE_QUALIFICATION_ENV_KEYS = [ "NEMOCLAW_E2E_EXPECTED_SHA", "NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG", "NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG_JSON", + "NEMOCLAW_E2E_MANAGED_IMAGE_REVISION", "NEMOCLAW_RUN_LIVE_E2E", "OPENSHELL_DOCKER_SUPERVISOR_IMAGE", ] as const; @@ -54,7 +55,7 @@ export function assertMcpBridgeManagedImageReceipt(options: { environment.NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG_JSON?.trim(); if (!selectedRevision && !exactCandidateCatalog) return; - const expectedRevision = selectedRevision ?? environment.NEMOCLAW_E2E_EXPECTED_SHA?.trim() ?? ""; + const expectedRevision = selectedRevision || environment.NEMOCLAW_E2E_EXPECTED_SHA?.trim() || ""; if (!/^[0-9a-f]{40}$/u.test(expectedRevision)) { throw new Error("managed-image MCP qualification requires an exact cohort revision"); } @@ -114,6 +115,9 @@ export function buildMcpBridgeOnboardEnv(options: { NEMOCLAW_PROVIDER: "custom", NEMOCLAW_SANDBOX_NAME: options.sandboxName, NEMOCLAW_RECREATE_SANDBOX: "1", + ...(options.agent === "langchain-deepagents-code" + ? { NEMOCLAW_TOOL_DISCLOSURE: "direct" } + : {}), }; } diff --git a/test/e2e/live/mcp-bridge-sandbox.ts b/test/e2e/live/mcp-bridge-sandbox.ts index bc622a55968..03c01a5335a 100644 --- a/test/e2e/live/mcp-bridge-sandbox.ts +++ b/test/e2e/live/mcp-bridge-sandbox.ts @@ -8,6 +8,7 @@ import { parseOpenShellPolicy } from "../../../src/lib/policy/merge"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import { assertExitZero, resultText } from "../fixtures/clients/command.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; +import { discoverHostAddress } from "../fixtures/host-address.ts"; import type { SandboxClient } from "../fixtures/clients/sandbox.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; @@ -117,25 +118,7 @@ export async function hostAddressForSandbox(_host: HostCliClient): Promise { - const probe = await host.command( - "bash", - [ - "-lc", - [ - 'ip_addr="$(ip route get 1.1.1.1 2>/dev/null | awk \'{for (i=1;i<=NF;i++) if ($i=="src") {print $(i+1); exit}}\')"', - 'if [ -n "$ip_addr" ]; then echo "$ip_addr"; exit 0; fi', - "ip_addr=\"$(hostname -I 2>/dev/null | awk '{print $1}')\"", - 'if [ -n "$ip_addr" ]; then echo "$ip_addr"; exit 0; fi', - "echo 127.0.0.1", - ].join("\n"), - ], - { - artifactName: "host-private-ip-for-mcp-rebinding", - env: buildAvailabilityProbeEnv(), - timeoutMs: 30_000, - }, - ); - return probe.stdout.trim().split(/\s+/)[0] || "127.0.0.1"; + return (await discoverHostAddress(host, "host-private-ip-for-mcp-rebinding")).address; } export { diff --git a/test/e2e/live/mcp-bridge-trusted-private.ts b/test/e2e/live/mcp-bridge-trusted-private.ts index 9f248992a09..934fd746565 100644 --- a/test/e2e/live/mcp-bridge-trusted-private.ts +++ b/test/e2e/live/mcp-bridge-trusted-private.ts @@ -22,6 +22,7 @@ import { } from "./mcp-bridge-sandbox.ts"; import { startFakeMcpHttpsServer } from "./mcp-bridge-servers.ts"; import { assertAuthenticatedMcpToolDiscovery } from "./mcp-bridge-tool-discovery.ts"; +import { startRoutedPrivateRelay } from "../fixtures/routed-private-relay.ts"; const SERVER_POLICY_KEY = "mcp_bridge_fake"; const REBIND_SERVER_NAME = "rebind"; @@ -66,7 +67,15 @@ export async function assertTrustedPrivateMcpRebindingDenied( cleanup.add(`remove ${options.artifactPrefix} trusted-private MCP bridge`, () => options.cleanupBridge(host, options.sandboxName, REBIND_SERVER_NAME, options.adapter), ); - const rebindMcpUrl = `https://${REBIND_HOSTNAME}:${rebindMcp.port}/mcp`; + const upstreamHost = await hostPrivateAddressForSandbox(host); + const relay = await startRoutedPrivateRelay({ + host, + sandboxName: options.sandboxName, + upstreamHost, + upstreamPort: rebindMcp.port, + }); + cleanup.add(`stop ${options.artifactPrefix} trusted-private MCP relay`, relay.close); + const rebindMcpUrl = `https://${REBIND_HOSTNAME}:${String(relay.port)}/mcp`; const hostsFixture = await setupDnsRebindingHostsFixture( host, options.sandboxName, @@ -83,7 +92,7 @@ export async function assertTrustedPrivateMcpRebindingDenied( url: options.survivingMcpUrl, }); const survivingPolicyBeforeAdd = survivingPolicyBeforeAddResult.policy; - const trustedPrivateAddress = await hostPrivateAddressForSandbox(host); + const trustedPrivateAddress = relay.address; expect(trustedPrivateAddress).not.toBe(REBIND_PUBLIC_IP); await remapDnsRebindingHostname( host, diff --git a/test/e2e/live/messaging-compatible-endpoint-helpers.ts b/test/e2e/live/messaging-compatible-endpoint-helpers.ts index e0c9b7f2a63..9ab97d53965 100644 --- a/test/e2e/live/messaging-compatible-endpoint-helpers.ts +++ b/test/e2e/live/messaging-compatible-endpoint-helpers.ts @@ -14,6 +14,7 @@ import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import { assertExitZero } from "../fixtures/clients/command.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; import { CLI_ENTRYPOINT } from "../fixtures/paths.ts"; +import { RuntimeProviderPrerequisite } from "../fixtures/runtime-provider.ts"; export function commandEnv(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { return { @@ -37,6 +38,33 @@ async function preCleanBestEffort(run: () => Promise): Promise { const GATEWAY_NAME = "nemoclaw"; const GATEWAY_PORT = resolveGatewayPortFromName(GATEWAY_NAME); +const GATEWAY_RESOURCE_NAME = "openshell-cluster-nemoclaw"; + +function selectedRuntimeProvider(host: HostCliClient): RuntimeProviderPrerequisite { + return new RuntimeProviderPrerequisite(host, (reason) => { + throw new Error(reason); + }); +} + +async function stopGatewayRuntimeResource( + host: HostCliClient, + artifactName: string, +): Promise { + const runtimeProvider = selectedRuntimeProvider(host); + const resources = await runtimeProvider.command( + ["container", "ps", "--filter", `name=^${GATEWAY_RESOURCE_NAME}$`, "--format", "{{.Names}}"], + { artifactName: `${artifactName}-list`, timeoutMs: 30_000 }, + ); + assertExitZero(resources, "list messaging-compatible gateway runtime resource"); + if (!resources.stdout.split(/\r?\n/u).some((name) => name.trim() === GATEWAY_RESOURCE_NAME)) { + return; + } + const stopped = await runtimeProvider.command(["container", "stop", GATEWAY_RESOURCE_NAME], { + artifactName, + timeoutMs: 90_000, + }); + assertExitZero(stopped, "stop messaging-compatible gateway runtime resource"); +} type GatewayPidState = | { kind: "absent" } @@ -195,51 +223,31 @@ export async function cleanupOwnedGatewayRuntimeStrict( artifactName: string, ): Promise { await stopOwnedGatewayPid(true); - const result = await host.command( - "bash", - [ - "-lc", - [ - "set -uo pipefail", - 'cid="$(docker ps -qf "name=openshell-cluster-nemoclaw" | head -1)" || exit $?', - 'if [ -n "$cid" ]; then docker stop "$cid" >/dev/null; fi', - ].join("\n"), - ], - { - artifactName, - env: commandEnv(), - timeoutMs: 90_000, - }, - ); - assertExitZero(result, "cleanup messaging-compatible owned gateway runtime"); + await stopGatewayRuntimeResource(host, artifactName); } export async function stopGatewayRuntime(host: HostCliClient, artifactName: string): Promise { await preCleanBestEffort(() => - host.command( - "bash", - [ - "-lc", - [ - "set +e", - 'openshell_bin="$1"', - '"$openshell_bin" forward stop 18789 >/dev/null 2>&1', - '"$openshell_bin" gateway stop -g nemoclaw >/dev/null 2>&1', - 'cid="$(docker ps -qf "name=openshell-cluster-nemoclaw" 2>/dev/null | head -1)"', - 'if [ -n "$cid" ]; then docker stop "$cid" >/dev/null 2>&1 || true; fi', - '"$openshell_bin" gateway remove nemoclaw >/dev/null 2>&1', - '"$openshell_bin" gateway destroy -g nemoclaw >/dev/null 2>&1', - "exit 0", - ].join("\n"), - "gateway-runtime-preclean", - host.openshellCommandPath, - ], - { - artifactName, + host.command(host.openshellCommandPath, ["forward", "stop", "18789"], { + artifactName: `${artifactName}-forward-stop`, + env: commandEnv(), + timeoutMs: 90_000, + }), + ); + await preCleanBestEffort(() => + host.command(host.openshellCommandPath, ["gateway", "stop", "-g", GATEWAY_NAME], { + artifactName: `${artifactName}-gateway-stop`, env: commandEnv(), timeoutMs: 90_000, - }, - ), + }), + ); + await preCleanBestEffort(() => stopGatewayRuntimeResource(host, artifactName)); + await preCleanBestEffort(() => + host.command(host.openshellCommandPath, ["gateway", "destroy", "-g", GATEWAY_NAME], { + artifactName: `${artifactName}-gateway-destroy`, + env: commandEnv(), + timeoutMs: 90_000, + }), ); await stopOwnedGatewayPid(false); } diff --git a/test/e2e/live/messaging-compatible-endpoint.test.ts b/test/e2e/live/messaging-compatible-endpoint.test.ts index 4289812ffb9..20e48675571 100644 --- a/test/e2e/live/messaging-compatible-endpoint.test.ts +++ b/test/e2e/live/messaging-compatible-endpoint.test.ts @@ -525,11 +525,13 @@ async function assertOpenClawAgentTurn( expect(leaked, `Proxy hop headers leaked to upstream: ${leaked.join(",")}`).toEqual([]); } -test("messaging compatible endpoint routes Telegram-enabled OpenClaw through inference.local", { +test( + "messaging compatible endpoint routes Telegram-enabled OpenClaw through inference.local", + { timeout: TEST_TIMEOUT_MS, meta: { e2ePhases: [ - "confirm Docker and register messaging cleanup", + "confirm the selected runtime and register messaging cleanup", "clear prior messaging state and start the compatible endpoint", "confirm host reachability to the compatible endpoint", "onboard Telegram-enabled OpenClaw", @@ -538,20 +540,12 @@ test("messaging compatible endpoint routes Telegram-enabled OpenClaw through inf "record authenticated traffic and proxy-header results", ], }, -}, async ({ artifacts, cleanup, host, progress, sandbox, skip }) => { - const docker = await host.command("docker", ["info"], { - artifactName: "prereq-docker-info-messaging-compatible-endpoint", - env: commandEnv(), - timeoutMs: 30_000, + }, + async ({ artifacts, cleanup, host, progress, runtimeProvider, sandbox }) => { + await runtimeProvider.requireAvailable({ + artifactName: "prereq-runtime-info-messaging-compatible-endpoint", + scenarioLabel: "messaging compatible endpoint", }); - if (docker.exitCode !== 0) { - if (process.env.GITHUB_ACTIONS === "true") { - throw new Error( - `Docker is required for messaging compatible endpoint E2E: ${resultText(docker)}`, - ); - } - skip("Docker is required for messaging compatible endpoint E2E"); - } await artifacts.target.declare({ id: "messaging-compatible-endpoint", @@ -670,7 +664,7 @@ test("messaging compatible endpoint routes Telegram-enabled OpenClaw through inf runner, endpointUrl, assertions: { - dockerRunning: docker.exitCode === 0, + runtimeProviderAvailable: true, mockReachable: hostReachability.exitCode === 0, onboardCompleted: onboard.exitCode === 0, providerRegistered: provider.exitCode === 0, @@ -682,4 +676,5 @@ test("messaging compatible endpoint routes Telegram-enabled OpenClaw through inf ), }, }); -}); + }, +); diff --git a/test/e2e/live/messaging-providers-helpers.ts b/test/e2e/live/messaging-providers-helpers.ts index 3de9fe71507..4d23f71a7c3 100644 --- a/test/e2e/live/messaging-providers-helpers.ts +++ b/test/e2e/live/messaging-providers-helpers.ts @@ -21,8 +21,10 @@ import { validateSandboxName, } from "../fixtures/clients/sandbox.ts"; import { expect } from "../fixtures/e2e-test.ts"; +import { rebindFixtureProviderPolicyEndpoint } from "../fixtures/gateway-providers.ts"; import { CLI_ENTRYPOINT, REPO_ROOT } from "../fixtures/paths.ts"; import { buildProcessTokenProbe } from "../fixtures/process-token-probe.ts"; +import { RuntimeProviderPrerequisite } from "../fixtures/runtime-provider.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; export { CLI_ENTRYPOINT, expectExitZero, REPO_ROOT }; @@ -235,7 +237,16 @@ export function assertDiscordGatewayCapture(captureFile: string, expectedToken: expect(identify?.tokenLooksPlaceholder, "Discord placeholder leaked").toBe(false); } -export type FakeDockerApiKind = "slack" | "telegram" | "wechat" | "discord-gateway"; +export type FakeDockerApiKind = + | "slack" + | "slack-app" + | "slack-bot" + | "slack-rest" + | "slack-websocket" + | "telegram" + | "wechat" + | "discord-gateway" + | "discord-message"; export type FakeDockerApi = { kind: FakeDockerApiKind; @@ -447,7 +458,14 @@ export function messagingEnv(): MessagingEnv { env.NEMOCLAW_SKIP_SLACK_AUTH_VALIDATION = "1"; } - return { env, tokens, telegramIds, telegramAllowlistKey, slackIds, wechatAccount }; + return { + env, + tokens, + telegramIds, + telegramAllowlistKey, + slackIds, + wechatAccount, + }; } export async function runSecondaryCleanup(run: () => Promise): Promise { @@ -653,7 +671,10 @@ export async function readOpenClawConfig( import json print(json.dumps(json.load(open('/sandbox/.openclaw/openclaw.json')))) PY`, - { artifactName: "read-openclaw-config-messaging-providers", redactionValues }, + { + artifactName: "read-openclaw-config-messaging-providers", + redactionValues, + }, ); expectExitZero(result, "read openclaw.json"); return JSON.parse(result.stdout.trim()) as OpenClawConfig; @@ -694,7 +715,10 @@ export async function sandboxOutput( artifactName: string, redactionValues: string[], ): Promise { - const result = await runSandboxShell(sandbox, script, { artifactName, redactionValues }); + const result = await runSandboxShell(sandbox, script, { + artifactName, + redactionValues, + }); expectExitZero(result, artifactName); return result.stdout.trim(); } @@ -720,7 +744,7 @@ if [ -n "$match" ]; then printf '%s\n' "$match"; else echo ABSENT; fi`; } async function captureFakeApiContainerDiagnostics( - host: HostCliClient, + runtimeProvider: RuntimeProviderPrerequisite, kind: FakeDockerApiKind, component: "api" | "api-proxy", container: string, @@ -728,7 +752,7 @@ async function captureFakeApiContainerDiagnostics( redactionValues: string[], ): Promise { await runSecondaryCleanup(async () => { - await runHost(host, "docker", ["inspect", "--format", "{{json .State}}", container], { + await runtimeProvider.command(["inspect", "--format", "{{json .State}}", container], { artifactName: `diagnose-fake-${kind}-${component}-state`, env, redactionValues, @@ -736,7 +760,7 @@ async function captureFakeApiContainerDiagnostics( }); }); await runSecondaryCleanup(async () => { - await runHost(host, "docker", ["logs", "--tail", "100", container], { + await runtimeProvider.command(["logs", "--tail", "100", container], { artifactName: `diagnose-fake-${kind}-${component}-logs`, env, redactionValues, @@ -747,19 +771,18 @@ async function captureFakeApiContainerDiagnostics( async function requireFakeApiProxyReady( host: HostCliClient, + runtimeProvider: RuntimeProviderPrerequisite, options: { kind: FakeDockerApiKind; proxyContainer: string; - bridgeAddress: string; + probeAddress: string; readinessPort: string; captureDiagnostics: () => Promise; env: NodeJS.ProcessEnv; redactionValues: string[]; }, ): Promise { - const running = await runHost( - host, - "docker", + const running = await runtimeProvider.command( ["inspect", "--format", "{{.State.Running}}", options.proxyContainer], { artifactName: `inspect-fake-${options.kind}-api-proxy-readiness`, @@ -773,7 +796,7 @@ async function requireFakeApiProxyReady( ? await runHost( host, "node", - ["-e", FAKE_API_PROXY_READINESS_SOURCE, options.bridgeAddress, options.readinessPort], + ["-e", FAKE_API_PROXY_READINESS_SOURCE, options.probeAddress, options.readinessPort], { artifactName: `probe-fake-${options.kind}-api-proxy-readiness`, env: options.env, @@ -791,9 +814,11 @@ async function requireFakeApiProxyReady( } type DockerContainerInspect = { + BoundingCaps?: unknown; Config?: { Env?: unknown; }; + EffectiveCaps?: unknown; Name?: unknown; HostConfig?: { CapDrop?: unknown; @@ -807,6 +832,10 @@ type DockerContainerInspect = { }; }; +function containerName(record: DockerContainerInspect): string | undefined { + return typeof record.Name === "string" ? record.Name.replace(/^\/+/u, "") : undefined; +} + function containerNetworks(record: DockerContainerInspect): string[] { const networks = record.NetworkSettings?.Networks; return networks !== null && typeof networks === "object" ? Object.keys(networks).sort() : []; @@ -860,22 +889,20 @@ function environmentContainsCredential(entries: string[], redactionValues: strin ); } -async function requireFakeApiDockerTopology( - host: HostCliClient, +async function requireFakeApiRuntimeTopology( + runtimeProvider: RuntimeProviderPrerequisite, options: { kind: FakeDockerApiKind; apiContainer: string; proxyContainer: string; network: string; - openshellBridgeAddress: string; + proxyPublishAddress: string; proxyPorts: readonly number[]; env: NodeJS.ProcessEnv; redactionValues: string[]; }, ): Promise { - const containerInspect = await runHost( - host, - "docker", + const containerInspect = await runtimeProvider.command( ["inspect", options.apiContainer, options.proxyContainer], { artifactName: `inspect-fake-${options.kind}-api-topology`, @@ -885,7 +912,7 @@ async function requireFakeApiDockerTopology( }, ); expectExitZero(containerInspect, `inspect fake ${options.kind} API topology`); - const networkInspect = await runHost(host, "docker", ["network", "inspect", options.network], { + const networkInspect = await runtimeProvider.command(["network", "inspect", options.network], { artifactName: `inspect-fake-${options.kind}-api-network`, env: options.env, redactionValues: options.redactionValues, @@ -899,11 +926,13 @@ async function requireFakeApiDockerTopology( containers = JSON.parse(containerInspect.stdout); networks = JSON.parse(networkInspect.stdout); } catch { - throw new Error(`fake ${options.kind} API Docker topology inspection returned invalid JSON`); + throw new Error( + `fake ${options.kind} API ${runtimeProvider.displayName} topology inspection returned invalid JSON`, + ); } const records = Array.isArray(containers) ? (containers as DockerContainerInspect[]) : []; - const api = records.find((record) => record.Name === `/${options.apiContainer}`); - const proxy = records.find((record) => record.Name === `/${options.proxyContainer}`); + const api = records.find((record) => containerName(record) === options.apiContainer); + const proxy = records.find((record) => containerName(record) === options.proxyContainer); const networkRecord = Array.isArray(networks) && networks.length === 1 ? networks[0] : undefined; const apiNetworks = api === undefined ? [] : containerNetworks(api); const proxyNetworks = proxy === undefined ? [] : containerNetworks(proxy); @@ -916,37 +945,51 @@ async function requireFakeApiDockerTopology( const inspectedProxyEnvironment = proxy?.Config?.Env; const proxyEnvironmentValid = isStringArray(inspectedProxyEnvironment); const proxyEnvironment = proxyEnvironmentValid ? inspectedProxyEnvironment : []; - const networkDriver = - networkRecord !== null && - typeof networkRecord === "object" && - typeof (networkRecord as { Driver?: unknown }).Driver === "string" - ? (networkRecord as { Driver: string }).Driver + const networkFields = + networkRecord !== null && typeof networkRecord === "object" + ? (networkRecord as { + Driver?: unknown; + Internal?: unknown; + driver?: unknown; + internal?: unknown; + }) : undefined; + const networkDriver = + runtimeProvider.id === "podman" ? networkFields?.driver : networkFields?.Driver; const networkInternal = - networkRecord !== null && - typeof networkRecord === "object" && - (networkRecord as { Internal?: unknown }).Internal === true; + runtimeProvider.id === "podman" + ? networkFields?.internal === true + : networkFields?.Internal === true; + const proxyCapabilitiesDropped = + runtimeProvider.id === "podman" + ? proxyCapabilityDrops.length > 0 && + proxy?.EffectiveCaps === null && + proxy?.BoundingCaps === null + : proxyCapabilityDrops.includes("ALL"); + const defaultNetwork = runtimeProvider.id === "podman" ? "podman" : "bridge"; if ( api === undefined || proxy === undefined || networkDriver !== "bridge" || !networkInternal || JSON.stringify(apiNetworks) !== JSON.stringify([options.network]) || - JSON.stringify(proxyNetworks) !== JSON.stringify(["bridge", options.network].sort()) || + JSON.stringify(proxyNetworks) !== JSON.stringify([defaultNetwork, options.network].sort()) || apiBindings.length !== 0 || JSON.stringify(observedContainerPorts) !== JSON.stringify(expectedContainerPorts) || proxyBindings.some( ({ hostAddress, hostPort }) => - hostAddress !== options.openshellBridgeAddress || !/^\d+$/u.test(hostPort), + hostAddress !== options.proxyPublishAddress || !/^\d+$/u.test(hostPort), ) || proxy?.HostConfig?.ReadonlyRootfs !== true || - !proxyCapabilityDrops.includes("ALL") || + !proxyCapabilitiesDropped || !proxySecurityOptions.includes("no-new-privileges") || !proxyEnvironmentValid || environmentContainsCredential(proxyEnvironment, options.redactionValues) || proxy?.HostConfig?.PidsLimit !== 32 ) { - throw new Error(`fake ${options.kind} API Docker topology did not preserve isolation`); + throw new Error( + `fake ${options.kind} API ${runtimeProvider.displayName} topology did not preserve isolation`, + ); } } @@ -959,12 +1002,20 @@ export async function startFakeDockerApi( nodeArgs?: readonly string[]; containerPrefix: string; portEnv: string; + portFileEnv?: string; captureFileEnv: string; expectedEnv: Record; redactionValues: string[]; env: NodeJS.ProcessEnv; }, ): Promise { + const runtimeProvider = new RuntimeProviderPrerequisite( + host, + (reason) => { + throw new Error(reason); + }, + options.env, + ); fs.mkdirSync(path.join(REPO_ROOT, ".tmp"), { recursive: true }); const dir = fs.mkdtempSync(path.join(REPO_ROOT, ".tmp", `fake-${options.kind}.`)); const captureFile = path.join(dir, "capture.jsonl"); @@ -978,52 +1029,54 @@ export async function startFakeDockerApi( await fs.promises.rm(dir, { recursive: true, force: true }); }); - const openshellNetwork = - options.env.OPENSHELL_DOCKER_NETWORK_NAME ?? - process.env.OPENSHELL_DOCKER_NETWORK_NAME ?? - DEFAULT_OPENSHELL_DOCKER_NETWORK; - const openshellNetworkInspect = await runHost( - host, - "docker", - ["network", "inspect", openshellNetwork], - { - artifactName: `inspect-fake-${options.kind}-openshell-network`, - env: options.env, - redactionValues: options.redactionValues, - timeoutMs: 30_000, - }, - ); - expectExitZero(openshellNetworkInspect, "inspect OpenShell Docker network"); - let openshellNetworkRecords: unknown; - try { - openshellNetworkRecords = JSON.parse(openshellNetworkInspect.stdout); - } catch { - throw new Error("OpenShell Docker network inspection returned invalid JSON"); - } - const openshellBridgeAddresses = - Array.isArray(openshellNetworkRecords) && openshellNetworkRecords.length === 1 - ? (( - openshellNetworkRecords[0] as { - Driver?: unknown; - IPAM?: { Config?: Array<{ Gateway?: unknown }> }; - } - ).IPAM?.Config?.flatMap((entry) => - typeof entry.Gateway === "string" && isIPv4(entry.Gateway) ? [entry.Gateway] : [], - ) ?? []) - : []; - const openshellBridgeAddress = - openshellBridgeAddresses.length === 1 ? openshellBridgeAddresses[0] : undefined; - if ( - (openshellNetworkRecords as Array<{ Driver?: unknown }> | undefined)?.[0]?.Driver !== - "bridge" || - typeof openshellBridgeAddress !== "string" - ) { - throw new Error("OpenShell Docker network must expose exactly one IPv4 bridge gateway"); + let proxyPublishAddress = "0.0.0.0"; + let proxyProbeAddress = "127.0.0.1"; + if (runtimeProvider.id === "docker") { + const openshellNetwork = + options.env.OPENSHELL_DOCKER_NETWORK_NAME ?? + process.env.OPENSHELL_DOCKER_NETWORK_NAME ?? + DEFAULT_OPENSHELL_DOCKER_NETWORK; + const openshellNetworkInspect = await runtimeProvider.command( + ["network", "inspect", openshellNetwork], + { + artifactName: `inspect-fake-${options.kind}-openshell-network`, + env: options.env, + redactionValues: options.redactionValues, + timeoutMs: 30_000, + }, + ); + expectExitZero(openshellNetworkInspect, "inspect OpenShell Docker network"); + let openshellNetworkRecords: unknown; + try { + openshellNetworkRecords = JSON.parse(openshellNetworkInspect.stdout); + } catch { + throw new Error("OpenShell Docker network inspection returned invalid JSON"); + } + const openshellBridgeAddresses = + Array.isArray(openshellNetworkRecords) && openshellNetworkRecords.length === 1 + ? (( + openshellNetworkRecords[0] as { + Driver?: unknown; + IPAM?: { Config?: Array<{ Gateway?: unknown }> }; + } + ).IPAM?.Config?.flatMap((entry) => + typeof entry.Gateway === "string" && isIPv4(entry.Gateway) ? [entry.Gateway] : [], + ) ?? []) + : []; + const openshellBridgeAddress = + openshellBridgeAddresses.length === 1 ? openshellBridgeAddresses[0] : undefined; + if ( + (openshellNetworkRecords as Array<{ Driver?: unknown }> | undefined)?.[0]?.Driver !== + "bridge" || + typeof openshellBridgeAddress !== "string" + ) { + throw new Error("OpenShell Docker network must expose exactly one IPv4 bridge gateway"); + } + proxyPublishAddress = openshellBridgeAddress; + proxyProbeAddress = openshellBridgeAddress; } - const networkCreate = await runHost( - host, - "docker", + const networkCreate = await runtimeProvider.command( ["network", "create", "--internal", network], { artifactName: `create-fake-${options.kind}-api-network`, @@ -1034,7 +1087,7 @@ export async function startFakeDockerApi( ); expectExitZero(networkCreate, `create fake ${options.kind} API network`); cleanup(`remove ${network}`, async () => { - const remove = await runHost(host, "docker", ["network", "rm", network], { + const remove = await runtimeProvider.command(["network", "rm", network], { artifactName: `cleanup-${network}`, env: options.env, redactionValues: options.redactionValues, @@ -1045,7 +1098,7 @@ export async function startFakeDockerApi( } }); - const dockerArgs = [ + const runtimeArgs = [ "run", "-d", "--name", @@ -1054,14 +1107,17 @@ export async function startFakeDockerApi( network, "-e", `${options.portEnv}=8080`, + ...(options.portFileEnv ? ["-e", `${options.portFileEnv}=/tmp/fake/port`] : []), "-e", `${options.captureFileEnv}=/tmp/fake/capture.jsonl`, ]; - if (options.kind === "slack") dockerArgs.push("-e", "FAKE_SLACK_API_WEBSOCKET_PORT=8081"); + if (options.kind === "slack") { + runtimeArgs.push("-e", "FAKE_SLACK_API_WEBSOCKET_PORT=8081"); + } for (const [key, value] of Object.entries(options.expectedEnv)) { - dockerArgs.push("-e", `${key}=${value}`); + runtimeArgs.push("-e", `${key}=${value}`); } - dockerArgs.push( + runtimeArgs.push( "-v", `${dir}:/tmp/fake`, "-v", @@ -1077,7 +1133,7 @@ export async function startFakeDockerApi( if (apiDiagnosticsCaptured) return; apiDiagnosticsCaptured = true; await captureFakeApiContainerDiagnostics( - host, + runtimeProvider, options.kind, "api", container, @@ -1086,19 +1142,23 @@ export async function startFakeDockerApi( ); }; cleanup(`remove ${container}`, async () => { - await captureApiDiagnostics(); - const remove = await runHost(host, "docker", ["rm", "-f", container], { - artifactName: `cleanup-${container}`, - env: options.env, - redactionValues: options.redactionValues, - timeoutMs: 60_000, - }); - if (remove.exitCode !== 0 && !/No such container:/iu.test(resultText(remove))) { - expectExitZero(remove, `remove fake ${options.kind} API container ${container}`); + try { + await captureApiDiagnostics(); + const remove = await runtimeProvider.command(["rm", "--force", container], { + artifactName: `cleanup-${container}`, + env: options.env, + redactionValues: options.redactionValues, + timeoutMs: 60_000, + }); + if (remove.exitCode !== 0 && !/No such container:/iu.test(resultText(remove))) { + expectExitZero(remove, `remove fake ${options.kind} API container ${container}`); + } + } finally { + fs.rmSync(dir, { recursive: true, force: true }); } }); - const start = await runHost(host, "docker", dockerArgs, { + const start = await runtimeProvider.command(runtimeArgs, { artifactName: `start-fake-${options.kind}-api`, env: options.env, redactionValues: options.redactionValues, @@ -1111,7 +1171,7 @@ export async function startFakeDockerApi( if (proxyDiagnosticsCaptured) return; proxyDiagnosticsCaptured = true; await captureFakeApiContainerDiagnostics( - host, + runtimeProvider, options.kind, "api-proxy", proxyContainer, @@ -1121,7 +1181,7 @@ export async function startFakeDockerApi( }; cleanup(`remove ${proxyContainer}`, async () => { await captureProxyDiagnostics(); - const remove = await runHost(host, "docker", ["rm", "-f", proxyContainer], { + const remove = await runtimeProvider.command(["rm", "--force", proxyContainer], { artifactName: `cleanup-${proxyContainer}`, env: options.env, redactionValues: options.redactionValues, @@ -1132,9 +1192,7 @@ export async function startFakeDockerApi( } }); - const proxyStart = await runHost( - host, - "docker", + const proxyStart = await runtimeProvider.command( [ "run", "-d", @@ -1142,7 +1200,7 @@ export async function startFakeDockerApi( proxyContainer, "--network", "bridge", - ...proxyPorts.flatMap((port) => ["-p", `${openshellBridgeAddress}::${String(port)}`]), + ...proxyPorts.flatMap((port) => ["-p", `${proxyPublishAddress}::${String(port)}`]), "--read-only", "--cap-drop", "ALL", @@ -1170,9 +1228,7 @@ export async function startFakeDockerApi( ); expectExitZero(proxyStart, `start fake ${options.kind} API proxy`); - const proxyConnect = await runHost( - host, - "docker", + const proxyConnect = await runtimeProvider.command( ["network", "connect", network, proxyContainer], { artifactName: `connect-fake-${options.kind}-api-proxy`, @@ -1183,21 +1239,19 @@ export async function startFakeDockerApi( ); expectExitZero(proxyConnect, `connect fake ${options.kind} API proxy`); - await requireFakeApiDockerTopology(host, { + await requireFakeApiRuntimeTopology(runtimeProvider, { kind: options.kind, apiContainer: container, proxyContainer, network, - openshellBridgeAddress, + proxyPublishAddress, proxyPorts, env: options.env, redactionValues: options.redactionValues, }); const publishedPort = async (containerPort: number, artifactName: string): Promise => { - const result = await runHost( - host, - "docker", + const result = await runtimeProvider.command( ["port", proxyContainer, `${String(containerPort)}/tcp`], { artifactName, @@ -1208,9 +1262,9 @@ export async function startFakeDockerApi( ); expectExitZero(result, `read fake ${options.kind} API proxy port`); const published = result.stdout.trim().match(/^(\d+\.\d+\.\d+\.\d+):(\d+)$/u); - if (published?.[1] !== openshellBridgeAddress || !published[2]) { + if (published?.[1] !== proxyPublishAddress || !published[2]) { throw new Error( - `fake ${options.kind} API proxy port did not bind only to the OpenShell bridge`, + `fake ${options.kind} API proxy port did not bind to the reviewed ${runtimeProvider.displayName} address`, ); } return published[2]; @@ -1223,10 +1277,10 @@ export async function startFakeDockerApi( FAKE_API_PROXY_READINESS_PORT, `port-fake-${options.kind}-api-proxy-readiness`, ); - await requireFakeApiProxyReady(host, { + await requireFakeApiProxyReady(host, runtimeProvider, { kind: options.kind, proxyContainer, - bridgeAddress: openshellBridgeAddress, + probeAddress: proxyProbeAddress, readinessPort: publishedReadinessPort, captureDiagnostics: async () => { await captureProxyDiagnostics(); @@ -1247,9 +1301,10 @@ export async function startFakeDockerApi( export async function applyRestRewritePolicy( host: HostCliClient, api: FakeDockerApi, + providerName: string, + credentialKey: string, env: NodeJS.ProcessEnv, redactionValues: string[], - providerName?: string, ): Promise { const result = await runHost( host, @@ -1278,35 +1333,84 @@ export async function applyRestRewritePolicy( }, ); expectExitZero(result, `apply ${api.kind} fake REST policy`); - if (!providerName) return; + await bindFixturePolicyEndpoint( + host, + api, + providerName, + credentialKey, + "rest", + env, + redactionValues, + ); +} - const binding = await runHost( +export async function applyWebSocketRewritePolicy( + host: HostCliClient, + api: FakeDockerApi, + providerName: string, + credentialKey: string, + env: NodeJS.ProcessEnv, + redactionValues: string[], +): Promise { + const result = await runHost( host, - "bash", + "openshell", [ - "-lc", - String.raw`set -eu -policy_file="$(mktemp)" -trap 'rm -f "$policy_file"' EXIT -"$1" policy get --base "$2" >"$policy_file" -node --import tsx "$5" "$policy_file" "$3" host.openshell.internal "$4" rest -"$1" policy set --policy "$policy_file" --wait "$2"`, - `bind-fake-${api.kind}-rest-policy`, - host.openshellCommandPath, + "policy", + "update", SANDBOX_NAME, - providerName, - api.port, - path.join(REPO_ROOT, "test/e2e/fixtures/hermes-discord-policy-binding.ts"), + "--add-endpoint", + `host.openshell.internal:${api.port}:read-write:websocket:enforce:websocket-credential-rewrite,allowed-ip=10.0.0.0/8,allowed-ip=172.16.0.0/12,allowed-ip=192.168.0.0/16`, + "--add-allow", + `host.openshell.internal:${api.port}:GET:/**`, + "--add-allow", + `host.openshell.internal:${api.port}:WEBSOCKET_TEXT:/**`, + "--binary", + "/usr/local/bin/node", + "--binary", + "/usr/bin/node", + "--wait", ], { - artifactName: `apply-${api.kind}-rest-policy-credential-binding`, - cwd: REPO_ROOT, + artifactName: `apply-${api.kind}-websocket-policy`, env, redactionValues, timeoutMs: 120_000, }, ); - expectExitZero(binding, `bind ${api.kind} fake REST policy credential`); + expectExitZero(result, `apply ${api.kind} fake WebSocket policy`); + await bindFixturePolicyEndpoint( + host, + api, + providerName, + credentialKey, + "websocket", + env, + redactionValues, + ); +} + +async function bindFixturePolicyEndpoint( + host: HostCliClient, + api: FakeDockerApi, + providerName: string, + credentialKey: string, + protocol: "rest" | "websocket", + env: NodeJS.ProcessEnv, + redactionValues: string[], +): Promise { + await rebindFixtureProviderPolicyEndpoint(host, SANDBOX_NAME, { + artifactName: `bind-${api.kind}-${protocol}-credential`, + credentialEnv: credentialKey, + endpoint: { + host: "host.openshell.internal", + port: api.port, + protocol, + }, + env, + providerName, + redactionValues, + }); } export function lastJsonLine( @@ -1328,7 +1432,7 @@ export async function runSlackApiRequest( sandbox: SandboxClient, port: string, apiPath: string, - authorization: string, + authorization: string | { envKey: string; aliasPrefix?: string }, redactionValues: string[], ): Promise { const result = await runSandboxNode( @@ -1336,7 +1440,19 @@ export async function runSlackApiRequest( ` import http from "node:http"; -const authorization = process.env.FAKE_SLACK_AUTH ?? ""; +let authorization = process.env.FAKE_SLACK_AUTH ?? ""; +const providerEnvKey = process.env.FAKE_SLACK_PROVIDER_ENV_KEY ?? ""; +if (providerEnvKey) { + const scoped = process.env[providerEnvKey] ?? ""; + const expected = new RegExp("^openshell:resolve:env:(v[0-9]{1,20}_" + providerEnvKey + ")$"); + const match = scoped.match(expected); + if (!match) throw new Error("missing current revision-scoped Slack provider placeholder"); + const aliasPrefix = process.env.FAKE_SLACK_ALIAS_PREFIX ?? ""; + const placeholder = aliasPrefix + ? aliasPrefix + "-OPENSHELL-RESOLVE-ENV-" + match[1] + : scoped; + authorization = "Bearer " + placeholder; +} const token = authorization.replace(/^Bearer\\s+/, ""); const data = new URLSearchParams({ token }).toString(); const req = http.request({ @@ -1369,7 +1485,14 @@ req.end(); env: { FAKE_SLACK_PORT: port, FAKE_SLACK_PATH: apiPath, - FAKE_SLACK_AUTH: authorization, + ...(typeof authorization === "string" + ? { FAKE_SLACK_AUTH: authorization } + : { + FAKE_SLACK_PROVIDER_ENV_KEY: authorization.envKey, + ...(authorization.aliasPrefix + ? { FAKE_SLACK_ALIAS_PREFIX: authorization.aliasPrefix } + : {}), + }), }, redactionValues, timeoutMs: 60_000, diff --git a/test/e2e/live/messaging-providers.test.ts b/test/e2e/live/messaging-providers.test.ts index 4cf25126f5e..59e2fccb15f 100644 --- a/test/e2e/live/messaging-providers.test.ts +++ b/test/e2e/live/messaging-providers.test.ts @@ -12,12 +12,13 @@ import fs from "node:fs"; import { testTimeoutOptions } from "../../helpers/timeouts"; -import { expect, test } from "../fixtures/e2e-test.ts"; +import { test } from "../fixtures/e2e-test.ts"; import { assertStockManagedImageReceipt } from "../fixtures/managed-image-receipt.ts"; import { accountBool, accountString, applyRestRewritePolicy, + applyWebSocketRewritePolicy, CLI_ENTRYPOINT, channelAccount, channelEnabled, @@ -39,6 +40,7 @@ import { rawTokenSurfaceProbe, readOpenClawConfig, runHost, + runDiscordGatewayClient, runSandboxShell, runSecondaryCleanup, runSlackApiRequest, @@ -68,11 +70,12 @@ test( "inspect providers placeholders and credential isolation", "probe Telegram and Discord policy rewrites", "exercise installed Slack, Telegram, and WeChat runtimes", + "prove Discord websocket credential rewrite", "inspect gateway health and optional live sends", ], }, }, - async ({ artifacts, cleanup, host, progress, sandbox, skip }) => { + async ({ artifacts, cleanup, host, progress, runtimeProvider, sandbox, skip }) => { if (!process.env.NVIDIA_INFERENCE_API_KEY) { skip("NVIDIA_INFERENCE_API_KEY is required for live messaging-provider E2E"); return; @@ -139,13 +142,10 @@ test( }), ); - const dockerInfo = await runHost(host, "docker", ["info"], { - artifactName: "prereq-docker-info-messaging-providers", - env: state.env, - redactionValues, - timeoutMs: 30_000, + await runtimeProvider.requireAvailable({ + artifactName: "prereq-runtime-provider-info-messaging-providers", + scenarioLabel: "messaging providers", }); - expectExitZero(dockerInfo, "Docker must be running"); progress.phase("install the all-channel OpenClaw sandbox"); const install = await runHost(host, "bash", ["install.sh", "--non-interactive"], { @@ -489,7 +489,6 @@ process.exit(Array.isArray(channels) && channels.some((c) => c?.channelId === "w /^openshell:resolve:env:v[0-9]+_TELEGRAM_BOT_TOKEN_AGENT_B$/u.test(extraB), "X4b: TELEGRAM_BOT_TOKEN_AGENT_B is a revision-scoped resolve placeholder", ); - check(extraA !== extraB, "X4b: extension keys resolve to distinct placeholders"); const startLog = await sandboxOutput( sandbox, @@ -505,12 +504,14 @@ process.exit(Array.isArray(channels) && channels.some((c) => c?.channelId === "w ); const config = await readOpenClawConfig(sandbox, redactionValues); - ([ - ["M6a", "telegram", "telegram"], - ["M6b", "discord", "discord"], - ["M6c", "slack", "slack"], - ["M6d", "whatsapp", "whatsapp"], - ] as const).forEach(([assertionId, channel, plugin]) => { + ( + [ + ["M6a", "telegram", "telegram"], + ["M6b", "discord", "discord"], + ["M6c", "slack", "slack"], + ["M6d", "whatsapp", "whatsapp"], + ] as const + ).forEach(([assertionId, channel, plugin]) => { check(channelEnabled(config, channel), `${assertionId}: channels.${channel}.enabled is true`); check( pluginEnabled(config, plugin), @@ -577,8 +578,8 @@ process.exit(Array.isArray(channels) && channels.some((c) => c?.channelId === "w check( Boolean( whatsappHealth && - typeof whatsappHealth === "object" && - (whatsappHealth as Record).enabled === false, + typeof whatsappHealth === "object" && + (whatsappHealth as Record).enabled === false, ), "M-WA8a: WhatsApp health monitor is disabled for unpaired QR session", ); @@ -635,12 +636,14 @@ process.exit(Array.isArray(channels) && channels.some((c) => c?.channelId === "w const parsedRuntime = JSON.parse(runtimeChannels) as { chat?: Record; }; - ([ - ["M6e", "telegram", "default"], - ["M6f", "discord", "default"], - ["M6g", "slack", "default"], - ["M6i", "openclaw-weixin", state.wechatAccount], - ] as const).forEach(([assertionId, channel, accountId]) => { + ( + [ + ["M6e", "telegram", "default"], + ["M6f", "discord", "default"], + ["M6g", "slack", "default"], + ["M6i", "openclaw-weixin", state.wechatAccount], + ] as const + ).forEach(([assertionId, channel, accountId]) => { const entry = parsedRuntime.chat?.[channel]; check( entry?.installed === true && @@ -870,10 +873,24 @@ req.setTimeout(30000, () => { req.destroy(); console.log("TIMEOUT"); }); } progress.phase("exercise installed Slack, Telegram, and WeChat runtimes"); - const fakeSlack = await startFakeDockerApi(host, cleanup.add.bind(cleanup), { - kind: "slack", + const fakeSlackBot = await startFakeDockerApi(host, cleanup.add.bind(cleanup), { + kind: "slack-bot", + imageScript: "fake-slack-api.cjs", + containerPrefix: "nemoclaw-fake-slack-bot", + portEnv: "FAKE_SLACK_API_PORT", + portFileEnv: "FAKE_SLACK_API_PORT_FILE", + captureFileEnv: "FAKE_SLACK_API_CAPTURE_FILE", + expectedEnv: { + FAKE_SLACK_API_EXPECTED_BOT_TOKEN: state.tokens.slackBot, + FAKE_SLACK_API_EXPECTED_APP_TOKEN: state.tokens.slackApp, + }, + env: state.env, + redactionValues, + }); + const fakeSlackApp = await startFakeDockerApi(host, cleanup.add.bind(cleanup), { + kind: "slack-app", imageScript: "fake-slack-api.cjs", - containerPrefix: "nemoclaw-fake-slack", + containerPrefix: "nemoclaw-fake-slack-app", portEnv: "FAKE_SLACK_API_PORT", captureFileEnv: "FAKE_SLACK_API_CAPTURE_FILE", expectedEnv: { @@ -885,25 +902,19 @@ req.setTimeout(30000, () => { req.destroy(); console.log("TIMEOUT"); }); }); await applyRestRewritePolicy( host, - fakeSlack, + fakeSlackBot, + `${SANDBOX_NAME}-slack-bridge`, + "SLACK_BOT_TOKEN", state.env, redactionValues, - `${SANDBOX_NAME}-slack-bridge`, ); - expect( - fakeSlack.alternatePort, - "fake Slack API must publish an independent app-token port", - ).toMatch(/^[1-9][0-9]*$/u); - const fakeSlackApp = { - ...fakeSlack, - port: fakeSlack.alternatePort!, - }; await applyRestRewritePolicy( host, fakeSlackApp, + `${SANDBOX_NAME}-slack-app`, + "SLACK_APP_TOKEN", state.env, redactionValues, - `${SANDBOX_NAME}-slack-app`, ); const slackBotPlaceholder = await sandboxOutput( @@ -926,7 +937,7 @@ req.setTimeout(30000, () => { req.destroy(); console.log("TIMEOUT"); }); const slackAuth = await runSlackApiRequest( sandbox, - fakeSlack.port, + fakeSlackBot.port, "/api/auth.test", `Bearer ${slackBotPlaceholder}`, redactionValues, @@ -936,7 +947,7 @@ req.setTimeout(30000, () => { req.destroy(); console.log("TIMEOUT"); }); `M-S15: Slack auth.test exercised revision-scoped placeholder rewrite (${slackAuth.slice(0, 200)})`, ); const slackAuthCapture = lastJsonLine( - fakeSlack.captureFile, + fakeSlackBot.captureFile, (row) => row.event === "request" && row.path === "/api/auth.test", ); check( @@ -950,7 +961,7 @@ req.setTimeout(30000, () => { req.destroy(); console.log("TIMEOUT"); }); const slackUnset = await runSlackApiRequest( sandbox, - fakeSlack.port, + fakeSlackBot.port, "/api/auth.test", "Bearer openshell:resolve:env:DEFINITELY_NOT_SET_XYZ", redactionValues, @@ -974,7 +985,7 @@ req.setTimeout(30000, () => { req.destroy(); console.log("TIMEOUT"); }); "M-S16: Slack Socket Mode HTTPS leg exercised revision-scoped placeholder rewrite", ); const slackAppCapture = lastJsonLine( - fakeSlack.captureFile, + fakeSlackApp.captureFile, (row) => row.event === "request" && row.path === "/api/apps.connections.open", ); check( @@ -991,7 +1002,7 @@ req.setTimeout(30000, () => { req.destroy(); console.log("TIMEOUT"); }); check(Boolean(allowedSlackUser), "M-S17: Slack allowlist has a user for the runtime proof"); const installedSlackProof = await runInstalledSlackRuntimeProof( sandbox, - fakeSlack, + fakeSlackBot, allowedSlackUser ?? "U0AR85ATALW", redactionValues, ); @@ -1010,7 +1021,7 @@ req.setTimeout(30000, () => { req.destroy(); console.log("TIMEOUT"); }); `M-S17c: OpenClaw 2026.7.1 Slack proof used the reviewed pipeline/runtime exports (${installedSlackProof.proof})`, ); const slackRuntimeCapture = lastJsonLine( - fakeSlack.captureFile, + fakeSlackBot.captureFile, (row) => row.event === "request" && row.path === "/api/chat.postMessage", ); check( @@ -1039,9 +1050,10 @@ req.setTimeout(30000, () => { req.destroy(); console.log("TIMEOUT"); }); await applyRestRewritePolicy( host, fakeTelegram, + `${SANDBOX_NAME}-telegram-bridge`, + "TELEGRAM_BOT_TOKEN", state.env, redactionValues, - `${SANDBOX_NAME}-telegram-bridge`, ); const telegramMockTarget = "42424242"; const telegramMockText = "NemoClaw OpenClaw Telegram plugin mock E2E"; @@ -1095,9 +1107,10 @@ req.setTimeout(30000, () => { req.destroy(); console.log("TIMEOUT"); }); await applyRestRewritePolicy( host, fakeWechat, + `${SANDBOX_NAME}-wechat-bridge`, + "WECHAT_BOT_TOKEN", state.env, redactionValues, - `${SANDBOX_NAME}-wechat-bridge`, ); const installedWechatProof = await runInstalledWechatRuntimeProof( sandbox, @@ -1136,6 +1149,58 @@ req.setTimeout(30000, () => { req.destroy(); console.log("TIMEOUT"); }); wechat: installedWechatProof, }); + progress.phase("prove Discord websocket credential rewrite"); + const fakeGateway = await startFakeDockerApi(host, cleanup.add.bind(cleanup), { + kind: "discord-gateway", + imageScript: "fake-discord-gateway.cjs", + containerPrefix: "nemoclaw-fake-discord-gateway", + portEnv: "FAKE_DISCORD_GATEWAY_PORT", + portFileEnv: "FAKE_DISCORD_GATEWAY_PORT_FILE", + captureFileEnv: "FAKE_DISCORD_GATEWAY_CAPTURE_FILE", + expectedEnv: { + FAKE_DISCORD_GATEWAY_EXPECTED_TOKEN: state.tokens.discord, + }, + env: state.env, + redactionValues, + }); + await applyWebSocketRewritePolicy( + host, + fakeGateway, + `${SANDBOX_NAME}-discord-bridge`, + "DISCORD_BOT_TOKEN", + state.env, + redactionValues, + ); + const gatewayProof = await runDiscordGatewayClient(sandbox, { + port: fakeGateway.port, + identifyToken: { kind: "revisioned-discord-env" }, + redactionValues, + }); + check( + gatewayProof.includes("UPGRADE"), + "M13d: native WebSocket upgrade reached fake Discord Gateway", + ); + check( + gatewayProof.includes("HELLO") && + gatewayProof.includes("IDENTIFY_SENT_PLACEHOLDER") && + gatewayProof.includes("READY") && + gatewayProof.includes("HEARTBEAT_ACK"), + "M13e: Discord HELLO, placeholder IDENTIFY, READY, heartbeat ACK completed", + ); + const gatewayIdentify = lastJsonLine( + fakeGateway.captureFile, + (row) => row.event === "identify", + ); + check(fs.existsSync(fakeGateway.captureFile), "M13f: fake Gateway capture file exists"); + const gatewayCaptureText = fs.readFileSync(fakeGateway.captureFile, "utf8"); + check( + gatewayIdentify?.tokenMatchesExpected === true && + gatewayIdentify?.tokenLooksPlaceholder === false && + !Object.prototype.hasOwnProperty.call(gatewayIdentify, "token") && + !gatewayCaptureText.includes(state.tokens.discord) && + !gatewayCaptureText.includes("openshell:resolve:env:"), + "M13f: fake Gateway proved placeholder-to-token rewrite without logging the raw token", + ); const gatewayPort = await sandboxOutput( sandbox, `node -e ' diff --git a/test/e2e/live/model-router-provider-routed-inference.test.ts b/test/e2e/live/model-router-provider-routed-inference.test.ts index e79fece551b..53c3791f0ed 100644 --- a/test/e2e/live/model-router-provider-routed-inference.test.ts +++ b/test/e2e/live/model-router-provider-routed-inference.test.ts @@ -70,7 +70,9 @@ function routedPongReason(raw: string): "ok" | string { return "ok"; } -test("model-router provider-routed onboard returns routed inference.local PONG", { +test( + "model-router provider-routed onboard returns routed inference.local PONG", + { meta: { e2ePhases: [ "confirm routed-provider prerequisites", @@ -81,25 +83,17 @@ test("model-router provider-routed onboard returns routed inference.local PONG", "record the routed inference contract result", ], }, -}, async ({ artifacts, cleanup, host, progress, sandbox, secrets, skip }) => { + }, + async ({ artifacts, cleanup, host, progress, runtimeProvider, sandbox, secrets }) => { expect( fs.existsSync(CLI_ENTRYPOINT), "run `npm run build:cli` before live repo CLI targets", ).toBe(true); - const docker = await host.command("docker", ["info"], { - artifactName: "prereq-docker-info-model-router-provider-routed", - env: buildAvailabilityProbeEnv(), - timeoutMs: 30_000, + await runtimeProvider.requireAvailable({ + artifactName: "prereq-runtime-info-model-router-provider-routed", + scenarioLabel: "provider-routed Model Router onboarding", }); - if (docker.exitCode !== 0) { - if (process.env.GITHUB_ACTIONS === "true") { - throw new Error( - `Docker is required for provider-routed Model Router onboarding: ${resultText(docker)}`, - ); - } - skip("Docker is required for provider-routed Model Router onboarding"); - } const apiKey = requireModelRouterPublicKey(secrets); @@ -107,7 +101,7 @@ test("model-router provider-routed onboard returns routed inference.local PONG", id: "model-router-provider-routed-inference", boundary: "direct-cli-onboard-and-sandbox-exec", contract: [ - "Docker is available before onboarding", + "the selected runtime is available before onboarding", "NVIDIA_API_KEY is present and nvapi-prefixed, then staged for the router's NVIDIA_INFERENCE_API_KEY credential", "nemoclaw onboard --fresh completes with NEMOCLAW_PROVIDER=routed", "host model-router health reports at least one healthy endpoint", @@ -220,10 +214,11 @@ test("model-router provider-routed onboard returns routed inference.local PONG", await artifacts.target.complete({ id: "model-router-provider-routed-inference", assertions: { - dockerRunning: docker.exitCode === 0, + runtimeProviderAvailable: true, onboardCompleted: onboard.exitCode === 0, modelRouterHealthy: hasHealthyEndpoint(lastHealth), routedPongCompletion: completionReason === "ok", }, }); -}); + }, +); diff --git a/test/e2e/live/native-runtime-qualification-case-executor.ts b/test/e2e/live/native-runtime-qualification-case-executor.ts index daedbc420c8..6d5a8b670e7 100644 --- a/test/e2e/live/native-runtime-qualification-case-executor.ts +++ b/test/e2e/live/native-runtime-qualification-case-executor.ts @@ -878,7 +878,15 @@ export async function executeNativeRuntimeQualificationCase(progress: TestProgre }, }); expect(bundle.identity.id).toBe("podman"); - expect(bundle.workload.profile.support).toBeNull(); + expect(bundle.workload.profile).toMatchObject({ + support: { + exactDigestReferences: true, + platforms: ["linux/amd64", "linux/arm64"], + }, + hostArchitectures: ["amd64", "arm64"], + managedImageSelectionPolicy: "require-managed", + legacyDockerfileBuilds: false, + }); const hostInspection = bundle.preflightDoctor.inspectHost(); if (hostInspection.status !== "ok") { throw new Error(`Podman host qualification failed: ${bounded(hostInspection.detail)}`); diff --git a/test/e2e/live/network-policy.test.ts b/test/e2e/live/network-policy.test.ts index bf3e2ca7b8f..467b9862e89 100644 --- a/test/e2e/live/network-policy.test.ts +++ b/test/e2e/live/network-policy.test.ts @@ -25,6 +25,7 @@ import { } from "../fixtures/clients/sandbox.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; import { CLI_DIST_ENTRYPOINT, CLI_ENTRYPOINT, REPO_ROOT } from "../fixtures/paths.ts"; +import { ensureConfiguredRuntimeProviderAvailable } from "../fixtures/runtime-provider.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; import { pollDeniedReasonLog } from "./network-policy-denied-log.ts"; import { requireInferenceLocalCompletionText } from "./network-policy-inference.ts"; @@ -32,10 +33,7 @@ import { runInteractivePolicyAdd } from "./network-policy-interactive.ts"; import { isTransientProviderValidationFailure } from "./network-policy-transient-provider.ts"; import { expectPackageDatabaseReadOnly } from "./package-database-read-only.ts"; import { parseVerifiedActivePolicyPresets } from "./policy-list-state.ts"; -import { - ensureDockerAvailable, - runRestrictedOnboardWithRetry, -} from "./restricted-onboard-helpers.ts"; +import { runRestrictedOnboardWithRetry } from "./restricted-onboard-helpers.ts"; const PERMISSIVE_POLICY = path.join( REPO_ROOT, @@ -498,7 +496,7 @@ test( timeout: TEST_TIMEOUT_MS, meta: { e2ePhases: [ - "confirm built CLI Docker OpenShell and credential", + "confirm built CLI selected runtime provider OpenShell and credential", "clear the sandbox and onboard restricted policy", "prove zero active presets, read-only package metadata, default denial, and the weather allowlist", "exercise package and SaaS policy presets", @@ -510,7 +508,7 @@ test( ], }, }, - async ({ artifacts, cleanup, host, progress, sandbox, secrets, skip }) => { + async ({ artifacts, cleanup, host, progress, runtimeProvider, sandbox, secrets, skip }) => { await artifacts.target.declare({ id: "network-policy", boundary: "live-sandbox-network-policy", @@ -537,17 +535,12 @@ test( "run `npm run build:cli` before live repo CLI targets", ).toBe(true); - const docker = await host.command("docker", ["info"], { - artifactName: "prereq-docker-info-network-policy", - env: buildAvailabilityProbeEnv(), - timeoutMs: 30_000, + await ensureConfiguredRuntimeProviderAvailable({ + artifactName: "prereq-runtime-provider-info-network-policy", + host, + scenarioLabel: "network-policy", + skip, }); - if (docker.exitCode !== 0) { - if (process.env.GITHUB_ACTIONS === "true") { - throw new Error(`Docker is required for network-policy live E2E: ${text(docker)}`); - } - skip("Docker is required for network-policy live E2E"); - } const openshellVersion = await host.command("openshell", ["--version"], { artifactName: "prereq-openshell-version-network-policy", @@ -663,7 +656,7 @@ test( await expectPackageDatabaseReadOnly({ artifactPrefix: "tc-net", env: baseEnv(), - host, + runtimeProvider, sandbox, sandboxName: SANDBOX_NAME, timeoutMs: SANDBOX_EXEC_TIMEOUT_MS, @@ -1128,7 +1121,7 @@ test( timeout: TEST_TIMEOUT_MS, meta: { e2ePhases: [ - "confirm built CLI Docker OpenShell and credential", + "confirm built CLI selected runtime provider OpenShell and credential", "clear the restricted-policy sandbox", "onboard default restricted OpenClaw", "confirm the restricted tier has zero active presets", @@ -1148,9 +1141,9 @@ test( "run `npm run build:cli` before live repo CLI scenarios", ).toBe(true); - await ensureDockerAvailable({ + await ensureConfiguredRuntimeProviderAvailable({ + artifactName: "prereq-runtime-provider-info-restricted-zero-presets", host, - artifactName: "prereq-docker-info-restricted-zero-presets", skip, scenarioLabel: "restricted-zero-presets", }); diff --git a/test/e2e/live/onboard-policy-preset-sequencing.test.ts b/test/e2e/live/onboard-policy-preset-sequencing.test.ts index 397f2b87849..a141ccfe832 100644 --- a/test/e2e/live/onboard-policy-preset-sequencing.test.ts +++ b/test/e2e/live/onboard-policy-preset-sequencing.test.ts @@ -64,8 +64,11 @@ test( ], }, }, - async ({ artifacts, cleanup, docker, host, progress }) => { - await docker.requireDocker(); + async ({ artifacts, cleanup, host, progress, runtimeProvider }) => { + await runtimeProvider.requireAvailable({ + artifactName: "prereq-runtime-provider-info", + scenarioLabel: "onboard policy preset sequencing", + }); progress.phase("start the local compatible-endpoint fake server"); const apiKey = `e2e-6042-${randomBytes(16).toString("hex")}`; diff --git a/test/e2e/live/onboard-repair.test.ts b/test/e2e/live/onboard-repair.test.ts index 39ef1544de6..d98bd44219e 100644 --- a/test/e2e/live/onboard-repair.test.ts +++ b/test/e2e/live/onboard-repair.test.ts @@ -131,11 +131,13 @@ async function waitSandboxAbsent(sandbox: SandboxClient, name: string): Promise< throw new Error(`${name} still exists after forced deletion`); } -test("onboard repair resumes missing sandbox and rejects conflicting resume inputs", { +test( + "onboard repair resumes missing sandbox and rejects conflicting resume inputs", + { timeout: LIVE_TIMEOUT_MS, meta: { e2ePhases: [ - "confirm Docker and start the compatible endpoint", + "confirm the selected runtime and start the compatible endpoint", "clear prior onboard-repair state", "interrupt onboarding after sandbox creation", "remove the recorded sandbox and resume repair", @@ -145,7 +147,8 @@ test("onboard repair resumes missing sandbox and rejects conflicting resume inpu "clear the repaired onboarding state", ], }, -}, async ({ artifacts, cleanup: cleanupRegistry, host, progress, sandbox, skip }) => { + }, + async ({ artifacts, cleanup: cleanupRegistry, host, progress, runtimeProvider, sandbox }) => { const corporateCa = createCorporateCaFixture("requests", "nemoclaw-repair-corporate-ca-"); cleanupRegistry.trackDisposable("remove corporate CA fixture", () => cleanupCorporateCaFixture(corporateCa), @@ -166,15 +169,10 @@ test("onboard repair resumes missing sandbox and rejects conflicting resume inpu ], }); - const docker = await host.command("docker", ["info"], { - artifactName: "phase-0-docker-info", - env: env(), - timeoutMs: 30_000, + await runtimeProvider.requireAvailable({ + artifactName: "phase-0-runtime-info", + scenarioLabel: "onboard repair", }); - if (docker.exitCode !== 0) { - if (process.env.GITHUB_ACTIONS === "true") throw new Error(resultText(docker)); - skip(`Docker is required: ${resultText(docker)}`); - } const fake = await startFakeOpenAiCompatibleServer({ host: "0.0.0.0", @@ -420,4 +418,5 @@ test("onboard repair resumes missing sandbox and rejects conflicting resume inpu await cleanup(host, sandbox); expect(fs.existsSync(SESSION_FILE)).toBe(false); await artifacts.target.complete({ id: "onboard-repair", status: "passed" }); -}); + }, +); diff --git a/test/e2e/live/onboard-resume.test.ts b/test/e2e/live/onboard-resume.test.ts index 16f9e8d166c..68617377d47 100644 --- a/test/e2e/live/onboard-resume.test.ts +++ b/test/e2e/live/onboard-resume.test.ts @@ -84,10 +84,10 @@ interface SessionStateComplete { >; } -interface SessionStatePostVerify { - status: "in_progress"; +interface SessionStateRetryableFailure { + status: "failed"; resumable: true; - machine: { state: "post_verify" }; + machine: { state: "failed" }; } interface MutableSessionState extends Record { @@ -171,7 +171,7 @@ test( ], }, }, - async ({ artifacts, cleanup, host, progress, sandbox }) => { + async ({ artifacts, cleanup, host, progress, runtimeProvider, sandbox }) => { const corporateCa = createCorporateCaFixture("host-anchor", "nemoclaw-resume-corporate-ca-"); cleanup.trackDisposable("remove corporate CA fixture", () => cleanupCorporateCaFixture(corporateCa), @@ -205,18 +205,10 @@ test( `bin/nemoclaw.js missing — ensure the workflow runs npm ci + npm run build:cli before this test`, ).toBe(true); - // Assertion: docker-running — `docker info` exits 0. Pass fixture allowlist - // env (includes PATH, HOME, etc.) so spawn can locate `docker`. - // The shell-probe boundary defaults to no env inheritance; fixture spawns - // must opt in via buildAvailabilityProbeEnv() to keep secret-passthrough - // explicit (NVIDIA_INFERENCE_API_KEY is NOT in the allowlist; we layer it explicitly - // in Phase 2 below). - const dockerInfo = await host.command("docker", ["info"], { - artifactName: "prereq-docker-info", - env: buildAvailabilityProbeEnv(), - timeoutMs: 30_000, + await runtimeProvider.requireAvailable({ + artifactName: "prereq-runtime-info", + scenarioLabel: "onboard resume", }); - expect(dockerInfo.exitCode, dockerInfo.stderr).toBe(0); // Assertion: openshell-installed — openshell CLI is on PATH (installed by // the live validation setup before this test runs). @@ -558,7 +550,9 @@ test( await artifacts.writeJson("phase-3-session-summary.json", completeSessionSummary(complete)); expect(complete.status).toBe("complete"); expect(complete.provider).toBe("compatible-endpoint"); - expect(([ + expect( + ( + [ "preflight", "gateway", "sandbox", @@ -567,7 +561,9 @@ test( "openclaw", "policies", "agent_setup", - ] as const).every((step) => ["complete", "skipped"].includes(complete.steps[step]?.status))).toBe(true); + ] as const + ).every((step) => ["complete", "skipped"].includes(complete.steps[step]?.status)), + ).toBe(true); // Assertion: registry-has-sandbox. expect(fs.existsSync(REGISTRY_FILE)).toBe(true); @@ -602,7 +598,7 @@ test( ); expect(unavailableResumeText).not.toContain(`Sandbox '${SANDBOX_NAME}' created`); - const paused = readSession(SESSION_FILE); + const paused = readSession(SESSION_FILE); await artifacts.writeJson("phase-3-5-session-route-unavailable.json", { status: paused.status, resumable: paused.resumable, @@ -637,9 +633,7 @@ test( const repairedResumeText = `${repairedResumeRun.stdout}\n${repairedResumeRun.stderr}`; expect(repairedResumeRun.exitCode, repairedResumeText).toBe(0); expect(repairedResumeText).toContain("is ready"); - expect(repairedResumeText).not.toContain( - `Deleting and recreating sandbox '${SANDBOX_NAME}'`, - ); + expect(repairedResumeText).not.toContain(`Deleting and recreating sandbox '${SANDBOX_NAME}'`); expect(repairedResumeText).not.toContain(`Sandbox '${SANDBOX_NAME}' created`); const repaired = readSession(SESSION_FILE); expect(repaired.status).toBe("complete"); @@ -673,9 +667,7 @@ test( implicitResumeText.includes("[reuse] Skipping"), implicitResumeText, ).toBe(true); - expect(implicitResumeText).not.toContain( - `Deleting and recreating sandbox '${SANDBOX_NAME}'`, - ); + expect(implicitResumeText).not.toContain(`Deleting and recreating sandbox '${SANDBOX_NAME}'`); expect(implicitResumeText).not.toContain(`Sandbox '${SANDBOX_NAME}' created`); markSessionInProgress(SESSION_FILE); diff --git a/test/e2e/live/openclaw-discord-pairing.test.ts b/test/e2e/live/openclaw-discord-pairing.test.ts index f2b27ee4179..bdc4a7fd842 100644 --- a/test/e2e/live/openclaw-discord-pairing.test.ts +++ b/test/e2e/live/openclaw-discord-pairing.test.ts @@ -19,10 +19,10 @@ import { writePairingArtifacts, } from "./openclaw-pairing-helpers.ts"; import { - dockerInfo, expectExitZero, expectSandboxReady, installSandboxOrSkipOnRateLimit, + requirePhase6RuntimeProvider, resultText, sandboxSh, shellQuote, @@ -45,7 +45,7 @@ test("OpenClaw Discord pairing request is shared with connect-shell approval", { "approve the Discord code through connect-shell", ], }, -}, async ({ artifacts, cleanup, host, progress, sandbox, secrets, skip }) => { +}, async ({ artifacts, cleanup, host, progress, runtimeProvider, sandbox, secrets, skip }) => { const apiKey = secrets.required("NVIDIA_INFERENCE_API_KEY"); const env = pairingEnv({ sandboxName: SANDBOX_NAME, @@ -81,8 +81,7 @@ test("OpenClaw Discord pairing request is shared with connect-shell approval", { ); await cleanupPairingSandbox(host, SANDBOX_NAME, env, redactions, "preclean-discord-pairing"); - const docker = await dockerInfo(host, env); - expect(docker.exitCode, resultText(docker)).toBe(0); + await requirePhase6RuntimeProvider(runtimeProvider, "OpenClaw Discord pairing"); progress.phase("install the Discord-enabled OpenClaw sandbox"); const install = await installSandboxOrSkipOnRateLimit( diff --git a/test/e2e/live/openclaw-inference-switch.test.ts b/test/e2e/live/openclaw-inference-switch.test.ts index c8d62d433d3..d35edd16057 100644 --- a/test/e2e/live/openclaw-inference-switch.test.ts +++ b/test/e2e/live/openclaw-inference-switch.test.ts @@ -867,11 +867,13 @@ async function runOpenClawInferenceSetWithRetry( }); } -test("openclaw-inference-switch: switches route and preserves live OpenClaw behavior", { +test( + "openclaw-inference-switch: switches route and preserves live OpenClaw behavior", + { timeout: TEST_TIMEOUT_MS, meta: { e2ePhases: [ - "confirm Docker and choose the baseline provider", + "confirm the selected runtime and choose the baseline provider", "clear existing inference-switch state", "install and onboard baseline OpenClaw", "prepare the switched provider and endpoint", @@ -881,7 +883,8 @@ test("openclaw-inference-switch: switches route and preserves live OpenClaw beha "apply sandbox retention and record the result", ], }, -}, async ({ artifacts, cleanup, host, progress, sandbox, secrets, skip }) => { + }, + async ({ artifacts, cleanup, host, progress, runtimeProvider, sandbox, secrets, skip }) => { await artifacts.target.declare({ id: "openclaw-inference-switch", boundary: "install-sh-openclaw-inference-set-and-live-agent-turn", @@ -890,7 +893,7 @@ test("openclaw-inference-switch: switches route and preserves live OpenClaw beha switchModel: SWITCH_MODEL, switchInferenceApi: SWITCH_INFERENCE_API, contracts: [ - "Docker is running and an authenticated compatible baseline endpoint is staged", + "the selected runtime is available and an authenticated compatible baseline endpoint is staged", "install.sh --non-interactive onboards an OpenClaw sandbox", "when selected, the mock baseline route completes one explicit authenticated fixture request", "nemoclaw inference set switches the running sandbox route", @@ -908,19 +911,10 @@ test("openclaw-inference-switch: switches route and preserves live OpenClaw beha "run `npm run build:cli` before live repo CLI targets", ).toBe(true); - const docker = await host.command("docker", ["info"], { - artifactName: "prereq-docker-info-openclaw-inference-switch", - env: buildAvailabilityProbeEnv(), - timeoutMs: 30_000, + await runtimeProvider.requireAvailable({ + artifactName: "prereq-runtime-info-openclaw-inference-switch", + scenarioLabel: "OpenClaw inference switch", }); - if (docker.exitCode !== 0) { - if (process.env.GITHUB_ACTIONS === "true") { - throw new Error( - `Docker is required for OpenClaw inference switch E2E: ${resultText(docker)}`, - ); - } - skip("Docker is required for OpenClaw inference switch E2E"); - } const useMockBaseline = SWITCH_PROVIDER === "compatible-anthropic-endpoint" && SWITCH_MOCK_ANTHROPIC === "1"; @@ -950,9 +944,12 @@ test("openclaw-inference-switch: switches route and preserves live OpenClaw beha const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-openclaw-switch-home-")); let mockProvider: MockAnthropicProvider | undefined; - cleanup.trackDisposable(`remove OpenClaw inference switch test home for ${SANDBOX_NAME}`, () => { + cleanup.trackDisposable( + `remove OpenClaw inference switch test home for ${SANDBOX_NAME}`, + () => { fs.rmSync(home, { recursive: true, force: true }); - }); + }, + ); cleanup.trackDisposable("close switched Anthropic provider", async () => { await mockProvider?.close(); }); @@ -1102,7 +1099,7 @@ test("openclaw-inference-switch: switches route and preserves live OpenClaw beha id: "openclaw-inference-switch", status: "passed", assertions: { - dockerRunning: docker.exitCode === 0, + runtimeProviderAvailable: true, installCompleted: install.exitCode === 0, inferenceSetCompleted: switchResult.exitCode === 0, gatewayRestartExpected, @@ -1115,4 +1112,5 @@ test("openclaw-inference-switch: switches route and preserves live OpenClaw beha openClawAgentPong: true, }, }); -}); + }, +); diff --git a/test/e2e/live/openclaw-pairing-helpers.ts b/test/e2e/live/openclaw-pairing-helpers.ts index 53374faeb28..b91971583b1 100644 --- a/test/e2e/live/openclaw-pairing-helpers.ts +++ b/test/e2e/live/openclaw-pairing-helpers.ts @@ -165,11 +165,12 @@ export async function startFakeSlackApi( botToken: string, appToken: string, redactions: string[], + transport: "rest" | "websocket", ): Promise { return startFakeDockerApi(host, cleanup.add.bind(cleanup), { - kind: "slack", + kind: transport === "rest" ? "slack-rest" : "slack-websocket", imageScript: "fake-slack-api.cjs", - containerPrefix: "nemoclaw-fake-slack-pairing", + containerPrefix: `nemoclaw-fake-slack-pairing-${transport}`, portEnv: "FAKE_SLACK_API_PORT", captureFileEnv: "FAKE_SLACK_API_CAPTURE_FILE", expectedEnv: { diff --git a/test/e2e/live/openclaw-skill-cli.test.ts b/test/e2e/live/openclaw-skill-cli.test.ts index 6f30402a03f..6d10d5d32b2 100644 --- a/test/e2e/live/openclaw-skill-cli.test.ts +++ b/test/e2e/live/openclaw-skill-cli.test.ts @@ -6,7 +6,6 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import { resultText, shellQuote } from "../fixtures/clients/command.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; import { @@ -115,11 +114,13 @@ async function expectSandboxShellZero( return result; } -test("openclaw-skill-cli: direct OpenClaw skills install/list/info/check roundtrip uses workspace path", { +test( + "openclaw-skill-cli: direct OpenClaw skills install/list/info/check roundtrip uses workspace path", + { timeout: INSTALL_TIMEOUT_MS + 10 * 60_000, meta: { e2ePhases: [ - "confirm built CLI Docker and hosted inference", + "confirm built CLI, selected runtime, and hosted inference", "clear the OpenClaw skill CLI sandbox", "install and onboard the OpenClaw sandbox", "confirm OpenClaw runtime directories", @@ -128,7 +129,8 @@ test("openclaw-skill-cli: direct OpenClaw skills install/list/info/check roundtr "record the workspace skill contract", ], }, -}, async ({ artifacts, cleanup, host, progress, sandbox, secrets, skip }) => { + }, + async ({ artifacts, cleanup, host, progress, runtimeProvider, sandbox, secrets, skip }) => { expect( fs.existsSync(CLI_ENTRYPOINT), "run `npm run build:cli` before live repo CLI targets", @@ -139,7 +141,7 @@ test("openclaw-skill-cli: direct OpenClaw skills install/list/info/check roundtr boundary: "install-sh-onboard-and-openclaw-skills-cli-in-sandbox", sandboxName: SANDBOX_NAME, contracts: [ - "Docker is available before install/onboard", + "the selected runtime is available before install/onboard", "NVIDIA_INFERENCE_API_KEY is staged as the compatible endpoint credential", "install.sh creates/recreates a real OpenClaw sandbox", "OPENCLAW_HOME, OPENCLAW_STATE_DIR, and OPENCLAW_WORKSPACE_DIR reach the sandbox runtime shell", @@ -154,17 +156,10 @@ test("openclaw-skill-cli: direct OpenClaw skills install/list/info/check roundtr const hosted = requireHostedInferenceConfig(secrets); const apiKey = hosted.apiKey; - const docker = await host.command("docker", ["info"], { - artifactName: "prereq-docker-info-openclaw-skill-cli", - env: buildAvailabilityProbeEnv(), - timeoutMs: 30_000, + await runtimeProvider.requireAvailable({ + artifactName: "prereq-runtime-info-openclaw-skill-cli", + scenarioLabel: "OpenClaw skill CLI", }); - if (docker.exitCode !== 0) { - if (process.env.GITHUB_ACTIONS === "true") { - throw new Error(`Docker is required for openclaw-skill-cli E2E: ${resultText(docker)}`); - } - skip("Docker is required for openclaw-skill-cli E2E"); - } const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-openclaw-skill-cli-home-")); const env = testEnv(home); @@ -291,4 +286,5 @@ test("openclaw-skill-cli: direct OpenClaw skills install/list/info/check roundtr installedSkill: SKILL_ID, expectedDiskPath: EXPECTED_WORKSPACE_SKILL_PATH, }); -}); + }, +); diff --git a/test/e2e/live/openclaw-slack-pairing.test.ts b/test/e2e/live/openclaw-slack-pairing.test.ts index ad02c20ac69..dec390d47f9 100644 --- a/test/e2e/live/openclaw-slack-pairing.test.ts +++ b/test/e2e/live/openclaw-slack-pairing.test.ts @@ -19,10 +19,10 @@ import { writePairingArtifacts, } from "./openclaw-pairing-helpers.ts"; import { - dockerInfo, expectExitZero, expectSandboxReady, installSandboxOrSkipOnRateLimit, + requirePhase6RuntimeProvider, resultText, trackSandboxCleanup, } from "./phase6-messaging-helpers.ts"; @@ -85,7 +85,7 @@ test("OpenClaw Slack Socket Mode pairing request is shared with connect-shell ap "approve the Slack code through connect-shell", ], }, -}, async ({ artifacts, cleanup, host, progress, sandbox, secrets, skip }) => { +}, async ({ artifacts, cleanup, host, progress, runtimeProvider, sandbox, secrets, skip }) => { const apiKey = secrets.required("NVIDIA_INFERENCE_API_KEY"); const env = pairingEnv({ sandboxName: SANDBOX_NAME, @@ -125,8 +125,7 @@ test("OpenClaw Slack Socket Mode pairing request is shared with connect-shell ap ); await cleanupPairingSandbox(host, SANDBOX_NAME, env, redactions, "preclean-slack-pairing"); - const docker = await dockerInfo(host, env); - expect(docker.exitCode, resultText(docker)).toBe(0); + await requirePhase6RuntimeProvider(runtimeProvider, "OpenClaw Slack pairing"); progress.phase("install the Slack-enabled OpenClaw sandbox"); const install = await installSandboxOrSkipOnRateLimit( @@ -167,6 +166,7 @@ test("OpenClaw Slack Socket Mode pairing request is shared with connect-shell ap SLACK_BOT_TOKEN, SLACK_APP_TOKEN, redactions, + "rest", ); const fakeSlackWebSocket = await startFakeSlackApi( host, @@ -175,6 +175,7 @@ test("OpenClaw Slack Socket Mode pairing request is shared with connect-shell ap SLACK_BOT_TOKEN, SLACK_APP_TOKEN, redactions, + "websocket", ); await applyFakePolicy({ host, diff --git a/test/e2e/live/openclaw-tui-chat-correlation.test.ts b/test/e2e/live/openclaw-tui-chat-correlation.test.ts index 9820fbaefbf..51bf8a260fb 100644 --- a/test/e2e/live/openclaw-tui-chat-correlation.test.ts +++ b/test/e2e/live/openclaw-tui-chat-correlation.test.ts @@ -28,7 +28,7 @@ import { import { expect, test } from "../fixtures/e2e-test.ts"; import { CLI_ENTRYPOINT, REPO_ROOT } from "../fixtures/paths.ts"; import type { NemoClawInstance } from "../fixtures/phases/onboarding.ts"; -import { ubuntuRepoDocker } from "../registry/matrix.ts"; +import { ubuntuRepoManagedRuntime } from "../registry/matrix.ts"; import { stripTerminalControl } from "../support/issue-4434-tui-capture.ts"; import { buildIssue6194OpenShellApprovalExpectScript, @@ -55,7 +55,7 @@ import { // expected-state probes; this test's regression-target probes are bespoke // websocket-trace assertions that don't fit the // `from(env) → from(state, instance)` model. -const ENVIRONMENT = ubuntuRepoDocker("cloud-openclaw"); +const ENVIRONMENT = ubuntuRepoManagedRuntime("cloud-openclaw"); const SANDBOX_NAME = "e2e-oc-tui-corr"; // OpenClaw 2026.7.1 is the post-fix regression-guard version for #2603 + #3145. @@ -267,7 +267,11 @@ function looksLikeEventCaptureFailure(repro: LiveIssue2603Trace): boolean { function issue2603AttemptOutcome( repro: LiveIssue2603Trace, index: number, -): Issue2603AttemptOutcome & { attempt: number; eventCount: number; chatEventCount: number } { +): Issue2603AttemptOutcome & { + attempt: number; + eventCount: number; + chatEventCount: number; +} { const failedAttempt = { attempt: index + 1, captureFailure: false, @@ -647,7 +651,9 @@ test( const expectScript = artifacts.pathFor("issue6194-openclaw-tui.expect"); const tuiSession = `${ISSUE6194_TUI_SESSION_PREFIX}-${instance.sandboxName}-${Date.now()}-${randomUUID()}`; precreateIssue6194Capture(captureFile); - writeFileSync(expectScript, buildIssue6194TuiExpectScript(), { mode: 0o700 }); + writeFileSync(expectScript, buildIssue6194TuiExpectScript(), { + mode: 0o700, + }); try { const tui = await host.command("expect", [expectScript], { artifactName: "issue6194-openclaw-tui-post-idle", diff --git a/test/e2e/live/openshell-credential-generation-window.test.ts b/test/e2e/live/openshell-credential-generation-window.test.ts index 5ffdefccb74..cfe655d012d 100644 --- a/test/e2e/live/openshell-credential-generation-window.test.ts +++ b/test/e2e/live/openshell-credential-generation-window.test.ts @@ -321,625 +321,635 @@ async function runFreshRequest( }>(result.stdout); } -test("openshell-credential-generation-window", { - timeout: 60 * 60_000, - meta: { - e2ePhases: [ - "start endpoints and onboard the credential-window sandbox", - "attach the MCP provider and observe its initial generation", - "prove a retained credential generation expires", - "rotate beyond the retained generation window", - "prove key and bridge removal revoke access", - "re-add the bridge and keep the old process revoked", - "rebuild the sandbox and confirm credential reuse", - "remove the MCP bridge and audit denied requests", - ], +test( + "openshell-credential-generation-window", + { + timeout: 60 * 60_000, + meta: { + e2ePhases: [ + "start endpoints and onboard the credential-window sandbox", + "attach the MCP provider and observe its initial generation", + "prove a retained credential generation expires", + "rotate beyond the retained generation window", + "prove key and bridge removal revoke access", + "re-add the bridge and keep the old process revoked", + "rebuild the sandbox and confirm credential reuse", + "remove the MCP bridge and audit denied requests", + ], + }, }, -}, async ({ artifacts, cleanup, host, progress, sandbox }) => { - expect(process.env.NEMOCLAW_OPENSHELL_EXACT_MAIN_PROOF).toBe("1"); - expect(CREDENTIAL_WINDOW_ROTATION_COUNT).toBeGreaterThan( - OPENSHELL_RETAINED_CREDENTIAL_GENERATIONS, - ); + async ({ artifacts, cleanup, host, progress, sandbox }) => { + expect(process.env.NEMOCLAW_OPENSHELL_EXACT_MAIN_PROOF).toBe("1"); + expect(CREDENTIAL_WINDOW_ROTATION_COUNT).toBeGreaterThan( + OPENSHELL_RETAINED_CREDENTIAL_GENERATIONS, + ); - const allSecrets = credentialWindowSecrets(); - const initialSecret = allSecrets[0]!; - const rotationSecrets = allSecrets.slice(1, CREDENTIAL_WINDOW_ROTATION_COUNT + 1); - const expirySecret = allSecrets.at(-2)!; - const restartSecret = allSecrets.at(-1)!; - artifacts.addRedactionValues([COMPATIBLE_KEY, ...allSecrets]); - await artifacts.target.declare({ - id: "openshell-credential-generation-window", - contracts: [ - "OpenShell f27ff150 retained credential generations", - "NemoClaw MCP detach, restart, and rebuild lifecycle", - ], - sourceRevision: "3dee5570a46076a57a3b056f35f35ebc0861ac85", - }); + const allSecrets = credentialWindowSecrets(); + const initialSecret = allSecrets[0]!; + const rotationSecrets = allSecrets.slice(1, CREDENTIAL_WINDOW_ROTATION_COUNT + 1); + const expirySecret = allSecrets.at(-2)!; + const restartSecret = allSecrets.at(-1)!; + artifacts.addRedactionValues([COMPATIBLE_KEY, ...allSecrets]); + await artifacts.target.declare({ + id: "openshell-credential-generation-window", + contracts: [ + "OpenShell f27ff150 retained credential generations", + "NemoClaw MCP detach, restart, and rebuild lifecycle", + ], + sourceRevision: "3dee5570a46076a57a3b056f35f35ebc0861ac85", + }); - const compatibleMock = await startCompatibleMock({ - apiKey: COMPATIBLE_KEY, - model: COMPATIBLE_MODEL, - }); - cleanup.add("stop credential-window compatible endpoint", () => compatibleMock.close()); - const fakeMcp = await startFakeMcpHttpsServer({ secret: initialSecret }); - cleanup.add("stop credential-window MCP endpoint", () => fakeMcp.close()); - const tunnel = await startPublicMcpHttpsTunnel({ - cleanup, - label: "credential-window MCP endpoint", - progress, - server: fakeMcp, - }); - const hostAddress = await hostAddressForSandbox(host); - const endpointUrl = `http://${hostAddress}:${compatibleMock.port}/v1`; - await host.cleanupSandbox(SANDBOX_NAME, { - artifactName: "precleanup-credential-window-sandbox", - timeoutMs: 15 * 60_000, - }); - cleanup.trackSandbox(host, SANDBOX_NAME, { - artifactName: "cleanup-credential-window-sandbox", - timeoutMs: 15 * 60_000, - }); - const onboard = await host.nemoclaw( - ["onboard", "--non-interactive", "--yes", "--yes-i-accept-third-party-software"], - { - artifactName: "onboard-credential-window-sandbox", - env: { - ...buildAvailabilityProbeEnv(), - COMPATIBLE_API_KEY: COMPATIBLE_KEY, - NVIDIA_INFERENCE_API_KEY: COMPATIBLE_KEY, - NEMOCLAW_AGENT: "openclaw", - NEMOCLAW_ENDPOINT_URL: endpointUrl, - NEMOCLAW_MODEL: COMPATIBLE_MODEL, - NEMOCLAW_COMPAT_MODEL: COMPATIBLE_MODEL, - NEMOCLAW_PREFERRED_API: "openai-completions", - NEMOCLAW_PROVIDER: "custom", - NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, - NEMOCLAW_RECREATE_SANDBOX: "1", + const compatibleMock = await startCompatibleMock({ + apiKey: COMPATIBLE_KEY, + model: COMPATIBLE_MODEL, + }); + cleanup.add("stop credential-window compatible endpoint", () => compatibleMock.close()); + const fakeMcp = await startFakeMcpHttpsServer({ secret: initialSecret }); + cleanup.add("stop credential-window MCP endpoint", () => fakeMcp.close()); + const tunnel = await startPublicMcpHttpsTunnel({ + cleanup, + label: "credential-window MCP endpoint", + progress, + server: fakeMcp, + }); + const hostAddress = await hostAddressForSandbox(host); + const endpointUrl = `http://${hostAddress}:${compatibleMock.port}/v1`; + await host.cleanupSandbox(SANDBOX_NAME, { + artifactName: "precleanup-credential-window-sandbox", + timeoutMs: 15 * 60_000, + }); + cleanup.trackSandbox(host, SANDBOX_NAME, { + artifactName: "cleanup-credential-window-sandbox", + timeoutMs: 15 * 60_000, + }); + const onboard = await host.nemoclaw( + ["onboard", "--non-interactive", "--yes", "--yes-i-accept-third-party-software"], + { + artifactName: "onboard-credential-window-sandbox", + env: { + ...buildAvailabilityProbeEnv(), + COMPATIBLE_API_KEY: COMPATIBLE_KEY, + NVIDIA_INFERENCE_API_KEY: COMPATIBLE_KEY, + NEMOCLAW_AGENT: "openclaw", + NEMOCLAW_ENDPOINT_URL: endpointUrl, + NEMOCLAW_MODEL: COMPATIBLE_MODEL, + NEMOCLAW_COMPAT_MODEL: COMPATIBLE_MODEL, + NEMOCLAW_PREFERRED_API: "openai-completions", + NEMOCLAW_PROVIDER: "custom", + NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, + NEMOCLAW_RECREATE_SANDBOX: "1", + }, + redactionValues: [COMPATIBLE_KEY, ...allSecrets], + timeoutMs: 20 * 60_000, }, - redactionValues: [COMPATIBLE_KEY, ...allSecrets], - timeoutMs: 20 * 60_000, - }, - ); - expectExitZero(onboard, "onboard credential-window sandbox"); + ); + expectExitZero(onboard, "onboard credential-window sandbox"); - progress.phase("attach the MCP provider and observe its initial generation"); - const add = await host.nemoclaw( - [ - SANDBOX_NAME, - "mcp", - "add", - SERVER_NAME, - "--url", - tunnel.url, - "--env", - CREDENTIAL_WINDOW_ENV_NAME, - ], - { - artifactName: "credential-window-mcp-add", - env: { - ...buildAvailabilityProbeEnv(), - [CREDENTIAL_WINDOW_ENV_NAME]: initialSecret, + progress.phase("attach the MCP provider and observe its initial generation"); + const add = await host.nemoclaw( + [ + SANDBOX_NAME, + "mcp", + "add", + SERVER_NAME, + "--url", + tunnel.url, + "--env", + CREDENTIAL_WINDOW_ENV_NAME, + ], + { + artifactName: "credential-window-mcp-add", + env: { + ...buildAvailabilityProbeEnv(), + [CREDENTIAL_WINDOW_ENV_NAME]: initialSecret, + }, + redactionValues: [COMPATIBLE_KEY, ...allSecrets], + timeoutMs: 4 * 60_000, }, - redactionValues: [COMPATIBLE_KEY, ...allSecrets], - timeoutMs: 4 * 60_000, - }, - ); - expectExitZero(add, "add credential-window MCP bridge"); - cleanup.add("remove credential-window MCP bridge", () => cleanupBridge(host)); - - const status = await host.nemoclaw([SANDBOX_NAME, "mcp", "status", SERVER_NAME, "--json"], { - artifactName: "credential-window-mcp-status", - env: buildAvailabilityProbeEnv(), - timeoutMs: 60_000, - }); - expectExitZero(status, "inspect credential-window MCP bridge"); - const providerName = (JSON.parse(status.stdout) as { provider: { name: string } }).provider.name; - expect(providerName).toMatch(/^e2e-cred-window-mcp-fake-[a-f0-9]{16}$/u); + ); + expectExitZero(add, "add credential-window MCP bridge"); + cleanup.add("remove credential-window MCP bridge", () => cleanupBridge(host)); - const originalRevision = await observeFreshRevision( - sandbox, - "credential-window-initial-fresh-revision", - ); - const resetControl = await sandbox.exec( - SANDBOX_NAME, - [ - "rm", - "-f", - CREDENTIAL_WINDOW_PATHS.control, - CREDENTIAL_WINDOW_PATHS.ready, - CREDENTIAL_WINDOW_PATHS.acknowledgement, - ], - { - artifactName: "credential-window-reset-control-files", - env: openshellEnv(), + const status = await host.nemoclaw([SANDBOX_NAME, "mcp", "status", SERVER_NAME, "--json"], { + artifactName: "credential-window-mcp-status", + env: buildAvailabilityProbeEnv(), timeoutMs: 60_000, - }, - ); - expectExitZero(resetControl, "reset credential-window control files"); - - progress.phase("prove a retained credential generation expires"); - const expiryAtMs = Date.now() + CREDENTIAL_WINDOW_EXPIRY_DELAY_MS; - fakeMcp.setSecret(expirySecret); - await updateProviderCredential( - sandbox, - providerName, - expirySecret, - expiryAtMs, - allSecrets, - "credential-window-install-expiring-generation", - ); - const expiryRevision = await observeDistinctFreshRevision( - sandbox, - originalRevision, - "credential-window-expiring-fresh-revision", - ); + }); + expectExitZero(status, "inspect credential-window MCP bridge"); + const providerName = (JSON.parse(status.stdout) as { provider: { name: string } }).provider + .name; + expect(providerName).toMatch(/^e2e-cred-window-mcp-fake-[a-f0-9]{16}$/u); - const expiryChildPromise = sandbox.exec( - SANDBOX_NAME, - ["nemoclaw-start", "node", "-e", buildCredentialWindowChildScript({ mcpUrl: tunnel.url })], - { - artifactName: "credential-window-expiry-child", - env: openshellEnv(), - redactionValues: [...allSecrets], - timeoutMs: 6 * 60_000, - }, - ); - let expiryChildRevision = ""; - let expiryChildResult: ShellProbeResult | undefined; - let restoredRevision = ""; - try { - expiryChildRevision = await waitForReadyRevision(sandbox); - expect(expiryChildRevision).toBe(expiryRevision); - await writeControl( + const originalRevision = await observeFreshRevision( sandbox, - CREDENTIAL_WINDOW_STEPS.allowedBeforeExpiry, - "credential-window-signal-before-expiry", + "credential-window-initial-fresh-revision", ); - await waitForAcknowledgement(sandbox, CREDENTIAL_WINDOW_STEPS.allowedBeforeExpiry, "allowed"); - expect( - requestEvidence( - fakeMcp, - credentialWindowRequestId(CREDENTIAL_WINDOW_STEPS.allowedBeforeExpiry), - expirySecret, - ), - ).toEqual({ - seen: true, - credentialRewritten: true, - placeholderAbsent: true, - }); + const resetControl = await sandbox.exec( + SANDBOX_NAME, + [ + "rm", + "-f", + CREDENTIAL_WINDOW_PATHS.control, + CREDENTIAL_WINDOW_PATHS.ready, + CREDENTIAL_WINDOW_PATHS.acknowledgement, + ], + { + artifactName: "credential-window-reset-control-files", + env: openshellEnv(), + timeoutMs: 60_000, + }, + ); + expectExitZero(resetControl, "reset credential-window control files"); - fakeMcp.setSecret(initialSecret); + progress.phase("prove a retained credential generation expires"); + const expiryAtMs = Date.now() + CREDENTIAL_WINDOW_EXPIRY_DELAY_MS; + fakeMcp.setSecret(expirySecret); await updateProviderCredential( sandbox, providerName, - initialSecret, - 0, + expirySecret, + expiryAtMs, allSecrets, - "credential-window-clear-expiry-with-current-generation", - ); - restoredRevision = await observeDistinctFreshRevision( - sandbox, - expiryRevision, - "credential-window-current-revision-before-expiry", + "credential-window-install-expiring-generation", ); - const currentDuringExpiryId = `${CREDENTIAL_WINDOW_REQUEST_PREFIX}:fresh-current-during-expiry`; - const currentDuringExpiry = await runFreshRequest( + const expiryRevision = await observeDistinctFreshRevision( sandbox, - tunnel.url, - currentDuringExpiryId, - allSecrets, - "credential-window-fresh-current-during-expiry", + originalRevision, + "credential-window-expiring-fresh-revision", ); - expect(currentDuringExpiry).toEqual({ - revision: restoredRevision, - status: 200, - }); - expect(requestEvidence(fakeMcp, currentDuringExpiryId, initialSecret)).toEqual({ - seen: true, - credentialRewritten: true, - placeholderAbsent: true, - }); - await expect - .poll(() => Date.now(), { - interval: 500, - timeout: CREDENTIAL_WINDOW_EXPIRY_DELAY_MS + 30_000, - message: "retained credential generation expiry deadline", - }) - .toBeGreaterThan(expiryAtMs); - await writeControl( - sandbox, - CREDENTIAL_WINDOW_STEPS.deniedAfterExpiry, - "credential-window-signal-after-expiry", + const expiryChildPromise = sandbox.exec( + SANDBOX_NAME, + ["nemoclaw-start", "node", "-e", buildCredentialWindowChildScript({ mcpUrl: tunnel.url })], + { + artifactName: "credential-window-expiry-child", + env: openshellEnv(), + redactionValues: [...allSecrets], + timeoutMs: 6 * 60_000, + }, ); - await waitForAcknowledgement(sandbox, CREDENTIAL_WINDOW_STEPS.deniedAfterExpiry, "denied"); - expect( - requestEvidence( - fakeMcp, - credentialWindowRequestId(CREDENTIAL_WINDOW_STEPS.deniedAfterExpiry), + let expiryChildRevision = ""; + let expiryChildResult: ShellProbeResult | undefined; + let restoredRevision = ""; + try { + expiryChildRevision = await waitForReadyRevision(sandbox); + expect(expiryChildRevision).toBe(expiryRevision); + await writeControl( + sandbox, + CREDENTIAL_WINDOW_STEPS.allowedBeforeExpiry, + "credential-window-signal-before-expiry", + ); + await waitForAcknowledgement(sandbox, CREDENTIAL_WINDOW_STEPS.allowedBeforeExpiry, "allowed"); + expect( + requestEvidence( + fakeMcp, + credentialWindowRequestId(CREDENTIAL_WINDOW_STEPS.allowedBeforeExpiry), + expirySecret, + ), + ).toEqual({ + seen: true, + credentialRewritten: true, + placeholderAbsent: true, + }); + + fakeMcp.setSecret(initialSecret); + await updateProviderCredential( + sandbox, + providerName, initialSecret, - ).seen, - ).toBe(false); - } finally { - await writeControl( - sandbox, - CREDENTIAL_WINDOW_STEPS.stop, - "credential-window-stop-expiry-child", - ).catch(() => - host.bestEffortCleanupSandbox(SANDBOX_NAME, { - artifactName: "credential-window-expiry-stop-fallback-destroy", - timeoutMs: 15 * 60_000, - }), - ); - expiryChildResult = await expiryChildPromise; - } + 0, + allSecrets, + "credential-window-clear-expiry-with-current-generation", + ); + restoredRevision = await observeDistinctFreshRevision( + sandbox, + expiryRevision, + "credential-window-current-revision-before-expiry", + ); + const currentDuringExpiryId = `${CREDENTIAL_WINDOW_REQUEST_PREFIX}:fresh-current-during-expiry`; + const currentDuringExpiry = await runFreshRequest( + sandbox, + tunnel.url, + currentDuringExpiryId, + allSecrets, + "credential-window-fresh-current-during-expiry", + ); + expect(currentDuringExpiry).toEqual({ + revision: restoredRevision, + status: 200, + }); + expect(requestEvidence(fakeMcp, currentDuringExpiryId, initialSecret)).toEqual({ + seen: true, + credentialRewritten: true, + placeholderAbsent: true, + }); + + await expect + .poll(() => Date.now(), { + interval: 500, + timeout: CREDENTIAL_WINDOW_EXPIRY_DELAY_MS + 30_000, + message: "retained credential generation expiry deadline", + }) + .toBeGreaterThan(expiryAtMs); + await writeControl( + sandbox, + CREDENTIAL_WINDOW_STEPS.deniedAfterExpiry, + "credential-window-signal-after-expiry", + ); + await waitForAcknowledgement(sandbox, CREDENTIAL_WINDOW_STEPS.deniedAfterExpiry, "denied"); + expect( + requestEvidence( + fakeMcp, + credentialWindowRequestId(CREDENTIAL_WINDOW_STEPS.deniedAfterExpiry), + initialSecret, + ).seen, + ).toBe(false); + } finally { + await writeControl( + sandbox, + CREDENTIAL_WINDOW_STEPS.stop, + "credential-window-stop-expiry-child", + ).catch(() => + host.bestEffortCleanupSandbox(SANDBOX_NAME, { + artifactName: "credential-window-expiry-stop-fallback-destroy", + timeoutMs: 15 * 60_000, + }), + ); + expiryChildResult = await expiryChildPromise; + } + + expect(expiryChildResult).toBeDefined(); + expectExitZero(expiryChildResult!, "retained-expiry credential-window child"); + expect(parseLastJsonLine(expiryChildResult!.stdout)).toEqual({ + revision: expiryChildRevision, + outcomes: [ + { + step: CREDENTIAL_WINDOW_STEPS.allowedBeforeExpiry, + outcome: "allowed", + }, + { + step: CREDENTIAL_WINDOW_STEPS.deniedAfterExpiry, + outcome: "denied", + }, + ], + }); - expect(expiryChildResult).toBeDefined(); - expectExitZero(expiryChildResult!, "retained-expiry credential-window child"); - expect(parseLastJsonLine(expiryChildResult!.stdout)).toEqual({ - revision: expiryChildRevision, - outcomes: [ + progress.phase("rotate beyond the retained generation window"); + const clearExpiryControl = await sandbox.exec( + SANDBOX_NAME, + [ + "rm", + "-f", + CREDENTIAL_WINDOW_PATHS.control, + CREDENTIAL_WINDOW_PATHS.ready, + CREDENTIAL_WINDOW_PATHS.acknowledgement, + ], { - step: CREDENTIAL_WINDOW_STEPS.allowedBeforeExpiry, - outcome: "allowed", + artifactName: "credential-window-reset-after-expiry", + env: openshellEnv(), + timeoutMs: 60_000, }, + ); + expectExitZero(clearExpiryControl, "reset credential-window controls after expiry proof"); + + const oldChildPromise = sandbox.exec( + SANDBOX_NAME, + ["nemoclaw-start", "node", "-e", buildCredentialWindowChildScript({ mcpUrl: tunnel.url })], { - step: CREDENTIAL_WINDOW_STEPS.deniedAfterExpiry, - outcome: "denied", + artifactName: "credential-window-old-child", + env: openshellEnv(), + redactionValues: [...allSecrets], + timeoutMs: 42 * 60_000, }, - ], - }); - - progress.phase("rotate beyond the retained generation window"); - const clearExpiryControl = await sandbox.exec( - SANDBOX_NAME, - [ - "rm", - "-f", - CREDENTIAL_WINDOW_PATHS.control, - CREDENTIAL_WINDOW_PATHS.ready, - CREDENTIAL_WINDOW_PATHS.acknowledgement, - ], - { - artifactName: "credential-window-reset-after-expiry", - env: openshellEnv(), - timeoutMs: 60_000, - }, - ); - expectExitZero(clearExpiryControl, "reset credential-window controls after expiry proof"); + ); + let oldChildRevision = ""; + let oldChildResult: ShellProbeResult | undefined; + let restartedRevision = ""; + const observedRevisions = [restoredRevision]; + try { + oldChildRevision = await waitForReadyRevision(sandbox); + expect(oldChildRevision).toBe(restoredRevision); + for (const [index, secret] of rotationSecrets.entries()) { + await rotateCredential(host, fakeMcp, secret, index + 1, allSecrets); + observedRevisions.push( + await observeFreshRevision(sandbox, `credential-window-fresh-revision-${index + 1}`), + ); + } + expect(new Set(observedRevisions).size).toBe(CREDENTIAL_WINDOW_ROTATION_COUNT + 1); + const currentRevision = observedRevisions.at(-1)!; + expect(currentRevision).not.toBe(oldChildRevision); + await artifacts.writeJson("credential-window-revisions.json", { + expiryAtMs, + expiryRevision, + oldChildRevision, + observedRevisions, + restoredRevision, + retainedGenerations: OPENSHELL_RETAINED_CREDENTIAL_GENERATIONS, + rotations: CREDENTIAL_WINDOW_ROTATION_COUNT, + }); + + const rotatedSecret = rotationSecrets.at(-1)!; + await writeControl( + sandbox, + CREDENTIAL_WINDOW_STEPS.fallbackAfterEviction, + "credential-window-signal-fallback-after-eviction", + ); + await waitForAcknowledgement( + sandbox, + CREDENTIAL_WINDOW_STEPS.fallbackAfterEviction, + "denied", + ); + expect( + requestEvidence( + fakeMcp, + credentialWindowRequestId(CREDENTIAL_WINDOW_STEPS.fallbackAfterEviction), + rotatedSecret, + ).seen, + ).toBe(false); + + const freshAfterEvictionId = `${CREDENTIAL_WINDOW_REQUEST_PREFIX}:fresh-after-eviction`; + const freshAfterEviction = await runFreshRequest( + sandbox, + tunnel.url, + freshAfterEvictionId, + allSecrets, + "credential-window-fresh-request-after-eviction", + ); + expect(freshAfterEviction).toEqual({ + revision: currentRevision, + status: 200, + }); + expect(requestEvidence(fakeMcp, freshAfterEvictionId, rotatedSecret)).toEqual({ + seen: true, + credentialRewritten: true, + placeholderAbsent: true, + }); + + progress.phase("prove key and bridge removal revoke access"); + await updateProviderCredential( + sandbox, + providerName, + "", + 0, + allSecrets, + "credential-window-remove-current-key", + ); + await expectFreshCredentialAbsent( + sandbox, + "credential-window-fresh-credential-absent-after-key-removal", + ); + await writeControl( + sandbox, + CREDENTIAL_WINDOW_STEPS.deniedAfterKeyRemoval, + "credential-window-signal-after-key-removal", + ); + await waitForAcknowledgement( + sandbox, + CREDENTIAL_WINDOW_STEPS.deniedAfterKeyRemoval, + "denied", + ); + expect( + requestEvidence( + fakeMcp, + credentialWindowRequestId(CREDENTIAL_WINDOW_STEPS.deniedAfterKeyRemoval), + rotatedSecret, + ).seen, + ).toBe(false); + + fakeMcp.setSecret(restartSecret); + await updateProviderCredential( + sandbox, + providerName, + restartSecret, + 0, + allSecrets, + "credential-window-restore-current-key-before-detach", + ); + const restoredKeyRevision = await observeDistinctFreshRevision( + sandbox, + currentRevision, + "credential-window-fresh-revision-after-key-restore", + ); + const freshAfterKeyRestoreId = `${CREDENTIAL_WINDOW_REQUEST_PREFIX}:fresh-after-key-restore`; + const freshAfterKeyRestore = await runFreshRequest( + sandbox, + tunnel.url, + freshAfterKeyRestoreId, + allSecrets, + "credential-window-fresh-request-after-key-restore", + ); + expect(freshAfterKeyRestore).toEqual({ + revision: restoredKeyRevision, + status: 200, + }); + expect(requestEvidence(fakeMcp, freshAfterKeyRestoreId, restartSecret)).toEqual({ + seen: true, + credentialRewritten: true, + placeholderAbsent: true, + }); + + const removeBeforeReadd = await host.nemoclaw([SANDBOX_NAME, "mcp", "remove", SERVER_NAME], { + artifactName: "credential-window-remove-before-readd", + env: buildAvailabilityProbeEnv(), + timeoutMs: 4 * 60_000, + }); + expectExitZero(removeBeforeReadd, "remove credential-window bridge before re-add"); + await expectFreshCredentialAbsent( + sandbox, + "credential-window-fresh-credential-absent-after-detach", + ); - const oldChildPromise = sandbox.exec( - SANDBOX_NAME, - ["nemoclaw-start", "node", "-e", buildCredentialWindowChildScript({ mcpUrl: tunnel.url })], - { - artifactName: "credential-window-old-child", - env: openshellEnv(), - redactionValues: [...allSecrets], - timeoutMs: 42 * 60_000, - }, - ); - let oldChildRevision = ""; - let oldChildResult: ShellProbeResult | undefined; - let restartedRevision = ""; - const observedRevisions = [restoredRevision]; - try { - oldChildRevision = await waitForReadyRevision(sandbox); - expect(oldChildRevision).toBe(restoredRevision); - for (const [index, secret] of rotationSecrets.entries()) { - await rotateCredential(host, fakeMcp, secret, index + 1, allSecrets); - observedRevisions.push( - await observeFreshRevision(sandbox, `credential-window-fresh-revision-${index + 1}`), + await writeControl( + sandbox, + CREDENTIAL_WINDOW_STEPS.deniedAfterDetach, + "credential-window-signal-after-detach", ); + await waitForAcknowledgement(sandbox, CREDENTIAL_WINDOW_STEPS.deniedAfterDetach, "denied"); + expect( + requestEvidence( + fakeMcp, + credentialWindowRequestId(CREDENTIAL_WINDOW_STEPS.deniedAfterDetach), + restartSecret, + ).seen, + ).toBe(false); + + progress.phase("re-add the bridge and keep the old process revoked"); + fakeMcp.setSecret(restartSecret); + const readd = await host.nemoclaw( + [ + SANDBOX_NAME, + "mcp", + "add", + SERVER_NAME, + "--url", + tunnel.url, + "--env", + CREDENTIAL_WINDOW_ENV_NAME, + ], + { + artifactName: "credential-window-readd-after-removal", + env: { + ...buildAvailabilityProbeEnv(), + [CREDENTIAL_WINDOW_ENV_NAME]: restartSecret, + }, + redactionValues: [...allSecrets], + timeoutMs: 4 * 60_000, + }, + ); + expectExitZero(readd, "re-add credential-window bridge"); + restartedRevision = await observeFreshRevision( + sandbox, + "credential-window-fresh-revision-after-restart", + ); + expect(restartedRevision).not.toBe(currentRevision); + expect(restartedRevision).not.toBe(restoredKeyRevision); + const freshAfterReaddId = `${CREDENTIAL_WINDOW_REQUEST_PREFIX}:fresh-after-readd`; + const freshAfterReadd = await runFreshRequest( + sandbox, + tunnel.url, + freshAfterReaddId, + allSecrets, + "credential-window-fresh-request-after-readd", + ); + expect(freshAfterReadd).toEqual({ + revision: restartedRevision, + status: 200, + }); + expect(requestEvidence(fakeMcp, freshAfterReaddId, restartSecret)).toEqual({ + seen: true, + credentialRewritten: true, + placeholderAbsent: true, + }); + await writeControl( + sandbox, + CREDENTIAL_WINDOW_STEPS.deniedAfterReadd, + "credential-window-signal-denied-after-readd", + ); + await waitForAcknowledgement(sandbox, CREDENTIAL_WINDOW_STEPS.deniedAfterReadd, "denied"); + expect( + requestEvidence( + fakeMcp, + credentialWindowRequestId(CREDENTIAL_WINDOW_STEPS.deniedAfterReadd), + restartSecret, + ).seen, + ).toBe(false); + } finally { + await writeControl( + sandbox, + CREDENTIAL_WINDOW_STEPS.stop, + "credential-window-stop-old-child", + ).catch(() => + host.bestEffortCleanupSandbox(SANDBOX_NAME, { + artifactName: "credential-window-stop-fallback-destroy", + timeoutMs: 15 * 60_000, + }), + ); + oldChildResult = await oldChildPromise; } - expect(new Set(observedRevisions).size).toBe(CREDENTIAL_WINDOW_ROTATION_COUNT + 1); - const currentRevision = observedRevisions.at(-1)!; - expect(currentRevision).not.toBe(oldChildRevision); - await artifacts.writeJson("credential-window-revisions.json", { - expiryAtMs, - expiryRevision, - oldChildRevision, - observedRevisions, - restoredRevision, - retainedGenerations: OPENSHELL_RETAINED_CREDENTIAL_GENERATIONS, - rotations: CREDENTIAL_WINDOW_ROTATION_COUNT, + + expect(oldChildResult).toBeDefined(); + expectExitZero(oldChildResult!, "old credential-window child"); + const childSummary = parseLastJsonLine(oldChildResult!.stdout); + expect(childSummary).toEqual({ + revision: oldChildRevision, + outcomes: [ + { + step: CREDENTIAL_WINDOW_STEPS.fallbackAfterEviction, + outcome: "denied", + }, + { step: CREDENTIAL_WINDOW_STEPS.deniedAfterKeyRemoval, outcome: "denied" }, + { step: CREDENTIAL_WINDOW_STEPS.deniedAfterDetach, outcome: "denied" }, + { + step: CREDENTIAL_WINDOW_STEPS.deniedAfterReadd, + outcome: "denied", + }, + ], }); - const rotatedSecret = rotationSecrets.at(-1)!; - await writeControl( + progress.phase("rebuild the sandbox and confirm credential reuse"); + const rebuild = await host.nemoclaw([SANDBOX_NAME, "rebuild", "--yes"], { + artifactName: "credential-window-rebuild-with-provider-reuse", + env: { + ...buildAvailabilityProbeEnv(), + COMPATIBLE_API_KEY: COMPATIBLE_KEY, + NVIDIA_INFERENCE_API_KEY: COMPATIBLE_KEY, + }, + redactionValues: [COMPATIBLE_KEY, ...allSecrets], + timeoutMs: 25 * 60_000, + }); + expectExitZero(rebuild, "rebuild credential-window sandbox without MCP host secret"); + const rebuiltRevision = await observeFreshRevision( sandbox, - CREDENTIAL_WINDOW_STEPS.fallbackAfterEviction, - "credential-window-signal-fallback-after-eviction", + "credential-window-fresh-revision-after-rebuild", ); - await waitForAcknowledgement(sandbox, CREDENTIAL_WINDOW_STEPS.fallbackAfterEviction, "denied"); - expect( - requestEvidence( - fakeMcp, - credentialWindowRequestId(CREDENTIAL_WINDOW_STEPS.fallbackAfterEviction), - rotatedSecret, - ).seen, - ).toBe(false); - - const freshAfterEvictionId = `${CREDENTIAL_WINDOW_REQUEST_PREFIX}:fresh-after-eviction`; - const freshAfterEviction = await runFreshRequest( + const freshAfterRebuildId = `${CREDENTIAL_WINDOW_REQUEST_PREFIX}:fresh-after-rebuild`; + const freshAfterRebuild = await runFreshRequest( sandbox, tunnel.url, - freshAfterEvictionId, + freshAfterRebuildId, allSecrets, - "credential-window-fresh-request-after-eviction", + "credential-window-fresh-request-after-rebuild", ); - expect(freshAfterEviction).toEqual({ - revision: currentRevision, + expect(freshAfterRebuild).toEqual({ + revision: rebuiltRevision, status: 200, }); - expect(requestEvidence(fakeMcp, freshAfterEvictionId, rotatedSecret)).toEqual({ + expect(requestEvidence(fakeMcp, freshAfterRebuildId, restartSecret)).toEqual({ seen: true, credentialRewritten: true, placeholderAbsent: true, }); - progress.phase("prove key and bridge removal revoke access"); - await updateProviderCredential( - sandbox, - providerName, - "", - 0, - allSecrets, - "credential-window-remove-current-key", - ); + progress.phase("remove the MCP bridge and audit denied requests"); + const remove = await host.nemoclaw([SANDBOX_NAME, "mcp", "remove", SERVER_NAME], { + artifactName: "credential-window-mcp-remove", + env: buildAvailabilityProbeEnv(), + timeoutMs: 4 * 60_000, + }); + expectExitZero(remove, "remove credential-window MCP bridge"); await expectFreshCredentialAbsent( sandbox, - "credential-window-fresh-credential-absent-after-key-removal", - ); - await writeControl( - sandbox, - CREDENTIAL_WINDOW_STEPS.deniedAfterKeyRemoval, - "credential-window-signal-after-key-removal", - ); - await waitForAcknowledgement(sandbox, CREDENTIAL_WINDOW_STEPS.deniedAfterKeyRemoval, "denied"); - expect( - requestEvidence( - fakeMcp, - credentialWindowRequestId(CREDENTIAL_WINDOW_STEPS.deniedAfterKeyRemoval), - rotatedSecret, - ).seen, - ).toBe(false); - - fakeMcp.setSecret(restartSecret); - await updateProviderCredential( - sandbox, - providerName, - restartSecret, - 0, - allSecrets, - "credential-window-restore-current-key-before-detach", + "credential-window-fresh-credential-absent-after-remove", ); - const restoredKeyRevision = await observeDistinctFreshRevision( - sandbox, - currentRevision, - "credential-window-fresh-revision-after-key-restore", - ); - const freshAfterKeyRestoreId = `${CREDENTIAL_WINDOW_REQUEST_PREFIX}:fresh-after-key-restore`; - const freshAfterKeyRestore = await runFreshRequest( - sandbox, - tunnel.url, - freshAfterKeyRestoreId, - allSecrets, - "credential-window-fresh-request-after-key-restore", - ); - expect(freshAfterKeyRestore).toEqual({ - revision: restoredKeyRevision, - status: 200, - }); - expect(requestEvidence(fakeMcp, freshAfterKeyRestoreId, restartSecret)).toEqual({ - seen: true, - credentialRewritten: true, - placeholderAbsent: true, - }); - - const removeBeforeReadd = await host.nemoclaw( - [SANDBOX_NAME, "mcp", "remove", SERVER_NAME], + const providerAfterRemove = await host.command( + host.openshellCommandPath, + ["provider", "get", providerName], { - artifactName: "credential-window-remove-before-readd", - env: buildAvailabilityProbeEnv(), - timeoutMs: 4 * 60_000, + artifactName: "credential-window-provider-absent-after-remove", + env: openshellEnv(), + timeoutMs: 60_000, }, ); - expectExitZero(removeBeforeReadd, "remove credential-window bridge before re-add"); - await expectFreshCredentialAbsent( - sandbox, - "credential-window-fresh-credential-absent-after-detach", - ); - - await writeControl( - sandbox, - CREDENTIAL_WINDOW_STEPS.deniedAfterDetach, - "credential-window-signal-after-detach", + expect(providerAfterRemove.exitCode).not.toBe(0); + expect(resultText(providerAfterRemove)).toMatch(/not found/iu); + const upstreamRequestIds = fakeMcp.requests.map((request) => requestId(request.body)); + expect(upstreamRequestIds).not.toContain( + credentialWindowRequestId(CREDENTIAL_WINDOW_STEPS.deniedAfterExpiry), ); - await waitForAcknowledgement(sandbox, CREDENTIAL_WINDOW_STEPS.deniedAfterDetach, "denied"); - expect( - requestEvidence( - fakeMcp, - credentialWindowRequestId(CREDENTIAL_WINDOW_STEPS.deniedAfterDetach), - restartSecret, - ).seen, - ).toBe(false); - - progress.phase("re-add the bridge and keep the old process revoked"); - fakeMcp.setSecret(restartSecret); - const readd = await host.nemoclaw( - [ - SANDBOX_NAME, - "mcp", - "add", - SERVER_NAME, - "--url", - tunnel.url, - "--env", - CREDENTIAL_WINDOW_ENV_NAME, - ], - { - artifactName: "credential-window-readd-after-removal", - env: { - ...buildAvailabilityProbeEnv(), - [CREDENTIAL_WINDOW_ENV_NAME]: restartSecret, - }, - redactionValues: [...allSecrets], - timeoutMs: 4 * 60_000, - }, + expect(upstreamRequestIds).not.toContain( + credentialWindowRequestId(CREDENTIAL_WINDOW_STEPS.fallbackAfterEviction), ); - expectExitZero(readd, "re-add credential-window bridge"); - restartedRevision = await observeFreshRevision( - sandbox, - "credential-window-fresh-revision-after-restart", + expect(upstreamRequestIds).not.toContain( + credentialWindowRequestId(CREDENTIAL_WINDOW_STEPS.deniedAfterKeyRemoval), ); - expect(restartedRevision).not.toBe(currentRevision); - expect(restartedRevision).not.toBe(restoredKeyRevision); - const freshAfterReaddId = `${CREDENTIAL_WINDOW_REQUEST_PREFIX}:fresh-after-readd`; - const freshAfterReadd = await runFreshRequest( - sandbox, - tunnel.url, - freshAfterReaddId, - allSecrets, - "credential-window-fresh-request-after-readd", + expect(upstreamRequestIds).not.toContain( + credentialWindowRequestId(CREDENTIAL_WINDOW_STEPS.deniedAfterDetach), ); - expect(freshAfterReadd).toEqual({ - revision: restartedRevision, - status: 200, - }); - expect(requestEvidence(fakeMcp, freshAfterReaddId, restartSecret)).toEqual({ - seen: true, - credentialRewritten: true, - placeholderAbsent: true, - }); - await writeControl( - sandbox, - CREDENTIAL_WINDOW_STEPS.deniedAfterReadd, - "credential-window-signal-denied-after-readd", + expect(upstreamRequestIds).not.toContain( + credentialWindowRequestId(CREDENTIAL_WINDOW_STEPS.deniedAfterReadd), ); - await waitForAcknowledgement(sandbox, CREDENTIAL_WINDOW_STEPS.deniedAfterReadd, "denied"); expect( - requestEvidence( - fakeMcp, - credentialWindowRequestId(CREDENTIAL_WINDOW_STEPS.deniedAfterReadd), - restartSecret, - ).seen, - ).toBe(false); - } finally { - await writeControl( - sandbox, - CREDENTIAL_WINDOW_STEPS.stop, - "credential-window-stop-old-child", - ).catch(() => - host.bestEffortCleanupSandbox(SANDBOX_NAME, { - artifactName: "credential-window-stop-fallback-destroy", - timeoutMs: 15 * 60_000, - }), - ); - oldChildResult = await oldChildPromise; - } - - expect(oldChildResult).toBeDefined(); - expectExitZero(oldChildResult!, "old credential-window child"); - const childSummary = parseLastJsonLine(oldChildResult!.stdout); - expect(childSummary).toEqual({ - revision: oldChildRevision, - outcomes: [ - { - step: CREDENTIAL_WINDOW_STEPS.fallbackAfterEviction, - outcome: "denied", - }, - { step: CREDENTIAL_WINDOW_STEPS.deniedAfterKeyRemoval, outcome: "denied" }, - { step: CREDENTIAL_WINDOW_STEPS.deniedAfterDetach, outcome: "denied" }, - { - step: CREDENTIAL_WINDOW_STEPS.deniedAfterReadd, - outcome: "denied", - }, - ], - }); - - progress.phase("rebuild the sandbox and confirm credential reuse"); - const rebuild = await host.nemoclaw([SANDBOX_NAME, "rebuild", "--yes"], { - artifactName: "credential-window-rebuild-with-provider-reuse", - env: { - ...buildAvailabilityProbeEnv(), - COMPATIBLE_API_KEY: COMPATIBLE_KEY, - NVIDIA_INFERENCE_API_KEY: COMPATIBLE_KEY, - }, - redactionValues: [COMPATIBLE_KEY, ...allSecrets], - timeoutMs: 25 * 60_000, - }); - expectExitZero(rebuild, "rebuild credential-window sandbox without MCP host secret"); - const rebuiltRevision = await observeFreshRevision( - sandbox, - "credential-window-fresh-revision-after-rebuild", - ); - const freshAfterRebuildId = `${CREDENTIAL_WINDOW_REQUEST_PREFIX}:fresh-after-rebuild`; - const freshAfterRebuild = await runFreshRequest( - sandbox, - tunnel.url, - freshAfterRebuildId, - allSecrets, - "credential-window-fresh-request-after-rebuild", - ); - expect(freshAfterRebuild).toEqual({ - revision: rebuiltRevision, - status: 200, - }); - expect(requestEvidence(fakeMcp, freshAfterRebuildId, restartSecret)).toEqual({ - seen: true, - credentialRewritten: true, - placeholderAbsent: true, - }); - - progress.phase("remove the MCP bridge and audit denied requests"); - const remove = await host.nemoclaw([SANDBOX_NAME, "mcp", "remove", SERVER_NAME], { - artifactName: "credential-window-mcp-remove", - env: buildAvailabilityProbeEnv(), - timeoutMs: 4 * 60_000, - }); - expectExitZero(remove, "remove credential-window MCP bridge"); - await expectFreshCredentialAbsent( - sandbox, - "credential-window-fresh-credential-absent-after-remove", - ); - const providerAfterRemove = await host.command( - host.openshellCommandPath, - ["provider", "get", providerName], - { - artifactName: "credential-window-provider-absent-after-remove", - env: openshellEnv(), - timeoutMs: 60_000, - }, - ); - expect(providerAfterRemove.exitCode).not.toBe(0); - expect(resultText(providerAfterRemove)).toMatch(/not found/iu); - const upstreamRequestIds = fakeMcp.requests.map((request) => requestId(request.body)); - expect(upstreamRequestIds).not.toContain( - credentialWindowRequestId(CREDENTIAL_WINDOW_STEPS.deniedAfterExpiry), - ); - expect(upstreamRequestIds).not.toContain( - credentialWindowRequestId(CREDENTIAL_WINDOW_STEPS.fallbackAfterEviction), - ); - expect(upstreamRequestIds).not.toContain( - credentialWindowRequestId(CREDENTIAL_WINDOW_STEPS.deniedAfterKeyRemoval), - ); - expect(upstreamRequestIds).not.toContain( - credentialWindowRequestId(CREDENTIAL_WINDOW_STEPS.deniedAfterDetach), - ); - expect(upstreamRequestIds).not.toContain( - credentialWindowRequestId(CREDENTIAL_WINDOW_STEPS.deniedAfterReadd), - ); - expect( - fakeMcp.requests.every( - (request: CredentialWindowRequest) => !request.auth.includes("openshell:resolve:env"), - ), - ).toBe(true); - await artifacts.target.complete({ - id: "openshell-credential-generation-window", - expiryRevision, - oldChildRevision, - rebuiltRevision, - restartedRevision, - rotations: CREDENTIAL_WINDOW_ROTATION_COUNT, - }); -}); + fakeMcp.requests.every( + (request: CredentialWindowRequest) => !request.auth.includes("openshell:resolve:env"), + ), + ).toBe(true); + await artifacts.target.complete({ + id: "openshell-credential-generation-window", + expiryRevision, + oldChildRevision, + rebuiltRevision, + restartedRevision, + rotations: CREDENTIAL_WINDOW_ROTATION_COUNT, + }); + }, +); diff --git a/test/e2e/live/package-database-read-only.ts b/test/e2e/live/package-database-read-only.ts index 16a403abed0..3a8ed3fab9f 100644 --- a/test/e2e/live/package-database-read-only.ts +++ b/test/e2e/live/package-database-read-only.ts @@ -1,15 +1,14 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { privilegedSandboxExecArgv } from "../../../src/lib/sandbox/privileged-exec.ts"; import { resultText, shellQuote } from "../fixtures/clients/command.ts"; -import type { HostCliClient } from "../fixtures/clients/host.ts"; import { type SandboxClient, trustedSandboxShellScript } from "../fixtures/clients/sandbox.ts"; +import type { RuntimeProviderPrerequisite } from "../fixtures/runtime-provider.ts"; type PackageDatabaseProbeOptions = { artifactPrefix: string; env: NodeJS.ProcessEnv; - host: HostCliClient; + runtimeProvider: RuntimeProviderPrerequisite; sandbox: SandboxClient; sandboxName: string; timeoutMs: number; @@ -23,9 +22,7 @@ export async function expectPackageDatabaseReadOnly( options: PackageDatabaseProbeOptions, ): Promise { const sentinel = `/var/lib/dpkg/nemoclaw-e2e-write-probe-${process.pid}`; - const prepare = await options.host.command( - "docker", - privilegedSandboxExecArgv( + const prepare = await options.runtimeProvider.execSandboxAsRoot( options.sandboxName, [ "sh", @@ -34,12 +31,10 @@ export async function expectPackageDatabaseReadOnly( "sh", sentinel, ], - false, - true, - ), { artifactName: `${options.artifactPrefix}-prepare-dpkg-landlock-sentinel`, env: options.env, + sanitizeEnvironment: true, timeoutMs: options.timeoutMs, }, ); @@ -76,12 +71,13 @@ printf 'DPKG_WRITE_DENIED\n' requireCondition(output.includes("CONTROL_WRITE_OK"), "writable control marker is missing"); requireCondition(output.includes("DPKG_WRITE_DENIED"), "Landlock denial marker is missing"); } finally { - const cleanup = await options.host.command( - "docker", - privilegedSandboxExecArgv(options.sandboxName, ["rm", "-f", "--", sentinel], false, true), + const cleanup = await options.runtimeProvider.execSandboxAsRoot( + options.sandboxName, + ["rm", "-f", "--", sentinel], { artifactName: `${options.artifactPrefix}-clean-dpkg-landlock-sentinel`, env: options.env, + sanitizeEnvironment: true, timeoutMs: options.timeoutMs, }, ); diff --git a/test/e2e/live/phase6-messaging-helpers.ts b/test/e2e/live/phase6-messaging-helpers.ts index 4bfd20ace28..e349734ef30 100644 --- a/test/e2e/live/phase6-messaging-helpers.ts +++ b/test/e2e/live/phase6-messaging-helpers.ts @@ -21,6 +21,7 @@ import { } from "../fixtures/clients/sandbox.ts"; import { expect } from "../fixtures/e2e-test.ts"; import { CLI_ENTRYPOINT, REPO_ROOT } from "../fixtures/paths.ts"; +import type { RuntimeProviderPrerequisite } from "../fixtures/runtime-provider.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; import { isNvidiaEndpointRateLimitFailure } from "./messaging-providers-helpers.ts"; @@ -323,13 +324,12 @@ export async function sandboxNode( ); } -export async function dockerInfo( - host: HostCliClient, - env: NodeJS.ProcessEnv, -): Promise { - return host.command("docker", ["info"], { - artifactName: "phase6-docker-info", - env, - timeoutMs: 30_000, +export async function requirePhase6RuntimeProvider( + runtimeProvider: RuntimeProviderPrerequisite, + scenarioLabel: string, +): Promise { + await runtimeProvider.requireAvailable({ + artifactName: "phase6-runtime-provider-info", + scenarioLabel, }); } diff --git a/test/e2e/live/podman-cpu-lifecycle.test.ts b/test/e2e/live/podman-cpu-lifecycle.test.ts index fa5668ec55a..ae38d9ad775 100644 --- a/test/e2e/live/podman-cpu-lifecycle.test.ts +++ b/test/e2e/live/podman-cpu-lifecycle.test.ts @@ -134,7 +134,15 @@ test( }); expect(doctor.detail).toContain("rootless server 5."); expect(bundle.identity.id).toBe("podman"); - expect(bundle.workload.profile.support).toBeNull(); + expect(bundle.workload.profile).toMatchObject({ + support: { + exactDigestReferences: true, + platforms: ["linux/amd64", "linux/arm64"], + }, + hostArchitectures: ["amd64", "arm64"], + managedImageSelectionPolicy: "require-managed", + legacyDockerfileBuilds: false, + }); expect(bundle.capabilities.hostLocalInference).toBe(false); const openshellBin = executableOnPath("openshell"); diff --git a/test/e2e/live/rebuild-hermes.test.ts b/test/e2e/live/rebuild-hermes.test.ts index ba5db53ba31..c89092e8f10 100644 --- a/test/e2e/live/rebuild-hermes.test.ts +++ b/test/e2e/live/rebuild-hermes.test.ts @@ -1035,6 +1035,10 @@ test( }, ); expectExitZero(writeExcludedHooksMarker, "write backup:false Hermes hooks marker"); + const sessionSummary = seedRegistryAndSession( + dashboardPort ?? fail("Hermes dashboard port allocation disappeared before registry seeding"), + seededOldSandboxImageState, + ); await cronRestore.seed(); const seededKanbanDb = await host.command("docker", inspectKanbanTaskArgs(SANDBOX_NAME), { artifactName: "phase-4-inspect-seeded-kanban-db", @@ -1069,10 +1073,6 @@ test( ); expectExitZero(preConfig, "read pre-rebuild Hermes config.yaml"); expect(preConfig.stdout).toContain("discord:"); - const sessionSummary = seedRegistryAndSession( - dashboardPort ?? fail("Hermes dashboard port allocation disappeared before registry seeding"), - seededOldSandboxImageState, - ); const seededRegistry = registrySandbox(); cleanupRegistryDashboardPort = seededRegistry.dashboardPort; expect( diff --git a/test/e2e/live/restricted-onboard-helpers.ts b/test/e2e/live/restricted-onboard-helpers.ts index f8a042bf3de..a4b51888c58 100644 --- a/test/e2e/live/restricted-onboard-helpers.ts +++ b/test/e2e/live/restricted-onboard-helpers.ts @@ -2,7 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 import type { ArtifactSink } from "../fixtures/artifacts.ts"; -import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; import { isTransientProviderValidationFailure } from "./network-policy-transient-provider.ts"; @@ -17,25 +16,6 @@ async function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } -export async function ensureDockerAvailable(opts: { - host: HostCliClient; - artifactName: string; - skip: SkipFn; - scenarioLabel: string; -}): Promise { - const docker = await opts.host.command("docker", ["info"], { - artifactName: opts.artifactName, - env: buildAvailabilityProbeEnv(), - timeoutMs: 30_000, - }); - if (docker.exitCode === 0) return; - const text = [docker.stdout, docker.stderr].filter(Boolean).join("\n"); - if (process.env.GITHUB_ACTIONS === "true") { - throw new Error(`Docker is required for ${opts.scenarioLabel} live E2E: ${text}`); - } - opts.skip(`Docker is required for ${opts.scenarioLabel} live E2E`); -} - export type RestrictedOnboardOptions = { host: HostCliClient; artifacts: ArtifactSink; diff --git a/test/e2e/live/sandbox-operations.test.ts b/test/e2e/live/sandbox-operations.test.ts index a42cf7cd619..02f36adf25c 100644 --- a/test/e2e/live/sandbox-operations.test.ts +++ b/test/e2e/live/sandbox-operations.test.ts @@ -36,9 +36,10 @@ import { } from "../fixtures/resource-limit-diagnostics.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; import { parseOpenClawAgentText } from "../fixtures/openclaw-agent-output.ts"; -import { ubuntuRepoDocker } from "../registry/matrix.ts"; +import type { RuntimeProviderPrerequisite } from "../fixtures/runtime-provider.ts"; +import { ubuntuRepoManagedRuntime } from "../registry/matrix.ts"; -const ENVIRONMENT = ubuntuRepoDocker("cloud-openclaw"); +const ENVIRONMENT = ubuntuRepoManagedRuntime("cloud-openclaw"); const SANDBOX_A = "e2e-sbx-a"; const SANDBOX_B = "e2e-sbx-b"; const CREDENTIAL_PROVIDER = "e2e-sandbox-tavily"; @@ -653,32 +654,31 @@ type GatewayRecoveryOutcome = async function assertGatewayRecovery( host: HostCliClient, + runtimeProvider: RuntimeProviderPrerequisite, sandboxName: string, ): Promise { - const running = await host.command( - "docker", - ["ps", "-q", "--filter", `name=${GATEWAY_CONTAINER}`], + const running = await runtimeProvider.command( + ["container", "ps", "--filter", `name=^${GATEWAY_CONTAINER}$`, "--format", "{{.Names}}"], { - artifactName: "tc-sbx-06-gateway-container-running", + artifactName: "tc-sbx-06-gateway-runtime-resource-running", env: buildAvailabilityProbeEnv(), timeoutMs: 15_000, }, ); - if (!running.stdout.trim()) { + if (!running.stdout.split(/\r?\n/u).some((name) => name.trim() === GATEWAY_CONTAINER)) { return "skipped-gateway-absent"; } - const kill = await host.command("docker", ["kill", GATEWAY_CONTAINER], { - artifactName: "tc-sbx-06-docker-kill-gateway", + const kill = await runtimeProvider.command(["container", "kill", GATEWAY_CONTAINER], { + artifactName: "tc-sbx-06-runtime-kill-gateway", env: buildAvailabilityProbeEnv(), timeoutMs: 30_000, }); expectExitZero(kill, "kill shared NemoClaw gateway container"); await new Promise((resolve) => setTimeout(resolve, 5_000)); - const afterKill = await host.command( - "docker", - ["inspect", "-f", "{{.State.Running}}", GATEWAY_CONTAINER], + const afterKill = await runtimeProvider.command( + ["container", "inspect", "--format", "{{.State.Running}}", GATEWAY_CONTAINER], { artifactName: "tc-sbx-06-gateway-container-after-kill", env: buildAvailabilityProbeEnv(), @@ -693,9 +693,8 @@ async function assertGatewayRecovery( env: buildAvailabilityProbeEnv(), timeoutMs: 10 * 60_000, }); - const afterStatus = await host.command( - "docker", - ["inspect", "-f", "{{.State.Running}}", GATEWAY_CONTAINER], + const afterStatus = await runtimeProvider.command( + ["container", "inspect", "--format", "{{.State.Running}}", GATEWAY_CONTAINER], { artifactName: "tc-sbx-06-gateway-container-after-status", env: buildAvailabilityProbeEnv(), @@ -832,7 +831,16 @@ test( ], }, }, - async ({ artifacts, cleanup, docker, environment, host, progress, sandbox, secrets }) => { + async ({ + artifacts, + cleanup, + environment, + host, + progress, + runtimeProvider, + sandbox, + secrets, + }) => { const hosted = requireHostedInferenceConfig(secrets); await artifacts.target.declare({ @@ -857,7 +865,10 @@ test( ], }); - await docker.requireDocker(); + await runtimeProvider.requireAvailable({ + artifactName: "prereq-runtime-provider-info", + scenarioLabel: "sandbox operations", + }); await environment.assertReady(ENVIRONMENT); cleanup.trackGateway(host, "nemoclaw", { @@ -902,7 +913,7 @@ test( await expectListed(host, SANDBOX_A, "tc-sbx-12-survivor-listed-after-destroy-b"); await assertAgentCanAnswer(host, SANDBOX_A, "tc-sbx-12-survivor-agent-after-destroy-b"); - const gatewayRecovery = await assertGatewayRecovery(host, SANDBOX_A); + const gatewayRecovery = await assertGatewayRecovery(host, runtimeProvider, SANDBOX_A); const finalDestroyCleanupMode = process.platform === "darwin" ? "macos-default" : "explicit-non-macos"; diff --git a/test/e2e/live/sandbox-survival.test.ts b/test/e2e/live/sandbox-survival.test.ts index f49118d9b24..8dbc45f0c29 100644 --- a/test/e2e/live/sandbox-survival.test.ts +++ b/test/e2e/live/sandbox-survival.test.ts @@ -29,6 +29,7 @@ import { requireHostedInferenceConfig } from "../fixtures/hosted-inference.ts"; import { REPO_ROOT } from "../fixtures/paths.ts"; import type { NemoClawInstance } from "../fixtures/phases/index.ts"; import type { SandboxMarker } from "../fixtures/phases/state-validation.ts"; +import type { RuntimeProviderPrerequisite } from "../fixtures/runtime-provider.ts"; const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-survival"; const MIN_OPENSHELL_VERSION = "0.0.24"; @@ -37,6 +38,8 @@ const MODEL = process.env.NEMOCLAW_MODEL ?? "nvidia/nemotron-3-super-120b-a12b"; const SURVIVAL_DIAGNOSTICS_SCRIPT = String.raw` set +e sandbox_name="$1" +shift +runtime_command=("$@") printf '%s\n' '== OpenShell sandbox status ==' openshell sandbox get "$sandbox_name" 2>&1 @@ -47,13 +50,11 @@ systemctl --user status nemoclaw-openshell-gateway --no-pager -l 2>&1 printf '%s\n' '== OpenShell gateway journal ==' journalctl --user -u nemoclaw-openshell-gateway -n 200 --no-pager 2>&1 -container_ids="$(docker ps -aq \ - --filter label=openshell.ai/managed-by=openshell \ +container_ids="$("\${runtime_command[@]}" container ps --all --quiet \ --filter "label=openshell.ai/sandbox-name=$sandbox_name")" printf '%s\n' '== matching containers ==' if [ -n "$container_ids" ]; then - docker ps -a --no-trunc \ - --filter label=openshell.ai/managed-by=openshell \ + "\${runtime_command[@]}" container ps --all --no-trunc \ --filter "label=openshell.ai/sandbox-name=$sandbox_name" \ --format '{{.ID}} {{.Names}} {{.Status}}' else @@ -62,7 +63,7 @@ fi for container_id in $container_ids; do printf '%s\n' "== container $container_id inspect ==" - docker inspect "$container_id" 2>&1 | node -e ' + "\${runtime_command[@]}" container inspect "$container_id" 2>&1 | node -e ' const fs = require("node:fs"); const row = JSON.parse(fs.readFileSync(0, "utf8"))[0] || {}; const prefix = "OPENSHELL_SANDBOX_COMMAND="; @@ -92,9 +93,9 @@ for container_id in $container_ids; do }) + "\n"); ' printf '%s\n' "== container $container_id host process tree ==" - docker top "$container_id" -eo pid,ppid,user,stat,comm 2>&1 + "\${runtime_command[@]}" container top "$container_id" -eo pid,ppid,user,stat,comm 2>&1 printf '%s\n' "== container $container_id runtime state ==" - docker exec "$container_id" sh -lc ' + "\${runtime_command[@]}" container exec "$container_id" sh -lc ' printf "%s\n" "== pid 1 ==" cat /proc/1/comm 2>/dev/null || true printf "\n%s\n" "== process tree ==" @@ -110,7 +111,7 @@ for container_id in $container_ids; do tail -n 300 /tmp/gateway.log 2>&1 || true ' 2>&1 printf '%s\n' "== container $container_id logs ==" - docker logs --tail 300 "$container_id" 2>&1 + "\${runtime_command[@]}" container logs --tail 300 "$container_id" 2>&1 done `; @@ -148,12 +149,21 @@ function installEnv(hostedEnv: NodeJS.ProcessEnv): NodeJS.ProcessEnv { async function captureSurvivalDiagnostics( host: HostCliClient, + runtimeProvider: RuntimeProviderPrerequisite, stage: string, redactionValues: string[], ): Promise { + const invocation = runtimeProvider.hostInvocation([]); await host.command( - "sh", - ["-lc", SURVIVAL_DIAGNOSTICS_SCRIPT, "sandbox-survival-diagnostics", SANDBOX_NAME], + "bash", + [ + "-lc", + SURVIVAL_DIAGNOSTICS_SCRIPT, + "sandbox-survival-diagnostics", + SANDBOX_NAME, + invocation.command, + ...invocation.args, + ], { artifactName: `sandbox-survival-${stage}-diagnostics`, env: buildAvailabilityProbeEnv(), @@ -182,7 +192,7 @@ test( timeout: 30 * 60_000, meta: { e2ePhases: [ - "confirm Docker and inference prerequisites", + "confirm the selected runtime and inference prerequisites", "install and register the OpenClaw sandbox", "prove baseline sandbox access and inference", "write persistent OpenClaw markers", @@ -200,6 +210,7 @@ test( provider, progress, runtime, + runtimeProvider, sandbox, secrets, skip, @@ -222,20 +233,15 @@ test( ], }); - const docker = await host.command("docker", ["info"], { - artifactName: "prereq-docker-info-sandbox-survival", - env: buildAvailabilityProbeEnv(), - timeoutMs: 30_000, + await runtimeProvider.requireAvailable({ + artifactName: "prereq-runtime-info-sandbox-survival", + scenarioLabel: "sandbox survival", }); - if (docker.exitCode !== 0) { - if (process.env.GITHUB_ACTIONS === "true") { - throw new Error(`Docker is required for sandbox survival E2E: ${resultText(docker)}`); - } - skip("Docker is required for sandbox survival E2E"); - } const endpointReachable = await provider.probeReachability( - trustedProviderEndpoint(hosted.endpointUrl, { allowedHosts: ["inference-api.nvidia.com"] }), + trustedProviderEndpoint(hosted.endpointUrl, { + allowedHosts: ["inference-api.nvidia.com"], + }), { artifactName: "prereq-inference-api-reachability", env: buildAvailabilityProbeEnv(), @@ -407,12 +413,12 @@ test( await stateValidation.expectSandboxMarkers(instance, markers, "pre-restart-marker-read"); progress.phase("restart the gateway and reconnect the sandbox"); - await captureSurvivalDiagnostics(host, "before-gateway-restart", [apiKey]); + await captureSurvivalDiagnostics(host, runtimeProvider, "before-gateway-restart", [apiKey]); await lifecycle.restartGatewayRuntime({ delayMs: 5_000, sandboxName: SANDBOX_NAME, }); - await captureSurvivalDiagnostics(host, "after-gateway-restart", [apiKey]); + await captureSurvivalDiagnostics(host, runtimeProvider, "after-gateway-restart", [apiKey]); await lifecycle.waitForGatewayConnected({ attempts: 60, intervalMs: 5_000, diff --git a/test/e2e/live/sessions-agents-cli.test.ts b/test/e2e/live/sessions-agents-cli.test.ts index bab8916ab49..79ca1898298 100644 --- a/test/e2e/live/sessions-agents-cli.test.ts +++ b/test/e2e/live/sessions-agents-cli.test.ts @@ -298,11 +298,13 @@ async function expectJsonCommand( return parseJsonEnvelope(result, args.join(" ")); } -test("sessions/agents host CLI routes to OpenClaw and preserves JSON envelopes", { +test( + "sessions/agents host CLI routes to OpenClaw and preserves JSON envelopes", + { timeout: TEST_TIMEOUT_MS, meta: { e2ePhases: [ - "confirm CLI Docker and OpenShell prerequisites", + "confirm CLI, selected runtime, and OpenShell prerequisites", "onboard the sessions and agents sandbox", "exercise main-agent session JSON and reset", "add and list the secondary agent", @@ -310,7 +312,8 @@ test("sessions/agents host CLI routes to OpenClaw and preserves JSON envelopes", "delete the secondary agent and confirm absence", ], }, -}, async ({ artifacts, cleanup, host, progress, sandbox, secrets, skip }) => { + }, + async ({ artifacts, cleanup, host, progress, runtimeProvider, sandbox, secrets, skip }) => { expect(fs.existsSync(CLI_ENTRYPOINT), "bin/nemoclaw.js missing").toBe(true); expect( fs.existsSync(CLI_DIST_ENTRYPOINT), @@ -332,14 +335,10 @@ test("sessions/agents host CLI routes to OpenClaw and preserves JSON envelopes", ], }); - const docker = await host.command("docker", ["info"], { - artifactName: "prereq-docker-info-sessions-agents-cli", - env: buildAvailabilityProbeEnv(), - timeoutMs: 30_000, + await runtimeProvider.requireAvailable({ + artifactName: "prereq-runtime-info-sessions-agents-cli", + scenarioLabel: "sessions and agents CLI", }); - expect(docker.exitCode, `Docker is required for sessions/agents E2E\n${resultText(docker)}`).toBe( - 0, - ); const hosted = requireHostedInferenceConfig(secrets); await ensureOpenshellAvailable(host); @@ -547,4 +546,5 @@ test("sessions/agents host CLI routes to OpenClaw and preserves JSON envelopes", agentEntries(agentsAfterDelete).some((entry) => entry.id === TEST_AGENT_ID), `agent '${TEST_AGENT_ID}' still visible after delete`, ).toBe(false); -}); + }, +); diff --git a/test/e2e/live/shields-config.test.ts b/test/e2e/live/shields-config.test.ts index efc8a87c126..d80fefc1e13 100644 --- a/test/e2e/live/shields-config.test.ts +++ b/test/e2e/live/shields-config.test.ts @@ -4,7 +4,7 @@ /** * * Preserves the real shields/config boundary from the former shell test: source - * install, OpenShell/Docker sandbox exec, host-root Docker tamper, chmod/chown + * install, OpenShell/runtime sandbox exec, host-root runtime tamper, chmod/chown * lock state, config redaction, audit JSONL, and the auto-restore timer. Local * helpers stay in this file because this is one focused security/policy * dependent, not a new shields fixture family. @@ -29,6 +29,7 @@ import { expect, test } from "../fixtures/e2e-test.ts"; import { requireHostedInferenceConfig } from "../fixtures/hosted-inference.ts"; import { REPO_ROOT } from "../fixtures/paths.ts"; import { pollUntil } from "../fixtures/polling.ts"; +import { RuntimeProviderPrerequisite } from "../fixtures/runtime-provider.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; import { failedStartupProcessControlCommands, @@ -105,14 +106,20 @@ async function sandboxShell( }); } -async function docker( +function selectedRuntimeProvider(host: HostCliClient): RuntimeProviderPrerequisite { + return new RuntimeProviderPrerequisite(host, (reason) => { + throw new Error(reason); + }); +} + +async function runtimeCommand( host: HostCliClient, args: string[], options: { artifactName: string; timeoutMs?: number; redactionValues?: string[] } = { - artifactName: "docker", + artifactName: "runtime", }, ): Promise { - return host.command("docker", args, { + return selectedRuntimeProvider(host).command(args, { artifactName: options.artifactName, env: commandEnv(), redactionValues: options.redactionValues, @@ -156,12 +163,12 @@ async function statPath( return { ...parsed, raw: result.stdout.trim() }; } -async function collectStartFailureDockerLogs( +async function collectStartFailureRuntimeLogs( host: HostCliClient, artifactPrefix: string, redactionValues: string[], ): Promise { - const lookup = await docker( + const lookup = await runtimeCommand( host, [ "ps", @@ -184,8 +191,8 @@ async function collectStartFailureDockerLogs( const result = lookup.exitCode !== 0 || !containerId ? lookup - : await docker(host, ["logs", "--tail", "200", containerId], { - artifactName: `${artifactPrefix}-failure-docker-logs`, + : await runtimeCommand(host, ["logs", "--tail", "200", containerId], { + artifactName: `${artifactPrefix}-failure-runtime-logs`, redactionValues, timeoutMs: 30_000, }); @@ -214,10 +221,10 @@ async function expectStopStartRecovery( const startFailureLogs = start.exitCode === 0 ? "" - : await collectStartFailureDockerLogs(host, artifactPrefix, redactionValues); + : await collectStartFailureRuntimeLogs(host, artifactPrefix, redactionValues); expect( start.exitCode, - [resultText(start), startFailureLogs && `Docker logs:\n${startFailureLogs}`] + [resultText(start), startFailureLogs && `Runtime logs:\n${startFailureLogs}`] .filter(Boolean) .join("\n"), ).toBe(0); @@ -263,7 +270,7 @@ async function expectLockedSandboxParent( artifactPrefix: string, ): Promise { const containerId = await findSandboxContainer(host); - const parent = await docker( + const parent = await runtimeCommand( host, ["exec", "--user", "0", containerId, "stat", "-c", "%a %U:%G", "/sandbox"], { artifactName: `${artifactPrefix}-sandbox-parent` }, @@ -279,7 +286,7 @@ async function expectCredentialsTraversalBoundary( ): Promise { const credentialsDir = "/sandbox/.openclaw/credentials"; const seededPath = `${credentialsDir}/.nemoclaw-permission-probe`; - const seeded = await docker( + const seeded = await runtimeCommand( host, ["exec", "--user", "0", containerId, "sh", "-c", `umask 077; : > ${seededPath}`], { artifactName: "phase-5a-seed-credential-permission-probe" }, @@ -306,7 +313,7 @@ async function expectCredentialsTraversalBoundary( expect(boundary.stdout).toContain(`${operation}=denied`); } } finally { - await docker(host, ["exec", "--user", "0", containerId, "rm", "-f", seededPath], { + await runtimeCommand(host, ["exec", "--user", "0", containerId, "rm", "-f", seededPath], { artifactName: "phase-5a-remove-credential-permission-probe", }); } @@ -344,7 +351,7 @@ async function preCleanSandbox( } async function findSandboxContainer(host: HostCliClient): Promise { - const result = await docker( + const result = await runtimeCommand( host, [ "ps", @@ -357,7 +364,7 @@ async function findSandboxContainer(host: HostCliClient): Promise { "-q", ], { - artifactName: "docker-ps-sandbox-container", + artifactName: "runtime-ps-sandbox-container", timeoutMs: 30_000, }, ); @@ -382,7 +389,7 @@ async function installedStartupCensus( "assert census is not None", "print(json.dumps({'count': census[0], 'pid': census[1]}))", ].join("\n"); - const result = await docker( + const result = await runtimeCommand( host, ["exec", "--user", "0", containerId, "python3", "-I", "-c", script], { artifactName, timeoutMs: 30_000 }, @@ -401,7 +408,7 @@ async function runInstalledFailedStartupUnlock( `plan_json=$(cat ${STATE_LOCK_PLAN_PATH})`, `exec timeout --signal=TERM --kill-after=5s 25m python3 -I ${CONFIG_GUARD_PATH} unlock-failed-startup --config-dir ${CONFIG_DIR} --plan-json "$plan_json"`, ].join("\n"); - return docker(host, ["exec", "--user", "0", containerId, "sh", "-c", script], { + return runtimeCommand(host, ["exec", "--user", "0", containerId, "sh", "-c", script], { artifactName, timeoutMs: 26 * 60_000, }); @@ -419,6 +426,7 @@ async function waitForChildlessStartup(host: HostCliClient, containerId: string) } function createPolicySetChildlessBoundaryShim( + host: HostCliClient, realOpenshellPath: string, containerId: string, startupPid: number, @@ -427,6 +435,7 @@ function createPolicySetChildlessBoundaryShim( const executable = path.join(directory, "openshell-childless-boundary.cjs"); const receipt = path.join(directory, "childless-boundary.json"); const processControl = failedStartupProcessControlCommands(containerId, startupPid); + const runtimeInvocation = selectedRuntimeProvider(host).hostInvocation([]); const childlessCensusScript = [ "import runpy, sys, time", `guard = runpy.run_path(${JSON.stringify(CONFIG_GUARD_PATH)})`, @@ -464,19 +473,22 @@ fs.writeFileSync(${JSON.stringify(receipt)}, JSON.stringify({ status: "arming" } flag: "wx", mode: 0o600, }); -const pause = spawnSync("docker", ${JSON.stringify(processControl.pauseSupervisor)}, { +const runtimeCommand = ${JSON.stringify(runtimeInvocation.command)}; +const runtimePrefix = ${JSON.stringify(runtimeInvocation.args)}; +const pause = spawnSync(runtimeCommand, [...runtimePrefix, ...${JSON.stringify(processControl.pauseSupervisor)}], { env: process.env, stdio: "inherit", }); if (pause.error || pause.status !== 0) process.exit(pause.status ?? 1); -const terminate = spawnSync("docker", ${JSON.stringify(processControl.terminateStartupChild)}, { +const terminate = spawnSync(runtimeCommand, [...runtimePrefix, ...${JSON.stringify(processControl.terminateStartupChild)}], { env: process.env, stdio: "inherit", }); if (terminate.error || terminate.status !== 0) process.exit(terminate.status ?? 1); const childless = spawnSync( - "docker", + runtimeCommand, [ + ...runtimePrefix, "exec", "--user", "0", @@ -503,9 +515,25 @@ async function readOriginalConfig( containerId: string, targetFile: string, ): Promise { + const invocation = selectedRuntimeProvider(host).hostInvocation([ + "container", + "exec", + "--user", + "0", + containerId, + "cat", + CONFIG_PATH, + ]); const result = await host.command( "bash", - ["-lc", `docker exec -u 0 ${containerId} cat ${CONFIG_PATH} > ${targetFile}`], + [ + "-lc", + 'output="$1"; shift; "$@" > "$output"', + "runtime-config-backup", + targetFile, + invocation.command, + ...invocation.args, + ], { artifactName: "phase-5b-backup-original-config", env: commandEnv(), @@ -541,7 +569,7 @@ test( timeout: TEST_TIMEOUT_MS, meta: { e2ePhases: [ - "confirm Docker and onboard the shields sandbox", + "confirm selected runtime and onboard the shields sandbox", "establish the mutable unified OpenClaw config", "lock config and workspace and inspect redaction", "restart OpenClaw with shields up", @@ -556,7 +584,7 @@ test( ], }, }, - async ({ artifacts, cleanup, host, progress, sandbox, secrets, skip }) => { + async ({ artifacts, cleanup, host, progress, runtimeProvider, sandbox, secrets }) => { await artifacts.target.declare({ id: "shields-config", boundary: "live-sandbox-shields-config", @@ -579,18 +607,10 @@ test( ], }); - const dockerInfo = await docker(host, ["info"], { - artifactName: "prereq-docker-info", - timeoutMs: 30_000, + await runtimeProvider.requireAvailable({ + artifactName: "prereq-runtime-info", + scenarioLabel: "shields-config", }); - if (dockerInfo.exitCode !== 0) { - if (process.env.GITHUB_ACTIONS === "true") { - throw new Error( - `Docker is required for shields-config live E2E: ${resultText(dockerInfo)}`, - ); - } - skip("Docker is required for shields-config live E2E"); - } const hosted = requireHostedInferenceConfig(secrets); const apiKey = hosted.apiKey; @@ -736,7 +756,7 @@ test( expect(dirAfterDoctor).toMatchObject({ mode: "2770", owner: "sandbox:sandbox" }); const containerId = await findSandboxContainer(host); - const gatewayWrite = await docker( + const gatewayWrite = await runtimeCommand( host, ["exec", "-u", "gateway", containerId, "sh", "-c", `printf ' ' >>${CONFIG_PATH}`], { @@ -912,26 +932,32 @@ test( const originalConfig = path.join(os.tmpdir(), `nemoclaw-shields-orig-${process.pid}.json`); await readOriginalConfig(host, containerId, originalConfig); try { - const tamper = await host.command( - "bash", + const tamper = await runtimeCommand( + host, [ - "-lc", + "container", + "exec", + "--user", + "0", + containerId, + "sh", + "-c", [ `had_immutable=false`, - `if docker exec -u 0 ${containerId} lsattr -d ${CONFIG_PATH} 2>/dev/null | awk '{print $1}' | grep -q i; then had_immutable=true; fi`, - `docker exec -u 0 ${containerId} sh -c 'chattr -i ${CONFIG_PATH} 2>/dev/null || true; chmod 644 ${CONFIG_PATH} && printf " " >> ${CONFIG_PATH} && chmod 444 ${CONFIG_PATH}'`, - `if [ "$had_immutable" = true ]; then docker exec -u 0 ${containerId} chattr +i ${CONFIG_PATH} >/dev/null 2>&1 || true; fi`, + `if lsattr -d ${CONFIG_PATH} 2>/dev/null | awk '{print $1}' | grep -q i; then had_immutable=true; fi`, + `chattr -i ${CONFIG_PATH} 2>/dev/null || true`, + `chmod 644 ${CONFIG_PATH} && printf " " >> ${CONFIG_PATH} && chmod 444 ${CONFIG_PATH}`, + `if [ "$had_immutable" = true ]; then chattr +i ${CONFIG_PATH} >/dev/null 2>&1 || true; fi`, ].join("\n"), ], { artifactName: "phase-5b-host-root-tamper", - env: commandEnv(), timeoutMs: 30_000, }, ); expect(tamper.exitCode, resultText(tamper)).toBe(0); - const afterTamper = await docker( + const afterTamper = await runtimeCommand( host, ["exec", containerId, "stat", "-c", "%a %U:%G", CONFIG_PATH], { @@ -955,11 +981,26 @@ test( expect(reUp.exitCode, resultText(reUp)).not.toBe(0); expect(resultText(reUp)).toContain("Refusing to re-seal"); } finally { - await host.command( + const invocation = selectedRuntimeProvider(host).hostInvocation([ + "container", + "exec", + "--interactive", + "--user", + "0", + containerId, + "sh", + "-c", + `chattr -i ${CONFIG_PATH} 2>/dev/null || true; chmod 644 ${CONFIG_PATH} && cat > ${CONFIG_PATH} && chmod 444 ${CONFIG_PATH} && chattr +i ${CONFIG_PATH} 2>/dev/null || true`, + ]); + const restore = await host.command( "bash", [ "-lc", - `docker exec -i -u 0 ${containerId} sh -c 'chattr -i ${CONFIG_PATH} 2>/dev/null || true; chmod 644 ${CONFIG_PATH} && cat > ${CONFIG_PATH} && chmod 444 ${CONFIG_PATH} && chattr +i ${CONFIG_PATH} 2>/dev/null || true' < ${originalConfig}`, + 'input="$1"; shift; exec "$@" < "$input"', + "runtime-config-restore", + originalConfig, + invocation.command, + ...invocation.args, ], { artifactName: "phase-5b-restore-original-config", @@ -967,6 +1008,7 @@ test( timeoutMs: 30_000, }, ); + expect(restore.exitCode, resultText(restore)).toBe(0); fs.rmSync(originalConfig, { force: true }); } @@ -984,15 +1026,20 @@ test( // "restart seal requires the exact shields-locked file posture" (which // stranded host state UNLOCKED while the tree stayed root-locked). The bytes // are untouched here, so it is a launderable perms drift, not content drift. - const permsDrift = await host.command( - "bash", + const permsDrift = await runtimeCommand( + host, [ - "-lc", - `docker exec -u 0 ${containerId} sh -c 'chattr -i ${CONFIG_HASH_PATH} 2>/dev/null || true; chmod 660 ${CONFIG_HASH_PATH} && chown sandbox:sandbox ${CONFIG_HASH_PATH}'`, + "container", + "exec", + "--user", + "0", + containerId, + "sh", + "-c", + `chattr -i ${CONFIG_HASH_PATH} 2>/dev/null || true; chmod 660 ${CONFIG_HASH_PATH} && chown sandbox:sandbox ${CONFIG_HASH_PATH}`, ], { artifactName: "phase-5c-config-hash-perms-only-drift", - env: commandEnv(), timeoutMs: 30_000, }, ); @@ -1229,7 +1276,7 @@ test( "prove installed failed-startup guard refuses a live child and supported shields down unlocks childless state", ); const recoveryContainerId = await findSandboxContainer(host); - const removeMarkers = await docker( + const removeMarkers = await runtimeCommand( host, ["exec", "--user", "0", recoveryContainerId, "rm", "-f", ...STARTUP_MARKER_PATHS], { artifactName: "phase-12-remove-startup-markers", timeoutMs: 30_000 }, @@ -1272,7 +1319,7 @@ test( ); cleanup.trackDisposable(`resume stopped supervisor for ${SANDBOX_NAME}`, async () => { await resumeSupervisorIfPaused(supervisorPaused, async () => { - const resume = await docker(host, processControl.resumeSupervisor, { + const resume = await runtimeCommand(host, processControl.resumeSupervisor, { artifactName: "cleanup-phase-12-resume-startup-supervisor", timeoutMs: 30_000, }); @@ -1292,6 +1339,7 @@ test( const realOpenshellPath = openshellResolution.stdout.trim(); expect(path.isAbsolute(realOpenshellPath), realOpenshellPath).toBe(true); const policyBoundary = createPolicySetChildlessBoundaryShim( + host, realOpenshellPath, recoveryContainerId, liveCensus.pid ?? 0, @@ -1330,7 +1378,7 @@ test( status: "childless", }); await waitForChildlessStartup(host, recoveryContainerId); - const unlockedPaths = await docker( + const unlockedPaths = await runtimeCommand( host, [ "exec", @@ -1351,7 +1399,7 @@ test( { mode: "2770", owner: "sandbox:sandbox" }, ]); - const resumeSupervisor = await docker(host, processControl.resumeSupervisor, { + const resumeSupervisor = await runtimeCommand(host, processControl.resumeSupervisor, { artifactName: "phase-12-resume-startup-supervisor", timeoutMs: 30_000, }); diff --git a/test/e2e/live/skill-agent.test.ts b/test/e2e/live/skill-agent.test.ts index 7e7204f3693..aaf4b641bee 100644 --- a/test/e2e/live/skill-agent.test.ts +++ b/test/e2e/live/skill-agent.test.ts @@ -108,7 +108,7 @@ test( timeout: 30 * 60_000, meta: { e2ePhases: [ - "confirm Docker and skill tooling", + "confirm the selected runtime and skill tooling", "onboard the OpenClaw skill sandbox", "inject and confirm the skill fixture", "ask the agent to consume the skill", @@ -116,7 +116,7 @@ test( ], }, }, - async ({ artifacts, cleanup, host, progress, sandbox, secrets, skip }) => { + async ({ artifacts, cleanup, host, progress, runtimeProvider, sandbox, secrets, skip }) => { expect( fs.existsSync(CLI_ENTRYPOINT), "run `npm run build:cli` before live repo CLI targets", @@ -129,17 +129,10 @@ test( `missing skill verify helper: ${VERIFY_SKILL_SCRIPT}`, ).toBe(true); - const docker = await host.command("docker", ["info"], { - artifactName: "prereq-docker-info-skill-agent", - env: buildAvailabilityProbeEnv(), - timeoutMs: 30_000, + await runtimeProvider.requireAvailable({ + artifactName: "prereq-runtime-info-skill-agent", + scenarioLabel: "skill-agent", }); - if (docker.exitCode !== 0) { - if (process.env.GITHUB_ACTIONS === "true") { - throw new Error(`Docker is required for skill-agent E2E: ${resultText(docker)}`); - } - skip("Docker is required for skill-agent E2E"); - } const hosted = requireHostedInferenceConfig(secrets); const apiKey = hosted.apiKey; @@ -148,7 +141,7 @@ test( id: "skill-agent", boundary: "direct-cli-onboard-sandbox-skill-and-agent-turn", contract: [ - "Docker is available before onboarding", + "the selected runtime is available before onboarding", "NVIDIA_INFERENCE_API_KEY is staged as the compatible endpoint credential", "nemoclaw onboard creates/recreates a real OpenClaw sandbox", "skill-smoke-fixture is injected into sandbox and home skill roots", @@ -382,7 +375,7 @@ test( id: "skill-agent", status: "passed", assertions: { - dockerRunning: docker.exitCode === 0, + runtimeProviderAvailable: true, onboardCompleted: onboard.exitCode === 0, skillInjected: addSkill.exitCode === 0, agentReturnedVerificationToken: agentOk, diff --git a/test/e2e/live/snapshot-commands.test.ts b/test/e2e/live/snapshot-commands.test.ts index 94e3d6d5632..e09bb138e6e 100644 --- a/test/e2e/live/snapshot-commands.test.ts +++ b/test/e2e/live/snapshot-commands.test.ts @@ -13,7 +13,6 @@ import { randomUUID } from "node:crypto"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import { resultText } from "../fixtures/clients/command.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; import { @@ -211,7 +210,7 @@ test( timeout: LIVE_TIMEOUT_MS, meta: { e2ePhases: [ - "confirm Docker and start hermetic inference", + "confirm the selected runtime and start hermetic inference", "onboard the snapshot sandbox", "create one snapshot", "destroy, freshly onboard, and restore workspace state", @@ -222,7 +221,7 @@ test( ], }, }, - async ({ artifacts, cleanup, host, progress, sandbox, skip }) => { + async ({ artifacts, cleanup, host, progress, runtimeProvider, sandbox }) => { await artifacts.target.declare({ id: "snapshot-commands", boundary: "install.sh + nemoclaw snapshot commands + openshell sandbox exec", @@ -239,17 +238,10 @@ test( ], }); - const dockerInfo = await host.command("docker", ["info"], { - artifactName: "phase-0-docker-info", - env: buildAvailabilityProbeEnv(), - timeoutMs: 30_000, + await runtimeProvider.requireAvailable({ + artifactName: "phase-0-runtime-info", + scenarioLabel: "snapshot commands", }); - if (dockerInfo.exitCode !== 0) { - if (process.env.GITHUB_ACTIONS === "true") { - throw new Error(`Docker is required for snapshot commands E2E: ${resultText(dockerInfo)}`); - } - skip(`Docker is required for snapshot commands E2E: ${resultText(dockerInfo)}`); - } const inference = await startFakeOpenAiCompatibleServer({ apiKey: INFERENCE_API_KEY, diff --git a/test/e2e/live/spark-express-vllm.test.ts b/test/e2e/live/spark-express-vllm.test.ts index 8c7ebf395da..46e28bf8c99 100644 --- a/test/e2e/live/spark-express-vllm.test.ts +++ b/test/e2e/live/spark-express-vllm.test.ts @@ -224,7 +224,7 @@ test("DGX Spark Express option 2 materializes the fixed vLLM profile and routes "prove sandbox inference and unrelated egress denial", ], }, -}, async ({ artifacts, cleanup, host, progress, sandbox, skip }) => { +}, async ({ artifacts, cleanup, host, progress, runtimeProvider, sandbox }) => { validateSandboxName(SANDBOX_NAME); assertLocalDockerEnvironment(process.env); const plan = vllmProfilePlan(); @@ -255,7 +255,7 @@ test("DGX Spark Express option 2 materializes the fixed vLLM profile and routes }); progress.phase("qualify the physical DGX Spark host"); - await requireLivePrerequisites(host, skip); + await requireLivePrerequisites(host, runtimeProvider); const platform = await host.command( "bash", [ diff --git a/test/e2e/live/state-backup-restore.test.ts b/test/e2e/live/state-backup-restore.test.ts index e8ee38e8e45..7b45cbcca1e 100644 --- a/test/e2e/live/state-backup-restore.test.ts +++ b/test/e2e/live/state-backup-restore.test.ts @@ -125,11 +125,13 @@ async function destroySandboxUntilAbsent( ); } -test("state-backup-restore: backup-workspace.sh restores workspace files and memory directory (#8006)", { +test( + "state-backup-restore: backup-workspace.sh restores workspace files and memory directory (#8006)", + { timeout: TEST_TIMEOUT_MS, meta: { e2ePhases: [ - "confirm Docker and the workspace backup script", + "confirm the selected runtime and the workspace backup script", "onboard the source sandbox", "write workspace and memory markers", "capture and inspect the host backup", @@ -138,13 +140,15 @@ test("state-backup-restore: backup-workspace.sh restores workspace files and mem "validate restored workspace and memory", ], }, -}, async ({ + }, + async ({ artifacts, cleanup, environment, host, onboard, progress, + runtimeProvider, sandbox, secrets, skip, @@ -154,19 +158,10 @@ test("state-backup-restore: backup-workspace.sh restores workspace files and mem secrets.required("NVIDIA_INFERENCE_API_KEY"); expect(fs.existsSync(path.join(REPO_ROOT, "scripts", "backup-workspace.sh"))).toBe(true); - const dockerInfo = await host.command("docker", ["info"], { - artifactName: "prereq-docker-info", - env: buildAvailabilityProbeEnv(), - timeoutMs: 30_000, + await runtimeProvider.requireAvailable({ + artifactName: "prereq-runtime-info", + scenarioLabel: "state backup and restore", }); - if (dockerInfo.exitCode !== 0) { - if (process.env.GITHUB_ACTIONS === "true") { - throw new Error( - `Docker is required for state-backup-restore live coverage: ${resultText(dockerInfo)}`, - ); - } - skip("Docker is required for state-backup-restore live coverage"); - } await artifacts.writeJson("contract.json", { sandboxName: SANDBOX_NAME, @@ -233,7 +228,7 @@ test("state-backup-restore: backup-workspace.sh restores workspace files and mem const ready = await environment.assertReady({ platform: "ubuntu-local", install: "repo-current", - runtime: "docker-running", + runtime: "managed-runtime-running", onboarding: "cloud-openclaw", }); @@ -445,4 +440,5 @@ test("state-backup-restore: backup-workspace.sh restores workspace files and mem } expect(memoryText).toContain("STATE=EXISTS"); expect(memoryText).toContain(`${markerContent}_daily`); -}); + }, +); diff --git a/test/e2e/live/telegram-injection.test.ts b/test/e2e/live/telegram-injection.test.ts index 9aa54d81771..b89baf16b4a 100644 --- a/test/e2e/live/telegram-injection.test.ts +++ b/test/e2e/live/telegram-injection.test.ts @@ -9,12 +9,12 @@ import { runSecondaryCleanup as bestEffortDiagnostic, CLI, COMMAND_TIMEOUT_MS, - dockerInfo, expectExitZero, expectSandboxReady, installSandboxOrSkipOnRateLimit, phase6Env, precleanSandbox, + requirePhase6RuntimeProvider, REPO_ROOT, redactionValues, resultText, @@ -192,7 +192,9 @@ async function assertSandboxProcessTableDoesNotExposeSecret( expect(result.stdout.trim(), resultText(result)).toBe("SECRET_ABSENT"); } -test("Telegram bridge-style message handling treats shell metacharacters as data", { +test( + "Telegram bridge-style message handling treats shell metacharacters as data", + { timeout: LIVE_TIMEOUT_MS, meta: { e2ePhases: [ @@ -203,7 +205,8 @@ test("Telegram bridge-style message handling treats shell metacharacters as data "confirm benign message passthrough", ], }, -}, async ({ artifacts, cleanup, host, progress, sandbox, secrets, skip }) => { + }, + async ({ artifacts, cleanup, host, progress, runtimeProvider, sandbox, secrets, skip }) => { const apiKey = secrets.required("NVIDIA_INFERENCE_API_KEY"); const env = phase6Env({ sandboxName: SANDBOX_NAME, @@ -237,8 +240,7 @@ test("Telegram bridge-style message handling treats shell metacharacters as data ); await precleanSandbox(host, SANDBOX_NAME, env, redactions, "preclean-telegram-injection"); - const docker = await dockerInfo(host, env); - expect(docker.exitCode, resultText(docker)).toBe(0); + await requirePhase6RuntimeProvider(runtimeProvider, "Telegram injection"); const install = await installSandboxOrSkipOnRateLimit( host, @@ -249,7 +251,13 @@ test("Telegram bridge-style message handling treats shell metacharacters as data "NVIDIA endpoint validation was rate-limited before Telegram injection assertions ran", ); expectExitZero(install, "install.sh --non-interactive"); - await expectSandboxReady(host, SANDBOX_NAME, env, redactions, "sandbox-list-telegram-injection"); + await expectSandboxReady( + host, + SANDBOX_NAME, + env, + redactions, + "sandbox-list-telegram-injection", + ); progress.phase("exercise command-substitution payloads"); for (const [label, marker, payload] of [ @@ -382,4 +390,5 @@ test("Telegram bridge-style message handling treats shell metacharacters as data timeoutMs: 60_000, }), ); -}); + }, +); diff --git a/test/e2e/live/token-rotation.test.ts b/test/e2e/live/token-rotation.test.ts index a4f40536382..c333a5a35d7 100644 --- a/test/e2e/live/token-rotation.test.ts +++ b/test/e2e/live/token-rotation.test.ts @@ -285,14 +285,14 @@ test( ...testTimeoutOptions(PHASE_TIMEOUT_MS), meta: { e2ePhases: [ - "confirm Docker and start hermetic inference", + "confirm the selected runtime and start hermetic inference", "install the sandbox and confirm provider hashes", "rotate only the Telegram provider", "reuse the sandbox and record rotation evidence", ], }, }, - async ({ artifacts, cleanup, host, progress, sandbox, skip }) => { + async ({ artifacts, cleanup, host, progress, runtimeProvider, sandbox }) => { expect( fs.existsSync(CLI_ENTRYPOINT), "run `npm run build:cli` before live repo CLI targets", @@ -300,17 +300,10 @@ test( assertTokenPairsDiffer(); - const docker = await host.command("docker", ["info"], { - artifactName: "prereq-docker-info-token-rotation", - env: buildAvailabilityProbeEnv(), - timeoutMs: 30_000, + await runtimeProvider.requireAvailable({ + artifactName: "prereq-runtime-info-token-rotation", + scenarioLabel: "token rotation", }); - if (docker.exitCode !== 0) { - if (process.env.GITHUB_ACTIONS === "true") { - throw new Error(`Docker is required for token rotation live E2E: ${resultText(docker)}`); - } - skip("Docker is required for token rotation live E2E"); - } const fakeOpenAI = await startFakeOpenAiCompatibleServer({ chatContent: "OK", @@ -439,8 +432,7 @@ test( // OpenClaw/plugin layers for the retained rotation; token values remain in // gateway providers and are never baked into this image. const cacheImageTag = `nemoclaw-token-rotation-cache:${process.pid}`; - const retainBuildCache = await host.command( - "docker", + const retainBuildCache = await runtimeProvider.command( ["image", "tag", sandboxImageTag(), cacheImageTag], { artifactName: "phase-1-retain-build-cache", @@ -450,7 +442,7 @@ test( ); expect(retainBuildCache.exitCode, resultText(retainBuildCache)).toBe(0); cleanup.trackDisposable("remove token-rotation build cache tag", async () => { - const remove = await host.command("docker", ["image", "rm", cacheImageTag], { + const remove = await runtimeProvider.command(["image", "rm", cacheImageTag], { artifactName: "cleanup-token-rotation-build-cache", env: buildAvailabilityProbeEnv(), timeoutMs: 30_000, diff --git a/test/e2e/live/tunnel-lifecycle-helpers.ts b/test/e2e/live/tunnel-lifecycle-helpers.ts index d377fe51ea5..1ca87367069 100644 --- a/test/e2e/live/tunnel-lifecycle-helpers.ts +++ b/test/e2e/live/tunnel-lifecycle-helpers.ts @@ -173,7 +173,7 @@ export const TUNNEL_LIFECYCLE_TEST_TIMEOUT_MS = TEST_TIMEOUT_MS; type TunnelLifecycleFixtures = Pick< E2ETargetFixtures, - "artifacts" | "cleanup" | "host" | "progress" | "secrets" + "artifacts" | "cleanup" | "host" | "progress" | "runtimeProvider" | "secrets" > & { skip: (note?: string) => never; }; @@ -221,6 +221,7 @@ export async function runTunnelLifecycleContract({ cleanup, host, progress, + runtimeProvider, secrets, skip, }: TunnelLifecycleFixtures): Promise { @@ -244,17 +245,10 @@ export async function runTunnelLifecycleContract({ registerTunnelLifecycleCleanup(cleanup, host); - const docker = await host.command("docker", ["info"], { + await runtimeProvider.requireAvailable({ artifactName: "prereq-docker-info-tunnel-lifecycle", - env: buildAvailabilityProbeEnv(), - timeoutMs: 30_000, + scenarioLabel: "tunnel lifecycle", }); - if (docker.exitCode !== 0) { - if (process.env.GITHUB_ACTIONS === "true") { - throw new Error(`Docker is required for tunnel lifecycle E2E: ${resultText(docker)}`); - } - skip("Docker is required for tunnel lifecycle E2E"); - } const cloudflared = await host.command("cloudflared", ["--version"], { artifactName: "prereq-cloudflared-version", diff --git a/test/e2e/mock-parity.json b/test/e2e/mock-parity.json index 264e8391214..37542a1b164 100644 --- a/test/e2e/mock-parity.json +++ b/test/e2e/mock-parity.json @@ -98,6 +98,7 @@ }, { "live": "test/e2e/live/native-runtime-qualification-case.test.ts", + "liveSources": ["test/e2e/live/native-runtime-qualification-case-executor.ts"], "fast": [ "test/e2e/support/native-runtime-qualification-case-helpers.test.ts", "test/e2e/support/native-runtime-qualification-producer-workflow.test.ts", @@ -115,6 +116,7 @@ }, { "live": "test/e2e/live/hermes-gpu-startup.test.ts", + "liveSources": ["test/e2e/live/hermes-gpu-startup-proof.ts"], "fast": [ "test/e2e/support/hermes-gpu-startup-fallback.test.ts", "test/e2e/support/hermes-gpu-startup-integrity.test.ts", @@ -129,6 +131,7 @@ }, { "live": "test/e2e/live/inference-routing.test.ts", + "liveSources": ["test/e2e/live/inference-routing-helpers.ts"], "fast": [ "nemoclaw/src/blueprint/runner-identity.test.ts", "nemoclaw/src/blueprint/runtime-identity.test.ts", @@ -223,14 +226,19 @@ }, { "live": "test/e2e/live/network-policy.test.ts", - "liveSources": ["test/e2e/live/policy-list-state.ts"], + "liveSources": [ + "test/e2e/live/package-database-read-only.ts", + "test/e2e/live/restricted-onboard-helpers.ts", + "test/e2e/live/policy-list-state.ts" + ], "fast": [ "src/lib/actions/sandbox/policy-channel-add-drift.test.ts", "test/channels/channels-add-preset.test.ts", "test/runtime/policy/policy-channel-agent-resolution.test.ts", "test/onboarding/validate-blueprint.test.ts", "test/e2e/support/e2e-cleanup-resources.test.ts", - "test/e2e/support/e2e-clients.test.ts" + "test/e2e/support/e2e-clients.test.ts", + "test/e2e/support/runtime-provider-fixture.test.ts" ] }, { @@ -251,20 +259,25 @@ "test/credentials/openshell-credential-generation-window.test.ts", "test/mcp/mcp-artifact-secret-scan.test.ts", "test/mcp/mcp-agent-matrix-artifact-proof.test.ts", - "test/e2e/support/mcp-workflow-boundary.test.ts" + "test/e2e/support/mcp-workflow-boundary.test.ts", + "test/e2e/support/runtime-provider-fixture.test.ts" ] }, { "live": "test/e2e/live/openclaw-tui-chat-correlation.test.ts", "fast": [ "test/e2e/support/openclaw-tui-ref-fidelity.test.ts", - "test/e2e/support/openclaw-tui-run-classification.test.ts" + "test/e2e/support/openclaw-tui-run-classification.test.ts", + "test/e2e/support/runtime-provider-fixture.test.ts" ] }, { "live": "test/e2e/live/dashboard-remote-bind.test.ts", "liveSources": ["test/e2e/live/dashboard-connect-handoff.ts"], - "fast": ["test/e2e/support/dashboard-connect-handoff.test.ts"] + "fast": [ + "test/e2e/support/dashboard-connect-handoff.test.ts", + "test/e2e/support/runtime-provider-fixture.test.ts" + ] }, { "live": "test/e2e/live/sandbox-rlimits-connect.test.ts", @@ -313,7 +326,10 @@ }, { "live": "test/e2e/live/onboard-policy-preset-sequencing.test.ts", - "fast": ["test/e2e/support/onboard-interactive-pty.test.ts"] + "fast": [ + "test/e2e/support/onboard-interactive-pty.test.ts", + "test/e2e/support/runtime-provider-fixture.test.ts" + ] }, { "live": "test/e2e/live/snapshot-commands.test.ts", @@ -460,6 +476,7 @@ ], "fast": [ "test/e2e/support/gpu-e2e-helpers.test.ts", + "test/e2e/support/runtime-provider-fixture.test.ts", "test/e2e/support/managed-image-cohort-contract.test.ts", "test/e2e/support/managed-image-receipt.test.ts", "test/e2e/support/stock-managed-image-workflow-boundary.test.ts", @@ -471,6 +488,7 @@ }, { "live": "test/e2e/live/hermes-discord.test.ts", + "liveSources": ["test/e2e/live/hermes-discord-proxy.ts"], "fast": [ "src/lib/messaging/channels/discord/credential-injection.test.ts", "src/lib/onboard/messaging-policy-presets.test.ts", @@ -520,7 +538,9 @@ }, { "live": "test/e2e/live/issue-4462-scope-upgrade-approval.test.ts", + "liveSources": ["test/e2e/live/issue-4462-admin-approval-helper.ts"], "fast": [ + "test/e2e/support/issue-4462-fixture-boundary.test.ts", "test/e2e/support/e2e-cleanup-resources.test.ts", "test/e2e/support/e2e-clients.test.ts" ] @@ -546,10 +566,14 @@ { "live": "test/e2e/live/mcp-bridge.test.ts", "liveSources": [ + "test/e2e/live/dns-rebinding-hosts-fixture.ts", "test/e2e/live/mcp-bridge-cleanup.ts", + "test/e2e/live/mcp-bridge-deepagents-config.ts", + "test/e2e/live/mcp-bridge-hermes-http.ts", "test/e2e/live/mcp-bridge-onboard-env.ts", "test/e2e/live/mcp-bridge-reliability.ts", "test/e2e/live/mcp-bridge-sandbox.ts", + "test/e2e/live/mcp-bridge-trusted-private.ts", "test/e2e/live/openshell-allowed-ips-rebinding.ts", "test/e2e/live/openshell-exact-main-runtime-contracts.ts" ], @@ -564,6 +588,7 @@ "test/e2e/support/e2e-redaction-entry.test.ts", "test/e2e/support/mcp-bridge-cleanup.test.ts", "test/e2e/support/mcp-bridge-hermes-lifecycle.test.ts", + "test/e2e/support/mcp-bridge-hermes-http.test.ts", "test/e2e/support/mcp-bridge-onboard-env.test.ts", "test/e2e/support/mcp-bridge-reliability.test.ts", "test/e2e/support/mcp-bridge-sandbox.test.ts", @@ -579,6 +604,7 @@ "fast": [ "test/e2e/support/e2e-cleanup-resources.test.ts", "test/e2e/support/e2e-clients.test.ts", + "test/e2e/support/messaging-compatible-endpoint-helpers.test.ts", "test/e2e/support/openclaw-agent-output.test.ts" ] }, @@ -586,6 +612,7 @@ "live": "test/e2e/live/messaging-providers.test.ts", "liveSources": [ "test/e2e/live/messaging-providers-helpers.ts", + "test/e2e/live/phase6-messaging-helpers.ts", "test/e2e/live/messaging-providers-wechat-runtime-proof.ts" ], "fast": [ @@ -626,7 +653,8 @@ "test/e2e/support/openclaw-discord-pairing-helpers.test.ts", "test/e2e/support/messaging-providers-runtime-proofs.test.ts", "test/e2e/support/e2e-cleanup-resources.test.ts", - "test/e2e/support/e2e-clients.test.ts" + "test/e2e/support/e2e-clients.test.ts", + "test/e2e/support/runtime-provider-fixture.test.ts" ] }, { @@ -693,6 +721,7 @@ "src/lib/actions/sandbox/rebuild-flow-lifecycle.test.ts", "test/e2e/support/e2e-cleanup-resources.test.ts", "test/e2e/support/e2e-clients.test.ts", + "test/e2e/support/rebuild-hermes-bootstrap.test.ts", "test/e2e/support/rebuild-hermes-env.test.ts", "test/e2e/support/rebuild-hermes-base-identity.test.ts", "test/e2e/support/rebuild-hermes-image-state.test.ts", @@ -873,11 +902,13 @@ }, { "live": "test/e2e/live/tunnel-lifecycle.test.ts", + "liveSources": ["test/e2e/live/tunnel-lifecycle-helpers.ts"], "fast": [ "test/e2e/support/e2e-progress-fixture.test.ts", "test/e2e/support/e2e-progress-outcome.test.ts", "test/e2e/support/e2e-semantic-phase-check.test.ts", - "test/e2e/support/workflow-e2e-progress.test.ts" + "test/e2e/support/workflow-e2e-progress.test.ts", + "test/e2e/support/runtime-provider-fixture.test.ts" ] }, { diff --git a/test/e2e/registry/definitions/baseline.ts b/test/e2e/registry/definitions/baseline.ts index 06f85f11d8e..7f1370a6177 100644 --- a/test/e2e/registry/definitions/baseline.ts +++ b/test/e2e/registry/definitions/baseline.ts @@ -8,6 +8,7 @@ import { macosRepoDocker, ubuntuRepoDocker, ubuntuRepoDockerLifecycle, + ubuntuRepoManagedRuntime, ubuntuRepoNoDocker, wslRepoDocker, } from "../matrix.ts"; @@ -16,6 +17,10 @@ import { type E2eExecutionMetadata, validateE2eExecutionMetadata, } from "../../../../tools/e2e/execution-coverage.mts"; +import { + E2E_GATEWAY_RUNTIMES, + type E2eGatewayRuntimeSupport, +} from "../../../../tools/e2e/gateway-runtime.mts"; interface CanonicalTargetInput { id: string; @@ -30,6 +35,7 @@ interface CanonicalTargetInput { requiredSecrets?: string[]; skippedCapabilities?: Array>; expectedFailure?: ExpectedFailureContract; + gatewayRuntimes: E2eGatewayRuntimeSupport; } function canonicalTarget(input: CanonicalTargetInput): TargetDefinition { @@ -53,7 +59,7 @@ function canonicalTarget(input: CanonicalTargetInput): TargetDefinition { if (input.expectedFailure) { builder = builder.expectedFailure(input.expectedFailure); } - const definition = builder.build(); + const definition = { ...builder.build(), gatewayRuntimes: input.gatewayRuntimes }; if (!input.executionCoverage) return definition; return { ...definition, @@ -76,21 +82,23 @@ const macosDockerSkipped = [ const canonicalTargetInputs: CanonicalTargetInput[] = [ { id: "ubuntu-repo-cloud-openclaw", + gatewayRuntimes: E2E_GATEWAY_RUNTIMES, manifestName: "openclaw-nvidia", - environment: ubuntuRepoDocker("cloud-openclaw"), + environment: ubuntuRepoManagedRuntime("cloud-openclaw"), expectedStateId: "cloud-openclaw-ready", suiteIds: ["smoke", "inference", "credentials"], - description: "Ubuntu repo checkout with Docker and cloud OpenClaw onboarding.", + description: "Ubuntu repo checkout with managed-runtime cloud OpenClaw onboarding.", executionCoverage: { agentRuntime: "openclaw", observableOutcome: "Repository install onboarding and hosted inference succeed", - environmentOrInferenceEndpoint: "Ubuntu Docker host; NVIDIA hosted inference", + environmentOrInferenceEndpoint: "Ubuntu managed-runtime host; NVIDIA hosted inference", unresolvedReason: "", }, requiredSecrets: ["NVIDIA_INFERENCE_API_KEY"], }, { id: "ubuntu-repo-cloud-hermes", + gatewayRuntimes: ["docker"], manifestName: "hermes-nvidia", environment: ubuntuRepoDocker("cloud-hermes"), expectedStateId: "cloud-hermes-ready", @@ -99,6 +107,7 @@ const canonicalTargetInputs: CanonicalTargetInput[] = [ }, { id: "ubuntu-repo-cloud-langchain-deepagents-code", + gatewayRuntimes: ["docker"], manifestName: "langchain-deepagents-code-nvidia", environment: ubuntuRepoDockerLifecycle( "cloud-langchain-deepagents-code", @@ -117,6 +126,7 @@ const canonicalTargetInputs: CanonicalTargetInput[] = [ }, { id: "gpu-repo-local-ollama-openclaw", + gatewayRuntimes: ["docker"], manifestName: "openclaw-ollama-gpu", environment: gpuRepoDockerCdi("local-ollama-openclaw"), expectedStateId: "local-ollama-openclaw-ready", @@ -125,6 +135,7 @@ const canonicalTargetInputs: CanonicalTargetInput[] = [ }, { id: "macos-repo-cloud-openclaw", + gatewayRuntimes: ["docker"], manifestName: "openclaw-nvidia-macos", environment: macosRepoDocker("cloud-openclaw"), expectedStateId: "macos-cli-ready-docker-optional", @@ -136,6 +147,7 @@ const canonicalTargetInputs: CanonicalTargetInput[] = [ }, { id: "wsl-repo-cloud-openclaw", + gatewayRuntimes: ["docker"], manifestName: "openclaw-nvidia-wsl", environment: wslRepoDocker("cloud-openclaw"), expectedStateId: "cloud-openclaw-ready", @@ -145,6 +157,7 @@ const canonicalTargetInputs: CanonicalTargetInput[] = [ }, { id: "brev-launchable-cloud-openclaw", + gatewayRuntimes: ["docker"], manifestName: "openclaw-nvidia-brev-launchable", environment: brevLaunchableRemote("cloud-openclaw"), expectedStateId: "cloud-openclaw-ready", @@ -154,6 +167,7 @@ const canonicalTargetInputs: CanonicalTargetInput[] = [ }, { id: "ubuntu-no-docker-preflight-negative", + gatewayRuntimes: ["docker"], manifestName: "openclaw-nvidia-no-docker-negative", environment: ubuntuRepoNoDocker("cloud-openclaw"), expectedStateId: "preflight-failure-no-sandbox", @@ -176,6 +190,7 @@ const canonicalTargetInputs: CanonicalTargetInput[] = [ // future `rebuild-from-old-version` lifecycle profile and is // intentionally out of scope here. id: "ubuntu-rebuild-openclaw", + gatewayRuntimes: ["docker"], manifestName: "openclaw-nvidia-rebuild", environment: ubuntuRepoDockerLifecycle("cloud-openclaw", "rebuild-current-version"), expectedStateId: "cloud-openclaw-ready", @@ -196,6 +211,7 @@ const canonicalTargetInputs: CanonicalTargetInput[] = [ // Docker container present (running, stopped, or a // `*-nemoclaw-gpu-backup-*` sibling). id: "ubuntu-repo-docker-post-reboot-recovery", + gatewayRuntimes: ["docker"], manifestName: "openclaw-nvidia-post-reboot-recovery", environment: ubuntuRepoDockerLifecycle("cloud-openclaw", "post-reboot-recovery"), expectedStateId: "post-reboot-recovery-ready", @@ -213,6 +229,7 @@ const canonicalTargetInputs: CanonicalTargetInput[] = [ }, { id: "ubuntu-repo-openai-compatible-openclaw", + gatewayRuntimes: ["docker"], manifestName: "openclaw-openai-compatible", environment: ubuntuRepoDocker("openai-compatible-openclaw"), expectedStateId: "cloud-openclaw-ready", @@ -221,6 +238,7 @@ const canonicalTargetInputs: CanonicalTargetInput[] = [ }, { id: "ubuntu-repo-cloud-openclaw-brave", + gatewayRuntimes: ["docker"], manifestName: "openclaw-nvidia-brave", environment: ubuntuRepoDocker("cloud-nvidia-openclaw-brave"), expectedStateId: "cloud-openclaw-ready", @@ -229,6 +247,7 @@ const canonicalTargetInputs: CanonicalTargetInput[] = [ }, { id: "ubuntu-repo-cloud-openclaw-telegram", + gatewayRuntimes: ["docker"], manifestName: "openclaw-nvidia-telegram", environment: ubuntuRepoDocker("cloud-nvidia-openclaw-telegram"), expectedStateId: "cloud-openclaw-ready", @@ -237,6 +256,7 @@ const canonicalTargetInputs: CanonicalTargetInput[] = [ }, { id: "ubuntu-repo-cloud-openclaw-discord", + gatewayRuntimes: ["docker"], manifestName: "openclaw-nvidia-discord", environment: ubuntuRepoDocker("cloud-nvidia-openclaw-discord"), expectedStateId: "cloud-openclaw-ready", @@ -245,6 +265,7 @@ const canonicalTargetInputs: CanonicalTargetInput[] = [ }, { id: "ubuntu-repo-cloud-openclaw-slack", + gatewayRuntimes: ["docker"], manifestName: "openclaw-nvidia-slack", environment: ubuntuRepoDocker("cloud-nvidia-openclaw-slack"), expectedStateId: "cloud-openclaw-ready", @@ -253,6 +274,7 @@ const canonicalTargetInputs: CanonicalTargetInput[] = [ }, { id: "ubuntu-repo-cloud-hermes-discord", + gatewayRuntimes: ["docker"], manifestName: "hermes-nvidia-discord", environment: ubuntuRepoDocker("cloud-nvidia-hermes-discord"), expectedStateId: "cloud-hermes-ready", @@ -261,6 +283,7 @@ const canonicalTargetInputs: CanonicalTargetInput[] = [ }, { id: "ubuntu-repo-cloud-hermes-slack", + gatewayRuntimes: ["docker"], manifestName: "hermes-nvidia-slack", environment: ubuntuRepoDocker("cloud-nvidia-hermes-slack"), expectedStateId: "cloud-hermes-ready", @@ -269,6 +292,7 @@ const canonicalTargetInputs: CanonicalTargetInput[] = [ }, { id: "ubuntu-repo-cloud-openclaw-resume", + gatewayRuntimes: ["docker"], manifestName: "openclaw-nvidia-resume", environment: ubuntuRepoDocker("cloud-nvidia-openclaw-resume-after-interrupt"), expectedStateId: "cloud-openclaw-ready", @@ -277,6 +301,7 @@ const canonicalTargetInputs: CanonicalTargetInput[] = [ }, { id: "ubuntu-repo-cloud-openclaw-repair", + gatewayRuntimes: ["docker"], manifestName: "openclaw-nvidia-repair", environment: ubuntuRepoDocker("cloud-nvidia-openclaw-repair-existing-config"), expectedStateId: "cloud-openclaw-ready", @@ -285,6 +310,7 @@ const canonicalTargetInputs: CanonicalTargetInput[] = [ }, { id: "ubuntu-repo-cloud-openclaw-double-same-provider", + gatewayRuntimes: ["docker"], manifestName: "openclaw-nvidia-double-same-provider", environment: ubuntuRepoDocker("cloud-nvidia-openclaw-double-same-provider"), expectedStateId: "cloud-openclaw-ready", @@ -293,6 +319,7 @@ const canonicalTargetInputs: CanonicalTargetInput[] = [ }, { id: "ubuntu-repo-cloud-openclaw-double-provider-switch", + gatewayRuntimes: ["docker"], manifestName: "openclaw-nvidia-double-provider-switch", environment: ubuntuRepoDocker("cloud-nvidia-openclaw-double-provider-switch"), expectedStateId: "cloud-openclaw-ready", @@ -301,6 +328,7 @@ const canonicalTargetInputs: CanonicalTargetInput[] = [ }, { id: "ubuntu-repo-cloud-openclaw-token-rotation", + gatewayRuntimes: ["docker"], manifestName: "openclaw-nvidia-token-rotation", environment: ubuntuRepoDocker("cloud-nvidia-openclaw-token-rotation"), expectedStateId: "cloud-openclaw-ready", @@ -309,6 +337,7 @@ const canonicalTargetInputs: CanonicalTargetInput[] = [ }, { id: "ubuntu-repo-cloud-openclaw-custom-policies", + gatewayRuntimes: ["docker"], manifestName: "openclaw-nvidia-custom-policies", environment: ubuntuRepoDocker("cloud-openclaw-custom-policies"), expectedStateId: "cloud-openclaw-custom-policies-ready", @@ -325,6 +354,7 @@ const canonicalTargetInputs: CanonicalTargetInput[] = [ }, { id: "ubuntu-invalid-nvidia-key-negative", + gatewayRuntimes: ["docker"], manifestName: "openclaw-nvidia-invalid-key", environment: ubuntuRepoDocker("cloud-openclaw-invalid-nvidia-key"), expectedStateId: "onboarding-failure-invalid-nvidia-key", @@ -339,6 +369,7 @@ const canonicalTargetInputs: CanonicalTargetInput[] = [ }, { id: "ubuntu-gateway-port-conflict-negative", + gatewayRuntimes: ["docker"], manifestName: "openclaw-nvidia-gateway-port-conflict", environment: ubuntuRepoDocker("cloud-openclaw-gateway-port-conflict"), expectedStateId: "onboarding-failure-gateway-port-conflict", @@ -353,8 +384,9 @@ const canonicalTargetInputs: CanonicalTargetInput[] = [ }, { id: "ubuntu-policy-custom-missing-presets-negative", + gatewayRuntimes: E2E_GATEWAY_RUNTIMES, manifestName: "openclaw-nvidia-policy-custom-missing-presets", - environment: ubuntuRepoDocker("cloud-openclaw-policy-custom-missing-presets"), + environment: ubuntuRepoManagedRuntime("cloud-openclaw-policy-custom-missing-presets"), expectedStateId: "onboarding-failure-policy-presets-required", onboardingAssertionIds: ["base-installed", "preflight-passed"], suiteIds: [], diff --git a/test/e2e/registry/matrix.ts b/test/e2e/registry/matrix.ts index 69ffb34dddb..7080fa82dea 100644 --- a/test/e2e/registry/matrix.ts +++ b/test/e2e/registry/matrix.ts @@ -12,6 +12,15 @@ export function ubuntuRepoDocker(onboarding: string): TargetEnvironment { }; } +export function ubuntuRepoManagedRuntime(onboarding: string): TargetEnvironment { + return { + platform: "ubuntu-local", + install: "repo-current", + runtime: "managed-runtime-running", + onboarding, + }; +} + export function gpuRepoDockerCdi(onboarding: string): TargetEnvironment { return { platform: "gpu-runner", install: "repo-current", runtime: "gpu-docker-cdi", onboarding }; } diff --git a/test/e2e/registry/run.ts b/test/e2e/registry/run.ts index 04383dc49fe..d821bd03864 100644 --- a/test/e2e/registry/run.ts +++ b/test/e2e/registry/run.ts @@ -6,6 +6,14 @@ import { fileURLToPath } from "node:url"; import type { E2eExecutionMetadata } from "../../../tools/e2e/execution-coverage.mts"; import { liveTargetTimeoutContract } from "../../../tools/e2e/onboard-timeout-contract.mts"; +import { + type E2eGatewayRuntime, + type E2eGatewayRuntimeSupport, + type E2eRuntimeProvider, + e2eRuntimeProviders, + runtimeCoverageVariant, + runtimeExecutionId, +} from "../../../tools/e2e/gateway-runtime.mts"; import { listTargets, requireTargets } from "./registry.ts"; import { resolveRunnerForTarget } from "./runner-routing.ts"; @@ -30,6 +38,9 @@ export interface LiveTargetInventoryEntry extends E2eExecutionMetadata { } export interface LiveTargetMatrixEntry extends LiveTargetInventoryEntry { + execution_id: string; + runtime_provider: E2eRuntimeProvider; + coverage_variant: string; runner: string; label: string; platform: string; @@ -86,12 +97,16 @@ function printList() { function liveMatrixEntry( target: TargetDefinition, support: LiveTargetSupport, + runtimeProvider: E2eRuntimeProvider, ): LiveTargetMatrixEntry { const { runner } = resolveRunnerForTarget(target); return { ...liveTargetInventoryEntry(target, support), + execution_id: runtimeExecutionId(target.id, "", runtimeProvider), + runtime_provider: runtimeProvider, + coverage_variant: runtimeCoverageVariant("", runtimeProvider), runner, - label: liveTargetTestTitle(target, support), + label: `${liveTargetTestTitle(target, support)} [${runtimeProvider}]`, platform: target.environment?.platform ?? "unknown", install: target.environment?.install ?? "unknown", runtime: target.environment?.runtime ?? "unknown", @@ -120,14 +135,29 @@ export function buildLiveTargetInventory(): LiveTargetInventoryEntry[] { return listTargets().map((target) => liveTargetInventoryEntry(target)); } -export function buildLiveTargetMatrix(ids: string[] = []): LiveTargetMatrixEntry[] { +export function liveTargetGatewayRuntimes(target: TargetDefinition): E2eGatewayRuntimeSupport { + return target.gatewayRuntimes ?? ["docker"]; +} + +export function buildLiveTargetMatrix( + ids: string[] = [], + gatewayRuntimes: readonly E2eGatewayRuntime[] = ["docker"], +): LiveTargetMatrixEntry[] { if (ids.length === 0) { return listTargets().flatMap((target) => { const support = liveTargetSupport(target); - return support.supported ? [liveMatrixEntry(target, support)] : []; + return support.supported + ? e2eRuntimeProviders(liveTargetGatewayRuntimes(target), gatewayRuntimes).map( + (runtimeProvider) => liveMatrixEntry(target, support, runtimeProvider), + ) + : []; }); } - return requireTargets(ids).map((target) => liveMatrixEntry(target, liveTargetSupport(target))); + return requireTargets(ids).flatMap((target) => + e2eRuntimeProviders(liveTargetGatewayRuntimes(target), gatewayRuntimes).map((runtimeProvider) => + liveMatrixEntry(target, liveTargetSupport(target), runtimeProvider), + ), + ); } function emitLiveMatrix(ids: string[]) { diff --git a/test/e2e/registry/runtime-support.ts b/test/e2e/registry/runtime-support.ts index 23f0dc48ac7..ad3477f6cab 100644 --- a/test/e2e/registry/runtime-support.ts +++ b/test/e2e/registry/runtime-support.ts @@ -10,7 +10,7 @@ import type { TargetDefinition } from "./types.ts"; const SUPPORTED_PLATFORMS = new Set(["ubuntu-local"]); const SUPPORTED_INSTALLS = new Set(["repo-current"]); -const SUPPORTED_RUNTIMES = new Set(["docker-running"]); +const SUPPORTED_RUNTIMES = new Set(["docker-running", "managed-runtime-running"]); const SUPPORTED_ONBOARDING = new Set([ "cloud-openclaw", "cloud-openclaw-policy-custom-missing-presets", diff --git a/test/e2e/registry/types.ts b/test/e2e/registry/types.ts index 4788d6e0f32..32990922d2c 100644 --- a/test/e2e/registry/types.ts +++ b/test/e2e/registry/types.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import type { E2eExecutionMetadata } from "../../../tools/e2e/execution-coverage.mts"; +import type { E2eGatewayRuntimeSupport } from "../../../tools/e2e/gateway-runtime.mts"; export type PhaseName = "environment" | "onboarding" | "state-validation" | "lifecycle" | "runtime"; @@ -208,6 +209,7 @@ export interface TargetDefinition { requiredSecrets?: string[]; skippedCapabilities?: Array>; expectedFailure?: ExpectedFailureContract; + gatewayRuntimes?: E2eGatewayRuntimeSupport; } // Legacy phase-action vocabulary retained for migration metadata. New live diff --git a/test/e2e/support/base-image-publication.test.ts b/test/e2e/support/base-image-publication.test.ts index f7127b05d2d..a932f776258 100644 --- a/test/e2e/support/base-image-publication.test.ts +++ b/test/e2e/support/base-image-publication.test.ts @@ -976,6 +976,20 @@ describe("base-image publication evidence", () => { ).rejects.toThrow(/not valid JSON/u); }); + it("omits authorization for public GitHub metadata requests", async () => { + let authorization: string | null = "unobserved"; + await expect( + githubRequest("/repos/NVIDIA/NemoClaw/pulls/9923", "unused-token", { + authenticated: false, + fetchImpl: async (_input, init) => { + authorization = new Headers(init.headers).get("authorization"); + return new Response(JSON.stringify({ ok: true }), { status: 200 }); + }, + }), + ).resolves.toEqual({ ok: true }); + expect(authorization).toBeNull(); + }); + it("loads directly with the Node strip-types runtime used by Actions (#7372)", () => { const modulePath = path.resolve( import.meta.dirname, diff --git a/test/e2e/support/channels-stop-start-googlechat.test.ts b/test/e2e/support/channels-stop-start-googlechat.test.ts index dfa3ef454db..0b0baa0ae57 100644 --- a/test/e2e/support/channels-stop-start-googlechat.test.ts +++ b/test/e2e/support/channels-stop-start-googlechat.test.ts @@ -24,6 +24,13 @@ type FixtureProviderDependencies = { }, ): string[]; }; +type FixtureLegacyProviderDependencies = { + upsertMessagingProviders( + tokenDefs: Parameters[0], + run: FixtureRunner, + options?: Parameters[2], + ): string[]; +}; type FixtureChannelDependencies = Pick< (typeof import("../../../src/lib/actions/sandbox/policy-channel-dependencies.ts"))["policyChannelDependencies"], @@ -329,6 +336,12 @@ describe("channels stop/start Google Chat live composition", () => { })); const run = runMock as unknown as FixtureRunner; const revalidateSandboxIdentity = vi.fn(); + const googlechatTokenDef = { + name: `${sandboxName}-googlechat-bridge`, + envKey: "GOOGLE_CHAT_ACCESS_TOKEN", + token: null, + providerType, + }; const restore = installGooglechatCredentialFixture(sandboxName, agent, { ensureProfiles, @@ -337,15 +350,7 @@ describe("channels stop/start Google Chat live composition", () => { run, }); const providerNames = providerDependencies.upsertMessagingProviders( - [ - delegatedTokenDef, - { - name: `${sandboxName}-googlechat-bridge`, - envKey: "GOOGLE_CHAT_ACCESS_TOKEN", - token: null, - providerType, - }, - ], + [delegatedTokenDef, googlechatTokenDef], run, { revalidateSandboxIdentity }, ); @@ -487,7 +492,6 @@ describe("channels stop/start Google Chat live composition", () => { root: "/repo", run, }); - providerDependencies.upsertMessagingProviders( [ { diff --git a/test/e2e/support/cli-artifact-packaging.test.ts b/test/e2e/support/cli-artifact-packaging.test.ts index 5984efe34ab..cd8a74e17f8 100644 --- a/test/e2e/support/cli-artifact-packaging.test.ts +++ b/test/e2e/support/cli-artifact-packaging.test.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { execFileSync, spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -18,7 +19,7 @@ const CATALOG_INPUT_WRITERS = { symlink: (catalog: string) => fs.symlinkSync("missing-catalog.json", catalog), } satisfies Record void>; -function runCliArtifactPackaging(catalogInput: CatalogInput) { +function runCliArtifactPackaging(catalogInput: CatalogInput, trustedCatalog = false) { const root = fs.mkdtempSync(path.join(os.tmpdir(), "cli-artifact-package-")); const workspace = path.join(root, "workspace"); const runnerTemp = path.join(root, "runner-temp"); @@ -95,6 +96,12 @@ exec ${JSON.stringify(systemTar)} "\${args[@]}" const catalog = path.join(dist, "e2e-managed-image-catalog.json"); CATALOG_INPUT_WRITERS[catalogInput](catalog); + const trustedCatalogPath = path.join(runnerTemp, "pr-managed-image-catalog.json"); + const trustedCatalogJson = '{"trusted":true}'; + trustedCatalog && fs.writeFileSync(trustedCatalogPath, `${trustedCatalogJson}\n`); + const trustedCatalogSha256 = trustedCatalog + ? createHash("sha256").update(fs.readFileSync(trustedCatalogPath)).digest("hex") + : ""; const result = spawnSync("bash", [path.resolve(CLI_ARTIFACT_PACKAGE_SCRIPT)], { cwd: workspace, @@ -104,6 +111,8 @@ exec ${JSON.stringify(systemTar)} "\${args[@]}" CANDIDATE_REPOSITORY: "NVIDIA/NemoClaw", CANDIDATE_SHA: candidateSha, GITHUB_OUTPUT: path.join(root, "github-output"), + MANAGED_IMAGE_CATALOG: trustedCatalog ? trustedCatalogJson : "", + MANAGED_IMAGE_CATALOG_SHA256: trustedCatalogSha256, PATH: `${toolDirectory}:${process.env.PATH ?? ""}`, RUN_ATTEMPT: "1", RUN_ID: "12345", @@ -115,6 +124,7 @@ exec ${JSON.stringify(systemTar)} "\${args[@]}" }); return { artifactExists: fs.existsSync(path.join(runnerTemp, "nemoclaw-cli-artifact")), + artifactPayload: path.join(runnerTemp, "nemoclaw-cli-artifact", "nemoclaw-cli.tar"), cleanup: () => fs.rmSync(root, { force: true, recursive: true }), output: `${result.stdout}${result.stderr}`, result, @@ -132,6 +142,18 @@ describe("CLI artifact packaging", () => { } }); + it("seals a trusted managed-image catalog into the exact candidate artifact", () => { + const fixture = runCliArtifactPackaging("absent", true); + try { + expect(fixture.result.status, fixture.output).toBe(0); + expect( + execFileSync("tar", ["-tf", fixture.artifactPayload], { encoding: "utf8" }).split("\n"), + ).toContain("dist/e2e-managed-image-catalog.json"); + } finally { + fixture.cleanup(); + } + }); + it.each(["file", "symlink"] as const)( "rejects a candidate-created managed-image catalog %s before artifact creation", (catalogInput) => { diff --git a/test/e2e/support/cli-artifact-workflow-boundary.test.ts b/test/e2e/support/cli-artifact-workflow-boundary.test.ts index 344b6f067c1..42d2dfc4028 100644 --- a/test/e2e/support/cli-artifact-workflow-boundary.test.ts +++ b/test/e2e/support/cli-artifact-workflow-boundary.test.ts @@ -59,6 +59,7 @@ type RestoreFixtureOptions = { | "missing-shared" | "non-dist" | "link" + | "managed-catalog" | "shared-module-directory" | "traversal"; buildIdentitySha?: string; @@ -140,6 +141,20 @@ function writeLinkArchive(context: ArchiveFixtureContext): void { }); } +const MANAGED_IMAGE_REVISION = "e".repeat(40); + +function writeManagedCatalogArchive(context: ArchiveFixtureContext): void { + writeCliArchive(context, (dist) => { + fs.writeFileSync( + path.join(dist, "e2e-managed-image-catalog.json"), + `${JSON.stringify({ + openclaw: { source: { revision: MANAGED_IMAGE_REVISION } }, + hermes: { source: { revision: MANAGED_IMAGE_REVISION } }, + })}\n`, + ); + }); +} + function writeCliDirectoryArchive(context: ArchiveFixtureContext): void { writeCliArchive(context, (dist) => { const entrypoint = path.join(dist, "nemoclaw.js"); @@ -194,6 +209,7 @@ function writeTraversalArchive(context: ArchiveFixtureContext): void { const ARCHIVE_FIXTURE_WRITERS = { "cli-directory": writeCliDirectoryArchive, link: writeLinkArchive, + "managed-catalog": writeManagedCatalogArchive, "missing-shared": writeMissingSharedArchive, "non-dist": writeNonDistArchive, @@ -352,6 +368,7 @@ function runRestoreValidation(options: RestoreFixtureOptions = {}) { PREEXISTING_DIST_WRITERS[options.preexistingDist ?? "none"](workspace); const githubOutput = path.join(root, "github-output"); + const githubEnv = path.join(root, "github-env"); const identityResult = spawnSync(IDENTITY_SCRIPT, [], { cwd: workspace, encoding: "utf8", @@ -394,6 +411,7 @@ function runRestoreValidation(options: RestoreFixtureOptions = {}) { ARTIFACT_NAME: identityOutputs.artifact_name, CANDIDATE_REPOSITORY: identityOutputs.candidate_repository, CANDIDATE_SHA: identityOutputs.candidate_sha, + GITHUB_ENV: githubEnv, GITHUB_WORKSPACE: workspace, PATH: `${toolDirectory}:${process.env.PATH ?? ""}`, PAYLOAD_SHA256: identityOutputs.payload_sha256, @@ -409,6 +427,7 @@ function runRestoreValidation(options: RestoreFixtureOptions = {}) { return { candidateSha, cleanup: () => fs.rmSync(root, { force: true, recursive: true }), + githubEnv: fs.existsSync(githubEnv) ? fs.readFileSync(githubEnv, "utf8") : "", output: `${identityResult.stdout}${identityResult.stderr}${ identitySucceeded ? `${restoreResult.stdout}${restoreResult.stderr}` : "" }`, @@ -574,6 +593,19 @@ describe("exact-commit CLI artifact restore", () => { } }); + it("exports restored managed-image catalog publication authority", () => { + const fixture = runRestoreValidation({ archive: "managed-catalog" }); + try { + expect(fixture.result.status, fixture.output).toBe(0); + expect(fixture.githubEnv).toBe( + `NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG=${fixture.workspace}/dist/e2e-managed-image-catalog.json\n` + + `NEMOCLAW_E2E_MANAGED_IMAGE_REVISION=${MANAGED_IMAGE_REVISION}\n`, + ); + } finally { + fixture.cleanup(); + } + }); + it("restores a binary payload when the host SHA-256 utility reports a different digest (#10569)", () => { const fixture = runRestoreValidation(); try { diff --git a/test/e2e/support/e2e-clients.test.ts b/test/e2e/support/e2e-clients.test.ts index 6c553196f23..f70ecc27ad0 100644 --- a/test/e2e/support/e2e-clients.test.ts +++ b/test/e2e/support/e2e-clients.test.ts @@ -484,7 +484,7 @@ describe("E2E fixture clients", () => { const containerGateway = new GatewayClient(containerHost, new SandboxClient(containerRunner)); const runtime = containerGateway.resolveHostRuntime(); containerRunner.exitCode = 0; - containerRunner.stdout = "abc123\n"; + containerRunner.stdout = "abc123\topenshell-cluster-nemoclaw\n"; await expect(runtime).resolves.toEqual({ kind: "container", id: "abc123" }); const statusRunner = new FakeRunner(); diff --git a/test/e2e/support/e2e-host-address.test.ts b/test/e2e/support/e2e-host-address.test.ts index b584a57014b..f03cd4e0755 100644 --- a/test/e2e/support/e2e-host-address.test.ts +++ b/test/e2e/support/e2e-host-address.test.ts @@ -44,4 +44,50 @@ describe("host address discovery", () => { /host address discovery failed/, ); }); + + it("uses the selected runtime provider's sandbox-to-host address", async () => { + const runner: CommandRunner = { + run: async () => { + throw new Error("provider-owned address must not run route discovery"); + }, + }; + + await expect( + discoverHostAddress( + new HostCliClient(runner), + "host-address", + { + NEMOCLAW_GATEWAY_RUNTIME: "podman", + OPENSHELL_PODMAN_SOCKET: "/tmp/podman.sock", + }, + "linux", + ), + ).resolves.toEqual({ source: "runtime-provider", address: "169.254.2.2", probe: null }); + }); + + it("preserves portable-profile route discovery", async () => { + const runner: CommandRunner = { + run: async (command) => ({ + command: [command.command, ...command.args], + exitCode: 0, + signal: null, + timedOut: false, + stdout: "route 10.0.0.2\n", + stderr: "", + artifacts: { stdout: "stdout.txt", stderr: "stderr.txt", result: "result.json" }, + }), + }; + + await expect( + discoverHostAddress( + new HostCliClient(runner), + "host-address", + { + NEMOCLAW_EXPERIMENTAL_PROFILE: "portable", + NEMOCLAW_GATEWAY_RUNTIME: "podman", + }, + "linux", + ), + ).resolves.toMatchObject({ source: "route", address: "10.0.0.2" }); + }); }); diff --git a/test/e2e/support/e2e-matrix.test.ts b/test/e2e/support/e2e-matrix.test.ts index 3b070764a50..54f31a7d985 100644 --- a/test/e2e/support/e2e-matrix.test.ts +++ b/test/e2e/support/e2e-matrix.test.ts @@ -110,15 +110,21 @@ describe("live E2E target matrix", () => { }); it("exposes execution coverage for every executable typed target (#9167)", () => { + expect(buildLiveTargetMatrix()).toEqual(buildLiveTargetMatrix([], ["docker"])); expect(buildLiveTargetMatrix()).toHaveLength(4); expectExecutableTypedTargetCoverage(); }); + it("keeps Docker-only typed fixtures out of the native Podman matrix", () => { + expect(buildLiveTargetMatrix([], ["podman"]).map((row) => row.id)).toEqual([ + "ubuntu-policy-custom-missing-presets-negative", + "ubuntu-repo-cloud-openclaw", + ]); + }); + it("assigns a 160-minute job timeout only to post-reboot recovery (#9622)", () => { expect( - Object.fromEntries( - buildLiveTargetMatrix().map((row) => [row.id, row.timeout_minutes]), - ), + Object.fromEntries(buildLiveTargetMatrix().map((row) => [row.id, row.timeout_minutes])), ).toEqual({ "ubuntu-policy-custom-missing-presets-negative": 45, "ubuntu-repo-cloud-langchain-deepagents-code": 45, diff --git a/test/e2e/support/e2e-operations-workflow-boundary.test.ts b/test/e2e/support/e2e-operations-workflow-boundary.test.ts index 9ceb9f150f1..7d6a388e4e6 100644 --- a/test/e2e/support/e2e-operations-workflow-boundary.test.ts +++ b/test/e2e/support/e2e-operations-workflow-boundary.test.ts @@ -33,7 +33,6 @@ describe("E2E operations workflow", testTimeoutOptions(15_000), () => { it("accepts the checked-in workflow", () => { expect(validateE2eOperationsWorkflowBoundary()).toEqual([]); }); - it("rejects a lookalike live cold-onboard performance artifact path (#6660)", () => { const workflow = readE2eOperationsWorkflow(); const upload = workflow.jobs.live.steps!.find((step) => step.name === "Upload E2E artifacts")!; @@ -50,7 +49,6 @@ describe("E2E operations workflow", testTimeoutOptions(15_000), () => { "live E2E must upload cold-onboard performance evidence", ); }); - it("requires the scorecard to wait for every reporting dependency", () => { const workflow = readE2eOperationsWorkflow(); workflow.jobs.scorecard.needs = [...(workflow.jobs.scorecard.needs as string[])]; @@ -60,7 +58,6 @@ describe("E2E operations workflow", testTimeoutOptions(15_000), () => { "scorecard needs must exactly match report-to-pr needs", ); }); - it("limits scorecard permissions to read access", () => { const workflow = readE2eOperationsWorkflow(); workflow.jobs.scorecard.permissions = { @@ -478,6 +475,7 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; "b", "c", "refs/heads/main", + 1, "::error::checkout_repository must be an owner/repository name\n", ], [ @@ -487,6 +485,7 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; "d", "c", "refs/heads/main", + 1, "::error::base_sha must match the PR base SHA\n", ], [ @@ -496,19 +495,21 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; "b", "d", "refs/heads/main", + 1, "::error::workflow_sha must match the trusted main workflow SHA\n", ], [ - "a matching workflow SHA from a non-main workflow ref", + "a matching workflow SHA from a same-repository workflow branch", "NVIDIA/NemoClaw", "a", "b", "c", "refs/heads/pr-controlled-workflow", - "::error::Manual PR E2E must be dispatched from trusted main\n", + 0, + "", ], ] as const)( - "rejects manual PR authentication for %s", + "handles manual PR authentication for %s", ( _caseName, requestedRepository, @@ -516,6 +517,7 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; requestedBaseCharacter, expectedWorkflowCharacter, workflowRef, + expectedStatus, expectedStderr, ) => { const apiHeadSha = "a".repeat(40); @@ -554,7 +556,7 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; }, ); - expect(result.status).toBe(1); + expect(result.status).toBe(expectedStatus); expect(result.stderr).toBe(expectedStderr); }, ); @@ -700,7 +702,6 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; const directory = mkdtempSync(join(tmpdir(), "nemoclaw-manual-pr-matrix-")); const output = join(directory, "output"); const summary = join(directory, "summary"); - try { writeFileSync(output, ""); writeFileSync(summary, ""); @@ -720,7 +721,6 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; const controllerTestMatrix = controllerOutput .find((line) => line.startsWith("test_matrix="))! .slice("test_matrix=".length); - writeFileSync(output, ""); const plannerResult = spawnSync( "bash", @@ -759,9 +759,15 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; const testMatrixLine = readFileSync(output, "utf8") .split("\n") .find((line) => line.startsWith("test_matrix="))!; - expect(JSON.parse(testMatrixLine.slice("test_matrix=".length))).toEqual( - JSON.parse(controllerTestMatrix), - ); + expect( + JSON.parse(testMatrixLine.slice("test_matrix=".length)).map( + ({ id, file, project }: { id: string; file: string; project: string }) => ({ + id, + file, + project, + }), + ), + ).toEqual(JSON.parse(controllerTestMatrix)); expect( (generateMatrix as unknown as { outputs: Record }).outputs.matrix, ).toBe("${{ steps.matrix.outputs.matrix }}"); @@ -769,7 +775,6 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; rmSync(directory, { recursive: true, force: true }); } }); - it.each([ ["inference-routing job", "inference-routing", ""], ["managed-image-protected-runtime job", "managed-image-protected-runtime", ""], diff --git a/test/e2e/support/e2e-phase-environment.test.ts b/test/e2e/support/e2e-phase-environment.test.ts index 043bde360d7..e693d46828c 100644 --- a/test/e2e/support/e2e-phase-environment.test.ts +++ b/test/e2e/support/e2e-phase-environment.test.ts @@ -10,7 +10,8 @@ import { describe, expect, expectTypeOf, it } from "vitest"; import { ArtifactSink } from "../fixtures/artifacts.ts"; import { type CommandRunner, HostCliClient } from "../fixtures/clients/index.ts"; import type { E2ETargetFixtures } from "../fixtures/e2e-test.ts"; -import { type DockerRuntimeReady, EnvironmentPhaseFixture } from "../fixtures/phases/index.ts"; +import { EnvironmentPhaseFixture, type RuntimeReady } from "../fixtures/phases/index.ts"; +import { RuntimeProviderPrerequisite } from "../fixtures/runtime-provider.ts"; import type { ShellProbeResult, ShellProbeRunOptions, @@ -89,11 +90,12 @@ describe("environment phase fixture", () => { runtime: "docker-running", onboarding: "cloud-openclaw", cliPath: "./bin/nemoclaw.js", - docker: { + runtimeProvider: { id: "docker-running", expectation: "required", + providerId: "docker", available: true, - } satisfies Partial, + } satisfies Partial, }); expect(runner.calls).toEqual([ { @@ -120,6 +122,51 @@ describe("environment phase fixture", () => { ]); }); + it("asserts the selected Podman provider for a managed runtime target", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); + runner.enqueue(shellResult(0, "Podman is available\n")); + const host = new HostCliClient(runner, { cliPath: "./bin/nemoclaw.js" }); + const runtimeProvider = new RuntimeProviderPrerequisite( + host, + (reason) => { + throw new Error(reason); + }, + { + HOME: "/home/runner", + PATH: "/usr/bin", + NEMOCLAW_GATEWAY_RUNTIME: "podman", + OPENSHELL_PODMAN_SOCKET: "/run/user/1001/podman/podman.sock", + XDG_RUNTIME_DIR: "/run/user/1001", + }, + ); + const environment = new EnvironmentPhaseFixture(host, undefined, runtimeProvider); + + const ready = await environment.assertReady({ + ...cloudOpenClawEnvironment, + runtime: "managed-runtime-running", + }); + + expect(ready.runtimeProvider).toMatchObject({ + id: "managed-runtime-running", + expectation: "required", + providerId: "podman", + available: true, + }); + expect(runner.calls[1]).toEqual({ + command: "podman", + args: ["--url", "unix:///run/user/1001/podman/podman.sock", "info"], + options: { + artifactName: "runtime-podman-info-managed-runtime-running", + env: expect.objectContaining({ + NEMOCLAW_GATEWAY_RUNTIME: "podman", + OPENSHELL_PODMAN_SOCKET: "/run/user/1001/podman/podman.sock", + }), + timeoutMs: 30_000, + }, + }); + }); + it("fails when a required Docker runtime is unavailable", async () => { const runner = new FakeRunner(); runner.enqueue(shellResult(0, "nemoclaw v0.0.0\n")); @@ -143,9 +190,10 @@ describe("environment phase fixture", () => { onboarding: "cloud-openclaw-no-docker", }); - expect(ready.docker).toMatchObject({ + expect(ready.runtimeProvider).toMatchObject({ id: "docker-missing", expectation: "missing", + providerId: "docker", available: false, }); }); @@ -162,9 +210,10 @@ describe("environment phase fixture", () => { onboarding: "cloud-openclaw-no-docker", }); - expect(ready.docker).toMatchObject({ + expect(ready.runtimeProvider).toMatchObject({ id: "docker-missing", expectation: "missing", + providerId: "docker", available: true, }); }); @@ -181,9 +230,10 @@ describe("environment phase fixture", () => { runtime: "macos-docker-optional", }); - expect(ready.docker).toMatchObject({ + expect(ready.runtimeProvider).toMatchObject({ id: "macos-docker-optional", expectation: "optional", + providerId: "docker", available: false, probeError: "spawn docker ENOENT", }); @@ -201,9 +251,10 @@ describe("environment phase fixture", () => { runtime: "macos-docker-optional", }); - expect(ready.docker).toMatchObject({ + expect(ready.runtimeProvider).toMatchObject({ id: "macos-docker-optional", expectation: "optional", + providerId: "docker", available: true, }); }); @@ -286,7 +337,7 @@ describe("environment phase fixture", () => { runtime: "gpu-docker-cdi", }); - expect(ready.docker).toMatchObject({ + expect(ready.runtimeProvider).toMatchObject({ id: "gpu-docker-cdi", expectation: "required", available: true, diff --git a/test/e2e/support/e2e-phase-lifecycle.test.ts b/test/e2e/support/e2e-phase-lifecycle.test.ts index 817e0fc0e7a..4b627ffeaa0 100644 --- a/test/e2e/support/e2e-phase-lifecycle.test.ts +++ b/test/e2e/support/e2e-phase-lifecycle.test.ts @@ -1,7 +1,6 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { execFileSync } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -165,6 +164,7 @@ describe("LifecyclePhaseFixture.simulate post-reboot-recovery (stop-original)", runner.enqueue(shellResult(0)); // forward stop runner.enqueue(shellResult(0)); // gateway stop runner.enqueue(shellResult(0)); // pid stop + runner.enqueue(shellResult(0, "gateway-id\topenshell-cluster-nemoclaw\n")); // discover runner.enqueue(shellResult(0)); // container stop runner.enqueue(shellResult(0)); // user service restart runner.enqueue(shellResult(0, "Connected to nemoclaw\n")); // openshell status @@ -176,31 +176,32 @@ describe("LifecyclePhaseFixture.simulate post-reboot-recovery (stop-original)", expect(result.profile).toBe("post-reboot-recovery"); expect(result.steps.map((step) => step.id)).toEqual([ - "docker-stop:openshell-cluster-e2e-cloud-oc", + "runtime-stop:openshell-cluster-e2e-cloud-oc", "gateway-restart:user-service", "gateway-connected:nemoclaw", - "docker-boot-start:openshell-cluster-e2e-cloud-oc", + "runtime-boot-start:openshell-cluster-e2e-cloud-oc", "sandbox-ready-after-boot:e2e-cloud-oc", "nemoclaw-status:e2e-cloud-oc", ]); expect(runner.calls.map((call) => `${call.command} ${call.args.join(" ")}`)).toEqual([ expect.stringContaining('bash -lc command -v "$1"'), expect.stringContaining("bash -lc set -eu"), - "docker ps -a --filter label=openshell.ai/sandbox-name=e2e-cloud-oc --format {{.Names}}", - "docker stop openshell-cluster-e2e-cloud-oc", + "docker container ps --all --filter label=openshell.ai/sandbox-name=e2e-cloud-oc --format {{.Names}}", + "docker container stop openshell-cluster-e2e-cloud-oc", "sh -lc command -v openshell >/dev/null 2>&1 && openshell forward stop 18789 || true", "sh -lc command -v openshell >/dev/null 2>&1 && openshell gateway stop -g nemoclaw || true", expect.stringContaining("sh -lc pid_file="), - expect.stringContaining("sh -lc cid="), + "docker container ps --format {{.ID}}\t{{.Names}}", + "docker container stop gateway-id", expect.stringContaining('systemctl --user cat "$service"'), "openshell status", - "docker start openshell-cluster-e2e-cloud-oc", + "docker container start openshell-cluster-e2e-cloud-oc", "openshell sandbox list", "nemoclaw e2e-cloud-oc status", ]); expect(cleanup.calls.map((call) => call.name)).toEqual([ "lifecycle.remove-staged-gateway-user-service", - "lifecycle.docker-start:openshell-cluster-e2e-cloud-oc", + "lifecycle.runtime-start:openshell-cluster-e2e-cloud-oc", ]); }); @@ -248,42 +249,42 @@ describe("LifecyclePhaseFixture.simulate post-reboot-recovery (stop-original)", .map((call) => `${call.command} ${call.args.join(" ")}`) .filter( (call) => - call === "docker start container-1" || + call === "docker container start container-1" || call === "openshell sandbox list" || call === "nemoclaw e2e-cloud-oc status", ), ).toEqual([ - "docker start container-1", + "docker container start container-1", "openshell sandbox list", "nemoclaw e2e-cloud-oc status", ]); expect(result.steps.slice(-3).map((step) => step.id)).toEqual([ - "docker-boot-start:container-1", + "runtime-boot-start:container-1", "sandbox-ready-after-boot:e2e-cloud-oc", "nemoclaw-status:e2e-cloud-oc", ]); expect(result.steps.at(-1)?.results[0]?.exitCode).toBe(0); }); - it("fails when no Docker container carries the OpenShell sandbox-name label", async () => { + it("fails when no managed runtime resource carries the OpenShell sandbox-name label", async () => { const runner = new FakeRunner(); const cleanup = new FakeCleanup(); const prepared = await preparedPostRebootFixture(runner, cleanup); runner.enqueue(shellResult(0, "\n")); // discover returns nothing await expect(prepared.simulate("post-reboot-recovery", instance())).rejects.toThrow( - /expected at least one Docker container labeled/, + /expected at least one managed runtime resource labeled/, ); }); - it("fails when docker discover returns non-zero", async () => { + it("fails when selected-runtime discovery returns non-zero", async () => { const runner = new FakeRunner(); const cleanup = new FakeCleanup(); const prepared = await preparedPostRebootFixture(runner, cleanup); runner.enqueue(shellResult(1, "Cannot connect to the Docker daemon")); await expect(prepared.simulate("post-reboot-recovery", instance())).rejects.toThrow( - /could not query Docker for label/, + /could not query the selected runtime provider for label/, ); }); @@ -334,25 +335,25 @@ describe("LifecyclePhaseFixture.simulate post-reboot-recovery (rename-to-gpu-bac ); expect(result.steps.map((step) => step.id.split("->")[0])).toContain( - "docker-rename:openshell-cluster-e2e-x", + "runtime-rename:openshell-cluster-e2e-x", ); const renameCall = runner.calls.find( - (call) => call.command === "docker" && call.args[0] === "rename", + (call) => call.command === "docker" && call.args.slice(0, 2).join(" ") === "container rename", ); expect(renameCall).toBeTruthy(); - expect(renameCall!.args[1]).toBe("openshell-cluster-e2e-x"); - expect(renameCall!.args[2]).toMatch(/^openshell-cluster-e2e-x-nemoclaw-gpu-backup-\d+$/); + expect(renameCall!.args[2]).toBe("openshell-cluster-e2e-x"); + expect(renameCall!.args[3]).toMatch(/^openshell-cluster-e2e-x-nemoclaw-gpu-backup-\d+$/); expect(runner.calls).toContainEqual( expect.objectContaining({ command: "docker", - args: ["start", renameCall!.args[2]], + args: ["container", "start", renameCall!.args[3]], }), ); // Cleanup queue now has both docker-start and docker-rename-back. expect(cleanup.calls.map((call) => call.name.split(":")[0])).toEqual([ - "lifecycle.docker-start", - "lifecycle.docker-rename-back", + "lifecycle.runtime-start", + "lifecycle.runtime-rename-back", ]); }); }); @@ -436,11 +437,9 @@ describe("LifecyclePhaseFixture gateway runtime restart helpers", () => { "sh -lc command -v openshell >/dev/null 2>&1 && openshell forward stop 18789 || true", "sh -lc command -v openshell >/dev/null 2>&1 && openshell gateway stop -g nemoclaw || true", expect.stringContaining("sh -lc pid_file="), - expect.stringContaining( - "docker ps --filter 'name=^/openshell-cluster-nemoclaw$' --format '{{.ID}}'", - ), + "docker container ps --format {{.ID}}\t{{.Names}}", expect.stringContaining("sh -lc pid_file="), - "docker ps -qf name=openshell-cluster-nemoclaw", + "docker container ps --format {{.ID}}\t{{.Names}}", "true ", expect.stringContaining("sh -lc set -eu"), "nemoclaw status", @@ -473,10 +472,10 @@ describe("LifecyclePhaseFixture gateway runtime restart helpers", () => { it("stops only the exact gateway container when a sandbox has the gateway-name prefix", async () => { const runner = new FakeRunner(); - runner.enqueue(shellResult(0, "12345\n")); // resolveHostRuntime pid probe runner.enqueue(shellResult(0)); // forward stop runner.enqueue(shellResult(0)); // gateway stop runner.enqueue(shellResult(0)); // pid stop + runner.enqueue(shellResult(0, "gateway-id\topenshell-cluster-nemoclaw\n")); // discover runner.enqueue(shellResult(0)); // container stop await fixture(runner, new FakeCleanup()).stopGatewayRuntime(); @@ -484,46 +483,16 @@ describe("LifecyclePhaseFixture gateway runtime restart helpers", () => { const containerStop = runner.calls.find( (call) => call.options?.artifactName === "lifecycle-gateway-container-stop", ); - expect(containerStop?.command).toBe("sh"); - expect(containerStop?.args.slice(0, 1)).toEqual(["-lc"]); - const containerStopScript = containerStop?.args[1] ?? ""; - - const fakeBin = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-gateway-docker-")); - const stopLog = path.join(fakeBin, "stopped.txt"); - const docker = path.join(fakeBin, "docker"); - fs.writeFileSync( - docker, - `#!/bin/sh -if [ "$1" = "ps" ]; then - shift - while [ "$#" -gt 0 ]; do - if [ "$1" = "--filter" ]; then filter="$2"; shift 2; else shift; fi - done - [ "$filter" = 'name=^/openshell-cluster-nemoclaw$' ] && printf '%s\\n' gateway-id -elif [ "$1" = "stop" ]; then - printf '%s\\n' "$2" >>"$DOCKER_STOP_LOG" -fi -`, - { mode: 0o755 }, + expect(containerStop?.command).toBe("docker"); + expect(containerStop?.args).toEqual(["container", "stop", "gateway-id"]); + const discovery = runner.calls.find( + (call) => call.options?.artifactName === "lifecycle-gateway-runtime-discover", ); - - try { - execFileSync("sh", ["-c", containerStopScript], { - env: { - ...process.env, - DOCKER_STOP_LOG: stopLog, - PATH: `${fakeBin}:/usr/bin:/bin`, - }, - }); - expect(fs.readFileSync(stopLog, "utf8")).toBe("gateway-id\n"); - } finally { - fs.rmSync(fakeBin, { force: true, recursive: true }); - } + expect(discovery?.args).toEqual(["container", "ps", "--format", "{{.ID}}\t{{.Names}}"]); }); it("can recover a PID runtime through sandbox-specific status", async () => { const runner = new FakeRunner(); - runner.enqueue(shellResult(75, "")); // no user service available runner.enqueue(shellResult(0, "status recovered\n")); const cleanup = new FakeCleanup(); @@ -537,7 +506,6 @@ fi ).resolves.toMatchObject({ exitCode: 0 }); expect(runner.calls.map((call) => `${call.command} ${call.args.join(" ")}`)).toEqual([ - expect.stringContaining("sh -lc set -eu"), "nemoclaw e2e-survival status", ]); }); diff --git a/test/e2e/support/e2e-phase-onboarding.test.ts b/test/e2e/support/e2e-phase-onboarding.test.ts index bfd6107d8cf..04607dd4a6f 100644 --- a/test/e2e/support/e2e-phase-onboarding.test.ts +++ b/test/e2e/support/e2e-phase-onboarding.test.ts @@ -6,9 +6,7 @@ import os from "node:os"; import path from "node:path"; import { describe, expect, expectTypeOf, it, vi } from "vitest"; -import { - ONBOARD_FINAL_HANDOFF_COMMAND_TIMEOUT_MS, -} from "../../../tools/e2e/onboard-timeout-contract.mts"; +import { ONBOARD_FINAL_HANDOFF_COMMAND_TIMEOUT_MS } from "../../../tools/e2e/onboard-timeout-contract.mts"; import { ArtifactSink } from "../fixtures/artifacts.ts"; import { type CommandRunner, HostCliClient } from "../fixtures/clients/index.ts"; import { DCODE_BASE_IMAGE, DCODE_BASE_IMAGE_ENV } from "../fixtures/dcode-base-image.ts"; @@ -125,9 +123,10 @@ function ready(overrides: Partial = {}): EnvironmentReady { runtime: "docker-running", onboarding: "cloud-openclaw", cliPath: "nemoclaw", - docker: { + runtimeProvider: { id: "docker-running", expectation: "required", + providerId: "docker", available: true, result: shellResult(0), }, @@ -371,10 +370,15 @@ describe("onboarding phase fixture", () => { await expect( onboard.from( ready({ - docker: { id: "docker-running", expectation: "required", available: false }, + runtimeProvider: { + id: "docker-running", + expectation: "required", + providerId: "docker", + available: false, + }, }), ), - ).rejects.toThrow(/requires an available Docker runtime/); + ).rejects.toThrow(/requires an available managed runtime provider/); }); it("rejects invalid sandbox names before cloud OpenClaw side effects", async () => { @@ -432,7 +436,12 @@ describe("onboarding phase fixture", () => { ready({ runtime: "docker-missing", onboarding: "cloud-openclaw-no-docker", - docker: { id: "docker-missing", expectation: "missing", available: true }, + runtimeProvider: { + id: "docker-missing", + expectation: "missing", + providerId: "docker", + available: true, + }, }), { sandboxName: "e2e-no-docker" }, ); @@ -558,7 +567,12 @@ describe("onboarding phase fixture", () => { ready({ runtime: "docker-missing", onboarding: "cloud-openclaw-no-docker", - docker: { id: "docker-missing", expectation: "missing", available: true }, + runtimeProvider: { + id: "docker-missing", + expectation: "missing", + providerId: "docker", + available: true, + }, }), { sandboxName: "e2e-no-docker" }, ); @@ -589,7 +603,12 @@ describe("onboarding phase fixture", () => { ready({ runtime: "docker-missing", onboarding: "cloud-openclaw-no-docker", - docker: { id: "docker-missing", expectation: "missing", available: false }, + runtimeProvider: { + id: "docker-missing", + expectation: "missing", + providerId: "docker", + available: false, + }, }), { sandboxName: "e2e-no-docker" }, ); @@ -630,7 +649,12 @@ describe("onboarding phase fixture", () => { ready({ runtime: "docker-missing", onboarding: "cloud-openclaw-no-docker", - docker: { id: "docker-missing", expectation: "missing", available: false }, + runtimeProvider: { + id: "docker-missing", + expectation: "missing", + providerId: "docker", + available: false, + }, }), { sandboxName: "e2e-no-docker" }, ); @@ -667,7 +691,12 @@ describe("onboarding phase fixture", () => { ready({ runtime: "docker-missing", onboarding: "cloud-openclaw-no-docker", - docker: { id: "docker-missing", expectation: "missing", available: false }, + runtimeProvider: { + id: "docker-missing", + expectation: "missing", + providerId: "docker", + available: false, + }, }), { sandboxName: "e2e-no-docker-ok" }, ), @@ -700,7 +729,12 @@ describe("onboarding phase fixture", () => { ready({ runtime: "docker-missing", onboarding: "cloud-openclaw-no-docker", - docker: { id: "docker-missing", expectation: "missing", available: false }, + runtimeProvider: { + id: "docker-missing", + expectation: "missing", + providerId: "docker", + available: false, + }, }), { sandboxName: "e2e-no-docker" }, ), @@ -759,16 +793,21 @@ describe("onboarding phase fixture", () => { await expect( onboard.from( ready({ - docker: { id: "docker-running", expectation: "required", available: false }, + runtimeProvider: { + id: "docker-running", + expectation: "required", + providerId: "docker", + available: false, + }, }), ), - ).rejects.toThrow(/requires an available Docker runtime/); + ).rejects.toThrow(/requires an available managed runtime provider/); expect(readJson(path.join(tmp, "onboarding.result.json"))).toMatchObject({ phase: "onboarding", status: "failed", onboarding: "cloud-openclaw", - error: "cloud-openclaw onboarding requires an available Docker runtime.", + error: "cloud-openclaw onboarding requires an available managed runtime provider.", }); } finally { fs.rmSync(tmp, { recursive: true, force: true }); diff --git a/test/e2e/support/e2e-report-to-pr-workflow-boundary.test.ts b/test/e2e/support/e2e-report-to-pr-workflow-boundary.test.ts index a2af3823f56..7430cee8c38 100644 --- a/test/e2e/support/e2e-report-to-pr-workflow-boundary.test.ts +++ b/test/e2e/support/e2e-report-to-pr-workflow-boundary.test.ts @@ -141,11 +141,17 @@ function executeGenerateMatrixWithPlannerOutput( const DEFAULT_TEST_MATRIX: CredentialFreeTestMatrixRow[] = [ { id: "alpha", + execution_id: "alpha-docker", + runtime_provider: "docker", + coverage_variant: "docker", file: "test/e2e/live/alpha.test.ts", project: "e2e-live", }, { id: "beta", + execution_id: "beta-docker", + runtime_provider: "docker", + coverage_variant: "docker", file: "test/e2e/live/beta.test.ts", project: "e2e-live", }, @@ -444,32 +450,32 @@ it.each([ label: "target", requestedLine: "**Requested targets:** `hermes-e2e`", }, -])("reports the canonical Hermes result for a retired dashboard $label selector", ({ - env, - requestedLine, -}) => { - const report = renderE2eReport({ - needs: { - "generate-matrix": { result: "success" }, - "hermes-e2e": { result: "success" }, - }, - env: { - EXPLICIT_ONLY_JOBS: "", - TEST_MATRIX: "[]", - JOB_PR_NUMBER: "42", - ...env, - }, - apiJobs: [{ conclusion: "success", name: "hermes-e2e", status: "completed" }], - apiJobsLoaded: true, - context: REPORT_CONTEXT, - }); +])( + "reports the canonical Hermes result for a retired dashboard $label selector", + ({ env, requestedLine }) => { + const report = renderE2eReport({ + needs: { + "generate-matrix": { result: "success" }, + "hermes-e2e": { result: "success" }, + }, + env: { + EXPLICIT_ONLY_JOBS: "", + TEST_MATRIX: "[]", + JOB_PR_NUMBER: "42", + ...env, + }, + apiJobs: [{ conclusion: "success", name: "hermes-e2e", status: "completed" }], + apiJobsLoaded: true, + context: REPORT_CONTEXT, + }); - expect(report.fatal).toBeUndefined(); - expect(report.body).toContain(requestedLine); - expect(report.body).toContain("| hermes-e2e | ✅ success | — |"); - expect(report.body).not.toContain("| hermes-dashboard |"); - expect(report.body).not.toContain("not reported"); -}); + expect(report.fatal).toBeUndefined(); + expect(report.body).toContain(requestedLine); + expect(report.body).toContain("| hermes-e2e | ✅ success | — |"); + expect(report.body).not.toContain("| hermes-dashboard |"); + expect(report.body).not.toContain("not reported"); + }, +); it("fails closed on an invalid test matrix without rendering a comment", () => { const report = renderE2eReport({ @@ -892,7 +898,14 @@ it( expect(generated.status, generated.stderr || generated.stdout).toBe(0); const outputs = parseSimpleOutput(fs.readFileSync(outputPath, "utf8")); const testMatrix = JSON.parse(outputs.test_matrix) as CredentialFreeTestMatrixRow[]; - expect(testMatrix).toEqual([selected]); + expect(testMatrix).toEqual([ + { + ...selected, + execution_id: `${selected.id}-docker`, + runtime_provider: "docker", + coverage_variant: "docker", + }, + ]); const { body, setFailed } = await executeReport({ apiJobs: [ diff --git a/test/e2e/support/e2e-semantic-phase-check.test.ts b/test/e2e/support/e2e-semantic-phase-check.test.ts index 20121fbb4bf..ca214508ec6 100644 --- a/test/e2e/support/e2e-semantic-phase-check.test.ts +++ b/test/e2e/support/e2e-semantic-phase-check.test.ts @@ -108,6 +108,9 @@ describe("semantic E2E phase checker", () => { testMatrix: [ { id: "vllm-docker-storage", + execution_id: "vllm-docker-storage-docker", + runtime_provider: "docker", + coverage_variant: "docker", file: "test/platform/images/vllm-docker-storage.test.ts", project: "integration", }, diff --git a/test/e2e/support/hermes-discord-policy-binding.test.ts b/test/e2e/support/hermes-discord-policy-binding.test.ts index 05b98436dee..625d5204f4a 100644 --- a/test/e2e/support/hermes-discord-policy-binding.test.ts +++ b/test/e2e/support/hermes-discord-policy-binding.test.ts @@ -6,16 +6,51 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { afterEach, beforeAll, describe, expect, it } from "vitest"; +import { afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import YAML from "yaml"; +import type { HostCliClient } from "../fixtures/clients/host.ts"; +import { rebindFixtureProviderPolicyEndpoint } from "../fixtures/gateway-providers.ts"; import { requireSuccessfulPolicyBoundaryBuild } from "../fixtures/hermes-discord-policy-boundary-build.ts"; +import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; +import { + HERMES_DISCORD_REST_PROOF_SOURCE, + isDiscordExternalAccessDenial, + verifyDiscordRestBoundary, +} from "../live/hermes-discord-proxy.ts"; const HELPER = path.resolve(import.meta.dirname, "../fixtures/hermes-discord-policy-binding.ts"); const TYPESCRIPT = path.resolve("node_modules/typescript/bin/tsc"); const POLICY_BOUNDARY_CONFIG = path.resolve("nemoclaw/tsconfig.shared.json"); const tempDirs: string[] = []; +describe("Discord external boundary classification", () => { + it("isolates the exact edge denial from credential-rewrite failures", () => { + expect(isDiscordExternalAccessDenial(403, "error code: 1010\n")).toBe(true); + expect(isDiscordExternalAccessDenial(403, "unresolved credential placeholder")).toBe(false); + expect(isDiscordExternalAccessDenial(401, "error code: 1010")).toBe(false); + }); + + it("records only exact external unavailability after the local rewrite proof", async () => { + const recordUnavailable = vi.fn(async () => undefined); + await expect( + verifyDiscordRestBoundary( + '{"statusCode":403,"body":"error code: 1010\\n"}\n', + recordUnavailable, + ), + ).resolves.toBeUndefined(); + expect(recordUnavailable).toHaveBeenCalledWith( + "Discord edge denied this runner before the API boundary (error 1010)", + ); + await expect( + verifyDiscordRestBoundary( + '{"statusCode":403,"body":"unresolved credential placeholder"}\n', + recordUnavailable, + ), + ).rejects.toThrow("Unexpected Discord users/@me response"); + }); +}); + function runBinding(policyFile: string, protocol = "websocket") { return spawnSync( process.execPath, @@ -34,6 +69,34 @@ function runBinding(policyFile: string, protocol = "websocket") { ); } +function runUnbind(policyFile: string) { + return spawnSync( + process.execPath, + [ + "--import", + "tsx", + HELPER, + "--unbind-provider", + policyFile, + "e2e-hermes-discord-discord-bridge", + ], + { encoding: "utf8", timeout: 15_000 }, + ); +} + +function successfulProbe(stdout = ""): ShellProbeResult { + return { + command: ["openshell"], + durationMs: 1, + exitCode: 0, + signal: null, + timedOut: false, + stdout, + stderr: "", + artifacts: { stdout: "stdout", stderr: "stderr", result: "result" }, + }; +} + function runBinaryAssertion(policyFile: string) { return spawnSync( process.execPath, @@ -63,6 +126,16 @@ describe("Hermes Discord E2E policy binding", () => { await requireSuccessfulPolicyBoundaryBuild(result); }); + it("uses only the fresh revision-scoped exec credential for the REST proof", () => { + expect(HERMES_DISCORD_REST_PROOF_SOURCE).toContain( + 'token = os.environ.get("DISCORD_BOT_TOKEN", "")', + ); + expect(HERMES_DISCORD_REST_PROOF_SOURCE).toContain( + 're.fullmatch(r"openshell:resolve:env:v[0-9]{1,20}_DISCORD_BOT_TOKEN", token)', + ); + expect(HERMES_DISCORD_REST_PROOF_SOURCE).not.toContain("/sandbox/.hermes/.env"); + }); + afterEach(() => { for (const tempDir of tempDirs.splice(0)) { fs.rmSync(tempDir, { force: true, recursive: true }); @@ -172,6 +245,119 @@ describe("Hermes Discord E2E policy binding", () => { }); }); + it("temporarily unbinds only the selected provider before attachment refresh", () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-unbind-policy-")); + tempDirs.push(tempDir); + const policyFile = path.join(tempDir, "policy.yaml"); + fs.writeFileSync( + policyFile, + YAML.stringify({ + version: 1, + network_policies: { + fake: { + endpoints: [ + { + host: "discord.com", + port: 443, + credential_binding: { provider: "e2e-hermes-discord-discord-bridge" }, + }, + { + host: "example.com", + port: 443, + credential_binding: { provider: "another-provider" }, + }, + ], + }, + }, + }), + ); + + const result = runUnbind(policyFile); + const endpoints = YAML.parse(fs.readFileSync(policyFile, "utf8")).network_policies.fake + .endpoints as Array>; + + expect(result.stderr).toBe(""); + expect(result.status).toBe(0); + expect(endpoints[0]).not.toHaveProperty("credential_binding"); + expect(endpoints[1]).toHaveProperty("credential_binding", { + provider: "another-provider", + }); + }); + + it("reattaches one provider without advancing its credential generation", async () => { + const providerName = "e2e-hermes-discord-discord-bridge"; + const originalPolicy = { + version: 1, + network_policies: { + discord: { + endpoints: [ + { + host: "discord.com", + port: 443, + credential_binding: { provider: providerName }, + }, + { + host: "host.openshell.internal", + port: 43117, + protocol: "websocket", + }, + { + host: "example.com", + port: 443, + credential_binding: { provider: "another-provider" }, + }, + ], + }, + }, + }; + const appliedPolicies: (typeof originalPolicy)[] = []; + const recordAppliedPolicy = (args: string[]) => { + const file = args.at(args.indexOf("--policy") + 1); + expect(file).toBeTruthy(); + appliedPolicies.push(YAML.parse(fs.readFileSync(file!, "utf8")) as typeof originalPolicy); + return Promise.resolve(successfulProbe()); + }; + const command = vi + .fn() + .mockResolvedValueOnce(successfulProbe(YAML.stringify(originalPolicy))) + .mockResolvedValueOnce(successfulProbe(providerName)) + .mockImplementationOnce(async (_command, args = []) => recordAppliedPolicy(args)); + const host = { + command, + openshellCommandPath: "/usr/local/bin/openshell", + } as unknown as HostCliClient; + + await rebindFixtureProviderPolicyEndpoint(host, "e2e-hermes-discord", { + artifactName: "hermes-discord-rebind", + credentialEnv: "DISCORD_BOT_TOKEN", + endpoint: { + host: "host.openshell.internal", + port: 43117, + protocol: "websocket", + }, + env: { + DISCORD_BOT_TOKEN: "test-fixture-token", + OPENSHELL_GATEWAY: "nemoclaw", + }, + providerName, + redactionValues: ["test-fixture-token"], + }); + + expect(command.mock.calls.map(([, args]) => args)).toEqual([ + ["policy", "get", "--base", "e2e-hermes-discord"], + ["sandbox", "provider", "list", "-g", "nemoclaw", "e2e-hermes-discord"], + ["policy", "set", "--policy", expect.any(String), "--wait", "e2e-hermes-discord"], + ]); + expect(appliedPolicies).toHaveLength(1); + + const reboundEndpoints = appliedPolicies[0]!.network_policies.discord.endpoints; + expect(reboundEndpoints[0]).toHaveProperty("credential_binding", { provider: providerName }); + expect(reboundEndpoints[1]).toHaveProperty("credential_binding", { provider: providerName }); + expect(reboundEndpoints[2]).toHaveProperty("credential_binding", { + provider: "another-provider", + }); + }); + it("rejects generic Python binaries in the Hermes fake Discord endpoint policy", () => { const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-hermes-discord-policy-")); tempDirs.push(tempDir); diff --git a/test/e2e/support/hermes-gpu-startup-proof.test.ts b/test/e2e/support/hermes-gpu-startup-proof.test.ts index 2062ba9bb18..85fa8945f14 100644 --- a/test/e2e/support/hermes-gpu-startup-proof.test.ts +++ b/test/e2e/support/hermes-gpu-startup-proof.test.ts @@ -9,9 +9,11 @@ import { assertHermesGpuStartupOutputContract, assertHermesManagedWorkloadAuthority, HERMES_GPU_FALLBACK_DISCLOSURE_FRAGMENTS, + normalizeImmutableImageContentId, } from "../live/hermes-gpu-startup-proof.ts"; const HEALTHY_NEW_GATEWAY = [ + "Container runtime: docker", "Starting OpenShell Docker-driver gateway...", "Docker-driver gateway is healthy", ].join("\n"); @@ -28,6 +30,7 @@ const NON_FALLBACK_DISCLOSURE_CASES = [ ["compatibility-only", HERMES_GPU_FALLBACK_DISCLOSURE_FRAGMENTS[4]], ] as const; const MANAGED_IMAGE_REFERENCE = `ghcr.io/nvidia/test@sha256:${"a".repeat(64)}`; +const MANAGED_IMAGE_CONTENT_ID = `sha256:${"c".repeat(64)}`; const OTHER_MANAGED_IMAGE_REFERENCE = `ghcr.io/nvidia/test@sha256:${"b".repeat(64)}`; const VALID_MANAGED_AUTHORITY = { agent: "hermes", @@ -40,7 +43,9 @@ describe("Hermes GPU startup output contract", () => { it.each(["native-success", "compatibility-only"] as const)( "accepts %s output without legacy Docker container progress text (#9362)", (route) => { - expect(() => assertHermesGpuStartupOutputContract(route, HEALTHY_NEW_GATEWAY)).not.toThrow(); + expect(() => + assertHermesGpuStartupOutputContract(route, "docker", HEALTHY_NEW_GATEWAY), + ).not.toThrow(); }, ); @@ -52,7 +57,7 @@ describe("Hermes GPU startup output contract", () => { ].join("\n"); expect(() => - assertHermesGpuStartupOutputContract("compatibility-fallback", output), + assertHermesGpuStartupOutputContract("compatibility-fallback", "docker", output), ).not.toThrow(); }); @@ -68,7 +73,7 @@ describe("Hermes GPU startup output contract", () => { ].join("\n"); expect(() => - assertHermesGpuStartupOutputContract("compatibility-fallback", output), + assertHermesGpuStartupOutputContract("compatibility-fallback", "docker", output), ).toThrow(); }, ); @@ -77,13 +82,25 @@ describe("Hermes GPU startup output contract", () => { "rejects fallback disclosure in %s output: %s (#9362)", (route, fragment) => { expect(() => - assertHermesGpuStartupOutputContract(route, `${HEALTHY_NEW_GATEWAY}\n${fragment}`), + assertHermesGpuStartupOutputContract( + route, + "docker", + `${HEALTHY_NEW_GATEWAY}\n${fragment}`, + ), ).toThrow(); }, ); }); describe("Hermes GPU managed-image authority proof", () => { + it("canonicalizes a Podman bare image content ID without changing canonical Docker IDs", () => { + expect(normalizeImmutableImageContentId("c".repeat(64))).toBe(MANAGED_IMAGE_CONTENT_ID); + expect(normalizeImmutableImageContentId(MANAGED_IMAGE_CONTENT_ID)).toBe( + MANAGED_IMAGE_CONTENT_ID, + ); + expect(normalizeImmutableImageContentId("not-an-image-id")).toBe("not-an-image-id"); + }); + it("accepts one immutable authority shared by the registry, contract, and receipt (#9362)", () => { expect( assertHermesManagedWorkloadAuthority( @@ -169,6 +186,26 @@ describe("Hermes GPU managed-image authority proof", () => { ).not.toThrow(); }); + it("accepts the provider content ID resolved from the exact digest-backed authority", () => { + expect(() => + assertHermesContainerImageAuthority( + MANAGED_IMAGE_CONTENT_ID, + MANAGED_IMAGE_REFERENCE, + MANAGED_IMAGE_CONTENT_ID, + ), + ).not.toThrow(); + }); + + it("accepts Podman's bare running-container content ID for the recorded authority", () => { + expect(() => + assertHermesContainerImageAuthority( + "c".repeat(64), + MANAGED_IMAGE_REFERENCE, + MANAGED_IMAGE_CONTENT_ID, + ), + ).not.toThrow(); + }); + it("rejects a running container outside the recorded authority (#9362)", () => { expect(() => assertHermesContainerImageAuthority("ghcr.io/nvidia/test:latest", MANAGED_IMAGE_REFERENCE), diff --git a/test/e2e/support/hermes-workflow-boundary.test.ts b/test/e2e/support/hermes-workflow-boundary.test.ts index be66dc51cf7..9331ed7362b 100644 --- a/test/e2e/support/hermes-workflow-boundary.test.ts +++ b/test/e2e/support/hermes-workflow-boundary.test.ts @@ -150,7 +150,7 @@ describe("Hermes GPU boundary", () => { const job = workflow.jobs[GPU]; job["runs-on"] = "ubuntu-latest"; job.if = "${{ always() }}"; - job.strategy["max-parallel"] = 2; + job.strategy["max-parallel"] = 9; job.strategy.matrix.include = [{ scenario: "native" }]; job.env.UNRELATED_SECRET = KEY; const run = step(job, "Run Hermes GPU startup live Vitest test"); @@ -159,8 +159,16 @@ describe("Hermes GPU boundary", () => { step(job, "Upload Hermes GPU startup artifacts").with.path = "wrong"; }, validateE2eWorkflowBoundary); - expect(errors.join("\n")).toMatch( - /GPU runner.*generate-matrix.*serialize.*secrets.*hosted Hermes.*artifact path.*hosted-compatible/s, + expect(errors).toEqual( + expect.arrayContaining([ + "hermes-gpu-startup job must run on the native RTX PRO 6000 GPU runner", + "hermes-gpu-startup job must use the trusted execution plan behind generate-matrix", + "hermes-gpu-startup must expand reviewed GPU scenarios by supported runtime", + "hermes-gpu-startup job env must not consume repository secrets", + "hermes-gpu-startup step 'Run Hermes GPU startup live Vitest test' must not run the hosted Hermes E2E test", + "hermes-gpu-startup upload needs a scenario artifact path", + "hermes-gpu-startup job must enable hosted-compatible inference mode", + ]), ); }); @@ -197,26 +205,24 @@ describe("Hermes GPU boundary", () => { ); }); - it.each(hermesTimeoutBoundaries)("requires 15-30 minutes of outer headroom for $jobName", ({ - jobName, - maximumTimeoutMinutes, - message, - minimumTimeoutMinutes, - }) => { - const insufficient = wfErrors((workflow) => { - workflow.jobs[jobName]["timeout-minutes"] = minimumTimeoutMinutes - 1; - }, validateE2eWorkflowBoundary); - const additional = wfErrors((workflow) => { - workflow.jobs[jobName]["timeout-minutes"] = minimumTimeoutMinutes + 1; - }, validateE2eWorkflowBoundary); - const excessive = wfErrors((workflow) => { - workflow.jobs[jobName]["timeout-minutes"] = maximumTimeoutMinutes + 1; - }, validateE2eWorkflowBoundary); + it.each(hermesTimeoutBoundaries)( + "requires 15-30 minutes of outer headroom for $jobName", + ({ jobName, maximumTimeoutMinutes, message, minimumTimeoutMinutes }) => { + const insufficient = wfErrors((workflow) => { + workflow.jobs[jobName]["timeout-minutes"] = minimumTimeoutMinutes - 1; + }, validateE2eWorkflowBoundary); + const additional = wfErrors((workflow) => { + workflow.jobs[jobName]["timeout-minutes"] = minimumTimeoutMinutes + 1; + }, validateE2eWorkflowBoundary); + const excessive = wfErrors((workflow) => { + workflow.jobs[jobName]["timeout-minutes"] = maximumTimeoutMinutes + 1; + }, validateE2eWorkflowBoundary); - expect(insufficient).toContain(message); - expect(additional).toEqual([]); - expect(excessive).toContain(message); - }); + expect(insufficient).toContain(message); + expect(additional).toEqual([]); + expect(excessive).toContain(message); + }, + ); it("rejects unconditional live secret in hermes-e2e mock run step", () => { const errors = wfErrors((workflow) => { diff --git a/test/e2e/support/hosted-inference.test.ts b/test/e2e/support/hosted-inference.test.ts index 43a924f1e4b..cd6c483c64d 100644 --- a/test/e2e/support/hosted-inference.test.ts +++ b/test/e2e/support/hosted-inference.test.ts @@ -415,6 +415,8 @@ printf '{"data":[]}' HOME: "/tmp/home", PATH: "/usr/bin", BUILDX_BUILDER: "external-builder", + CONTAINERS_CONF: "/tmp/native-podman-containers.conf", + CONTAINERS_STORAGE_CONF: "/tmp/native-podman-storage.conf", NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1", NEMOCLAW_OPENSHELL_CHANNEL: "dev", NVIDIA_INFERENCE_API_KEY: "repo-hosted-key", @@ -423,6 +425,8 @@ printf '{"data":[]}' expect(env.NEMOCLAW_E2E_USE_HOSTED_INFERENCE).toBe("1"); expect(env.NEMOCLAW_OPENSHELL_CHANNEL).toBe("dev"); + expect(env.CONTAINERS_CONF).toBe("/tmp/native-podman-containers.conf"); + expect(env.CONTAINERS_STORAGE_CONF).toBe("/tmp/native-podman-storage.conf"); expect(env).not.toHaveProperty("NVIDIA_INFERENCE_API_KEY"); expect(env).not.toHaveProperty("RANDOM_NON_SECRET"); expect(env).not.toHaveProperty("BUILDX_BUILDER"); diff --git a/test/e2e/support/issue-4462-fixture-boundary.test.ts b/test/e2e/support/issue-4462-fixture-boundary.test.ts index d92aa9994ab..09e359e7ff0 100644 --- a/test/e2e/support/issue-4462-fixture-boundary.test.ts +++ b/test/e2e/support/issue-4462-fixture-boundary.test.ts @@ -6,6 +6,7 @@ import { spawnSync } from "node:child_process"; import { describe, expect, it } from "vitest"; import { ISSUE_4462_PAIRING_SEED_PY } from "../fixtures/issue-4462-pairing-seed.ts"; +import { ISSUE_4462_SCOPE_UPGRADE_PHASES } from "../live/issue-4462-admin-approval-helper.ts"; const BEHAVIOR_HARNESS_PY = String.raw` import base64 @@ -179,5 +180,8 @@ describe("scope-upgrade approval live fixture", () => { expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); expect(result.stdout.trim()).toBe("ISSUE_4462_FIXTURE_BEHAVIOR_OK"); + expect(ISSUE_4462_SCOPE_UPGRADE_PHASES[0]).toBe( + "confirm configured runtime availability and clear the scope-upgrade sandbox", + ); }); }); diff --git a/test/e2e/support/managed-image-receipt.test.ts b/test/e2e/support/managed-image-receipt.test.ts index d945ab71f72..5bd7b76361c 100644 --- a/test/e2e/support/managed-image-receipt.test.ts +++ b/test/e2e/support/managed-image-receipt.test.ts @@ -417,6 +417,13 @@ describe("stock E2E managed-image receipt assertion", () => { E2E_MANAGED_IMAGE_REVISION: REVISION, }), ).toBe(true); + expect( + shouldAssertStockManagedImageReceipt( + "node", + ["/release/bin/nemoclaw.js", "onboard", "--help"], + { E2E_MANAGED_IMAGE_REVISION: REVISION }, + ), + ).toBe(false); expect( shouldAssertStockManagedImageReceipt("/workspace/bin/nemoclaw.js", ["onboard"], { E2E_MANAGED_IMAGE_REVISION: REVISION, diff --git a/test/e2e/support/manual-pr-credential-authorization.test.ts b/test/e2e/support/manual-pr-credential-authorization.test.ts index e29009fdf08..681fd74842b 100644 --- a/test/e2e/support/manual-pr-credential-authorization.test.ts +++ b/test/e2e/support/manual-pr-credential-authorization.test.ts @@ -73,14 +73,14 @@ describe("manual PR E2E credential authorization", () => { expectedAllowed: false, }, { - caseName: "a non-main workflow ref with otherwise matching identities", + caseName: "a same-repository feature branch with matching identities", checkoutRepository: "NVIDIA/NemoClaw", nvidiaOwned: true, workflowRepository: "NVIDIA/NemoClaw", workflowRef: "refs/heads/pr-controlled-workflow", checkoutShaMatches: true, workflowShaMatches: true, - expectedAllowed: false, + expectedAllowed: true, }, ])( "sets E2E credential access to $expectedAllowed for $caseName (#9047)", diff --git a/test/e2e/support/mcp-bridge-hermes-http.test.ts b/test/e2e/support/mcp-bridge-hermes-http.test.ts index 048271e06f8..5f9cfb3f74b 100644 --- a/test/e2e/support/mcp-bridge-hermes-http.test.ts +++ b/test/e2e/support/mcp-bridge-hermes-http.test.ts @@ -104,9 +104,9 @@ describe("Hermes MCP HTTP failure diagnostics", () => { expect(message).not.toContain(secret); } - expect(() => assertHermesMcpHttpResponse(httpResult(200), [])).toThrowError( - /fixture result token/u, - ); + expect(() => + assertHermesMcpHttpResponse(httpResult(200, `missing ${secret}`), [secret]), + ).toThrowError(/fixture result token.*redacted response body: missing \[REDACTED\]/u); expect(() => assertHermesMcpHttpResponse(httpResult(200, "", `${HERMES_MCP_HTTP_STATUS_MARKER}500\n`), []), ).toThrowError(/exactly one HTTP status marker/u); diff --git a/test/e2e/support/mcp-bridge-onboard-env.test.ts b/test/e2e/support/mcp-bridge-onboard-env.test.ts index 0a26db4c0cd..0661b95ab06 100644 --- a/test/e2e/support/mcp-bridge-onboard-env.test.ts +++ b/test/e2e/support/mcp-bridge-onboard-env.test.ts @@ -256,6 +256,7 @@ describe("MCP bridge onboarding environment", () => { expect(() => assertMcpBridgeManagedImageReceipt({ environment: { + E2E_MANAGED_IMAGE_REVISION: "", GITHUB_ACTIONS: "true", NEMOCLAW_E2E_EXPECTED_SHA: revision, NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG: catalogPath, diff --git a/test/e2e/support/mcp-bridge-sandbox.test.ts b/test/e2e/support/mcp-bridge-sandbox.test.ts index c49f9e88295..fa99f4b3db4 100644 --- a/test/e2e/support/mcp-bridge-sandbox.test.ts +++ b/test/e2e/support/mcp-bridge-sandbox.test.ts @@ -69,11 +69,13 @@ function denialResult( }; } -async function captureRestoreScript(hostBackupPath: string, sandboxBackupPath: string) { +async function captureRestoreCommand(hostBackupPath: string, sandboxBackupPath: string) { + let restoreArgs: string[] = []; let restoreScript = ""; const host = { command: async (_command: string, args: string[]) => { restoreScript = args[1] ?? ""; + restoreArgs = args.slice(2); return denialResult(); }, } as unknown as HostCliClient; @@ -83,7 +85,7 @@ async function captureRestoreScript(hostBackupPath: string, sandboxBackupPath: s hostBackupPath, sandboxBackupPath, }); - return restoreScript; + return { restoreArgs, restoreScript }; } describe("MCP curl policy denial classification", SUITE_OPTIONS, () => { @@ -92,7 +94,7 @@ describe("MCP curl policy denial classification", SUITE_OPTIONS, () => { const host = { command: async (_command: string, args: string[]) => { probeScript = args[1] ?? ""; - return { ...denialResult(), stdout: "10.20.30.40\n" }; + return { ...denialResult(), stdout: "route 10.20.30.40\n" }; }, } as unknown as HostCliClient; @@ -386,7 +388,10 @@ network_policies: }); it("restores host DNS strictly while treating the ephemeral sandbox as best effort", async () => { - const restoreScript = await captureRestoreScript("/tmp/host-backup", "/tmp/sandbox-backup"); + const { restoreScript } = await captureRestoreCommand( + "/tmp/host-backup", + "/tmp/sandbox-backup", + ); expect(restoreScript).toContain("set -uo pipefail"); expect(restoreScript).not.toContain("set -euo pipefail"); @@ -395,7 +400,10 @@ network_policies: expect(restoreScript).toContain("host_restore_failed=1"); expect(restoreScript).toContain('if [ "$host_restore_failed" -ne 0 ]; then exit 1; fi'); expect(restoreScript).toContain("for attempt in 1 2 3; do"); - expect(restoreScript).toContain('docker exec --user 0 -i "$container_id"'); + expect(restoreScript).toContain('runtime_command=("$@")'); + expect(restoreScript).toContain( + '"${runtime_command[@]}" container exec --user 0 --interactive "$container_id"', + ); expect(restoreScript).toContain( "::warning::could not restore ephemeral sandbox /etc/hosts; cleanup will destroy the sandbox", ); @@ -418,16 +426,20 @@ network_policies: '#!/bin/sh\n[ "${FAKE_SUDO_STATUS:-0}" -eq 0 ] || exit "$FAKE_SUDO_STATUS"\ncat > "$FAKE_HOSTS_PATH"\n', ); writeExecutable("cmp", '#!/bin/sh\nexit "${FAKE_CMP_STATUS:-0}"\n'); - writeExecutable( - "docker", - '#!/bin/sh\nif [ "$1" = ps ]; then echo fake-container; exit 0; fi\nif [ "$1" = exec ]; then cat >/dev/null; exit "${FAKE_DOCKER_EXEC_STATUS:-0}"; fi\nexit 64\n', - ); writeExecutable("sleep", "#!/bin/sh\nexit 0\n"); try { - const restoreScript = await captureRestoreScript(hostBackupPath, sandboxBackupPath); + const { restoreArgs, restoreScript } = await captureRestoreCommand( + hostBackupPath, + sandboxBackupPath, + ); + const runtimeCommand = restoreArgs[1] ?? "missing-runtime-command"; + writeExecutable( + runtimeCommand, + '#!/bin/sh\nif [ "$1" = container ] && [ "$2" = ps ]; then echo fake-container; exit 0; fi\nif [ "$1" = container ] && [ "$2" = exec ]; then cat >/dev/null; exit "${FAKE_RUNTIME_EXEC_STATUS:-0}"; fi\nexit 64\n', + ); const runRestore = (extraEnv: Record = {}) => - spawnSync("/bin/bash", ["-c", restoreScript], { + spawnSync("/bin/bash", ["-c", restoreScript, ...restoreArgs], { encoding: "utf8", env: { ...process.env, @@ -458,7 +470,7 @@ network_policies: expect(fs.existsSync(sandboxBackupPath)).toBe(true); resetBackups(); - const sandboxFailure = runRestore({ FAKE_DOCKER_EXEC_STATUS: "1" }); + const sandboxFailure = runRestore({ FAKE_RUNTIME_EXEC_STATUS: "1" }); expect(sandboxFailure.status, sandboxFailure.stderr).toBe(0); expect(sandboxFailure.stderr).toContain( "::warning::could not restore ephemeral sandbox /etc/hosts; cleanup will destroy the sandbox", diff --git a/test/e2e/support/messaging-compatible-endpoint-helpers.test.ts b/test/e2e/support/messaging-compatible-endpoint-helpers.test.ts index b4098382d32..acebffab638 100644 --- a/test/e2e/support/messaging-compatible-endpoint-helpers.test.ts +++ b/test/e2e/support/messaging-compatible-endpoint-helpers.test.ts @@ -48,7 +48,7 @@ describe("messaging compatible endpoint helper coverage", () => { await expect( cleanupOwnedGatewayRuntimeStrict(host, "strict-invalid-gateway-pid"), ).rejects.toThrow(/PID file is invalid or unreadable/u); - expect(command).toHaveBeenCalledTimes(1); + expect(command).toHaveBeenCalledTimes(4); } finally { vi.unstubAllEnvs(); process.kill(pid!, "SIGKILL"); @@ -117,7 +117,7 @@ describe("messaging compatible endpoint helper coverage", () => { })(), ).rejects.toThrow(/HTTP 429/); - expect(calls).toHaveLength(3); + expect(calls).toHaveLength(6); expect(calls[0]?.command).toBe("node"); expect(calls[0]?.args[0]).toMatch(/bin\/nemoclaw\.js$/); expect(calls[0]?.args.slice(1)).toEqual(["e2e-msg-compat-missing", "destroy", "--yes"]); @@ -125,10 +125,26 @@ describe("messaging compatible endpoint helper coverage", () => { command: "openshell", args: ["sandbox", "delete", "e2e-msg-compat-missing"], }); - expect(calls[2]?.command).toBe("bash"); - expect(calls[2]?.args[0]).toBe("-lc"); - expect(calls[2]?.args[1]).toContain('"$openshell_bin" gateway destroy -g nemoclaw'); - expect(calls[2]?.args.at(-1)).toBe("openshell"); + expect(calls[2]).toEqual({ + command: "openshell", + args: ["forward", "stop", "18789"], + }); + expect(calls[3]).toEqual({ + command: "openshell", + args: ["gateway", "stop", "-g", "nemoclaw"], + }); + expect(calls[4]?.args).toEqual([ + "container", + "ps", + "--filter", + "name=^openshell-cluster-nemoclaw$", + "--format", + "{{.Names}}", + ]); + expect(calls[5]).toEqual({ + command: "openshell", + args: ["gateway", "destroy", "-g", "nemoclaw"], + }); }); }); diff --git a/test/e2e/support/messaging-providers-runtime-proofs.test.ts b/test/e2e/support/messaging-providers-runtime-proofs.test.ts index 8ebd6554bd6..2b5df450742 100644 --- a/test/e2e/support/messaging-providers-runtime-proofs.test.ts +++ b/test/e2e/support/messaging-providers-runtime-proofs.test.ts @@ -128,6 +128,7 @@ function fakeDockerInspect( missingProxyControl?: MissingProxyControl; proxyEnvironment: readonly string[]; publishedAddress?: string; + runtimeProviderId: "docker" | "podman"; }, ): string { const [apiRun, proxyRun] = calls.filter((call) => call[0] === "run") as [string[], string[]]; @@ -137,12 +138,19 @@ function fakeDockerInspect( const connectedProxyNetworks = calls .filter((call) => call[0] === "network" && call[1] === "connect" && call[3] === proxyContainer) .map((call) => call[2]!); - const proxyNetworks = [optionValue(proxyRun, "--network"), ...connectedProxyNetworks].filter( - (network) => options.missingProxyControl !== "internal-network" || network !== apiNetwork, - ); + const proxyNetworks = [optionValue(proxyRun, "--network"), ...connectedProxyNetworks] + .filter( + (network) => options.missingProxyControl !== "internal-network" || network !== apiNetwork, + ) + .map((network) => + options.runtimeProviderId === "podman" && network === "bridge" ? "podman" : network, + ); + const proxyDropsAllCapabilities = options.missingProxyControl !== "--cap-drop"; + const inspectName = (name: string): string => + options.runtimeProviderId === "podman" ? name : `/${name}`; return JSON.stringify([ { - Name: "/" + apiContainer, + Name: inspectName(apiContainer), HostConfig: {}, NetworkSettings: { Networks: { [apiNetwork]: {} }, @@ -153,13 +161,31 @@ function fakeDockerInspect( }, }, { + BoundingCaps: + options.runtimeProviderId === "podman" + ? proxyDropsAllCapabilities + ? null + : ["CAP_CHOWN"] + : undefined, Config: { Env: [...optionValues(proxyRun, "-e"), ...options.proxyEnvironment], }, - Name: "/" + proxyContainer, + EffectiveCaps: + options.runtimeProviderId === "podman" + ? proxyDropsAllCapabilities + ? null + : ["CAP_CHOWN"] + : undefined, + Name: inspectName(proxyContainer), HostConfig: { CapDrop: - options.missingProxyControl === "--cap-drop" ? [] : optionValues(proxyRun, "--cap-drop"), + options.runtimeProviderId === "podman" + ? proxyDropsAllCapabilities + ? ["CAP_CHOWN"] + : [] + : proxyDropsAllCapabilities + ? optionValues(proxyRun, "--cap-drop") + : [], PidsLimit: options.missingProxyControl === "--pids-limit" ? undefined @@ -203,6 +229,7 @@ function fakeDockerHost( const containers = new Set(); const networks = new Set(); const networkInspect = options.networkInspect ?? OPENSHELL_NETWORK_INSPECT; + let runtimeProviderId: "docker" | "podman" = "docker"; let proxyRunning = options.proxyRunning !== false; const dockerCommand = (args: string[]) => { const executedArgs = [...args]; @@ -219,7 +246,15 @@ function fakeDockerHost( executedArgs[2] === "openshell-docker" ? networkInspect : JSON.stringify([ - { Driver: "bridge", Internal: createdNetwork?.includes("--internal") === true }, + runtimeProviderId === "podman" + ? { + driver: "bridge", + internal: createdNetwork?.includes("--internal") === true, + } + : { + Driver: "bridge", + Internal: createdNetwork?.includes("--internal") === true, + }, ]), ); } @@ -250,6 +285,7 @@ function fakeDockerHost( missingProxyControl: options.missingProxyControl, proxyEnvironment: options.proxyEnvironment ?? [], publishedAddress: options.publishedAddress, + runtimeProviderId, }), ) : executedArgs[2] === "{{json .State}}" @@ -290,13 +326,17 @@ function fakeDockerHost( commandOptions?: { artifactName?: string }, ) => { commands.push({ command, args: [...args] }); - expect(["docker", "node"]).toContain(command); + expect(["docker", "node", "podman"]).toContain(command); + runtimeProviderId = + command === "podman" ? "podman" : command === "docker" ? "docker" : runtimeProviderId; + const runtimeArgs = + command === "podman" && args[0] === "--url" ? args.slice(2) : args; const result = command === "node" ? options.proxyReady === false ? failedCommand("proxy could not reach the upstream API") : successfulCommand() - : dockerCommand(args); + : dockerCommand(runtimeArgs); const artifactNames = commandOptions?.artifactName === undefined ? [] : [commandOptions.artifactName]; for (const artifactName of artifactNames) { @@ -341,7 +381,11 @@ function expectFakeDiscordDiagnosticArtifacts(artifacts: Map): v expect(artifacts.get("diagnose-fake-discord-gateway-api-logs")).toContain("api diagnostic logs"); } -function startFakeDiscordApi(host: HostCliClient, cleanup: CleanupAction[]) { +function startFakeDiscordApi( + host: HostCliClient, + cleanup: CleanupAction[], + env: NodeJS.ProcessEnv = {}, +) { return startFakeDockerApi(host, (name, run) => cleanup.push({ name, run }), { kind: "discord-gateway", imageScript: "fake-discord-gateway.cjs", @@ -350,7 +394,7 @@ function startFakeDiscordApi(host: HostCliClient, cleanup: CleanupAction[]) { captureFileEnv: "FAKE_DISCORD_GATEWAY_CAPTURE_FILE", expectedEnv: { FAKE_DISCORD_GATEWAY_EXPECTED_TOKEN: "fixture-discord-token" }, redactionValues: ["fixture-discord-token"], - env: {}, + env, }); } @@ -547,6 +591,49 @@ describe("messaging provider installed-runtime proofs", () => { } }); + it("publishes the isolated proxy through rootless Podman without binding its bridge gateway", async () => { + const { calls, host } = fakeDockerHost({ networkInspect: "[]" }); + const cleanup: CleanupAction[] = []; + + try { + const api = await startFakeDiscordApi(host, cleanup, { + NEMOCLAW_GATEWAY_RUNTIME: "podman", + OPENSHELL_PODMAN_SOCKET: "/run/user/1001/podman/podman.sock", + }); + const proxyRun = calls.filter((args) => args[0] === "run").at(-1)!; + const publications = optionValues(proxyRun, "-p"); + + expect(publications).toContain("0.0.0.0::8080"); + expect(publications).toContain(`0.0.0.0::${String(FAKE_API_PROXY_READINESS_PORT)}`); + expect(publications.some((entry) => entry.startsWith(`${OPENSHELL_BRIDGE_ADDRESS}::`))).toBe( + false, + ); + expect(calls).not.toContainEqual(["network", "inspect", "openshell-docker"]); + expect(api.port).toBe("32100"); + } finally { + await runCleanup(cleanup); + } + }); + + it("rejects effective proxy capabilities reported by rootless Podman", async () => { + const { host } = fakeDockerHost({ + missingProxyControl: "--cap-drop", + networkInspect: "[]", + }); + const cleanup: CleanupAction[] = []; + + try { + await expect( + startFakeDiscordApi(host, cleanup, { + NEMOCLAW_GATEWAY_RUNTIME: "podman", + OPENSHELL_PODMAN_SOCKET: "/run/user/1001/podman/podman.sock", + }), + ).rejects.toThrow(/Podman topology did not preserve isolation/u); + } finally { + await runCleanup(cleanup); + } + }); + it("rejects ambiguous OpenShell IPv4 gateways and cleans its temporary directory", async () => { const networkInspect = JSON.stringify([ { diff --git a/test/e2e/support/native-podman-setup-action.test.ts b/test/e2e/support/native-podman-setup-action.test.ts new file mode 100644 index 00000000000..a59834abc09 --- /dev/null +++ b/test/e2e/support/native-podman-setup-action.test.ts @@ -0,0 +1,12 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { validateNativePodmanSetupAction } from "../../../tools/e2e/workflow-boundary.mts"; + +describe("native Podman E2E setup boundary", () => { + it("provides Podman authority without impersonating Docker", () => { + expect(validateNativePodmanSetupAction()).toEqual([]); + }); +}); diff --git a/test/e2e/support/native-runtime-qualification.test.ts b/test/e2e/support/native-runtime-qualification.test.ts index 9e16813848e..0c974a689cb 100644 --- a/test/e2e/support/native-runtime-qualification.test.ts +++ b/test/e2e/support/native-runtime-qualification.test.ts @@ -154,7 +154,7 @@ describe("native runtime qualification contract", () => { ); }); - it("accepts candidate prerequisites only for the expected candidate commit and target-branch base SHA without activating Podman", () => { + it("accepts candidate prerequisites only for the expected candidate commit and target-branch base SHA", () => { expect(consumeNativeRuntimeCandidateEvidence(candidateEvidence(), SOURCE_REVISION)).toEqual({ schemaVersion: 1, candidateId: "podman-cpu-lifecycle", @@ -165,7 +165,7 @@ describe("native runtime qualification contract", () => { expect(() => consumeNativeRuntimeCandidateEvidence(candidateEvidence(), "b".repeat(40)), ).toThrow("does not match source"); - expect(CURRENT_RUNTIME_PROVIDER_BUNDLES).not.toHaveProperty("podman"); + expect(CURRENT_RUNTIME_PROVIDER_BUNDLES).toHaveProperty("podman"); }); it("consumes complete evidence only against externally resolved protected identities", () => { @@ -183,7 +183,7 @@ describe("native runtime qualification contract", () => { source: expectedProtectedSource(), }); expect(Object.isFrozen(authority.source.artifact)).toBe(true); - expect(CURRENT_RUNTIME_PROVIDER_BUNDLES).not.toHaveProperty("podman"); + expect(CURRENT_RUNTIME_PROVIDER_BUNDLES).toHaveProperty("podman"); }); it.each([ @@ -197,19 +197,16 @@ describe("native runtime qualification contract", () => { { headSha: "f".repeat(40), baseSha: "e".repeat(40) }, "externally expected protected source", ], - ])( - "rejects an internally consistent but wrong %s evidence pair", - (_label, source, error) => { - expect(() => - consumeNativeRuntimeQualificationEvidence( - PODMAN_PROTECTED_HOST_LOCAL_INFERENCE_QUALIFICATION, - qualificationEvidence(source), - expectedProtectedSource(), - nativeQualificationReceiptReader, - ), - ).toThrow(error); - }, - ); + ])("rejects an internally consistent but wrong %s evidence pair", (_label, source, error) => { + expect(() => + consumeNativeRuntimeQualificationEvidence( + PODMAN_PROTECTED_HOST_LOCAL_INFERENCE_QUALIFICATION, + qualificationEvidence(source), + expectedProtectedSource(), + nativeQualificationReceiptReader, + ), + ).toThrow(error); + }); it("rejects missing immutable GitHub artifact identity", () => { const source = { diff --git a/test/e2e/support/pr-managed-image-workflow-boundary.test.ts b/test/e2e/support/pr-managed-image-workflow-boundary.test.ts deleted file mode 100644 index 3d435fa913f..00000000000 --- a/test/e2e/support/pr-managed-image-workflow-boundary.test.ts +++ /dev/null @@ -1,67 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { describe, expect, it } from "vitest"; - -import { - readE2eOperationsWorkflow, - validateE2eOperationsWorkflow, -} from "../../../tools/e2e/operations-workflow-boundary.mts"; - -describe("manual PR managed-image workflow boundary", () => { - it.each([ - ["workflow", (workflow: ReturnType) => workflow], - [ - "generate-matrix job", - (workflow: ReturnType) => workflow.jobs["generate-matrix"], - ], - ])("rejects obsolete catalog inputs inherited from the %s environment", (_scope, target) => { - const workflow = readE2eOperationsWorkflow(); - target(workflow).env = { - MANAGED_IMAGE_CATALOG: "candidate catalog", - MANAGED_IMAGE_CATALOG_SHA256: "candidate digest", - }; - - expect(validateE2eOperationsWorkflow(workflow)).toContain( - "Manual PR CLI packaging must not accept obsolete managed-image catalog authority", - ); - }); - - it("rejects an obsolete staged catalog reference in CLI packaging", () => { - const workflow = readE2eOperationsWorkflow(); - const packageStep = workflow.jobs["generate-matrix"].steps!.find( - (step) => step.name === "Package exact-commit CLI", - )!; - packageStep.run = `${packageStep.run ?? ""}\ncat pr-managed-image-catalog.json\n`; - - expect(validateE2eOperationsWorkflow(workflow)).toContain( - "Manual PR CLI packaging must not accept obsolete managed-image catalog authority", - ); - }); - - it("rejects restoration of the obsolete manual PR catalog resolver", () => { - const workflow = readE2eOperationsWorkflow(); - const matrixJob = workflow.jobs["generate-matrix"]; - matrixJob.outputs!.managed_image_catalog = - "${{ steps.resolve_pr_managed_image_catalog.outputs.catalog }}"; - const checkoutIndex = matrixJob.steps!.findIndex( - (step) => step.name === "Check out E2E candidate", - ); - matrixJob.steps!.splice(checkoutIndex, 0, { - id: "resolve_pr_managed_image_catalog", - name: "Resolve exact PR managed-image catalog", - shell: "bash", - run: "node tools/e2e/pr-managed-image-publication.mts catalog.json", - }); - const packageStep = matrixJob.steps!.find((step) => step.name === "Package exact-commit CLI")!; - packageStep.env!.MANAGED_IMAGE_CATALOG = - "${{ steps.resolve_pr_managed_image_catalog.outputs.catalog }}"; - - expect(validateE2eOperationsWorkflow(workflow)).toEqual( - expect.arrayContaining([ - "Manual PR E2E must not resolve an exact candidate managed-image catalog", - "Manual PR CLI packaging must not accept obsolete managed-image catalog authority", - ]), - ); - }); -}); diff --git a/test/e2e/support/rebuild-hermes-bootstrap.test.ts b/test/e2e/support/rebuild-hermes-bootstrap.test.ts index 85cc2c92068..80e1641c742 100644 --- a/test/e2e/support/rebuild-hermes-bootstrap.test.ts +++ b/test/e2e/support/rebuild-hermes-bootstrap.test.ts @@ -467,6 +467,9 @@ describe("rebuild-Hermes direct bootstrap", () => { expect(liveSource).not.toContain('"--cleanup-gateway"'); expect(liveSource).not.toMatch(/host\.command\(\s*["']openshell["']/u); expect(liveSource).not.toMatch(/^\s*['"`]openshell\s/mu); + expect(liveSource.indexOf("const sessionSummary = seedRegistryAndSession(")).toBeLessThan( + liveSource.indexOf("await cronRestore.seed();"), + ); }); it("retains the eight-phase rebuild contract with truthful bootstrap coverage (#7144)", () => { diff --git a/test/e2e/support/runtime-matrix.test.ts b/test/e2e/support/runtime-matrix.test.ts new file mode 100644 index 00000000000..f98161a3e5d --- /dev/null +++ b/test/e2e/support/runtime-matrix.test.ts @@ -0,0 +1,64 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + buildE2eWorkflowPlan, + renderE2eWorkflowPlanSummary, +} from "../../../tools/e2e/workflow-plan.mts"; + +const BOTH_RUNTIMES = { gatewayRuntimes: ["docker", "podman"] as const }; + +describe("E2E runtime matrix", () => { + it("expands one managed target across runtimes without duplicating single-run contracts", () => { + const managed = buildE2eWorkflowPlan({ jobs: "cloud-inference" }, BOTH_RUNTIMES); + const dockerOnly = buildE2eWorkflowPlan({ jobs: "gateway-guard-recovery" }, BOTH_RUNTIMES); + const runtimeAgnostic = buildE2eWorkflowPlan({ jobs: "spark-install" }, BOTH_RUNTIMES); + const managedRows = managed.catalogueMatrices["nvidia-inference"]; + + expect(managedRows).toEqual([ + expect.objectContaining({ + id: "cloud-inference", + execution_id: "cloud-inference-default-docker", + runtime_provider: "docker", + coverage_variant: "default-docker", + }), + expect.objectContaining({ + id: "cloud-inference", + execution_id: "cloud-inference-default-podman", + runtime_provider: "podman", + coverage_variant: "default-podman", + }), + ]); + expect(dockerOnly.catalogueMatrices["nvidia-inference"]).toEqual([ + expect.objectContaining({ id: "gateway-guard-recovery", runtime_provider: "docker" }), + ]); + expect(runtimeAgnostic.catalogueMatrices["nvidia-inference"]).toEqual([ + expect.objectContaining({ id: "spark-install", runtime_provider: "none" }), + ]); + expect(managed.coverageMatrix.map((row) => row.variant)).toEqual([ + "default-docker", + "default-podman", + ]); + expect(renderE2eWorkflowPlanSummary(dockerOnly)).toContain( + "| `gateway-guard-recovery` | podman | docker |", + ); + expect(renderE2eWorkflowPlanSummary(runtimeAgnostic)).not.toContain( + "| `spark-install` | podman |", + ); + }); + + it.each([ + "bootstrap-install-smoke", + "concurrent-gateway-ports", + "llama-cpp-generic-gpu", + "rebuild-hermes-stale-base", + ])("keeps the explicit Docker contract %s out of Podman fanout", (target) => { + const plan = buildE2eWorkflowPlan({ jobs: target }, BOTH_RUNTIMES); + const rows = Object.values(plan.catalogueMatrices).flat(); + + expect(rows).toEqual([expect.objectContaining({ id: target, runtime_provider: "docker" })]); + expect(renderE2eWorkflowPlanSummary(plan)).toContain(`| \`${target}\` | podman | docker |`); + }); +}); diff --git a/test/e2e/support/runtime-provider-fixture.test.ts b/test/e2e/support/runtime-provider-fixture.test.ts new file mode 100644 index 00000000000..a55df4f566d --- /dev/null +++ b/test/e2e/support/runtime-provider-fixture.test.ts @@ -0,0 +1,169 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import type { HostCliClient } from "../fixtures/clients/host.ts"; +import { + ensureConfiguredRuntimeProviderAvailable, + RuntimeProviderPrerequisite, +} from "../fixtures/runtime-provider.ts"; +import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; + +function successfulProbe(): ShellProbeResult { + return { + command: [], + exitCode: 0, + signal: null, + timedOut: false, + stdout: "ready", + stderr: "", + artifacts: { stdout: "", stderr: "", result: "" }, + }; +} + +function successfulProbeWithStdout(stdout: string): ShellProbeResult { + return { ...successfulProbe(), stdout }; +} + +describe("configured E2E runtime provider fixture", () => { + it("preserves native Podman runtime authority for CLI child processes", () => { + const env = buildAvailabilityProbeEnv({ + HOME: "/home/runner", + PATH: "/usr/bin", + DBUS_SESSION_BUS_ADDRESS: "unix:path=/run/user/1001/bus", + NEMOCLAW_E2E_MANAGED_IMAGE_REVISION: "a".repeat(40), + NEMOCLAW_GATEWAY_RUNTIME: "podman", + OPENSHELL_PODMAN_SOCKET: "/run/user/1001/podman/podman.sock", + XDG_RUNTIME_DIR: "/run/user/1001", + }); + + expect(env).toMatchObject({ + DBUS_SESSION_BUS_ADDRESS: "unix:path=/run/user/1001/bus", + NEMOCLAW_E2E_MANAGED_IMAGE_REVISION: "a".repeat(40), + NEMOCLAW_GATEWAY_RUNTIME: "podman", + OPENSHELL_PODMAN_SOCKET: "/run/user/1001/podman/podman.sock", + XDG_RUNTIME_DIR: "/run/user/1001", + }); + }); + + it("probes native Podman through the workflow-owned socket authority", async () => { + const command = vi.fn().mockResolvedValue(successfulProbe()); + const environment = { + HOME: "/home/runner", + PATH: "/reviewed/bin:/usr/bin", + NEMOCLAW_GATEWAY_RUNTIME: "podman", + OPENSHELL_PODMAN_SOCKET: "/run/user/1001/podman/podman.sock", + XDG_RUNTIME_DIR: "/run/user/1001", + }; + + await ensureConfiguredRuntimeProviderAvailable({ + artifactName: "provider-info", + environment, + host: { command } as unknown as HostCliClient, + scenarioLabel: "Hermes GPU response validation", + skip: (reason) => { + throw new Error(reason); + }, + }); + + expect(command).toHaveBeenCalledWith( + "podman", + ["--url", "unix:///run/user/1001/podman/podman.sock", "info"], + expect.objectContaining({ + artifactName: "provider-info", + env: expect.objectContaining({ + NEMOCLAW_GATEWAY_RUNTIME: "podman", + OPENSHELL_PODMAN_SOCKET: "/run/user/1001/podman/podman.sock", + XDG_RUNTIME_DIR: "/run/user/1001", + }), + timeoutMs: 30_000, + }), + ); + expect(command.mock.calls[0]?.[2]?.env?.PATH).toContain(environment.PATH); + }); + + it("keeps the portable profile on its existing Docker compatibility probe", async () => { + const command = vi.fn().mockResolvedValue(successfulProbe()); + + await ensureConfiguredRuntimeProviderAvailable({ + artifactName: "portable-provider-info", + environment: { + HOME: "/home/runner", + PATH: "/usr/bin", + NEMOCLAW_EXPERIMENTAL_PROFILE: "portable", + NEMOCLAW_GATEWAY_RUNTIME: "podman", + }, + host: { command } as unknown as HostCliClient, + scenarioLabel: "portable-profile", + skip: (reason) => { + throw new Error(reason); + }, + }); + + expect(command).toHaveBeenCalledWith( + "docker", + ["info"], + expect.objectContaining({ artifactName: "portable-provider-info" }), + ); + }); + + it("executes fixture-owned root probes through the selected Podman resource", async () => { + const command = vi + .fn() + .mockResolvedValueOnce(successfulProbeWithStdout(`${"a".repeat(64)}\n`)) + .mockResolvedValueOnce(successfulProbe()); + const runtime = new RuntimeProviderPrerequisite( + { command } as unknown as HostCliClient, + (reason) => { + throw new Error(reason); + }, + { + HOME: "/home/runner", + PATH: "/usr/bin", + NEMOCLAW_GATEWAY_RUNTIME: "podman", + OPENSHELL_PODMAN_SOCKET: "/run/user/1001/podman/podman.sock", + }, + ); + + await runtime.execSandboxAsRoot("alpha", ["id", "-u"], { + artifactName: "privileged-id", + sanitizeEnvironment: true, + }); + + expect(command).toHaveBeenNthCalledWith( + 1, + "podman", + [ + "--url", + "unix:///run/user/1001/podman/podman.sock", + "container", + "ps", + "--all", + "--no-trunc", + "--filter", + "label=openshell.ai/sandbox-name=alpha", + "--format", + "{{.ID}}", + ], + expect.objectContaining({ artifactName: "privileged-id-resource" }), + ); + expect(command).toHaveBeenNthCalledWith( + 2, + "podman", + expect.arrayContaining([ + "--url", + "unix:///run/user/1001/podman/podman.sock", + "container", + "exec", + "--user", + "root", + "a".repeat(64), + "id", + "-u", + ]), + expect.objectContaining({ artifactName: "privileged-id" }), + ); + }); +}); diff --git a/test/e2e/support/sandbox-name-workflow-boundary.test.ts b/test/e2e/support/sandbox-name-workflow-boundary.test.ts index fffbd2d96e0..fa2d2300ce4 100644 --- a/test/e2e/support/sandbox-name-workflow-boundary.test.ts +++ b/test/e2e/support/sandbox-name-workflow-boundary.test.ts @@ -26,10 +26,7 @@ describe("live workflow sandbox name boundary", () => { it("rejects overlong and unresolved optional-lane sandbox identities (#8497)", () => { const workflow = readYaml(".github/workflows/e2e.yaml"); workflow.jobs["hermes-gpu-startup"]!.strategy!.matrix = { - include: [ - { agent: "openclaw", sandbox_name: "e2e-overlong-optional-lane" }, - { agent: "hermes" }, - ], + scenario: ["e2e-overlong-optional-lane", {}], }; expect(validateWorkflowSandboxNames(workflow)).toEqual( diff --git a/test/e2e/support/security-posture.test.ts b/test/e2e/support/security-posture.test.ts index 992f837de06..4e2afe4168e 100644 --- a/test/e2e/support/security-posture.test.ts +++ b/test/e2e/support/security-posture.test.ts @@ -8,14 +8,14 @@ import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; +import type { RuntimeProviderPrivilegedSandboxCommandResult } from "../../../src/lib/onboard/runtime-provider/contract.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; import type { SandboxClient } from "../fixtures/clients/sandbox.ts"; import { assertSecurityPosture, - dockerRuntimeEndpointArgs, OPENSHELL_SUPERVISOR_CAPABILITY_MASK, + PODMAN_OPENSHELL_SUPERVISOR_CAPABILITY_MASK, type ProcessSecurityIdentity, - parseOpenShellContainerId, parseSplitProcessSecurityReport, SPLIT_PROCESS_SECURITY_PROBE, type SplitProcessSecurityReport, @@ -28,11 +28,8 @@ import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; const ZERO_CAPABILITIES = "0000000000000000"; const SUPERVISOR_EXECUTABLE = "/opt/openshell/bin/openshell-sandbox"; -const CONTAINER_ID = "a".repeat(64); const SANDBOX_NAME = "secure-sandbox"; -const SANDBOX_ID = "sandbox-id"; -const CONTAINER_NAME = `openshell-default--${SANDBOX_NAME}-${SANDBOX_ID}`; -const PORTABLE_DOCKER_HOST = "unix:///run/user/1000/podman/podman.sock"; +const RESOURCE_HANDLE = "opaque-runtime-resource"; const CONTROLLED_PROC_HARNESS = String.raw`import contextlib import grp import json @@ -546,6 +543,13 @@ describe("security posture fixture", () => { expect(validateSplitProcessSecurityReport(report)).toEqual(report); }); + it("accepts a root-only OpenShell supervisor without an engine-added sandbox group", () => { + const report = validReport(); + report.supervisor.status.groups = ["0"]; + + expect(validateSplitProcessSecurityReport(report)).toEqual(report); + }); + it.each([ { agent: "OpenClaw", @@ -567,26 +571,25 @@ describe("security posture fixture", () => { ], sandboxGid: 999, }, - ])("accepts the observed $agent process tree with one direct supervisor and secure descendants", ({ - observedProcEntries, - processes, - sandboxGid, - }) => { - const report = validReport(); - report.observedProcEntries = observedProcEntries; - report.sandboxUid = 998; - report.sandboxGid = sandboxGid; - report.supervisor.status.groups = ["0", String(sandboxGid)]; - setCurrentChildSupervisors( - report, - processes.map(({ pid, ppid, startTime }) => - validNemoclawStartProcess({ pid, ppid, sandboxGid, sandboxUid: 998, startTime }), - ), - ); + ])( + "accepts the observed $agent process tree with one direct supervisor and secure descendants", + ({ observedProcEntries, processes, sandboxGid }) => { + const report = validReport(); + report.observedProcEntries = observedProcEntries; + report.sandboxUid = 998; + report.sandboxGid = sandboxGid; + report.supervisor.status.groups = ["0", String(sandboxGid)]; + setCurrentChildSupervisors( + report, + processes.map(({ pid, ppid, startTime }) => + validNemoclawStartProcess({ pid, ppid, sandboxGid, sandboxUid: 998, startTime }), + ), + ); - expect(validateSplitProcessSecurityReport(report)).toEqual(report); - expect(parseSplitProcessSecurityReport(JSON.stringify(report))).toEqual(report); - }); + expect(validateSplitProcessSecurityReport(report)).toEqual(report); + expect(parseSplitProcessSecurityReport(JSON.stringify(report))).toEqual(report); + }, + ); it("accepts a nested canonical descendant tree", () => { const report = validReport(); @@ -636,28 +639,21 @@ describe("security posture fixture", () => { name: "group identity", }, { - error: /supervisor Groups expected exactly 0 1000/u, - mutate: (report) => { - report.supervisor.status.groups = ["0"]; - }, - name: "missing sandbox supplementary group", - }, - { - error: /supervisor Groups expected exactly 0 1000/u, + error: /supervisor Groups expected exactly 0 or 0 1000/u, mutate: (report) => { report.supervisor.status.groups = ["0", "44"]; }, name: "wrong sandbox supplementary group", }, { - error: /supervisor Groups expected exactly 0 1000/u, + error: /supervisor Groups expected exactly 0 or 0 1000/u, mutate: (report) => { report.supervisor.status.groups = ["0", "1000", "44"]; }, name: "extra supplementary group", }, { - error: /supervisor Groups expected exactly 0 1000/u, + error: /supervisor Groups expected exactly 0 or 0 1000/u, mutate: (report) => { report.supervisor.status.groups = ["0", "0"]; }, @@ -734,31 +730,31 @@ describe("security posture fixture", () => { ); }, }, - ])("rejects a census with $directCount direct nemoclaw-start child supervisors", ({ - directCount, - mutate, - }) => { - const report = validReport(); - mutate(report); + ])( + "rejects a census with $directCount direct nemoclaw-start child supervisors", + ({ directCount, mutate }) => { + const report = validReport(); + mutate(report); - expect(() => validateSplitProcessSecurityReport(report)).toThrow( - new RegExp(`found ${directCount}`, "u"), - ); - }); + expect(() => validateSplitProcessSecurityReport(report)).toThrow( + new RegExp(`found ${directCount}`, "u"), + ); + }, + ); - it.each([ - "202", - "303", - ])("rejects a repeated nemoclaw-start PID with reported start time %s", (startTime) => { - const report = validReport(); - const duplicatePid = structuredClone(report.childSupervisors[0]!); - duplicatePid.startTime = startTime; - report.childSupervisors.push(duplicatePid); + it.each(["202", "303"])( + "rejects a repeated nemoclaw-start PID with reported start time %s", + (startTime) => { + const report = validReport(); + const duplicatePid = structuredClone(report.childSupervisors[0]!); + duplicatePid.startTime = startTime; + report.childSupervisors.push(duplicatePid); - expect(() => validateSplitProcessSecurityReport(report)).toThrow( - /PID 42 appeared more than once/u, - ); - }); + expect(() => validateSplitProcessSecurityReport(report)).toThrow( + /PID 42 appeared more than once/u, + ); + }, + ); it.each([ { @@ -864,21 +860,18 @@ describe("security posture fixture", () => { }); }); - it.each([ - "capInh", - "capPrm", - "capEff", - "capBnd", - "capAmb", - ] as const)("rejects every nemoclaw-start process with a nonzero %s set", (field) => { - reportsWithEachNemoclawStartProcessFirst().forEach((report) => { - report.childSupervisors[0]!.status[field] = "0000000000000001"; + it.each(["capInh", "capPrm", "capEff", "capBnd", "capAmb"] as const)( + "rejects every nemoclaw-start process with a nonzero %s set", + (field) => { + reportsWithEachNemoclawStartProcessFirst().forEach((report) => { + report.childSupervisors[0]!.status[field] = "0000000000000001"; - expect(() => validateSplitProcessSecurityReport(report)).toThrow( - new RegExp(`nemoclaw-start process\\.${field} expected 0`, "u"), - ); - }); - }); + expect(() => validateSplitProcessSecurityReport(report)).toThrow( + new RegExp(`nemoclaw-start process\\.${field} expected 0`, "u"), + ); + }); + }, + ); it("rejects malformed and overflowing split-process reports", () => { expect(() => parseSplitProcessSecurityReport("not-json")).toThrow(/emitted invalid JSON/u); @@ -946,193 +939,92 @@ describe("security posture fixture", () => { ); }); - it("selects one exact OpenShell container identity", () => { - const row = `${CONTAINER_ID}\t${CONTAINER_NAME}\t${SANDBOX_ID}\tdefault\n`; - - expect(parseOpenShellContainerId(row, SANDBOX_NAME)).toBe(CONTAINER_ID); - }); - - it("derives Docker discovery from the privileged execution endpoint", () => { - expect(dockerRuntimeEndpointArgs(["exec", "--user", "root"])).toEqual([]); - expect( - dockerRuntimeEndpointArgs(["--host", PORTABLE_DOCKER_HOST, "exec", "--user", "root"]), - ).toEqual(["--host", PORTABLE_DOCKER_HOST]); - expect(() => dockerRuntimeEndpointArgs(["--host", "", "exec"])).toThrow( - /supported runtime endpoint/u, - ); - expect(() => dockerRuntimeEndpointArgs(["--context", "remote", "exec"])).toThrow( - /supported runtime endpoint/u, - ); - }); - - it.each([ - ["", /found 0/u], - [ - `${CONTAINER_ID}\t${CONTAINER_NAME}\t${SANDBOX_ID}\tdefault\n${"b".repeat(64)}\t${CONTAINER_NAME}\t${SANDBOX_ID}\tdefault`, - /found 2/u, - ], - [ - `abc\t${CONTAINER_NAME}\t${SANDBOX_ID}\tdefault`, - /unexpected OpenShell Docker container identity/u, - ], - [ - `${CONTAINER_ID}\twrong-name\t${SANDBOX_ID}\tdefault`, - /unexpected OpenShell Docker container identity/u, - ], - [ - `${CONTAINER_ID}\t${CONTAINER_NAME}\t${SANDBOX_ID}\tother`, - /unexpected OpenShell Docker container identity/u, - ], - [ - `${CONTAINER_ID}\t${CONTAINER_NAME}\tunsafe/id\tdefault`, - /unexpected OpenShell Docker container identity/u, - ], - ])("rejects a container selection that is absent, ambiguous, or inexact", (output, error) => { - expect(() => parseOpenShellContainerId(output, SANDBOX_NAME)).toThrow(error); - }); - it.each([ - ["direct Docker", []], - ["portable container runtime", ["--host", PORTABLE_DOCKER_HOST]], - ])("checks the split-process report through %s before the remaining posture", async (_runtime, dockerEndpointArgs) => { - vi.stubEnv("NEMOCLAW_E2E_SECURITY_POSTURE", "1"); - vi.stubEnv("NEMOCLAW_E2E_EXPECT_OPENSHELL_SPLIT_PROCESS", "1"); - vi.stubEnv("DOCKER_HOST", "unix:///run/trusted-docker.sock"); - vi.stubEnv("DOCKER_CONTEXT", "untrusted-context"); - vi.stubEnv("DOCKER_CONFIG", "/tmp/untrusted-docker-config"); - vi.stubEnv("DOCKER_TLS_VERIFY", "1"); - vi.stubEnv("DOCKER_CERT_PATH", "/tmp/untrusted-docker-certs"); - const report = validReport(); - const directChildSupervisor = report.childSupervisors[0]!; - setCurrentChildSupervisors(report, [ - validNemoclawStartProcess({ - pid: 43, - ppid: directChildSupervisor.pid, - startTime: "203", - }), - directChildSupervisor, - ]); - const containerRow = `${CONTAINER_ID}\t${CONTAINER_NAME}\t${SANDBOX_ID}\tdefault\n`; - const command = vi - .fn() - .mockResolvedValueOnce(successfulProbe("uid=1000 gid=1000\n")) - .mockResolvedValueOnce(successfulProbe(containerRow)) - .mockResolvedValueOnce(successfulProbe(JSON.stringify(report))); - const execShell = vi.fn(async () => successfulProbe()); - const host = { command } as unknown as HostCliClient; - const sandbox = { execShell } as unknown as SandboxClient; - const privilegedProbeArgs = [ - ...dockerEndpointArgs, - "exec", - "--env", - "LD_PRELOAD=", - "--env", - "PYTHONPATH=", - "--user", - "root", - CONTAINER_ID, - "/usr/bin/python3", - "-I", - "-c", - SPLIT_PROCESS_SECURITY_PROBE, - ]; - const privilegedExecArgv = vi.fn( - ( - _sandboxName: string, - _command: string[], - _stdin?: boolean, - _sanitizeEnvironment?: boolean, - _expectedContainerId?: string, - ) => privilegedProbeArgs, - ); - - const summary = await assertSecurityPosture(host, sandbox, SANDBOX_NAME, "openclaw", { - privilegedExecArgv, - }); + { capabilityMask: OPENSHELL_SUPERVISOR_CAPABILITY_MASK, providerId: "docker" }, + { capabilityMask: PODMAN_OPENSHELL_SUPERVISOR_CAPABILITY_MASK, providerId: "podman" }, + ])( + "checks the split-process report through the selected $providerId provider", + async ({ capabilityMask, providerId }) => { + vi.stubEnv("NEMOCLAW_E2E_SECURITY_POSTURE", "1"); + vi.stubEnv("NEMOCLAW_E2E_EXPECT_OPENSHELL_SPLIT_PROCESS", "1"); + const report = validReport(); + report.supervisor.status.capBnd = capabilityMask; + report.supervisor.status.capEff = capabilityMask; + report.supervisor.status.capPrm = capabilityMask; + const directChildSupervisor = report.childSupervisors[0]!; + setCurrentChildSupervisors(report, [ + validNemoclawStartProcess({ + pid: 43, + ppid: directChildSupervisor.pid, + startTime: "203", + }), + directChildSupervisor, + ]); + const command = vi + .fn() + .mockResolvedValueOnce(successfulProbe("uid=1000 gid=1000\n")); + const execShell = vi.fn(async () => successfulProbe()); + const host = { command } as unknown as HostCliClient; + const sandbox = { execShell } as unknown as SandboxClient; + const resolvePrivilegedTarget = vi.fn(() => ({ + providerId, + resourceHandle: RESOURCE_HANDLE, + })); + const executePrivilegedCommand = vi.fn((): RuntimeProviderPrivilegedSandboxCommandResult => ({ + status: 0, + signal: null, + stdout: Buffer.from(JSON.stringify(report), "utf8"), + stderr: Buffer.alloc(0), + })); + + const summary = await assertSecurityPosture(host, sandbox, SANDBOX_NAME, "openclaw", { + executePrivilegedCommand, + resolvePrivilegedTarget, + }); - expect(summary).toEqual({ - configureGuard: true, - hostNonRoot: true, - rcFilesLocked: true, - runtimeProxyEnvLocked: true, - splitProcess: { - childSupervisor: directChildSupervisor, - supervisor: report.supervisor, - }, - startupLogClean: true, - }); - expect(command).toHaveBeenCalledTimes(3); - expect(command).toHaveBeenNthCalledWith( - 2, - "docker", - [ - ...dockerEndpointArgs, - "ps", - "--no-trunc", - "--filter", - "label=openshell.ai/managed-by=openshell", - "--filter", - `label=openshell.ai/sandbox-name=${SANDBOX_NAME}`, - "--format", - '{{.ID}}\t{{.Names}}\t{{.Label "openshell.ai/sandbox-id"}}\t{{.Label "openshell.ai/sandbox-workspace"}}', - ], - expect.objectContaining({ artifactName: "security-posture-container-identity" }), - ); - expect(command).toHaveBeenNthCalledWith( - 3, - "docker", - privilegedProbeArgs, - expect.objectContaining({ artifactName: "security-posture-split-processes" }), - ); - expect(privilegedExecArgv).toHaveBeenNthCalledWith( - 1, - SANDBOX_NAME, - ["/usr/bin/python3", "-I", "-c", SPLIT_PROCESS_SECURITY_PROBE], - false, - true, - ); - expect(privilegedExecArgv).toHaveBeenNthCalledWith( - 2, - SANDBOX_NAME, - ["/usr/bin/python3", "-I", "-c", SPLIT_PROCESS_SECURITY_PROBE], - false, - true, - CONTAINER_ID, - ); - [1, 2].forEach((callIndex) => { - const dockerEnv = command.mock.calls[callIndex]?.[2]?.env; - expect(dockerEnv).toMatchObject({ DOCKER_HOST: "unix:///run/trusted-docker.sock" }); - expect(dockerEnv).not.toHaveProperty("DOCKER_CONTEXT"); - expect(dockerEnv).not.toHaveProperty("DOCKER_CONFIG"); - expect(dockerEnv).not.toHaveProperty("DOCKER_TLS_VERIFY"); - expect(dockerEnv).not.toHaveProperty("DOCKER_CERT_PATH"); - }); - expect(execShell).toHaveBeenCalledTimes(4); - }); + expect(summary).toEqual({ + configureGuard: true, + hostNonRoot: true, + rcFilesLocked: true, + runtimeProxyEnvLocked: true, + splitProcess: { + childSupervisor: directChildSupervisor, + supervisor: report.supervisor, + }, + startupLogClean: true, + }); + expect(command).toHaveBeenCalledTimes(1); + expect(resolvePrivilegedTarget).toHaveBeenCalledTimes(2); + expect(executePrivilegedCommand).toHaveBeenCalledWith( + SANDBOX_NAME, + ["/usr/bin/python3", "-I", "-c", SPLIT_PROCESS_SECURITY_PROBE], + { + expectedResourceHandle: RESOURCE_HANDLE, + sanitizeEnvironment: true, + timeout: 30_000, + }, + ); + expect(execShell).toHaveBeenCalledTimes(4); + }, + ); - it("rejects container runtime endpoint drift before privileged inspection", async () => { + it("rejects runtime provider resource identity drift during privileged inspection", async () => { vi.stubEnv("NEMOCLAW_E2E_SECURITY_POSTURE", "1"); vi.stubEnv("NEMOCLAW_E2E_EXPECT_OPENSHELL_SPLIT_PROCESS", "1"); - const containerRow = `${CONTAINER_ID}\t${CONTAINER_NAME}\t${SANDBOX_ID}\tdefault\n`; const command = vi .fn() - .mockResolvedValueOnce(successfulProbe("uid=1000 gid=1000\n")) - .mockResolvedValueOnce(successfulProbe(containerRow)); + .mockResolvedValueOnce(successfulProbe("uid=1000 gid=1000\n")); const execShell = vi.fn(); - let invocation = 0; - const privilegedExecArgv = vi.fn( - ( - _sandboxName: string, - _command: string[], - _stdin?: boolean, - _sanitizeEnvironment?: boolean, - _expectedContainerId?: string, - ) => [ - "--host", - invocation++ === 0 ? "unix:///run/podman-a.sock" : "unix:///run/podman-b.sock", - "exec", - ], - ); + const resolvePrivilegedTarget = vi + .fn() + .mockReturnValueOnce({ providerId: "podman", resourceHandle: "first" }) + .mockReturnValueOnce({ providerId: "podman", resourceHandle: "second" }); + const executePrivilegedCommand = vi.fn((): RuntimeProviderPrivilegedSandboxCommandResult => ({ + status: 0, + signal: null, + stdout: Buffer.from(JSON.stringify(validReport()), "utf8"), + stderr: Buffer.alloc(0), + })); await expect( assertSecurityPosture( @@ -1140,36 +1032,32 @@ describe("security posture fixture", () => { { execShell } as unknown as SandboxClient, SANDBOX_NAME, "openclaw", - { privilegedExecArgv }, + { executePrivilegedCommand, resolvePrivilegedTarget }, ), - ).rejects.toThrow(/runtime endpoint changed/u); + ).rejects.toThrow(/runtime provider resource identity changed/u); - expect(command).toHaveBeenCalledTimes(2); + expect(command).toHaveBeenCalledTimes(1); + expect(executePrivilegedCommand).toHaveBeenCalledOnce(); expect(execShell).not.toHaveBeenCalled(); }); - it("rejects Docker environment drift before privileged inspection", async () => { + it("rejects a failed provider-owned privileged security probe", async () => { vi.stubEnv("NEMOCLAW_E2E_SECURITY_POSTURE", "1"); vi.stubEnv("NEMOCLAW_E2E_EXPECT_OPENSHELL_SPLIT_PROCESS", "1"); - vi.stubEnv("DOCKER_HOST", "unix:///run/docker-a.sock"); - const containerRow = `${CONTAINER_ID}\t${CONTAINER_NAME}\t${SANDBOX_ID}\tdefault\n`; const command = vi .fn() - .mockResolvedValueOnce(successfulProbe("uid=1000 gid=1000\n")) - .mockImplementationOnce(async () => { - vi.stubEnv("DOCKER_HOST", "unix:///run/docker-b.sock"); - return successfulProbe(containerRow); - }); + .mockResolvedValueOnce(successfulProbe("uid=1000 gid=1000\n")); const execShell = vi.fn(); - const privilegedExecArgv = vi.fn( - ( - _sandboxName: string, - _command: string[], - _stdin?: boolean, - _sanitizeEnvironment?: boolean, - _expectedContainerId?: string, - ) => ["exec"], - ); + const resolvePrivilegedTarget = vi.fn(() => ({ + providerId: "podman", + resourceHandle: RESOURCE_HANDLE, + })); + const executePrivilegedCommand = vi.fn((): RuntimeProviderPrivilegedSandboxCommandResult => ({ + status: 125, + signal: null, + stdout: Buffer.alloc(0), + stderr: Buffer.from("provider probe failed", "utf8"), + })); await expect( assertSecurityPosture( @@ -1177,12 +1065,12 @@ describe("security posture fixture", () => { { execShell } as unknown as SandboxClient, SANDBOX_NAME, "openclaw", - { privilegedExecArgv }, + { executePrivilegedCommand, resolvePrivilegedTarget }, ), - ).rejects.toThrow(/privileged Docker environment changed/u); + ).rejects.toThrow(/provider probe failed/u); - expect(privilegedExecArgv).toHaveBeenCalledTimes(1); - expect(command).toHaveBeenCalledTimes(2); + expect(command).toHaveBeenCalledTimes(1); + expect(executePrivilegedCommand).toHaveBeenCalledOnce(); expect(execShell).not.toHaveBeenCalled(); }); diff --git a/test/e2e/support/standard-profile-workflow-boundary.test.ts b/test/e2e/support/standard-profile-workflow-boundary.test.ts index 163de831b1a..d69b8c7c83e 100644 --- a/test/e2e/support/standard-profile-workflow-boundary.test.ts +++ b/test/e2e/support/standard-profile-workflow-boundary.test.ts @@ -208,7 +208,9 @@ describe("standard E2E execution profile", () => { CANDIDATE_REPOSITORY: "NVIDIA/NemoClaw", CANDIDATE_SHA: "a".repeat(40), CATALOGUE_ID: "hermes-inference-switch", + COVERAGE_VARIANT: "anthropic-podman", ENV: "/dev/null", + EXECUTION_ID: "hermes-inference-switch-anthropic-podman", GITHUB_ENV: githubEnvironment, GITHUB_OUTPUT: githubOutput, GITHUB_WORKSPACE_VALUE: directory, @@ -217,6 +219,7 @@ describe("standard E2E execution profile", () => { INSTALL_MODE: "credential-free", LC_ALL: "C", PATH: process.env.PATH ?? "", + RUNTIME_PROVIDER: "podman", SHARD: "anthropic", TARGET_ID: "hermes-inference-switch", TEST_FILE: "test/e2e/live/hermes-inference-switch.test.ts", @@ -231,11 +234,12 @@ describe("standard E2E execution profile", () => { expect(valid.status, valid.stderr).toBe(0); expect(fs.readFileSync(githubOutput, "utf8")).toBe( "artifact_directory=e2e-artifacts/live/hermes-inference-switch/anthropic\n" + - "upload_name=e2e-hermes-inference-switch-anthropic\n", + "upload_name=e2e-hermes-inference-switch-anthropic-podman\n", ); expect(fs.readFileSync(githubEnvironment, "utf8")).toBe( `E2E_ARTIFACT_DIR=${directory}/e2e-artifacts/live/hermes-inference-switch/anthropic\n` + - "NEMOCLAW_E2E_SHARD=anthropic\n", + "NEMOCLAW_E2E_SHARD=anthropic\n" + + "NEMOCLAW_GATEWAY_RUNTIME=podman\n", ); const unsafe = spawnSync("bash", [...shellArguments, planScript], { @@ -266,7 +270,10 @@ describe("standard E2E execution profile", () => { ARTIFACT_DIRECTORY: artifactDirectory, CANDIDATE_REPOSITORY: "NVIDIA/NemoClaw", CANDIDATE_SHA: "a".repeat(40), + COVERAGE_VARIANT: "default-podman", + EXECUTION_ID: "snapshot-commands-default-podman", JOB_STATUS: "success", + RUNTIME_PROVIDER: "podman", RUN_ATTEMPT: "2", RUN_ID: "123", TARGET_ID: "snapshot-commands", @@ -293,6 +300,9 @@ describe("standard E2E execution profile", () => { ).toEqual({ kind: "nemoclaw-e2e-evidence-v1", targetId: "snapshot-commands", + executionId: "snapshot-commands-default-podman", + coverageVariant: "default-podman", + runtimeProvider: "podman", candidate: { repository: "NVIDIA/NemoClaw", sha: "a".repeat(40) }, workflow: { repository: "NVIDIA/NemoClaw", diff --git a/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts b/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts index 7ae8b69ec2d..be196e73ce4 100644 --- a/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts +++ b/test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts @@ -230,16 +230,13 @@ describe("E2E artifact uploads", () => { expect(validateUploadE2eArtifactsInvocations(workflow)).toEqual( expect.arrayContaining([ - "messaging-providers upload-e2e-artifacts invocation must not override its contract", - "messaging-providers upload-e2e-artifacts must use the action defaults", - "messaging-providers default upload caller must declare a valid E2E_TARGET_ID", + "messaging-providers upload-e2e-artifacts must preserve its explicit name/path contract", "hermes-gpu-startup upload-e2e-artifacts must preserve its explicit name/path contract", "mcp-bridge upload-e2e-artifacts invocation must remain gated by its reviewed pre-upload checks", "openshell-gateway-auth-contract upload-e2e-artifacts invocation must remain gated by its reviewed pre-upload checks", "shared-e2e must not declare E2E_EXECUTION_PROFILE", "shared-e2e must not declare E2E_JOB", "shared-e2e upload-e2e-artifacts invocation must not override its contract", - "shared-e2e default upload caller E2E_TARGET_ID must be '${{ matrix.id }}'", "messaging-providers upload-e2e-artifacts invocation must follow artifact producers and precede only Docker auth cleanup", ]), ); diff --git a/test/e2e/support/workflow-plan-test-assertions.ts b/test/e2e/support/workflow-plan-test-assertions.ts new file mode 100644 index 00000000000..cba73c4e544 --- /dev/null +++ b/test/e2e/support/workflow-plan-test-assertions.ts @@ -0,0 +1,30 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + buildE2eWorkflowPlan, + releaseRequiredWorkflowJobs, + selectedWorkflowJobs, +} from "../../../tools/e2e/workflow-plan.mts"; + +export function expectedWorkflowPlanCiOutput( + plan: ReturnType, +): string { + return [ + `matrix=${JSON.stringify(plan.matrix)}`, + `test_matrix=${JSON.stringify(plan.testMatrix)}`, + `catalogue_standard_matrix=${JSON.stringify(plan.catalogueMatrices.standard)}`, + `catalogue_nvidia_api_matrix=${JSON.stringify(plan.catalogueMatrices["nvidia-api"])}`, + `catalogue_nvidia_inference_matrix=${JSON.stringify(plan.catalogueMatrices["nvidia-inference"])}`, + `catalogue_github_read_matrix=${JSON.stringify(plan.catalogueMatrices["github-read"])}`, + `catalogue_brave_nvidia_inference_matrix=${JSON.stringify(plan.catalogueMatrices["brave-nvidia-inference"])}`, + `gateway_runtimes=${JSON.stringify(plan.gatewayRuntimes)}`, + `runtime_providers_by_job=${JSON.stringify(plan.runtimeProvidersByJob)}`, + `selected_jobs=${JSON.stringify(plan.selectedJobs)}`, + `selected_workflow_jobs=${JSON.stringify(selectedWorkflowJobs(plan))}`, + `hermes_selected=${plan.hermesSelected}`, + `explicit_only_jobs=${plan.explicitOnlyJobs.join(",")}`, + `release_required_jobs=${JSON.stringify(releaseRequiredWorkflowJobs())}`, + "", + ].join("\n"); +} diff --git a/test/e2e/support/workflow-plan.test.ts b/test/e2e/support/workflow-plan.test.ts index 58b3c89ab88..d401e0de046 100644 --- a/test/e2e/support/workflow-plan.test.ts +++ b/test/e2e/support/workflow-plan.test.ts @@ -10,6 +10,7 @@ import { describe, expect, it } from "vitest"; import { credentialFreeTestCoverage, + credentialFreeTestMatrix, discoverCredentialFreeTests, } from "../../../tools/e2e/credential-free-tests.mts"; import { E2E_AGENT_RUNTIMES } from "../../../tools/e2e/execution-coverage.mts"; @@ -37,9 +38,10 @@ import { REPO_ROOT } from "../fixtures/paths.ts"; import { listTargets } from "../registry/registry.ts"; import { buildLiveTargetMatrix } from "../registry/run.ts"; import { liveTargetSupport } from "../registry/runtime-support.ts"; +import { expectedWorkflowPlanCiOutput } from "./workflow-plan-test-assertions.ts"; const PLANNER_CLI = path.join(REPO_ROOT, "tools", "e2e", "workflow-plan.mts"); -const TSX = path.join(REPO_ROOT, "node_modules", ".bin", "tsx"); +const PLANNER_CLI_PREFIX = ["--import", "tsx", PLANNER_CLI]; function firstId(rows: readonly T[], label: string): string { expect(rows, `expected at least one ${label}`).not.toHaveLength(0); @@ -53,24 +55,6 @@ function retiredControllerSelectorIds(): string[] { return retiredIds; } -function expectedCiOutput(plan: ReturnType): string { - return [ - `matrix=${JSON.stringify(plan.matrix)}`, - `test_matrix=${JSON.stringify(plan.testMatrix)}`, - `catalogue_standard_matrix=${JSON.stringify(plan.catalogueMatrices.standard)}`, - `catalogue_nvidia_api_matrix=${JSON.stringify(plan.catalogueMatrices["nvidia-api"])}`, - `catalogue_nvidia_inference_matrix=${JSON.stringify(plan.catalogueMatrices["nvidia-inference"])}`, - `catalogue_github_read_matrix=${JSON.stringify(plan.catalogueMatrices["github-read"])}`, - `catalogue_brave_nvidia_inference_matrix=${JSON.stringify(plan.catalogueMatrices["brave-nvidia-inference"])}`, - `selected_jobs=${JSON.stringify(plan.selectedJobs)}`, - `selected_workflow_jobs=${JSON.stringify(selectedWorkflowJobs(plan))}`, - `hermes_selected=${plan.hermesSelected}`, - `explicit_only_jobs=${plan.explicitOnlyJobs.join(",")}`, - `release_required_jobs=${JSON.stringify(releaseRequiredWorkflowJobs())}`, - "", - ].join("\n"); -} - function prCandidatePlan( plan: ReturnType, ): ReturnType { @@ -87,9 +71,11 @@ function expectExplicitCatalogueCoverage(): void { describe("E2E workflow plan", () => { it("defaults to every release-required target and tagged credential-free test", () => { const plan = buildE2eWorkflowPlan(); - + expect(plan).toEqual(buildE2eWorkflowPlan({}, { gatewayRuntimes: ["docker"] })); expect(plan.matrix).toEqual(buildLiveTargetMatrix()); - expect(plan.testMatrix).toEqual(discoverCredentialFreeTests()); + expect(plan.testMatrix).toEqual( + credentialFreeTestMatrix(discoverCredentialFreeTests(), ["docker"]), + ); expect(Object.values(plan.catalogueMatrices).flat()).toHaveLength(E2E_TARGET_CATALOGUE.length); expect( plan.coverageMatrix.reduce>((counts, row) => { @@ -110,6 +96,28 @@ describe("E2E workflow plan", () => { }), ]); expect(plan.hermesSelected).toBe(true); + expect(plan.coverageMatrix).toHaveLength(92); + expect(selectedWorkflowJobs(plan)).toEqual([ + "catalogue-brave-nvidia-inference", + "catalogue-github-read", + "catalogue-nvidia-api", + "catalogue-nvidia-inference", + "catalogue-standard", + "cloud-onboard", + "hermes-e2e", + "hermes-gpu-startup", + "live", + "managed-image-multiarch-startup", + "managed-image-protected-runtime", + "mcp-bridge", + "mcp-bridge-dev", + "messaging-providers", + "openclaw-plugin-runtime-exdev", + "openshell-credential-generation-window", + "openshell-gateway-auth-contract", + "shared-e2e", + "staging-brev-launchable", + ]); expect(plan.explicitOnlyJobs).toEqual([ "staging-brev-launchable-identity", "external-gateway-health", @@ -121,6 +129,42 @@ describe("E2E workflow plan", () => { expect(releaseRequiredWorkflowJobs()).not.toContain("llama-cpp-dgx-spark-qualification"); }); + it("selects only native Podman-eligible executions when explicitly requested", () => { + const plan = buildE2eWorkflowPlan({}, { gatewayRuntimes: ["podman"] }); + const catalogueIds = Object.values(plan.catalogueMatrices) + .flat() + .map((row) => row.id); + expect(plan.matrix.map((row) => row.id)).toEqual([ + "ubuntu-policy-custom-missing-presets-negative", + "ubuntu-repo-cloud-openclaw", + ]); + expect(plan.testMatrix).toEqual([]); + expect(catalogueIds).toHaveLength(52); + expect(catalogueIds).not.toEqual( + expect.arrayContaining([ + "bootstrap-install-smoke", + "gateway-guard-recovery", + "rebuild-hermes", + "rebuild-openclaw", + ]), + ); + expect(catalogueIds.some((id) => id.startsWith("openshell-gateway-upgrade-"))).toBe(false); + expect(selectedWorkflowJobs(plan)).toEqual([ + "catalogue-brave-nvidia-inference", + "catalogue-github-read", + "catalogue-nvidia-api", + "catalogue-nvidia-inference", + "catalogue-standard", + "cloud-onboard", + "hermes-e2e", + "hermes-gpu-startup", + "live", + "mcp-bridge", + "mcp-bridge-dev", + "messaging-providers", + "openshell-credential-generation-window", + ]); + }); it("omits only targets whose optional credential is unavailable", () => { const plan = withoutUnavailableOptionalCredentialTargets(buildE2eWorkflowPlan(), new Set()); const braveRows = plan.catalogueMatrices["brave-nvidia-inference"].map((row) => row.id); @@ -307,6 +351,12 @@ describe("E2E workflow plan", () => { expect(() => validateE2eTargetCatalogue([{ ...target, artifactLayout: "unreviewed" as never }]), ).toThrow("invalid artifact layout"); + expect(() => validateE2eTargetCatalogue([{ ...target, gatewayRuntimes: [] }])).toThrow( + "invalid gateway runtime support", + ); + expect(() => + validateE2eTargetCatalogue([{ ...target, gatewayRuntimes: ["docker", "docker"] as never }]), + ).toThrow("invalid gateway runtime support"); expect(() => validateE2eTargetCatalogue([{ ...target, artifactLayout: "flat-shard", shard: "default" }]), ).toThrow("flat artifact layout requires a named shard"); @@ -332,7 +382,12 @@ describe("E2E workflow plan", () => { ], [ "bootstrap-install-smoke", - { profile: "nvidia-inference", restoreCli: false, compatibleApiKey: true }, + { + profile: "nvidia-inference", + restoreCli: false, + compatibleApiKey: true, + gatewayRuntimes: ["docker"], + }, ], [ "hermes-discord", @@ -473,7 +528,32 @@ describe("E2E workflow plan", () => { }, ); - expect(readFileSync(output, "utf8")).toBe(expectedCiOutput(plan)); + expect(readFileSync(output, "utf8")).toBe(expectedWorkflowPlanCiOutput(plan)); + expect(readFileSync(summary, "utf8")).toBe(renderE2eWorkflowPlanSummary(plan)); + } finally { + rmSync(directory, { force: true, recursive: true }); + } + }); + + it("uses the explicit Podman planner in the CI output path", () => { + const directory = mkdtempSync(path.join(tmpdir(), "nemoclaw-workflow-plan-podman-")); + const output = path.join(directory, "github-output"); + const summary = path.join(directory, "summary.md"); + const plan = buildE2eWorkflowPlan({}, { gatewayRuntimes: ["podman"] }); + try { + writeE2eWorkflowPlanCiOutput( + {}, + { + GITHUB_OUTPUT: output, + GITHUB_STEP_SUMMARY: summary, + INFERENCE_MODE: "mock", + NEMOCLAW_E2E_CREDENTIALS_ALLOWED: "true", + NEMOCLAW_E2E_EXPECTED_SHA: "a".repeat(40), + NEMOCLAW_GATEWAY_RUNTIMES: "podman", + }, + ); + + expect(readFileSync(output, "utf8")).toBe(expectedWorkflowPlanCiOutput(plan)); expect(readFileSync(summary, "utf8")).toBe(renderE2eWorkflowPlanSummary(plan)); } finally { rmSync(directory, { force: true, recursive: true }); @@ -641,6 +721,10 @@ describe("E2E workflow plan", () => { expect(plan).toEqual({ ...fullPlan, selectedJobs: [...fullPlan.selectedJobs, "jetson-nvmap-gpu"], + runtimeProvidersByJob: { + ...fullPlan.runtimeProvidersByJob, + "jetson-nvmap-gpu": ["none"], + }, }); }); @@ -775,7 +859,7 @@ describe("E2E workflow plan", () => { const summary = path.join(directory, "summary.md"); const plan = buildE2eWorkflowPlan({ jobs: "bootstrap-install-smoke" }); try { - const result = spawnSync(TSX, [PLANNER_CLI, "--ci-output"], { + const result = spawnSync(process.execPath, [...PLANNER_CLI_PREFIX, "--ci-output"], { cwd: REPO_ROOT, encoding: "utf8", env: { @@ -792,7 +876,7 @@ describe("E2E workflow plan", () => { }); expect(result.status, result.stderr).toBe(0); - expect(readFileSync(output, "utf8")).toBe(expectedCiOutput(plan)); + expect(readFileSync(output, "utf8")).toBe(expectedWorkflowPlanCiOutput(plan)); expect(readFileSync(summary, "utf8")).toBe( renderE2eWorkflowPlanSummary(plan, { includeCoverageAudit: false }), ); @@ -808,7 +892,7 @@ describe("E2E workflow plan", () => { const activeJobs = "cloud-onboard,security-posture"; const plan = buildE2eWorkflowPlan({ jobs: activeJobs }); try { - const result = spawnSync(TSX, [PLANNER_CLI, "--ci-output"], { + const result = spawnSync(process.execPath, [...PLANNER_CLI_PREFIX, "--ci-output"], { cwd: REPO_ROOT, encoding: "utf8", env: { @@ -825,7 +909,7 @@ describe("E2E workflow plan", () => { }); expect(result.status, result.stderr).toBe(0); - expect(readFileSync(output, "utf8")).toBe(expectedCiOutput(plan)); + expect(readFileSync(output, "utf8")).toBe(expectedWorkflowPlanCiOutput(plan)); expect(readFileSync(summary, "utf8")).toBe( renderE2eWorkflowPlanSummary(plan, { includeCoverageAudit: false }), ); @@ -841,6 +925,7 @@ describe("E2E workflow plan", () => { const output = path.join(directory, "github-output"); const summary = path.join(directory, "summary.md"); const plan: ReturnType = { + gatewayRuntimes: ["docker"], matrix: [], testMatrix: [], catalogueMatrices: { @@ -852,11 +937,12 @@ describe("E2E workflow plan", () => { }, coverageMatrix: [], selectedJobs: [], + runtimeProvidersByJob: {}, hermesSelected: false, explicitOnlyJobs: readFreeStandingJobsInventory().explicitOnlyJobs, }; try { - const result = spawnSync(TSX, [PLANNER_CLI, "--ci-output"], { + const result = spawnSync(process.execPath, [...PLANNER_CLI_PREFIX, "--ci-output"], { cwd: REPO_ROOT, encoding: "utf8", env: { @@ -872,7 +958,7 @@ describe("E2E workflow plan", () => { }); expect(result.status, result.stderr).toBe(0); - expect(readFileSync(output, "utf8")).toBe(expectedCiOutput(plan)); + expect(readFileSync(output, "utf8")).toBe(expectedWorkflowPlanCiOutput(plan)); expect(readFileSync(summary, "utf8")).toBe(renderE2eWorkflowPlanSummary(plan)); } finally { rmSync(directory, { force: true, recursive: true }); @@ -884,6 +970,7 @@ describe("E2E workflow plan", () => { "emits an empty shared plan for the Jetson dispatch %s selector (#8142)", (selector) => { expect(buildE2eWorkflowPlan({ [selector]: "jetson-nvmap-gpu" })).toEqual({ + gatewayRuntimes: ["docker"], matrix: [], testMatrix: [], catalogueMatrices: { @@ -895,6 +982,7 @@ describe("E2E workflow plan", () => { }, coverageMatrix: [], selectedJobs: ["jetson-nvmap-gpu"], + runtimeProvidersByJob: { "jetson-nvmap-gpu": ["none"] }, hermesSelected: false, explicitOnlyJobs: readFreeStandingJobsInventory().explicitOnlyJobs, }); @@ -906,6 +994,7 @@ describe("E2E workflow plan", () => { const output = path.join(directory, "github-output"); const summary = path.join(directory, "summary.md"); const plan: ReturnType = { + gatewayRuntimes: ["docker"], matrix: [], testMatrix: [], catalogueMatrices: { @@ -917,11 +1006,12 @@ describe("E2E workflow plan", () => { }, coverageMatrix: [], selectedJobs: [], + runtimeProvidersByJob: {}, hermesSelected: false, explicitOnlyJobs: readFreeStandingJobsInventory().explicitOnlyJobs, }; try { - const result = spawnSync(TSX, [PLANNER_CLI, "--ci-output"], { + const result = spawnSync(process.execPath, [...PLANNER_CLI_PREFIX, "--ci-output"], { cwd: REPO_ROOT, encoding: "utf8", env: { @@ -937,7 +1027,7 @@ describe("E2E workflow plan", () => { }); expect(result.status, result.stderr).toBe(0); - expect(readFileSync(output, "utf8")).toBe(expectedCiOutput(plan)); + expect(readFileSync(output, "utf8")).toBe(expectedWorkflowPlanCiOutput(plan)); expect(readFileSync(summary, "utf8")).toBe(renderE2eWorkflowPlanSummary(plan)); } finally { rmSync(directory, { force: true, recursive: true }); @@ -947,7 +1037,7 @@ describe("E2E workflow plan", () => { it("rejects the retired bootstrap job outside a PR controller checkout", () => { const directory = mkdtempSync(path.join(tmpdir(), "nemoclaw-workflow-plan-cli-")); try { - const result = spawnSync(TSX, [PLANNER_CLI, "--ci-output"], { + const result = spawnSync(process.execPath, [...PLANNER_CLI_PREFIX, "--ci-output"], { cwd: REPO_ROOT, encoding: "utf8", env: { @@ -1073,7 +1163,7 @@ describe("E2E workflow plan", () => { }, ); - expect(readFileSync(output, "utf8")).toBe(expectedCiOutput(plan)); + expect(readFileSync(output, "utf8")).toBe(expectedWorkflowPlanCiOutput(plan)); expect(readFileSync(summary, "utf8")).toBe( renderE2eWorkflowPlanSummary(plan, { includeCoverageAudit: false }), ); @@ -1104,6 +1194,18 @@ describe("E2E workflow plan", () => { } }); + it("rejects an unsupported gateway runtime before writing CI output", () => { + expect(() => + writeE2eWorkflowPlanCiOutput( + {}, + { + INFERENCE_MODE: "mock", + NEMOCLAW_GATEWAY_RUNTIME: "containerd", + }, + ), + ).toThrow("Invalid gateway runtimes: containerd"); + }); + it("requires changed-file evidence for push planning", () => { expect(() => writeE2eWorkflowPlanCiOutput( @@ -1133,10 +1235,12 @@ describe("E2E workflow plan", () => { expect(output.trim().split("\n")).toHaveLength(1); const parsed = JSON.parse(output); expect(Object.keys(parsed)).toEqual([ + "gatewayRuntimes", "matrix", "testMatrix", "catalogueMatrices", "selectedJobs", + "runtimeProvidersByJob", "hermesSelected", "explicitOnlyJobs", "coverageMatrix", @@ -1145,21 +1249,25 @@ describe("E2E workflow plan", () => { }); it("renders the selected targets and workflow jobs as a readable plan", () => { - const filtered = spawnSync(TSX, [PLANNER_CLI, "--summary", "--jobs", "hermes-e2e"], { - cwd: REPO_ROOT, - encoding: "utf8", - timeout: 30_000, - }); + const filtered = spawnSync( + process.execPath, + [...PLANNER_CLI_PREFIX, "--summary", "--jobs", "hermes-e2e"], + { + cwd: REPO_ROOT, + encoding: "utf8", + timeout: 30_000, + }, + ); expect(filtered.status, filtered.stderr).toBe(0); expect(filtered.stdout).toBe(`## E2E Execution Plan | Target or job | Agent runtime | Observable outcome | Environment or inference endpoint | Source | Unresolved reason | | --- | --- | --- | --- | --- | --- | -| \`hermes-e2e\` | hermes | Install onboarding health inference lifecycle dashboard and security succeed | Ubuntu; mock or NVIDIA hosted inference | retained-workflow | | +| \`hermes-e2e / docker\` | hermes | Install onboarding health inference lifecycle dashboard and security succeed | Ubuntu; mock or NVIDIA hosted inference | retained-workflow | | `); - const complete = spawnSync(TSX, [PLANNER_CLI, "--summary"], { + const complete = spawnSync(process.execPath, [...PLANNER_CLI_PREFIX, "--summary"], { cwd: REPO_ROOT, encoding: "utf8", timeout: 30_000, @@ -1167,26 +1275,26 @@ describe("E2E workflow plan", () => { expect(complete.status, complete.stderr).toBe(0); expect(complete.stdout).toContain( - "| `cloud-onboard` | openclaw | Public install onboarding hosted inference and security checks succeed | Ubuntu; NVIDIA hosted inference | retained-workflow | |", + "| `cloud-onboard / docker` | openclaw | Public install onboarding hosted inference and security checks succeed | Ubuntu; NVIDIA hosted inference | retained-workflow | |", ); expect(complete.stdout).toContain( - "| `ubuntu-repo-cloud-openclaw` | openclaw | Repository install onboarding and hosted inference succeed | Ubuntu Docker host; NVIDIA hosted inference | typed-registry | |", + "| `ubuntu-repo-cloud-openclaw / docker` | openclaw | Repository install onboarding and hosted inference succeed | Ubuntu managed-runtime host; NVIDIA hosted inference | typed-registry | |", ); expect(complete.stdout).toContain( - "| `vllm-docker-storage` | none | vLLM storage gate accepts and rejects the intended host states | Native Linux Docker host; no inference endpoint | shared-e2e | |", + "| `vllm-docker-storage / docker` | none | vLLM storage gate accepts and rejects the intended host states | Native Linux Docker host; no inference endpoint | shared-e2e | |", ); expect(complete.stdout).toContain( - "| `channels-add-remove` | openclaw | Messaging: adds and removes Telegram configuration | Ubuntu; no inference endpoint | catalogue | |", + "| `channels-add-remove / default-docker` | openclaw | Messaging: adds and removes Telegram configuration | Ubuntu; no inference endpoint | catalogue | |", ); expect(complete.stdout).toContain( - "| `model-router-provider-routed-inference` | openclaw | Inference: Model Router returns a provider-routed response | Ubuntu; NVIDIA API and Model Router | catalogue | |", + "| `model-router-provider-routed-inference / default-docker` | openclaw | Inference: Model Router returns a provider-routed response | Ubuntu; NVIDIA API and Model Router | catalogue | |", ); expect(complete.stdout).toContain( - "| `spark-install` | unresolved | Install: leaves NemoClaw and OpenShell usable after standard installation | Ubuntu; NVIDIA hosted inference | catalogue | The test asserts CLI usability but does not assert an agent runtime |", + "| `spark-install / default-runtime-agnostic` | unresolved | Install: leaves NemoClaw and OpenShell usable after standard installation | Ubuntu; NVIDIA hosted inference | catalogue | The test asserts CLI usability but does not assert an agent runtime |", ); expect(complete.stdout).toContain("### Repeated outcomes with distinct evidence"); expect(complete.stdout).toContain( - "| Repository install onboarding and hosted inference succeed | `ubuntu-repo-cloud-langchain-deepagents-code`, `ubuntu-repo-cloud-openclaw` | agent runtime |", + "| Repository install onboarding and hosted inference succeed | `ubuntu-repo-cloud-langchain-deepagents-code / docker`, `ubuntu-repo-cloud-openclaw / docker` | agent runtime and environment or inference endpoint |", ); expect(complete.stdout).toContain("### Intentional exclusions"); expect(complete.stdout).toContain( @@ -1214,8 +1322,8 @@ describe("E2E workflow plan", () => { it("reports CLI failures as workflow annotations", () => { const result = spawnSync( - TSX, - [PLANNER_CLI, "--jobs", "hermes-e2e", "--targets", "definitely-unknown-e2e-target"], + process.execPath, + [...PLANNER_CLI_PREFIX, "--jobs", "hermes-e2e", "--targets", "definitely-unknown-e2e-target"], { cwd: REPO_ROOT, encoding: "utf8", timeout: 30_000 }, ); @@ -1231,7 +1339,7 @@ describe("E2E workflow plan", () => { const summary = path.join(directory, "summary.md"); const plan = buildE2eWorkflowPlan({ jobs: testId }); try { - const result = spawnSync(TSX, [PLANNER_CLI, "--ci-output"], { + const result = spawnSync(process.execPath, [...PLANNER_CLI_PREFIX, "--ci-output"], { cwd: REPO_ROOT, encoding: "utf8", env: { @@ -1246,7 +1354,7 @@ describe("E2E workflow plan", () => { }); expect(result.status, result.stderr).toBe(0); - expect(readFileSync(output, "utf8")).toBe(expectedCiOutput(plan)); + expect(readFileSync(output, "utf8")).toBe(expectedWorkflowPlanCiOutput(plan)); expect(readFileSync(summary, "utf8")).toBe( renderE2eWorkflowPlanSummary(plan, { includeCoverageAudit: false }), ); @@ -1262,7 +1370,7 @@ describe("E2E workflow plan", () => { const target = "ubuntu-repo-cloud-langchain-deepagents-code"; const plan = buildE2eWorkflowPlan({ jobs: "cloud-onboard", targets: target }); try { - const result = spawnSync(TSX, [PLANNER_CLI, "--ci-output"], { + const result = spawnSync(process.execPath, [...PLANNER_CLI_PREFIX, "--ci-output"], { cwd: REPO_ROOT, encoding: "utf8", env: { @@ -1277,7 +1385,7 @@ describe("E2E workflow plan", () => { }); expect(result.status, result.stderr).toBe(0); - expect(readFileSync(output, "utf8")).toBe(expectedCiOutput(plan)); + expect(readFileSync(output, "utf8")).toBe(expectedWorkflowPlanCiOutput(plan)); expect(readFileSync(summary, "utf8")).toBe( renderE2eWorkflowPlanSummary(plan, { includeCoverageAudit: false }), ); diff --git a/test/helpers/destroy-flow-test-harness.ts b/test/helpers/destroy-flow-test-harness.ts index 13689869242..290bbb9e0d2 100644 --- a/test/helpers/destroy-flow-test-harness.ts +++ b/test/helpers/destroy-flow-test-harness.ts @@ -6,7 +6,7 @@ import { createRequire } from "node:module"; import { expect, type MockInstance, vi } from "vitest"; import type { SandboxDestroyExecutionResult } from "../../src/lib/actions/sandbox/destroy-execution"; import type { PreparedManagedLlamaCppRuntimeCleanup } from "../../src/lib/inference/local-model-profile/cleanup"; -import type { ManagedHermesStateVolumeCleanupResult } from "../../src/lib/onboard/managed-workload/hermes-state-volume"; +import type { ManagedAgentStateVolumeCleanupResult } from "../../src/lib/onboard/managed-workload/hermes-state-volume"; import type { Session } from "../../src/lib/state/onboard-session"; import type { RetainedSandboxRecoveryRecord } from "../../src/lib/state/onboard-session/retained-sandbox-recovery"; import type { SandboxEntry, SandboxWorkloadReceipt } from "../../src/lib/state/registry"; @@ -43,7 +43,7 @@ export type DestroyHarness = { cleanupManagedLlamaCppRuntimeForSandboxSpy: MockInstance; preparePortableDestroyAuthoritySpy: MockInstance; promptSpy: MockInstance; - removeManagedHermesStateVolumeSpy: MockInstance; + removeManagedAgentStateVolumesSpy: MockInstance; removeSandboxSpy: MockInstance; resolveRetainedSandboxRecoverySpy: MockInstance; retirePortableLifecycleReceiptSpy: MockInstance; @@ -101,7 +101,7 @@ type DestroyHarnessOptions = { hostLocalInferenceReceipt?: string | null; hostLocalInferenceProvenance?: SandboxEntry["hostLocalInferenceProvenance"]; liveListOutput?: string; - managedHermesStateVolumeCleanupResult?: ManagedHermesStateVolumeCleanupResult; + managedAgentStateVolumeCleanupResults?: readonly ManagedAgentStateVolumeCleanupResult[]; onPrepareManagedLlamaCppRuntimeCleanup?: () => void; preparedManagedLlamaCppRuntimeCleanup?: PreparedManagedLlamaCppRuntimeCleanup | null; mcpAddState?: "prepared"; @@ -566,9 +566,9 @@ export function createDestroyHarness(options: DestroyHarnessOptions = {}): Destr vi.spyOn(sandboxProviderCleanup, "emitProviderDetachResidualHint").mockImplementation( () => undefined, ); - const removeManagedHermesStateVolumeSpy = vi - .spyOn(sandboxProviderCleanup, "removeManagedHermesStateVolume") - .mockReturnValue(options.managedHermesStateVolumeCleanupResult ?? { status: "not-applicable" }); + const removeManagedAgentStateVolumesSpy = vi + .spyOn(sandboxProviderCleanup, "removeManagedAgentStateVolumes") + .mockReturnValue(options.managedAgentStateVolumeCleanupResults ?? []); const stopNimByNameSpy = vi.spyOn(nim, "stopNimContainerByName").mockImplementation(() => { if (options.stopInferenceError !== undefined) { throw new Error(options.stopInferenceError); @@ -685,7 +685,7 @@ export function createDestroyHarness(options: DestroyHarnessOptions = {}): Destr portableDestroyRevalidateSpy, portableDestroyVerifyAbsentSpy, promptSpy, - removeManagedHermesStateVolumeSpy, + removeManagedAgentStateVolumesSpy, removeSandboxSpy, resolveRetainedSandboxRecoverySpy, retirePortableLifecycleReceiptSpy, diff --git a/test/helpers/docker-state-mutation-harness.ts b/test/helpers/docker-state-mutation-harness.ts index 4a578ee255a..7335cb2268b 100644 --- a/test/helpers/docker-state-mutation-harness.ts +++ b/test/helpers/docker-state-mutation-harness.ts @@ -155,6 +155,8 @@ export interface DockerStateMutationHarnessOptions { readonly loseReleaseResponseOnce?: boolean; readonly signalHelperOnce?: boolean; readonly stateMountType?: "bind" | "volume"; + readonly podmanAmbiguousMounts?: boolean; + readonly podmanMaterializedImageMount?: boolean; readonly timeoutResponseCopyOnce?: boolean; } @@ -171,6 +173,66 @@ export interface DockerStateMutationHarnessState { supervisorStopped: boolean; } +function additionalInspectionMounts( + options: DockerStateMutationHarnessOptions, + state: DockerStateMutationHarnessState, +): Record[] { + const mounts: Record[] = []; + if (options.podmanMaterializedImageMount) { + mounts.push( + { + Type: "image", + Source: "ghcr.io/nvidia/openshell/supervisor:0.0.106", + Destination: "/opt/openshell/bin", + Mode: "", + RW: false, + Propagation: "rprivate", + }, + { + Type: "bind", + Source: + "/run/user/1000/containers/storage/overlay-containers/" + + `${DOCKER_STATE_MUTATION_RUNTIME_ID}/userdata/overlay/example/merge`, + Destination: "/opt/openshell/bin", + Mode: "", + RW: true, + Propagation: "rprivate", + }, + ); + } + if (options.podmanAmbiguousMounts) { + mounts.push( + { + Type: "bind", + Source: "/srv/first", + Destination: "/sandbox/ambiguous", + Mode: "", + RW: true, + Propagation: "rprivate", + }, + { + Type: "bind", + Source: "/srv/second", + Destination: "/sandbox/ambiguous", + Mode: "", + RW: true, + Propagation: "rprivate", + }, + ); + } + if (state.overlayProc) { + mounts.push({ + Type: "bind", + Source: "/proc", + Destination: "/proc", + Mode: "", + RW: true, + Propagation: "rprivate", + }); + } + return mounts; +} + function createContainerStateMutationHarness( providerId: "docker" | "podman", options: DockerStateMutationHarnessOptions = {}, @@ -188,7 +250,7 @@ function createContainerStateMutationHarness( : "/var/lib/openshell/alpha/hermes", mountType: stateMountType, sandboxId: SANDBOX_ID, - pidMode: "", + pidMode: providerId === "podman" ? "private" : "", privileged: false, overlayProc: false, supervisorStopped: false, @@ -257,8 +319,17 @@ function createContainerStateMutationHarness( }; const capture = vi.fn((_executable, args, _timeout, input) => { - const commandStart = args.findIndex((value) => value === "ps" || value === "container"); + const commandStart = args.findIndex( + (value) => value === "ps" || value === "container" || value === "info", + ); const command = commandStart < 0 ? [] : args.slice(commandStart); + if (command[0] === "info") { + return { + status: 0, + stdout: "/run/user/1000/containers/storage\n", + stderr: "", + }; + } if (command[0] === "ps") { return { status: 0, stdout: `${DOCKER_STATE_MUTATION_RUNTIME_ID}\n`, stderr: "" }; } @@ -273,7 +344,7 @@ function createContainerStateMutationHarness( false, false, state.runtimePid, - "openshell", + providerId === "podman" ? "true" : "openshell", "alpha", state.sandboxId, state.pidMode, @@ -297,18 +368,7 @@ function createContainerStateMutationHarness( RW: true, Propagation: "rprivate", }, - ...(state.overlayProc - ? [ - { - Type: "bind", - Source: "/proc", - Destination: "/proc", - Mode: "", - RW: true, - Propagation: "rprivate", - }, - ] - : []), + ...additionalInspectionMounts(options, state), ], ]), stderr: "", @@ -653,6 +713,10 @@ function createContainerStateMutationHarness( providerId, providerDisplayName: "Podman", engineOperation: "state-mutation", + runtimeIdInspectField: "ID", + privatePidMode: "private", + managedLabelKey: "openshell.managed", + managedLabelValue: "true", }); const sandbox: SandboxEntry = { name: "alpha", diff --git a/test/helpers/growth-guardrail-checks.ts b/test/helpers/growth-guardrail-checks.ts index f0679df4af8..e712b6e028e 100644 --- a/test/helpers/growth-guardrail-checks.ts +++ b/test/helpers/growth-guardrail-checks.ts @@ -83,15 +83,22 @@ function scriptKind(file: string): ts.ScriptKind { return /\.[cm]?js$/i.test(file) ? ts.ScriptKind.JS : ts.ScriptKind.TS; } +const parsedTestSources = new Map>(); + +function parseTestSource(file: string, source: string): ts.SourceFile { + const kind = scriptKind(file); + const cached = parsedTestSources.get(source)?.get(kind); + if (cached) return cached; + const sourceFile = ts.createSourceFile(file, source, ts.ScriptTarget.Latest, true, kind); + const byKind = parsedTestSources.get(source) ?? new Map(); + byKind.set(kind, sourceFile); + parsedTestSources.set(source, byKind); + return sourceFile; +} + function countIfStatements(file: string, source: string | null): number { if (source === null) return 0; - const sourceFile = ts.createSourceFile( - file, - source, - ts.ScriptTarget.Latest, - true, - scriptKind(file), - ); + const sourceFile = parseTestSource(file, source); let count = 0; function visit(node: ts.Node): void { if (ts.isIfStatement(node)) count += 1; @@ -175,13 +182,7 @@ function thinCallbackForwardingLoop(node: ts.FunctionLikeDeclaration): LoopState function countTestLoops(file: string, source: string | null): number { if (source === null) return 0; - const sourceFile = ts.createSourceFile( - file, - source, - ts.ScriptTarget.Latest, - true, - scriptKind(file), - ); + const sourceFile = parseTestSource(file, source); type LexicalScope = ts.SourceFile | ts.Block; const localFunctions = new Map>(); function enclosingScope(node: ts.Node): LexicalScope | null { diff --git a/test/helpers/hermes-shields-provider-consumer-harness.ts b/test/helpers/hermes-shields-provider-consumer-harness.ts index 557a8d6f85c..9f5f53da60c 100644 --- a/test/helpers/hermes-shields-provider-consumer-harness.ts +++ b/test/helpers/hermes-shields-provider-consumer-harness.ts @@ -83,6 +83,8 @@ export type HermesShieldsProviderConsumerHarness = { supportSpy: MockInstance; transitionSpy: MockInstance; verifyLockSpy: MockInstance; + preflightStateDirLockSpy: MockInstance; + verifyLockedStateDirPostureSpy: MockInstance; verifyStateDirMutablePostureSpy: MockInstance; cleanup: () => void; }; @@ -278,6 +280,12 @@ export function createHermesShieldsProviderConsumerHarness( const verifyLockSpy = vi .spyOn(verifyLock, "verifyShieldsLockState") .mockReturnValue({ issues: [] }); + const preflightStateDirLockSpy = vi + .spyOn(stateDirLock, "preflightStateDirLock") + .mockReturnValue([]); + const verifyLockedStateDirPostureSpy = vi + .spyOn(stateDirLock, "verifyLockedStateDirPosture") + .mockReturnValue([]); const verifyStateDirMutablePostureSpy = vi .spyOn(stateDirLock, "verifyStateDirMutablePosture") .mockImplementation(() => []); @@ -304,9 +312,13 @@ export function createHermesShieldsProviderConsumerHarness( vi.spyOn(policy, "verifyAppliedPolicyDocument").mockImplementation(() => undefined), registrySpy, vi - .spyOn(privilegedExec, "privilegedSandboxExecArgv") - .mockImplementation((_sandboxName: unknown, command: unknown) => command as string[]), + .spyOn(privilegedExec, "capturePrivilegedSandboxCommand") + .mockImplementation((_sandboxName: unknown, command: unknown) => + Buffer.from(dockerExec.dockerExecFileSync(command as string[])), + ), verifyLockSpy, + preflightStateDirLockSpy, + verifyLockedStateDirPostureSpy, verifyStateDirMutablePostureSpy, vi.spyOn(console, "log").mockImplementation(() => undefined), vi.spyOn(console, "error").mockImplementation(() => undefined), @@ -391,6 +403,8 @@ export function createHermesShieldsProviderConsumerHarness( supportSpy, transitionSpy, verifyLockSpy, + preflightStateDirLockSpy, + verifyLockedStateDirPostureSpy, verifyStateDirMutablePostureSpy, }; } diff --git a/test/helpers/hermes-unsafe-config-shields-harness.ts b/test/helpers/hermes-unsafe-config-shields-harness.ts index 74a1bf748d9..1893346de35 100644 --- a/test/helpers/hermes-unsafe-config-shields-harness.ts +++ b/test/helpers/hermes-unsafe-config-shields-harness.ts @@ -299,8 +299,9 @@ export function createHermesUnsafeConfigHarness( lifecycleGeneration: "legacy-generation", workload: { kind: "managed-image" }, })); - vi.spyOn(privilegedExec, "privilegedSandboxExecArgv").mockImplementation( - (_sandboxName: unknown, cmd: unknown) => cmd as string[], + vi.spyOn(privilegedExec, "capturePrivilegedSandboxCommand").mockImplementation( + (_sandboxName: unknown, cmd: unknown) => + Buffer.from(dockerExec.dockerExecFileSync(cmd as string[])), ); vi.spyOn(stateDirLock, "applyStateDirLockMode").mockReturnValue([]); vi.spyOn(stateDirLock, "preflightStateDirLock").mockReturnValue([]); diff --git a/test/helpers/installer-readiness-stubs.ts b/test/helpers/installer-readiness-stubs.ts index 38531aa3c5a..49578c760c4 100644 --- a/test/helpers/installer-readiness-stubs.ts +++ b/test/helpers/installer-readiness-stubs.ts @@ -42,9 +42,11 @@ export function writeFailedOnboardSession(home: string): void { export function writeInstallerReadinessModuleStubs(readinessDir: string): void { const onboardDir = path.join(path.dirname(readinessDir), "onboard"); const experimentalDir = path.join(onboardDir, "experimental"); + const runtimeProviderDir = path.join(onboardDir, "runtime-provider"); fs.mkdirSync(readinessDir, { recursive: true }); fs.mkdirSync(onboardDir, { recursive: true }); fs.mkdirSync(experimentalDir, { recursive: true }); + fs.mkdirSync(runtimeProviderDir, { recursive: true }); fs.writeFileSync( `${readinessDir}/host.js`, `exports.createHostReadinessReport = (_options, collection) => ({ host: collection.assess() });\n`, @@ -99,6 +101,15 @@ export function writeInstallerReadinessModuleStubs(readinessDir: string): void { `${experimentalDir}/portable-profile.js`, `exports.isPortableExperimentalProfile = (env = process.env) => env.NEMOCLAW_EXPERIMENTAL_PROFILE === "portable";\n`, ); + fs.writeFileSync( + `${runtimeProviderDir}/selection.js`, + `exports.resolveConfiguredRuntimeProvider = () => ({ + gateway: { + supported: true, + prepareHostRuntime: () => ({ sandboxHostAddress: null }), + }, +});\n`, + ); } export function runStorageRemediationInstallerPreflight({ diff --git a/test/helpers/managed-image-buildless-e2e.ts b/test/helpers/managed-image-buildless-e2e.ts index b9e73149d87..e16701b71e5 100644 --- a/test/helpers/managed-image-buildless-e2e.ts +++ b/test/helpers/managed-image-buildless-e2e.ts @@ -458,6 +458,10 @@ runner.run = (command, options = {}) => { } return createdSandbox.run(command) ?? { status: 0, stdout: "", stderr: "" }; }; +const doctorHostCommand = require(${source("src/lib/actions/sandbox/doctor-host-command.ts")}); +replace(doctorHostCommand, "captureHostCommand", (command, args) => + runner.run([command, ...args]), +); runner.runFile = (file, args = []) => runner.run([file, ...args]); runner.runCapture = (command) => { const normalized = normalize(command); diff --git a/test/helpers/mcp-bridge-adapter-deepagents-fixture.ts b/test/helpers/mcp-bridge-adapter-deepagents-fixture.ts index 6d869d0a57b..b6a4a5d62c1 100644 --- a/test/helpers/mcp-bridge-adapter-deepagents-fixture.ts +++ b/test/helpers/mcp-bridge-adapter-deepagents-fixture.ts @@ -47,6 +47,7 @@ export function runDeepAgentsConfigCommand( initialLegacyConfig?: Record | string, initialLegacyMode = 0o600, managedOptions: DeepAgentsManagedFixtureOptions = {}, + runtimeEnvironment: NodeJS.ProcessEnv = {}, ): DeepAgentsConfigCommandResult { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-deepagents-mcp-")); const configPath = path.join(tmp, ".deepagents", ".nemoclaw-mcp.json"); @@ -90,7 +91,16 @@ export function runDeepAgentsConfigCommand( 'runtime_kind = "auto" # NEMOCLAW_DEEPAGENTS_RUNTIME_TEST_ANCHOR', `runtime_kind = "${runtimeKind}" # NEMOCLAW_DEEPAGENTS_RUNTIME_TEST_ANCHOR`, ); - const result = spawnSync("bash", ["-c", fixtureCommand], { encoding: "utf-8", timeout: 5000 }); + const canonicalEnvironment = Object.fromEntries( + [...command.matchAll(/openshell:resolve:env:([A-Za-z_][A-Za-z0-9_]*)/gu)].map( + ([, name]) => [name!, `openshell:resolve:env:${name!}`], + ), + ); + const result = spawnSync("bash", ["-c", fixtureCommand], { + encoding: "utf-8", + env: { ...process.env, ...canonicalEnvironment, ...runtimeEnvironment }, + timeout: 5000, + }); const configExists = fs.existsSync(configPath); const legacyConfigExists = fs.existsSync(legacyConfigPath); const configIsFifo = configExists && fs.lstatSync(configPath).isFIFO(); diff --git a/test/helpers/rebuild-flow-generic-harness.ts b/test/helpers/rebuild-flow-generic-harness.ts index 2b1fdb774c4..3bf33fc14ac 100644 --- a/test/helpers/rebuild-flow-generic-harness.ts +++ b/test/helpers/rebuild-flow-generic-harness.ts @@ -7,6 +7,7 @@ import { vi } from "vitest"; import type { RebuildRecreateOnboardOpts } from "../../src/lib/actions/sandbox/rebuild-gpu-opt-out"; import { agentDefs, + agentForwardStop, agentRuntime, buildContextFingerprint, createHarnessTempDir, @@ -193,6 +194,7 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): .spyOn(rebuildFlowHelpers, "warnUnpreservedUserManagedFiles") .mockImplementation(() => undefined); vi.spyOn(resolve, "resolveOpenshell").mockReturnValue(null); + vi.spyOn(agentForwardStop, "settleAgentForwardPortsForRebuild").mockReturnValue(true); vi.spyOn(agentDefs, "loadAgent").mockReturnValue(agentDef); vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue( agentDef.name === "openclaw" ? null : ({ name: agentDef.name } as never), diff --git a/test/helpers/rebuild-flow-harness.ts b/test/helpers/rebuild-flow-harness.ts index 3c045b64f09..f5d77d55208 100644 --- a/test/helpers/rebuild-flow-harness.ts +++ b/test/helpers/rebuild-flow-harness.ts @@ -26,6 +26,7 @@ delete require.cache[requireDist.resolve(rebuildModulePath)]; export const agentDefs = requireDist("../../agent/defs.js"); export const agentOnboard = requireDist("../../agent/onboard.js"); export const agentRuntime = requireDist("../../agent/runtime.js"); +export const agentForwardStop = requireDist("../../tunnel/agent-forward-stop.js"); export const buildContextFingerprint = requireDist( "../../adapters/fs/build-context-fingerprint.js", ); diff --git a/test/helpers/runtime-provider-bundle.ts b/test/helpers/runtime-provider-bundle.ts index ee2e5fd8e41..37468d0d21b 100644 --- a/test/helpers/runtime-provider-bundle.ts +++ b/test/helpers/runtime-provider-bundle.ts @@ -108,6 +108,7 @@ export function createInMemoryRuntimeProviderBundle({ status: "ok", detail: "ready", }), + validateSandboxGpu: () => undefined, preflightLifecycle: () => null, }, gateway: { @@ -115,6 +116,32 @@ export function createInMemoryRuntimeProviderBundle({ supported: true, launcher: gatewayLauncher, inspectLegacyContainer: false, + prepareHostRuntime: () => ({ + providerId, + openShellDriver: "memory", + bindAddress: "127.0.0.1", + grpcHost: "127.0.0.1", + sshGatewayHost: "127.0.0.1", + portCheckHost: "127.0.0.1", + socketPath: null, + requiredServerIpSans: [], + sandboxHostAddress: null, + usesHostGatewayRoute: false, + resourceOwnership: { label: "test.managed", value: providerId }, + gatewayConfig: { + sandboxNamespace: "scoped", + hostGatewayIp: null, + includeSupervisorBin: true, + processOwnership: "scoped-namespace", + }, + network: { + sandboxSourceCidrs: () => [], + inspect: () => undefined, + usesHostGatewayRoute: () => false, + run: () => ({ status: 0 }), + ensureProbeImageCached: () => ({ ok: true, alreadyCached: true }), + }, + }), }, workload: { providerId, @@ -151,6 +178,18 @@ export function createInMemoryRuntimeProviderBundle({ providerId, supported: true, channelStopTransport: "openshell", + privilegedSandboxControl: { + resolveTarget: ({ sandboxName }) => ({ + providerId, + resourceHandle: `in-memory:${sandboxName}`, + }), + execute: () => ({ + status: 0, + signal: null, + stdout: Buffer.alloc(0), + stderr: Buffer.alloc(0), + }), + }, start(input: RuntimeProviderLifecycleInput) { state.running.add(input.sandboxName); event("start", input.sandboxName); @@ -230,6 +269,7 @@ export function createInMemoryRuntimeProviderBundle({ { operation: "sandbox-lifecycle", engineId: "memory", displayName: "In-memory" }, { operation: "workload-cleanup", engineId: "memory", displayName: "In-memory" }, ], + capture: () => ({ status: 0, stdout: "", stderr: "" }), }, }; } diff --git a/test/helpers/shields-flow-harness.ts b/test/helpers/shields-flow-harness.ts index f90a46fe897..9a3c541b41d 100644 --- a/test/helpers/shields-flow-harness.ts +++ b/test/helpers/shields-flow-harness.ts @@ -388,17 +388,49 @@ export function createShieldsFlowHarness( vi.spyOn(privilegedExec, "isDirectSandboxFallbackUnavailableError").mockReturnValue( Boolean(options.directSandboxUnavailable), ); - vi.spyOn(privilegedExec, "privilegedSandboxExecArgv").mockImplementation( - (_sandboxName: unknown, cmd: unknown) => - options.directSandboxUnavailable - ? throwHarnessError(directSandboxUnavailableError) - : [ - "exec", - "--user", - "root", - "openshell-openclaw", - ...(Array.isArray(cmd) ? cmd.map(String) : []), - ], + const privilegedArgv = (cmd: unknown) => + options.directSandboxUnavailable + ? throwHarnessError(directSandboxUnavailableError) + : [ + "exec", + "--user", + "root", + "openshell-openclaw", + ...(Array.isArray(cmd) ? cmd.map(String) : []), + ]; + vi.spyOn(privilegedExec, "capturePrivilegedSandboxCommand").mockImplementation( + (_sandboxName: unknown, cmd: unknown, rawOptions: unknown) => + Buffer.from( + dockerExec.dockerExecFileSync(privilegedArgv(cmd), { + stdio: ["ignore", "pipe", "pipe"], + timeout: + rawOptions && typeof rawOptions === "object" && "timeout" in rawOptions + ? Number((rawOptions as { timeout?: unknown }).timeout) + : undefined, + }), + ), + ); + vi.spyOn(privilegedExec, "executePrivilegedSandboxCommand").mockImplementation( + (_sandboxName: unknown, cmd: unknown, rawOptions: unknown) => { + const result = dockerExec.dockerSpawnSync(privilegedArgv(cmd), { + encoding: "utf-8", + input: + rawOptions && typeof rawOptions === "object" && "input" in rawOptions + ? (rawOptions as { input?: unknown }).input + : undefined, + timeout: + rawOptions && typeof rawOptions === "object" && "timeout" in rawOptions + ? Number((rawOptions as { timeout?: unknown }).timeout) + : undefined, + }); + return { + status: result.status, + signal: result.signal, + stdout: Buffer.from(String(result.stdout ?? "")), + stderr: Buffer.from(String(result.stderr ?? "")), + ...(result.error ? { error: result.error } : {}), + }; + }, ); const dockerSpawnCalls: Array<{ args: string[]; timeout: number | undefined }> = []; vi.spyOn(dockerExec, "dockerSpawnSync").mockImplementation( diff --git a/test/inference/managed/managed-gateway-control-deadline.test.ts b/test/inference/managed/managed-gateway-control-deadline.test.ts index 4a154a9cb37..420b934d95d 100644 --- a/test/inference/managed/managed-gateway-control-deadline.test.ts +++ b/test/inference/managed/managed-gateway-control-deadline.test.ts @@ -332,9 +332,10 @@ def advance(duration, timeout=None): class ScriptedSocket: - def __init__(self, chunks, request_delay): + def __init__(self, chunks, request_delay, close_error=False): self.chunks = list(chunks) self.request_delay = request_delay + self.close_error = close_error self.timeout = None self.closed = False @@ -357,6 +358,8 @@ class ScriptedSocket: def close(self): self.closed = True + if self.close_error: + raise OSError("transport already closed") class ScriptedConnection(real_connection): @@ -365,6 +368,7 @@ class ScriptedConnection(real_connection): self.sock = ScriptedSocket( active["chunks"], active["request_delay"], + active.get("close_error", False), ) @@ -393,6 +397,12 @@ results = { "request_delay": 0.0, "chunks": [(0.0, complete)], }), + "healthy_close_race": check({ + "connect_delay": 0.0, + "request_delay": 0.0, + "chunks": [(0.0, complete)], + "close_error": True, + }), "unauthorized": check({ "connect_delay": 0.0, "request_delay": 0.0, @@ -455,6 +465,7 @@ describe("managed gateway recovery deadline", () => { it("applies one recovery deadline to every HTTP health check phase (#8262)", () => { expect(runHarness(HTTP_DEADLINE_HARNESS)).toEqual({ healthy: true, + healthy_close_race: true, slow_body: false, slow_connect: false, slow_headers: false, diff --git a/test/inference/managed/managed-gateway-control.test.ts b/test/inference/managed/managed-gateway-control.test.ts index 6ac1597b37c..99cd19ce982 100644 --- a/test/inference/managed/managed-gateway-control.test.ts +++ b/test/inference/managed/managed-gateway-control.test.ts @@ -1158,7 +1158,7 @@ with tempfile.TemporaryDirectory() as root: lambda _reader, _supervisor: diagnostic_output_events ) control._control = lambda *_args: (_ for _ in ()).throw( - control.ControlError("SUPERVISOR_UNAVAILABLE", stage="await-replacement") + control.ControlError("GATEWAY_FAILED", stage="await-replacement") ) staged_stderr = io.StringIO() try: @@ -1443,7 +1443,7 @@ describe("managed gateway root control", () => { staged_diagnostic: [ 1, [ - "SUPERVISOR_UNAVAILABLE", + "GATEWAY_FAILED", "NEMOCLAW_CONTROL_STAGE=await-replacement", "NEMOCLAW_SUPERVISOR_PID=40", "NEMOCLAW_GATEWAY_PID=44", diff --git a/test/inference/managed/managed-gateway-namespace-health.test.ts b/test/inference/managed/managed-gateway-namespace-health.test.ts new file mode 100644 index 00000000000..a31254f7443 --- /dev/null +++ b/test/inference/managed/managed-gateway-namespace-health.test.ts @@ -0,0 +1,42 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import path from "node:path"; +import { expect, it } from "vitest"; + +const HELPER = path.join(import.meta.dirname, "../../..", "scripts", "managed-gateway-control.py"); + +const SAME_NAMESPACE_HARNESS = String.raw` +import importlib.util +import json +import os +import sys + +spec = importlib.util.spec_from_file_location("managed_control_namespace", sys.argv[1]) +control = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = control +spec.loader.exec_module(control) + +setns_calls = [] +control.os.setns = lambda *_args: setns_calls.append(True) +control._http_healthy = lambda *_args: True +with control.ProcReader() as reader: + identity = reader.capture(os.getpid()) + result = control._http_healthy_in_gateway_namespace(reader, identity, 18642, "/health") + +print(json.dumps([result, len(setns_calls)])) +`; + +it.runIf(process.platform === "linux")( + "probes directly when controller and gateway share a network namespace", + () => { + const result = spawnSync("python3", ["-c", SAME_NAMESPACE_HARNESS, HELPER], { + encoding: "utf-8", + timeout: 10_000, + }); + + expect(result.status, result.stderr).toBe(0); + expect(JSON.parse(result.stdout)).toEqual([true, 0]); + }, +); diff --git a/test/inference/managed/managed-image-protected-runtime-contract.test.ts b/test/inference/managed/managed-image-protected-runtime-contract.test.ts index d338b17447a..7f00e365dae 100644 --- a/test/inference/managed/managed-image-protected-runtime-contract.test.ts +++ b/test/inference/managed/managed-image-protected-runtime-contract.test.ts @@ -31,10 +31,12 @@ import { managedImageOpenShellCommittedProbe, managedImageOpenShellProbe, parseManagedImageOpenShellE2eInputs, + protectedManagedStateRootDriverConfig, removeManagedImageGatewayStateIfSafe, resolveManagedImageOnboardModule, } from "../../../scripts/checks/run-managed-image-openshell-e2e.ts"; import { resolveOnboardManagedBootstrapLaunch } from "../../../src/lib/onboard/managed-workload/onboard-orchestration.js"; +import type { RuntimeProviderBundle } from "../../../src/lib/onboard/runtime-provider/contract.ts"; const IMAGE = `localhost:5000/nemoclaw-managed-protected/openclaw@sha256:${"a".repeat(64)}`; const VALID_SANDBOX = "managed-openclaw"; @@ -91,6 +93,26 @@ function createManagedImageCommandRunner( } describe("protected managed-image runtime contract", () => { + it("projects declared managed state roots through the selected provider driver", () => { + const mount = { + type: "volume" as const, + source: "nemoclaw-hermes-state-v1-alpha", + target: "/sandbox/.hermes", + read_only: false, + }; + const provider = { + workload: { managedStateMountDriverId: "docker" }, + } as Pick; + + expect(JSON.parse(protectedManagedStateRootDriverConfig(provider, [mount])!)).toEqual({ + docker: { mounts: [mount] }, + }); + expect(protectedManagedStateRootDriverConfig(provider, [])).toBeNull(); + expect(() => + protectedManagedStateRootDriverConfig({ workload: {} } as typeof provider, [mount]), + ).toThrow("provider-owned mount projection"); + }); + it("binds the rollback failure adapter to the canonical managed-bootstrap state root", async () => { const stateRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-protected-rollback-")); const journalRoot = path.join(stateRoot, "managed-bootstrap"); @@ -139,6 +161,7 @@ describe("protected managed-image runtime contract", () => { }, }, } as never, + sandboxName: "alpha", stateRoot: "/tmp/nemoclaw-state", bootstrapIdentity: "bootstrap-identity", request: {} as never, @@ -169,6 +192,15 @@ describe("protected managed-image runtime contract", () => { }, ); + it("loads managed state-volume operations through the existing onboard boundary", () => { + expect(MANAGED_IMAGE_ONBOARD.managedWorkloadOnboard.prepareManagedStateVolumes).toBeTypeOf( + "function", + ); + expect(MANAGED_IMAGE_ONBOARD.managedWorkloadOnboard.removeManagedStateVolumes).toBeTypeOf( + "function", + ); + }); + it("rejects a missing protected OpenShell operation with a precise contract error (#8759)", () => { expect(() => resolveManagedImageOnboardModule({ diff --git a/test/installer-integration/install-preflight-docker-bootstrap.test.ts b/test/installer-integration/install-preflight-docker-bootstrap.test.ts index 6ebdc5994e0..153f0b81de3 100644 --- a/test/installer-integration/install-preflight-docker-bootstrap.test.ts +++ b/test/installer-integration/install-preflight-docker-bootstrap.test.ts @@ -92,6 +92,53 @@ ensure_docker }; } + it.each([ + ["managed Docker", {}, ["PORTABLE_OVERRIDE", "ENSURE_DOCKER", "ENSURE_BUILD_DEPS"]], + [ + "native managed Podman", + { NEMOCLAW_GATEWAY_RUNTIME: "podman" }, + ["PORTABLE_OVERRIDE", "ENSURE_BUILD_DEPS"], + ], + [ + "portable experimental Podman compatibility", + { NEMOCLAW_EXPERIMENTAL_PROFILE: "portable", NEMOCLAW_GATEWAY_RUNTIME: "podman" }, + ["PORTABLE_OVERRIDE", "ENSURE_DOCKER", "ENSURE_BUILD_DEPS"], + ], + ] as const)("selects host bootstrap for %s", (_name, environment, expected) => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-runtime-bootstrap-")); + const result = spawnSync( + "bash", + [ + "--noprofile", + "--norc", + "-c", + ` +source "$INSTALLER_UNDER_TEST" >/dev/null +maybe_offer_express_install() { :; } +ensure_station_express_host() { :; } +prepare_portable_experimental_runtime_override() { printf 'PORTABLE_OVERRIDE\\n'; } +ensure_docker() { printf 'ENSURE_DOCKER\\n'; } +ensure_openshell_build_deps() { printf 'ENSURE_BUILD_DEPS\\n'; } +prepare_installer_host +`, + ], + { + cwd: tmp, + encoding: "utf-8", + env: { + HOME: tmp, + PATH: TEST_SYSTEM_PATH, + INSTALLER_UNDER_TEST: INSTALLER_PAYLOAD, + ...environment, + }, + }, + ); + const output = `${result.stdout}${result.stderr}`; + + expect(result.status, output).toBe(0); + expect(result.stdout.trim().split("\n")).toEqual(expected); + }); + it("reports when Docker is reachable for a non-docker-group Linux user", () => { const { result, sudoLog } = runEnsureDockerWithStubs({ dockerScript: `#!/usr/bin/env bash diff --git a/test/installer-integration/install-preflight.test.ts b/test/installer-integration/install-preflight.test.ts index 2909c1ccfde..2ec867c1c5d 100644 --- a/test/installer-integration/install-preflight.test.ts +++ b/test/installer-integration/install-preflight.test.ts @@ -186,7 +186,7 @@ if [ -n "\${1:-}" ] && [ -f "$1" ]; then exec ${JSON.stringify(process.execPath)} "$@" fi if [ "$1" = "-e" ]; then - exit 1 + exit 0 fi echo "unexpected node invocation: $*" >&2 exit 99 @@ -248,7 +248,7 @@ if [ "$1" = "--version" ]; then exit 0 fi if [ "$1" = "-e" ]; then - exit 1 + exit 0 fi echo "unexpected node invocation: $*" >&2 exit 99 @@ -1052,7 +1052,7 @@ if [ -n "\${1:-}" ] && [ -f "$1" ]; then exec ${JSON.stringify(process.execPath)} "$@" fi if [ "$1" = "-e" ]; then - exit 1 + exit 0 fi exit 99 `, @@ -1154,7 +1154,7 @@ if [ -n "\${1:-}" ] && [ -f "$1" ]; then exec ${JSON.stringify(process.execPath)} "$@" fi if [ "$1" = "-e" ]; then - exit 1 + exit 0 fi exit 99 `, @@ -2320,7 +2320,7 @@ if [ "$1" = "-v" ] || [ "$1" = "--version" ]; then echo "v22.19.0"; exit 0; fi if [ -n "\${1:-}" ] && [ -f "$1" ]; then exec ${JSON.stringify(process.execPath)} "$@" fi -if [ "$1" = "-e" ]; then exit 1; fi +if [ "$1" = "-e" ]; then exit 0; fi exit 99`, ); diff --git a/test/mcp/mcp-bridge-destroy-marker-recovery.test.ts b/test/mcp/mcp-bridge-destroy-marker-recovery.test.ts index 92e693793de..e7e6876fc26 100644 --- a/test/mcp/mcp-bridge-destroy-marker-recovery.test.ts +++ b/test/mcp/mcp-bridge-destroy-marker-recovery.test.ts @@ -575,7 +575,8 @@ bridge.removeMcpBridge("stuck-sandbox", "github", { force: true, allowResidual: `; const result = runNodeScript(home, script); expect(result.status).toBe(0); - expect(result.stderr).toContain("adapter cleanup failed (injected)"); + expect(result.stderr).toContain("MCP force cleanup reported"); + expect(result.stderr).not.toContain("adapter cleanup failed (injected)"); const jsonMarker = "<>"; const parsed = JSON.parse( result.stdout.slice(result.stdout.indexOf(jsonMarker) + jsonMarker.length), diff --git a/test/mcp/mcp-provider-ownership.test.ts b/test/mcp/mcp-provider-ownership.test.ts index 02c0acda9e5..06fe77ba0b3 100644 --- a/test/mcp/mcp-provider-ownership.test.ts +++ b/test/mcp/mcp-provider-ownership.test.ts @@ -608,7 +608,8 @@ bridge.removeMcpBridge("alpha", "fake", { force: true }).then( bridgePresent: boolean; }; expect(payload.message).toContain("registry entry was preserved"); - expect(result.stderr).toContain("Expected stable provider ID"); + expect(result.stderr).toContain("MCP force cleanup reported"); + expect(result.stderr).not.toContain("Expected stable provider ID"); expect(payload.calls.some((call) => call === "provider get alpha-mcp-fake")).toBe(true); expect(payload.bridgePresent).toBe(true); }); diff --git a/test/mcp/mcp-tool-discovery-image-contract.test.ts b/test/mcp/mcp-tool-discovery-image-contract.test.ts index edc4b8c1210..ae38aab5581 100644 --- a/test/mcp/mcp-tool-discovery-image-contract.test.ts +++ b/test/mcp/mcp-tool-discovery-image-contract.test.ts @@ -204,7 +204,7 @@ describe("MCP tool discovery image contract", () => { // source-shape-contract: security -- Exact reviewed runtime digests reject substituted executable and license artifacts before managed image construction. it.each([ { - expectedHash: "dac2be15941e6d3ee5092719d712d122b2cfbe0a159fdb70c11cb7bafa597d33", + expectedHash: "b62843823ffc1d72acdaece960f3536b9e2ef0b97677d3d566db5973cd431279", relativePath: "managed-startup-image-runtime.bundle", }, { diff --git a/test/onboarding/config-set-prompt-error.test.ts b/test/onboarding/config-set-prompt-error.test.ts index a30c2cba234..3d5ec18af22 100644 --- a/test/onboarding/config-set-prompt-error.test.ts +++ b/test/onboarding/config-set-prompt-error.test.ts @@ -100,8 +100,14 @@ async function runConfigSetWithPrompt(prompt: () => Promise) { validateOpenClawConfigCandidate: () => [], }); installMock(privilegedExecPath, { - privilegedSandboxExecArgv: () => ["docker", "exec", "container-id"], - resolveDirectSandboxContainer: () => "container-id", + capturePrivilegedSandboxCommand: () => Buffer.alloc(0), + executePrivilegedSandboxCommand: () => ({ + status: 0, + signal: null, + stdout: Buffer.alloc(0), + stderr: Buffer.alloc(0), + }), + resolvePrivilegedSandboxTarget: () => ({ resourceHandle: "container-id" }), withPrivilegedSandboxExecutionLease: ( _sandboxName: string, _operation: string, diff --git a/test/onboarding/config-set.test.ts b/test/onboarding/config-set.test.ts index 542887fef7b..ba1da934cd3 100644 --- a/test/onboarding/config-set.test.ts +++ b/test/onboarding/config-set.test.ts @@ -24,7 +24,9 @@ const { hermesCompatHashRecoveryError, isHermesCompatHashRecoveryError, } = require("../../src/lib/sandbox/config"); -const { selectDirectSandboxContainer } = require("../../src/lib/sandbox/privileged-exec"); +const { + selectDockerPrivilegedSandboxTarget: selectDirectSandboxContainer, +} = require("../../src/lib/onboard/runtime-provider/docker-privileged-sandbox-identity"); type MutableScalar = string | number | boolean | null | undefined; type MutableValue = MutableScalar | MutableMap | MutableValue[]; diff --git a/test/onboarding/effective-policy-contracts.test.ts b/test/onboarding/effective-policy-contracts.test.ts index a18d4e4bf4f..b95ec3f9eb3 100644 --- a/test/onboarding/effective-policy-contracts.test.ts +++ b/test/onboarding/effective-policy-contracts.test.ts @@ -526,15 +526,22 @@ describe("effective built-in policy contracts", () => { }); it("composes Hermes-specific messaging mutation and runtime identity rules", () => { - const effective = composePresets(["discord", "slack", "wechat"], "hermes"); + const effective = composePresets(["telegram", "discord", "slack", "wechat", "teams"], "hermes"); + const telegram = requireNetworkPolicy(effective, "telegram"); const discord = requireNetworkPolicy(effective, "discord"); const slack = requireNetworkPolicy(effective, "slack"); const wechat = requireNetworkPolicy(effective, "wechat_bridge"); + const teams = requireNetworkPolicy(effective, "teams"); expectDistinctSlackCredentialSelectors(slack); - for (const policy of [discord, slack, wechat]) { + for (const policy of [telegram, discord, slack, wechat, teams]) { expect(binaries(policy)).toEqual( - expect.arrayContaining(["/usr/bin/python3*", "/opt/hermes/.venv/bin/python"]), + expect.arrayContaining([ + "/usr/bin/python3*", + "/usr/bin/python3.13", + "/opt/hermes/.venv/bin/python3", + "/opt/hermes/.venv/bin/python", + ]), ); } for (const host of ["gateway.discord.gg", "*.discord.gg"]) { diff --git a/test/onboarding/onboard-dashboard.test.ts b/test/onboarding/onboard-dashboard.test.ts index 3b5d4c00a7a..bc43de27f34 100644 --- a/test/onboarding/onboard-dashboard.test.ts +++ b/test/onboarding/onboard-dashboard.test.ts @@ -667,7 +667,7 @@ describe("onboard dashboard helpers", () => { expect(output).toMatch(/Browser:\n\s+https?:\/\/\S+/); expect(output).not.toContain("#token="); expect(output).not.toContain("dashboard-url --quiet"); - expect(output).toContain("then run: openclaw tui"); + expect(output).toContain("then run the configured interactive agent command"); }); it("offers launch first and keeps connect in the OpenClaw ready summary (#6006)", () => { @@ -680,7 +680,7 @@ describe("onboard dashboard helpers", () => { "", " Or open a sandbox shell first:", " nemoclaw my-gpt-claw connect", - " then run: openclaw tui", + " then run the configured interactive agent command", ].join("\n"), ); expect(output.indexOf("nemoclaw launch my-gpt-claw")).toBeLessThan( @@ -701,7 +701,7 @@ describe("onboard dashboard helpers", () => { "", " Or open a sandbox shell first:", " nemohermes my-hermes connect", - " then run: hermes", + " then run the configured interactive agent command", ].join("\n"), ); expect(output).not.toContain("openclaw tui"); @@ -720,7 +720,7 @@ describe("onboard dashboard helpers", () => { "", " Or open a sandbox shell first:", " nemoclaw my-dcode connect", - " then run: dcode", + " then run the configured interactive agent command", ].join("\n"), ); expect(output).not.toContain("openclaw tui"); diff --git a/test/onboarding/onboard-gateway-runtime.test.ts b/test/onboarding/onboard-gateway-runtime.test.ts index 59ee286c5c9..895bd5ed26e 100644 --- a/test/onboarding/onboard-gateway-runtime.test.ts +++ b/test/onboarding/onboard-gateway-runtime.test.ts @@ -85,18 +85,6 @@ describe("onboard gateway runtime helpers", () => { expect(linuxEnv.OPENSHELL_CLUSTER_IMAGE).toBeUndefined(); expect(linuxEnv.OPENSHELL_DOCKER_SUPERVISOR_IMAGE).toContain(":0.0.37"); - const darwinEnv = getDockerDriverGatewayEnv("openshell 0.0.37", "darwin"); - expect(darwinEnv.OPENSHELL_DRIVERS).toBe("docker"); - expect(darwinEnv.OPENSHELL_BIND_ADDRESS).toBe("127.0.0.1"); - expect(darwinEnv.OPENSHELL_GRPC_ENDPOINT).toBe("https://127.0.0.1:8080"); - expect(darwinEnv.OPENSHELL_LOCAL_TLS_DIR).toContain( - path.join("nemoclaw", "openshell-docker-gateway", "tls"), - ); - expect(darwinEnv.OPENSHELL_SSH_GATEWAY_HOST).toBe("127.0.0.1"); - expect(darwinEnv.OPENSHELL_DOCKER_SUPERVISOR_IMAGE).toContain(":0.0.37"); - expect(darwinEnv.OPENSHELL_DOCKER_SUPERVISOR_BIN).toBeUndefined(); - expect(darwinEnv.OPENSHELL_VM_DRIVER_STATE_DIR).toBeUndefined(); - const originalOverlayFix = process.env.NEMOCLAW_DISABLE_OVERLAY_FIX; process.env.NEMOCLAW_DISABLE_OVERLAY_FIX = "1"; try { diff --git a/test/onboarding/onboard-messaging.test.ts b/test/onboarding/onboard-messaging.test.ts index 320aa3c3462..9310b20004e 100644 --- a/test/onboarding/onboard-messaging.test.ts +++ b/test/onboarding/onboard-messaging.test.ts @@ -450,10 +450,8 @@ const { createSandbox } = require(${onboardPath}); NEMOCLAW_NON_INTERACTIVE: "1", }, }); - assert.equal(result.status, 0, result.stderr); const payload = parseStdoutJson(result.stdout); - assert.ok(payload.createCommand.command.includes("sandbox create")); assert.match(payload.createCommand.command, /--provider my-assistant-slack-bridge/); assert.match(payload.createCommand.command, /--provider my-assistant-slack-app/); @@ -463,6 +461,8 @@ const { createSandbox } = require(${onboardPath}); assert.deepEqual(payload.slackBinaryPaths, [ "/usr/local/bin/hermes", "/usr/bin/python3*", + "/usr/bin/python3.13", + "/opt/hermes/.venv/bin/python3", "/opt/hermes/.venv/bin/python", ]); assert.ok( diff --git a/test/onboarding/openshell-0.0.101-migration-review.test.ts b/test/onboarding/openshell-0.0.101-migration-review.test.ts index cab5284f4d1..276c6186a61 100644 --- a/test/onboarding/openshell-0.0.101-migration-review.test.ts +++ b/test/onboarding/openshell-0.0.101-migration-review.test.ts @@ -9,6 +9,9 @@ import { describe, expect, it } from "vitest"; import { buildDockerDriverGatewayConfigToml } from "../../src/lib/onboard/docker-driver-gateway-config.js"; import { PORTABLE_HOST_GATEWAY_IP } from "../../src/lib/onboard/experimental/portable-profile.js"; +import { CURRENT_RUNTIME_PROVIDER_BUNDLES } from "../../src/lib/onboard/runtime-provider/current.js"; +import { prepareNativePodmanGatewayHostRuntime } from "../../src/lib/onboard/runtime-provider/podman-runtime-surfaces.js"; +import { requireRuntimeProviderBundle } from "../../src/lib/onboard/runtime-provider/registry.js"; const repoRoot = path.resolve(import.meta.dirname, "../.."); const review = fs.readFileSync( @@ -200,21 +203,51 @@ describe("OpenShell 0.0.101 migration review", () => { OPENSHELL_EGRESS_ADAPTER: "unreviewed", OPENSHELL_VM_RUNTIME: "unreviewed", }; - const dockerToml = buildDockerDriverGatewayConfigToml({ + const dockerEnv = { ...untrustedNewSurfaceInputs, OPENSHELL_DRIVERS: "vm", OPENSHELL_GRPC_ENDPOINT: "https://127.0.0.1:8080", OPENSHELL_DOCKER_NETWORK_NAME: "openshell-docker", OPENSHELL_DOCKER_SUPERVISOR_IMAGE: "supervisor:test", - }); - const podmanToml = buildDockerDriverGatewayConfigToml({ + }; + const dockerProvider = requireRuntimeProviderBundle( + "docker", + CURRENT_RUNTIME_PROVIDER_BUNDLES, + ); + expect(dockerProvider.gateway.supported).toBe(true); + const dockerGateway = dockerProvider.gateway as Extract< + typeof dockerProvider.gateway, + { readonly supported: true } + >; + const dockerToml = buildDockerDriverGatewayConfigToml( + dockerEnv, + undefined, + undefined, + "nemoclaw", + dockerGateway.prepareHostRuntime({ + environment: process.env, + platform: process.platform, + }), + ); + const podmanEnv = { ...untrustedNewSurfaceInputs, OPENSHELL_DRIVERS: "podman", OPENSHELL_GRPC_ENDPOINT: `https://${PORTABLE_HOST_GATEWAY_IP}:8080`, OPENSHELL_DOCKER_NETWORK_NAME: "openshell-podman", OPENSHELL_DOCKER_SUPERVISOR_IMAGE: "supervisor:test", OPENSHELL_PODMAN_SOCKET: "/run/user/1001/podman/podman.sock", - }); + }; + const podmanToml = buildDockerDriverGatewayConfigToml( + podmanEnv, + undefined, + undefined, + "nemoclaw", + prepareNativePodmanGatewayHostRuntime({ + environment: process.env, + platform: "linux", + socketPath: podmanEnv.OPENSHELL_PODMAN_SOCKET, + }), + ); expect(dockerToml).toContain('compute_drivers = ["docker"]'); expect(dockerToml).toContain("[openshell.drivers.docker]"); diff --git a/test/onboarding/openshell-0.0.99-migration-review.test.ts b/test/onboarding/openshell-0.0.99-migration-review.test.ts index 0f605c74eef..f860b165199 100644 --- a/test/onboarding/openshell-0.0.99-migration-review.test.ts +++ b/test/onboarding/openshell-0.0.99-migration-review.test.ts @@ -171,6 +171,7 @@ describe("OpenShell 0.0.99 migration review", () => { }, }, } as never, + sandboxName: "alpha", stateRoot: "/tmp/nemoclaw-state", bootstrapIdentity: "bootstrap-identity", request: {} as never, diff --git a/test/package-contract/cli/config-set-prompt-eof.test.ts b/test/package-contract/cli/config-set-prompt-eof.test.ts index 6d2d366d3f1..b6d46ce2268 100644 --- a/test/package-contract/cli/config-set-prompt-eof.test.ts +++ b/test/package-contract/cli/config-set-prompt-eof.test.ts @@ -83,8 +83,14 @@ function runConfigSetWithInput(input: string) { " }),", "});", "install(" + PRIVILEGED_EXEC_PATH + ", {", - ' privilegedSandboxExecArgv: () => ["docker", "exec", "container-id"],', - ' resolveDirectSandboxContainer: () => "container-id",', + " capturePrivilegedSandboxCommand: () => Buffer.alloc(0),", + " executePrivilegedSandboxCommand: () => ({", + " status: 0,", + " signal: null,", + " stdout: Buffer.alloc(0),", + " stderr: Buffer.alloc(0),", + " }),", + ' resolvePrivilegedSandboxTarget: () => ({ resourceHandle: "container-id" }),', " withPrivilegedSandboxExecutionLease: (_sandboxName, _operation, callback) => callback(),", "});", "", diff --git a/test/package-contract/installer-host-preflight.test.ts b/test/package-contract/installer-host-preflight.test.ts index d7ce6c4769f..1734d1ebd5c 100644 --- a/test/package-contract/installer-host-preflight.test.ts +++ b/test/package-contract/installer-host-preflight.test.ts @@ -38,8 +38,10 @@ function runInstallerHostAdmissionTest( forcedRejection?: { findingIds: string[]; capabilityIds: string[] }, options: { experimentalProfile?: string; + gatewayRuntime?: string; gatewayManagementMode?: string; portableProfileArtifact?: "present" | "missing"; + providerPreparationFailure?: string; } = {}, ) { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-installer-host-admission-")); @@ -48,10 +50,12 @@ function runInstallerHostAdmissionTest( const onboardDir = path.join(sourceRoot, "dist", "lib", "onboard"); const experimentalDir = path.join(onboardDir, "experimental"); const readinessDir = path.join(sourceRoot, "dist", "lib", "readiness"); + const runtimeProviderDir = path.join(onboardDir, "runtime-provider"); fs.mkdirSync(fakeBin); fs.mkdirSync(onboardDir, { recursive: true }); fs.mkdirSync(experimentalDir, { recursive: true }); fs.mkdirSync(readinessDir, { recursive: true }); + fs.mkdirSync(runtimeProviderDir, { recursive: true }); fs.writeFileSync( path.join(onboardDir, "preflight.js"), @@ -87,6 +91,22 @@ exports.loadGatewayManagementDeclaration = () => ({ for (const [artifactPath, contents] of portableProfileArtifacts) { fs.writeFileSync(artifactPath, contents); } + fs.writeFileSync( + path.join(runtimeProviderDir, "selection.js"), + `const preparationFailure = ${JSON.stringify(options.providerPreparationFailure ?? null)}; +exports.resolveConfiguredRuntimeProvider = () => ({ + gateway: { + supported: true, + prepareHostRuntime: () => { + if (preparationFailure) throw new Error(preparationFailure); + return { + sandboxHostAddress: + process.env.NEMOCLAW_GATEWAY_RUNTIME === "podman" ? "169.254.2.2" : null, + }; + }, + }, +});\n`, + ); fs.writeFileSync( path.join(readinessDir, "host.js"), `exports.createHostReadinessReport = (_options, collection) => { @@ -155,6 +175,7 @@ exports.evaluateOnboardReadinessAdmission = (report, options) => { const { NEMOCLAW_EXPERIMENTAL_PROFILE: _experimentalProfile, + NEMOCLAW_GATEWAY_RUNTIME: _gatewayRuntime, TEST_GATEWAY_MANAGEMENT_MODE: _gatewayManagementMode, ...inheritedEnv } = process.env; @@ -167,6 +188,7 @@ exports.evaluateOnboardReadinessAdmission = (report, options) => { ...(options.experimentalProfile ? { NEMOCLAW_EXPERIMENTAL_PROFILE: options.experimentalProfile } : {}), + ...(options.gatewayRuntime ? { NEMOCLAW_GATEWAY_RUNTIME: options.gatewayRuntime } : {}), TEST_GATEWAY_MANAGEMENT_MODE: options.gatewayManagementMode ?? "", }; @@ -226,6 +248,34 @@ describe("installer host preflight package contract", () => { expect(output).toMatch(/The detected container runtime is unsupported\./); }); + it("admits the unsupported host classifier through the selected managed runtime provider", () => { + const { output, result } = runInstallerHostAdmissionTest( + { + runtime: "podman", + isUnsupportedRuntime: true, + }, + undefined, + { gatewayRuntime: "podman" }, + ); + + expect(result.status, output).toBe(0); + expect(output).not.toMatch(/Host preflight found issues/); + }); + + it("fails closed when selected provider host preparation throws", () => { + const { output, result } = runInstallerHostAdmissionTest( + { runtime: "podman", isUnsupportedRuntime: true }, + undefined, + { + gatewayRuntime: "podman", + providerPreparationFailure: "native Podman address preparation failed", + }, + ); + + expect(result.status).toBe(1); + expect(output).toContain("native Podman address preparation failed"); + }); + it("keeps an unsupported runtime blocked without the portable classifier artifact (#9007)", () => { const { output, result } = runInstallerHostAdmissionTest( { diff --git a/test/process-recovery/process-recovery-managed-controller.test.ts b/test/process-recovery/process-recovery-managed-controller.test.ts index e07da6c342b..1b1c2571b9f 100644 --- a/test/process-recovery/process-recovery-managed-controller.test.ts +++ b/test/process-recovery/process-recovery-managed-controller.test.ts @@ -354,6 +354,7 @@ describe("managed gateway recovery controller", () => { ])( "enforces managed recovery for $label", ({ + label, recoverResults, expectedResult, expectedActions, @@ -413,6 +414,7 @@ beta 127.0.0.1 18789 12345 running`; name: "beta", agent: "openclaw", dashboardPort: 18789, + ...(label === "PID 1 supervisor" ? { openshellDriver: "podman" } : {}), }); vi.spyOn(openshellRuntime, "captureOpenshell").mockReturnValue({ status: 0, diff --git a/test/process-recovery/process-recovery-primitives.test.ts b/test/process-recovery/process-recovery-primitives.test.ts index ef4b3f2fcbd..cdd3fa9cb2e 100644 --- a/test/process-recovery/process-recovery-primitives.test.ts +++ b/test/process-recovery/process-recovery-primitives.test.ts @@ -288,12 +288,11 @@ describe("waitForManagedGatewaySupervisor", () => { }); describe("executeGatewaySupervisorAction", () => { - const controlPath = "/usr/local/bin/nemoclaw-gateway-control"; const targetContainerId = "a".repeat(64); it("sanitizes a temporarily unavailable direct container into the retry marker", () => { const privilegedExec = requireSource("../../src/lib/sandbox/privileged-exec.ts"); - vi.spyOn(privilegedExec, "privilegedSandboxExecArgv").mockImplementation(() => { + vi.spyOn(privilegedExec, "resolvePrivilegedSandboxTarget").mockImplementation(() => { throw new Error("temporary direct-container discovery detail"); }); vi.spyOn(privilegedExec, "isDirectSandboxFallbackUnavailableError").mockReturnValue(true); @@ -307,7 +306,7 @@ describe("executeGatewaySupervisorAction", () => { it("keeps other privileged-control refusals terminal and classified", () => { const privilegedExec = requireSource("../../src/lib/sandbox/privileged-exec.ts"); - vi.spyOn(privilegedExec, "privilegedSandboxExecArgv").mockImplementation(() => { + vi.spyOn(privilegedExec, "resolvePrivilegedSandboxTarget").mockImplementation(() => { throw new Error( "OpenShell container identity changed for sandbox 'new-clone'; refusing privileged execution against a different container.", ); @@ -324,7 +323,7 @@ describe("executeGatewaySupervisorAction", () => { it("emits the managed-control identity marker for a pinned container refusal (#9364)", () => { const privilegedExec = requireSource("../../src/lib/sandbox/privileged-exec.ts"); - vi.spyOn(privilegedExec, "privilegedSandboxExecArgv").mockImplementation(() => { + vi.spyOn(privilegedExec, "resolvePrivilegedSandboxTarget").mockImplementation(() => { throw new Error( "OpenShell container identity changed for sandbox 'new-clone'; refusing privileged execution against a different container.", ); @@ -341,21 +340,17 @@ describe("executeGatewaySupervisorAction", () => { }); it("binds an exact Docker restart transition to the selected container (#8726)", () => { - const dockerExec = requireSource("../../src/lib/adapters/docker/exec.ts"); const privilegedExec = requireSource("../../src/lib/sandbox/privileged-exec.ts"); - vi.spyOn(privilegedExec, "privilegedSandboxExecArgv").mockReturnValue([ - "exec", - "--user", - "root", - targetContainerId, - controlPath, - "probe", - "b".repeat(64), - ]); - vi.spyOn(dockerExec, "dockerSpawnSync").mockReturnValue({ + vi.spyOn(privilegedExec, "resolvePrivilegedSandboxTarget").mockReturnValue({ + resourceHandle: targetContainerId, + }); + vi.spyOn(privilegedExec, "executePrivilegedSandboxCommand").mockReturnValue({ status: 1, - stdout: "", - stderr: `Error response from daemon: Container ${targetContainerId} is restarting, wait until the container is running`, + signal: null, + stdout: Buffer.alloc(0), + stderr: Buffer.from( + `Error response from daemon: Container ${targetContainerId} is restarting, wait until the container is running`, + ), } as never); expect(executeGatewaySupervisorAction("new-clone", "probe", 100)).toEqual({ @@ -374,21 +369,17 @@ describe("executeGatewaySupervisorAction", () => { ])( "does not bind %s as a Docker restart transition (#8726)", (_case, status, stdout, id, suffix) => { - const dockerExec = requireSource("../../src/lib/adapters/docker/exec.ts"); const privilegedExec = requireSource("../../src/lib/sandbox/privileged-exec.ts"); - vi.spyOn(privilegedExec, "privilegedSandboxExecArgv").mockReturnValue([ - "exec", - "--user", - "root", - targetContainerId, - controlPath, - "probe", - "b".repeat(64), - ]); - vi.spyOn(dockerExec, "dockerSpawnSync").mockReturnValue({ + vi.spyOn(privilegedExec, "resolvePrivilegedSandboxTarget").mockReturnValue({ + resourceHandle: targetContainerId, + }); + vi.spyOn(privilegedExec, "executePrivilegedSandboxCommand").mockReturnValue({ status, - stdout, - stderr: `Error response from daemon: Container ${id} is restarting, wait until the container is running${suffix}`, + signal: null, + stdout: Buffer.from(stdout), + stderr: Buffer.from( + `Error response from daemon: Container ${id} is restarting, wait until the container is running${suffix}`, + ), } as never); expect(executeGatewaySupervisorAction("new-clone", "probe", 100)).toEqual({ @@ -733,7 +724,6 @@ describe("executeSandboxExecCommand", () => { it("honors the sandbox-exec timeout without falling back to SSH", () => { const childProcess = requireSource("node:child_process"); - const dockerExec = requireSource("../../src/lib/adapters/docker/exec.ts"); const privilegedExec = requireSource("../../src/lib/sandbox/privileged-exec.ts"); const timeoutError = Object.assign(new Error("timed out"), { code: "ETIMEDOUT" }); const spawn = vi.spyOn(childProcess, "spawnSync").mockReturnValue({ @@ -742,21 +732,15 @@ describe("executeSandboxExecCommand", () => { stderr: "", error: timeoutError, } as never); - vi.spyOn(privilegedExec, "privilegedSandboxExecArgv").mockReturnValue([ - "exec", - "--user", - "root", - "openshell-alpha", - "sh", - "-c", - "marked-command", - ]); - const dockerSpawnSync = vi.spyOn(dockerExec, "dockerSpawnSync").mockReturnValue({ - status: null, - stdout: "", - stderr: "", - error: timeoutError, - } as never); + const executePrivileged = vi + .spyOn(privilegedExec, "executePrivilegedSandboxCommand") + .mockReturnValue({ + status: null, + signal: null, + stdout: Buffer.alloc(0), + stderr: Buffer.alloc(0), + error: timeoutError, + } as never); const previousTimeout = process.env.NEMOCLAW_SANDBOX_EXEC_TIMEOUT_MS; process.env.NEMOCLAW_SANDBOX_EXEC_TIMEOUT_MS = "50"; @@ -768,7 +752,9 @@ describe("executeSandboxExecCommand", () => { expect(result).toBeNull(); expect(spawn.mock.calls.some(([command]) => command === "ssh")).toBe(false); expect(spawn.mock.calls[0]?.[2]).toEqual(expect.objectContaining({ timeout: 50 })); - expect(dockerSpawnSync.mock.calls[0]?.[1]).toEqual(expect.objectContaining({ timeout: 50 })); + expect(executePrivileged.mock.calls[0]?.[2]).toEqual( + expect.objectContaining({ sanitizeEnvironment: true, timeout: 50 }), + ); } finally { previousTimeout === undefined ? delete process.env.NEMOCLAW_SANDBOX_EXEC_TIMEOUT_MS @@ -797,7 +783,6 @@ describe("executeSandboxExecCommand", () => { it("rejects a non-frame preamble and surfaces a missing trusted fallback identity", () => { const childProcess = requireSource("node:child_process"); - const dockerExec = requireSource("../../src/lib/adapters/docker/exec.ts"); const privilegedExec = requireSource("../../src/lib/sandbox/privileged-exec.ts"); vi.spyOn(childProcess, "spawnSync").mockReturnValue({ status: 0, @@ -807,14 +792,12 @@ describe("executeSandboxExecCommand", () => { ].join("\n"), stderr: "", } as never); - const privilegedArgv = vi.spyOn(privilegedExec, "privilegedSandboxExecArgv"); - const dockerSpawnSync = vi.spyOn(dockerExec, "dockerSpawnSync"); + const executePrivileged = vi.spyOn(privilegedExec, "executePrivilegedSandboxCommand"); expect(() => withFakeOpenshellBinary(() => executeSandboxExecCommand("hermes-box", "echo RUNNING")), ).toThrow(/No NemoClaw registry entry found.*refusing privileged exec/); - expect(privilegedArgv).toHaveBeenCalledTimes(1); - expect(dockerSpawnSync).not.toHaveBeenCalled(); + expect(executePrivileged).toHaveBeenCalledTimes(1); }); it("keeps the Hermes validator source out of the host shell payload", () => { @@ -842,29 +825,20 @@ describe("executeSandboxExecCommand", () => { it("falls back to local Docker root exec when OpenShell exec output has no marker", () => { const childProcess = requireSource("node:child_process"); - const dockerExec = requireSource("../../src/lib/adapters/docker/exec.ts"); const privilegedExec = requireSource("../../src/lib/sandbox/privileged-exec.ts"); vi.spyOn(childProcess, "spawnSync").mockReturnValue({ status: 0, stdout: "OpenShell transport preamble\n", stderr: "", } as never); - const privilegedArgv = vi - .spyOn(privilegedExec, "privilegedSandboxExecArgv") - .mockReturnValue([ - "exec", - "--user", - "root", - "openshell-hermes-box-generated", - "sh", - "-c", - "marked-command", - ]); - const dockerSpawnSync = vi.spyOn(dockerExec, "dockerSpawnSync").mockReturnValue({ - status: 0, - stdout: "__NEMOCLAW_SANDBOX_EXEC_STARTED__\nSECRET_BOUNDARY_OK\n", - stderr: "", - } as never); + const executePrivileged = vi + .spyOn(privilegedExec, "executePrivilegedSandboxCommand") + .mockReturnValue({ + status: 0, + signal: null, + stdout: Buffer.from("__NEMOCLAW_SANDBOX_EXEC_STARTED__\nSECRET_BOUNDARY_OK\n"), + stderr: Buffer.alloc(0), + } as never); const priorSecret = process.env.TEST_MCP_RAW_TOKEN; const priorGateway = process.env.OPENSHELL_GATEWAY; @@ -881,37 +855,22 @@ describe("executeSandboxExecCommand", () => { : (process.env.OPENSHELL_GATEWAY = priorGateway); expect(result).toEqual({ status: 0, stdout: "SECRET_BOUNDARY_OK", stderr: "" }); - expect(privilegedArgv).toHaveBeenCalledWith("hermes-box", [ - "sh", - "-c", - expect.stringContaining("echo SECRET_BOUNDARY_OK"), - ]); - expect(dockerSpawnSync.mock.calls[0]?.[0]).toEqual([ - "exec", - "--user", - "root", - "openshell-hermes-box-generated", - "sh", - "-c", - "marked-command", - ]); - const dockerOptions = dockerSpawnSync.mock.calls[0]?.[1] as { env?: NodeJS.ProcessEnv }; - expect(dockerOptions.env?.TEST_MCP_RAW_TOKEN).toBeUndefined(); - expect(dockerOptions.env?.OPENSHELL_GATEWAY).toBe("nemoclaw-19080"); - expect(dockerOptions.env?.PATH).toBe(process.env.PATH); + expect(executePrivileged).toHaveBeenCalledWith( + "hermes-box", + ["sh", "-c", expect.stringContaining("echo SECRET_BOUNDARY_OK")], + expect.objectContaining({ sanitizeEnvironment: true }), + ); }); it("does not let Docker fallback satisfy a strict provider credential proof", () => { const childProcess = requireSource("node:child_process"); - const dockerExec = requireSource("../../src/lib/adapters/docker/exec.ts"); const privilegedExec = requireSource("../../src/lib/sandbox/privileged-exec.ts"); const spawn = vi.spyOn(childProcess, "spawnSync").mockReturnValue({ status: 1, stdout: "OpenShell transport failed before the child marker\n", stderr: "gateway unavailable\n", } as never); - const privilegedArgv = vi.spyOn(privilegedExec, "privilegedSandboxExecArgv"); - const dockerSpawnSync = vi.spyOn(dockerExec, "dockerSpawnSync"); + const executePrivileged = vi.spyOn(privilegedExec, "executePrivilegedSandboxCommand"); const result = withFakeOpenshellBinary(() => executeSandboxExecCommand("hermes-box", '[ -z "${FAKE_MCP_SECRET+x}" ]', undefined, { @@ -920,8 +879,7 @@ describe("executeSandboxExecCommand", () => { ); expect(result).toBeNull(); - expect(privilegedArgv).not.toHaveBeenCalled(); - expect(dockerSpawnSync).not.toHaveBeenCalled(); + expect(executePrivileged).not.toHaveBeenCalled(); const args = spawn.mock.calls[0]?.[1] as string[]; const shellPayload = args.at(-1) ?? ""; expect(shellPayload).not.toMatch(/[\r\n]/); diff --git a/test/repository/layer-import-boundaries.test.ts b/test/repository/layer-import-boundaries.test.ts index 8cc3db8229a..6b3d2e4a15f 100644 --- a/test/repository/layer-import-boundaries.test.ts +++ b/test/repository/layer-import-boundaries.test.ts @@ -6,7 +6,10 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; -import { findLayerImportBoundaryViolations } from "../../scripts/checks/layer-import-boundaries.mts"; +import { + findLayerImportBoundaryViolations, + findManagedRuntimeBoundaryViolations, +} from "../../scripts/checks/layer-import-boundaries.mts"; const REPO_ROOT = path.join(import.meta.dirname, "../.."); let fixtureCounter = 0; @@ -43,6 +46,10 @@ describe("CLI layer import boundaries (#6245)", () => { expect(findLayerImportBoundaryViolations()).toEqual([]); }); + it("keeps managed runtime orchestration provider-neutral (#9145)", () => { + expect(findManagedRuntimeBoundaryViolations()).toEqual([]); + }); + it("collects TypeScript import-equals references (#6245)", () => { const violations = scanFixture( fixturePath("src/lib/domain", "import-equals"), @@ -181,38 +188,37 @@ describe("CLI layer import boundaries (#6245)", () => { ); }); - it.each([ - "Command", - "NemoClawCommand", - ])("rejects an unrelated local %s class as a command base (#6245)", (baseName) => { - const violations = scanFixture( - fixturePath("src/commands", "local-command-base"), - `class ${baseName} {}\nexport default class Example extends ${baseName} {}\n`, - ); + it.each(["Command", "NemoClawCommand"])( + "rejects an unrelated local %s class as a command base (#6245)", + (baseName) => { + const violations = scanFixture( + fixturePath("src/commands", "local-command-base"), + `class ${baseName} {}\nexport default class Example extends ${baseName} {}\n`, + ); - expect(violations).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - detail: "command files must define exactly one registered oclif command class; found 0", - }), - ]), - ); - }); + expect(violations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + detail: "command files must define exactly one registered oclif command class; found 0", + }), + ]), + ); + }, + ); - it.each([ - ".mts", - ".cts", - ".tsx", - ])("scans production %s modules for protected-layer violations (#6245)", (extension) => { - const violations = scanFixture( - fixturePath("src/lib/actions", "module-extension", extension), - 'import { Command } from "@oclif/core";\n', - ); + it.each([".mts", ".cts", ".tsx"])( + "scans production %s modules for protected-layer violations (#6245)", + (extension) => { + const violations = scanFixture( + fixturePath("src/lib/actions", "module-extension", extension), + 'import { Command } from "@oclif/core";\n', + ); - expect(violations).toEqual( - expect.arrayContaining([expect.objectContaining({ rule: "actions-no-oclif" })]), - ); - }); + expect(violations).toEqual( + expect.arrayContaining([expect.objectContaining({ rule: "actions-no-oclif" })]), + ); + }, + ); it("recognizes alternate-extension action modules outside the actions directory (#6245)", () => { const violations = scanFixture( @@ -311,18 +317,17 @@ describe("CLI layer import boundaries (#6245)", () => { } }); - it.each([ - ".test.mts", - ".spec.cts", - ".test.tsx", - ])("excludes %s test modules from the production scan (#6245)", (extension) => { - expect( - scanFixture( - fixturePath("src/lib/actions", "test-module-extension", extension), - 'import { Command } from "@oclif/core";\n', - ), - ).toEqual([]); - }); + it.each([".test.mts", ".spec.cts", ".test.tsx"])( + "excludes %s test modules from the production scan (#6245)", + (extension) => { + expect( + scanFixture( + fixturePath("src/lib/actions", "test-module-extension", extension), + 'import { Command } from "@oclif/core";\n', + ), + ).toEqual([]); + }, + ); it("does not recurse through a symbolic-link loop (#6245)", () => { const fixtureRoot = fs.mkdtempSync( diff --git a/test/runtime/policy/hermes-slack-policy-reconciliation.test.ts b/test/runtime/policy/hermes-slack-policy-reconciliation.test.ts index b92f5fde398..7aa1f471154 100644 --- a/test/runtime/policy/hermes-slack-policy-reconciliation.test.ts +++ b/test/runtime/policy/hermes-slack-policy-reconciliation.test.ts @@ -122,6 +122,8 @@ exit 1 expect(binaries).toEqual([ "/usr/local/bin/hermes", "/usr/bin/python3*", + "/usr/bin/python3.13", + "/opt/hermes/.venv/bin/python3", "/opt/hermes/.venv/bin/python", ]); expect(binaries).not.toContain("/usr/local/bin/node"); diff --git a/test/runtime/policy/policy-openclaw-npm-compatibility.test.ts b/test/runtime/policy/policy-openclaw-npm-compatibility.test.ts index 2adebf7a3e0..1c9c17c90ab 100644 --- a/test/runtime/policy/policy-openclaw-npm-compatibility.test.ts +++ b/test/runtime/policy/policy-openclaw-npm-compatibility.test.ts @@ -340,6 +340,7 @@ process.stdout.write("\\n__RESULT__" + JSON.stringify({ expect(payload.policy).toEqual(drifted); expect(payload.registry).not.toHaveProperty("policies"); expect(calls.some((call) => call.startsWith("policy set "))).toBe(false); - expect(stderr).toContain("differs from both the reviewed baseline"); + expect(stderr).toContain("validation failed"); + expect(stderr).not.toContain("differs from both the reviewed baseline"); }); }); diff --git a/test/runtime/policy/shields-up-runtime-perms.test.ts b/test/runtime/policy/shields-up-runtime-perms.test.ts index e9d9568f599..63a6b0d8ace 100644 --- a/test/runtime/policy/shields-up-runtime-perms.test.ts +++ b/test/runtime/policy/shields-up-runtime-perms.test.ts @@ -129,8 +129,20 @@ Module._load = function patchedLoad(request, parent, isMain) { } if (request === "../sandbox/privileged-exec") { return { - privilegedSandboxExecArgv(_sandboxName, cmd) { - return [...cmd]; + capturePrivilegedSandboxCommand(_sandboxName, cmd) { + const docker = Module._load("../adapters/docker/exec", parent, isMain); + return Buffer.from(docker.dockerExecFileSync([...cmd])); + }, + executePrivilegedSandboxCommand(_sandboxName, cmd) { + const docker = Module._load("../adapters/docker/exec", parent, isMain); + const result = docker.dockerSpawnSync([...cmd]); + return { + status: result.status, + signal: result.signal, + stdout: Buffer.from(result.stdout || ""), + stderr: Buffer.from(result.stderr || ""), + ...(result.error ? { error: result.error } : {}), + }; }, withPrivilegedSandboxExecutionLease(_sandboxName, _operation, fn) { return fn(); @@ -222,19 +234,19 @@ describe("shields-up state-dir lock preserves sandbox-group access + runtime ses expect(HERMES_STATE_LOCK_PLAN.writableSubpaths).toEqual(["profiles/dashboard-home"]); }); - it.each([ - "extensions", - "agent", - ])("surfaces a recursive guard refusal for a symlinked %s root", (root) => { - const unsafePath = `/sandbox/.openclaw/${root}`; - const result = runLockAgentConfigProbe({ stateDirIssuePath: unsafePath }); + it.each(["extensions", "agent"])( + "surfaces a recursive guard refusal for a symlinked %s root", + (root) => { + const unsafePath = `/sandbox/.openclaw/${root}`; + const result = runLockAgentConfigProbe({ stateDirIssuePath: unsafePath }); - expect(result.status).toBe(2); - expect(result.stderr).toContain("Config not locked"); - expect(result.stderr).toContain("state-dir guard lock [state-root-symlink]"); - expect(result.stderr).toContain(unsafePath); - expect(result.stderr).toContain("state-dir roots must not be symlinks"); - }); + expect(result.status).toBe(2); + expect(result.stderr).toContain("Config not locked"); + expect(result.stderr).toContain("state-dir guard lock [state-root-symlink]"); + expect(result.stderr).toContain(unsafePath); + expect(result.stderr).toContain("state-dir roots must not be symlinks"); + }, + ); it("preserves the top-level seal when recursive containment refuses", () => { const result = runLockAgentConfigProbe({ diff --git a/test/runtime/sandbox/repro-2681-group-writable.test.ts b/test/runtime/sandbox/repro-2681-group-writable.test.ts index 08a454a146d..772c3952f6f 100644 --- a/test/runtime/sandbox/repro-2681-group-writable.test.ts +++ b/test/runtime/sandbox/repro-2681-group-writable.test.ts @@ -196,7 +196,24 @@ function withMockedDockerExecFileSync( filename: privilegedExecPath, loaded: true, exports: { - privilegedSandboxExecArgv: (_sandboxName: string, cmd: readonly string[]) => [...cmd], + capturePrivilegedSandboxCommand: (_sandboxName: string, cmd: readonly string[]) => + Buffer.from(dockerExecModule.dockerExecFileSync([...cmd])), + executePrivilegedSandboxCommand: (_sandboxName: string, cmd: readonly string[]) => { + const result = dockerExecModule.dockerSpawnSync([...cmd]) as { + status: number | null; + signal: NodeJS.Signals | null; + stdout: string; + stderr: string; + error?: Error; + }; + return { + status: result.status, + signal: result.signal, + stdout: Buffer.from(result.stdout), + stderr: Buffer.from(result.stderr), + ...(result.error ? { error: result.error } : {}), + }; + }, withPrivilegedSandboxExecutionLease: ( _sandboxName: string, _operation: string, @@ -628,9 +645,7 @@ describe("mutable agent config permissions", () => { ).toBe(false); }); - it.each( - ["run-state-dir-transition", "apply-shields-transition", "finish-shields-transition"], - )( + it.each(["run-state-dir-transition", "apply-shields-transition", "finish-shields-transition"])( "shields-down restores Hermes sticky group-writable config root without group-writable config files [%s]", (action) => { const commands: string[][] = []; @@ -844,8 +859,20 @@ Module._load = function patchedLoad(request, parent, isMain) { } if (request === "../sandbox/privileged-exec") { return { - privilegedSandboxExecArgv(_sandboxName, cmd) { - return [...cmd]; + capturePrivilegedSandboxCommand(_sandboxName, cmd) { + const docker = Module._load("../adapters/docker/exec", parent, isMain); + return Buffer.from(docker.dockerExecFileSync([...cmd])); + }, + executePrivilegedSandboxCommand(_sandboxName, cmd) { + const docker = Module._load("../adapters/docker/exec", parent, isMain); + const result = docker.dockerSpawnSync([...cmd]); + return { + status: result.status, + signal: result.signal, + stdout: Buffer.from(result.stdout || ""), + stderr: Buffer.from(result.stderr || ""), + ...(result.error ? { error: result.error } : {}), + }; }, withPrivilegedSandboxExecutionLease(_sandboxName, _operation, fn) { return fn(); diff --git a/test/security/config-set-nested-ssrf.test.ts b/test/security/config-set-nested-ssrf.test.ts index e24d44d174c..f33ba38e244 100644 --- a/test/security/config-set-nested-ssrf.test.ts +++ b/test/security/config-set-nested-ssrf.test.ts @@ -34,8 +34,14 @@ function installMockPrivilegedExec( exports: { // Routing is covered by privileged-exec tests; this suite exercises // config validation and write behavior without requiring real Docker. - privilegedSandboxExecArgv: (_sandboxName: string, cmd: readonly string[]) => [...cmd], - resolveDirectSandboxContainer: () => "container-id", + capturePrivilegedSandboxCommand: () => Buffer.alloc(0), + executePrivilegedSandboxCommand: () => ({ + status: 0, + signal: null, + stdout: Buffer.alloc(0), + stderr: Buffer.alloc(0), + }), + resolvePrivilegedSandboxTarget: () => ({ resourceHandle: "container-id" }), withPrivilegedSandboxExecutionLease: ( _sandboxName: string, _operation: string, diff --git a/test/state/runtime-state-mutation-hermes-publisher.test.ts b/test/state/runtime-state-mutation-hermes-publisher.test.ts index 3c041640e6c..84ce8dc555c 100644 --- a/test/state/runtime-state-mutation-hermes-publisher.test.ts +++ b/test/state/runtime-state-mutation-hermes-publisher.test.ts @@ -317,6 +317,7 @@ with tempfile.TemporaryDirectory() as temporary: publisher._verify_final_posture = real_verify_final_posture publisher._verify_top_posture = lambda posture: None +publisher.pwd.getpwnam = lambda _name: type("User", (), {"pw_uid": os.getuid()})() class VerificationResult: ok = False @@ -661,6 +662,7 @@ describe("Hermes runtime state mutation publisher", () => { expect.stringMatching(/\/sandbox\/\.hermes\/\.env$/), expect.stringMatching(/\/sandbox\/\.hermes\/config\.yaml$/), ], + mutable_service_uids: [expect.any(Number)], }, }; expect(failures).toMatchObject([ diff --git a/test/state/state-dir-guard-mutable-posture.test.ts b/test/state/state-dir-guard-mutable-posture.test.ts index 84537d53fb0..cbe797ce009 100644 --- a/test/state/state-dir-guard-mutable-posture.test.ts +++ b/test/state/state-dir-guard-mutable-posture.test.ts @@ -61,6 +61,9 @@ module._verify_dir = verify_dir_with_swap result = module.run_guard( action, config_dir, identity, module.parse_agent_state_lock_plan(plan_json), mutable_top_level_files=tuple(sys.argv[5:]), + mutable_service_uids=(os.getuid(),) + if os.environ.get("NEMOCLAW_TEST_MUTABLE_SERVICE_OWNER") == "1" + else (), ) for issue in result.issues: print(json.dumps(issue.as_json())) @@ -134,7 +137,7 @@ describe("read-only recursive mutable posture observation", () => { [configPath], ); - expect(observed.status).toBe(0); + expect(observed.status, JSON.stringify(observed.lines)).toBe(0); expect(observed.lines).toContainEqual({ type: "test-observation", inodeFlagMutationCalls: 0, @@ -197,6 +200,84 @@ describe("read-only recursive mutable posture observation", () => { }, ); + it("accepts Hermes service-owned mutable modes without changing them", () => { + const { root, configDir } = fixture(); + const skillsDir = path.join(configDir, "skills"); + const skillDir = path.join(skillsDir, "bundled-skill"); + const skillFile = path.join(skillDir, "SKILL.md"); + const pairingDir = path.join(configDir, "pairing"); + const pairingFile = path.join(pairingDir, "state.json"); + fs.mkdirSync(skillDir, { recursive: true }); + fs.mkdirSync(pairingDir); + fs.writeFileSync(skillFile, "skill\n", { mode: 0o644 }); + fs.writeFileSync(pairingFile, "pairing\n", { mode: 0o600 }); + fs.chmodSync(root, 0o755); + fs.chmodSync(configDir, 0o700); + fs.chmodSync(skillsDir, 0o775); + fs.chmodSync(skillDir, 0o755); + fs.chmodSync(pairingDir, 0o700); + const before = [skillsDir, skillDir, skillFile, pairingDir, pairingFile].map((entry) => + fs.lstatSync(entry), + ); + + const observed = runGuard("verify-mutable", configDir, { + NEMOCLAW_TEST_MUTABLE_SERVICE_OWNER: "1", + }); + + expect(observed.status, JSON.stringify(observed.lines)).toBe(0); + expect(observed.lines.at(-1)).toEqual( + expect.objectContaining({ + type: "result", + action: "verify-mutable", + status: "ok", + issueCount: 0, + }), + ); + expect(fs.lstatSync(skillsDir)).toMatchObject({ ino: before[0]?.ino, mode: before[0]?.mode }); + expect(fs.lstatSync(skillDir)).toMatchObject({ ino: before[1]?.ino, mode: before[1]?.mode }); + expect(fs.lstatSync(skillFile)).toMatchObject({ ino: before[2]?.ino, mode: before[2]?.mode }); + expect(fs.lstatSync(pairingDir)).toMatchObject({ ino: before[3]?.ino, mode: before[3]?.mode }); + expect(fs.lstatSync(pairingFile)).toMatchObject({ ino: before[4]?.ino, mode: before[4]?.mode }); + }); + + it("rejects world-writable Hermes service state without changing it", () => { + const { configDir } = fixture(); + const skillsDir = path.join(configDir, "skills"); + const skillFile = path.join(skillsDir, "state.json"); + fs.mkdirSync(skillsDir); + fs.writeFileSync(skillFile, "state\n", { mode: 0o666 }); + fs.chmodSync(skillsDir, 0o777); + fs.chmodSync(skillFile, 0o666); + const dirBefore = fs.lstatSync(skillsDir); + const fileBefore = fs.lstatSync(skillFile); + + const observed = runGuard("verify-mutable", configDir, { + NEMOCLAW_TEST_MUTABLE_SERVICE_OWNER: "1", + }); + + expect(observed.status).toBe(1); + expect(observed.lines).toEqual( + expect.arrayContaining( + [skillsDir, skillFile].map((entry) => + expect.objectContaining({ + type: "issue", + code: "verification-mode-mismatch", + path: entry, + }), + ), + ), + ); + expect(fs.lstatSync(skillsDir)).toMatchObject({ + ino: dirBefore.ino, + mode: dirBefore.mode, + }); + expect(observeFixtureFile(skillFile)).toMatchObject({ + ino: fileBefore.ino, + mode: fileBefore.mode, + content: "state\n", + }); + }); + it("reports nested skills and pairing drift without changing either entry (#9485)", () => { const { root, configDir } = fixture(); const skillState = path.join(configDir, "skills", "pairing", "state.json"); diff --git a/test/state/state-dir-guard-verification.test.ts b/test/state/state-dir-guard-verification.test.ts index 147a3aca6c9..b99b390b643 100644 --- a/test/state/state-dir-guard-verification.test.ts +++ b/test/state/state-dir-guard-verification.test.ts @@ -150,7 +150,56 @@ print(json.dumps({ })) `; +const VERIFY_READ_ONLY_ACTIONS = String.raw` +import importlib.util, json, os, sys, tempfile +spec = importlib.util.spec_from_file_location("nemoclaw_state_dir_guard", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) +identity = module.Identity(root_uid=os.getuid(), root_gid=os.getgid(), sandbox_uid=os.getuid(), sandbox_gid=os.getgid()) +plan = module.parse_agent_state_lock_plan(json.dumps({"version":1,"readOnlyRoots":["skills"],"confidentialRoots":[],"readOnlyPrefixes":[],"confidentialPrefixes":[],"writableSubpaths":[]})) + +def verify(action, root_mode, file_mode): + with tempfile.TemporaryDirectory() as root: + root = os.path.realpath(root) + config = os.path.join(root, ".agent") + skills = os.path.join(config, "skills") + os.makedirs(skills, mode=root_mode) + os.chmod(skills, root_mode) + source = os.path.join(skills, "skill.py") + with open(source, "w", encoding="utf-8") as stream: + stream.write("pass\n") + os.chmod(source, file_mode) + before = (os.stat(skills).st_mode, os.stat(source).st_mode) + result = module.run_guard(action, config, identity, plan) + after = (os.stat(skills).st_mode, os.stat(source).st_mode) + return {"ok": result.ok, "unchanged": before == after, "codes": [issue.code for issue in result.issues]} + +print(json.dumps({ + "locked": verify("verify-lock", 0o755, 0o640), + "locked_drift": verify("verify-lock", 0o2770, 0o660), +})) +`; + describe("state directory guard verification", () => { + it("verifies the complete recursive locked posture without changing metadata", () => { + const result = spawnSync("python3", ["-I", "-c", VERIFY_READ_ONLY_ACTIONS, GUARD_PATH], { + encoding: "utf-8", + }); + + expect(result.status, `${result.stderr}\n${result.stdout}`).toBe(0); + const outcomes = JSON.parse(result.stdout) as Record< + string, + { ok: boolean; unchanged: boolean; codes: string[] } + >; + expect(outcomes.locked).toEqual({ ok: true, unchanged: true, codes: [] }); + expect(outcomes.locked_drift).toMatchObject({ + ok: false, + unchanged: true, + codes: expect.arrayContaining(["verification-mode-mismatch"]), + }); + }); + it("rejects locked high-risk files that lost sandbox group access (#8304)", () => { const result = spawnSync("python3", ["-I", "-c", VERIFY_HIGH_RISK_MODES, GUARD_PATH], { encoding: "utf-8", diff --git a/tools/e2e/base-image-publication.mts b/tools/e2e/base-image-publication.mts index c468368d879..17078532afe 100644 --- a/tools/e2e/base-image-publication.mts +++ b/tools/e2e/base-image-publication.mts @@ -142,6 +142,7 @@ export function writePublicationRunOutputs(path: string, run: PublicationRun): v } export interface GithubRequestOptions { + authenticated?: boolean; additionalRepository?: string; fetchImpl?: (input: string, init: RequestInit) => Promise; sleep?: (milliseconds: number) => Promise; @@ -811,6 +812,7 @@ export async function githubRequest( const now = options.now ?? Date.now; const attempts = options.attempts ?? REQUEST_ATTEMPTS; const timeoutMs = options.timeoutMs ?? REQUEST_TIMEOUT_MS; + const authenticated = options.authenticated ?? true; if (!Number.isSafeInteger(attempts) || attempts < 1 || attempts > REQUEST_ATTEMPTS) { throw new Error(`request attempts must be between 1 and ${REQUEST_ATTEMPTS}`); } @@ -824,7 +826,7 @@ export async function githubRequest( response = await fetchImpl(`${API_ROOT}${path}`, { headers: { Accept: "application/vnd.github+json", - Authorization: `Bearer ${token}`, + ...(authenticated ? { Authorization: `Bearer ${token}` } : {}), "User-Agent": "NemoClaw-base-image-publication-gate", "X-GitHub-Api-Version": "2022-11-28", }, diff --git a/tools/e2e/check-semantic-phases.mts b/tools/e2e/check-semantic-phases.mts index e2d561344da..4b2552abbbc 100644 --- a/tools/e2e/check-semantic-phases.mts +++ b/tools/e2e/check-semantic-phases.mts @@ -7,6 +7,7 @@ import { Writable } from "node:stream"; import { fileURLToPath } from "node:url"; import ts from "typescript"; +import type {} from "vitest"; import { createVitest } from "vitest/node"; import { REPO_ROOT } from "../../test/e2e/fixtures/paths.ts"; @@ -21,7 +22,7 @@ import { } from "./workflow-boundary.mts"; import { buildE2eWorkflowPlan } from "./workflow-plan.mts"; -declare module "@vitest/runner" { +declare module "vitest" { interface TaskMeta { e2ePhases?: readonly string[]; } diff --git a/tools/e2e/credential-free-tests.mts b/tools/e2e/credential-free-tests.mts index 18d337bc76c..5a4b8204099 100644 --- a/tools/e2e/credential-free-tests.mts +++ b/tools/e2e/credential-free-tests.mts @@ -8,18 +8,33 @@ import { fileURLToPath } from "node:url"; import { moduleTagDeclarations, stripModuleTagDeclarations } from "./module-tags.mts"; import { type E2eExecutionMetadata, validateE2eExecutionMetadata } from "./execution-coverage.mts"; +import { + type E2eGatewayRuntime, + type E2eGatewayRuntimeSupport, + type E2eRuntimeProvider, + e2eRuntimeProviders, + runtimeCoverageVariant, + runtimeExecutionId, + supportsE2eGatewayRuntime, +} from "./gateway-runtime.mts"; export const CREDENTIAL_FREE_TEST_TAG = "e2e/credential-free"; export const SHARED_E2E_JOB_ID = "shared-e2e"; export type CredentialFreeTestProject = "e2e-live" | "integration"; -export type CredentialFreeTestMatrixRow = { +export type CredentialFreeTestDefinitionRow = { id: string; file: string; project: CredentialFreeTestProject; }; +export type CredentialFreeTestMatrixRow = CredentialFreeTestDefinitionRow & { + execution_id: string; + runtime_provider: E2eRuntimeProvider; + coverage_variant: string; +}; + export type CredentialFreeTestModule = { file: string; project: CredentialFreeTestProject; @@ -45,25 +60,43 @@ const CREDENTIAL_FREE_TEST_COVERAGE = { observableOutcome: "Buildless onboarding selects exact managed images for every agent", environmentOrInferenceEndpoint: "Mocked integration environment; no inference endpoint", unresolvedReason: "", + gatewayRuntimes: ["docker"], }, "vllm-docker-storage": { agentRuntime: "none", observableOutcome: "vLLM storage gate accepts and rejects the intended host states", environmentOrInferenceEndpoint: "Native Linux Docker host; no inference endpoint", unresolvedReason: "", + gatewayRuntimes: ["docker"], }, -} as const satisfies Readonly>; +} as const satisfies Readonly< + Record +>; export function credentialFreeTestCoverage(id: string): E2eExecutionMetadata { if (!Object.hasOwn(CREDENTIAL_FREE_TEST_COVERAGE, id)) { throw new Error(`Credential-free test ${id} requires execution coverage metadata`); } - const metadata = ( - CREDENTIAL_FREE_TEST_COVERAGE as Readonly> - )[id]; + const { gatewayRuntimes: _gatewayRuntimes, ...metadata } = + CREDENTIAL_FREE_TEST_COVERAGE[id as keyof typeof CREDENTIAL_FREE_TEST_COVERAGE]; return validateE2eExecutionMetadata(metadata, `Credential-free test ${id}`); } +export function credentialFreeTestSupportsGatewayRuntime( + id: string, + runtime: E2eGatewayRuntime, +): boolean { + return supportsE2eGatewayRuntime(credentialFreeTestGatewayRuntimes(id), runtime); +} + +export function credentialFreeTestGatewayRuntimes(id: string): E2eGatewayRuntimeSupport { + if (!Object.hasOwn(CREDENTIAL_FREE_TEST_COVERAGE, id)) { + throw new Error(`Credential-free test ${id} requires execution coverage metadata`); + } + return CREDENTIAL_FREE_TEST_COVERAGE[id as keyof typeof CREDENTIAL_FREE_TEST_COVERAGE] + .gatewayRuntimes; +} + export function credentialFreeTestProjectForFile( file: string, ): CredentialFreeTestProject | undefined { @@ -141,7 +174,7 @@ export function stripCredentialFreeTestDeclarations(source: string): string { export function credentialFreeTestRowFromModule( module: CredentialFreeTestModule, -): CredentialFreeTestMatrixRow { +): CredentialFreeTestDefinitionRow { validateTestFile(module.file, module.project); const tags = credentialFreeTestTags(module.source, module.file); if (tags.length !== 1) { @@ -160,7 +193,7 @@ export function credentialFreeTestRowFromModule( export function discoverCredentialFreeTestRows( modules: readonly CredentialFreeTestModule[], -): CredentialFreeTestMatrixRow[] { +): CredentialFreeTestDefinitionRow[] { const rows = modules.map(credentialFreeTestRowFromModule).sort((left, right) => { return ( left.id.localeCompare(right.id) || @@ -240,9 +273,11 @@ export function listVitestCredentialFreeTestModules( }); } -const discoveryCache = new Map(); +const discoveryCache = new Map(); -export function discoverCredentialFreeTests(repoRoot = REPO_ROOT): CredentialFreeTestMatrixRow[] { +export function discoverCredentialFreeTests( + repoRoot = REPO_ROOT, +): CredentialFreeTestDefinitionRow[] { const resolvedRoot = fs.realpathSync(repoRoot); const cached = discoveryCache.get(resolvedRoot); if (cached) return cached.map((row) => ({ ...row })); @@ -251,6 +286,23 @@ export function discoverCredentialFreeTests(repoRoot = REPO_ROOT): CredentialFre return rows.map((row) => ({ ...row })); } +export function credentialFreeTestMatrix( + rows: readonly CredentialFreeTestDefinitionRow[], + gatewayRuntimes: readonly E2eGatewayRuntime[], +): CredentialFreeTestMatrixRow[] { + return rows.flatMap((row) => { + const support = + CREDENTIAL_FREE_TEST_COVERAGE[row.id as keyof typeof CREDENTIAL_FREE_TEST_COVERAGE] + .gatewayRuntimes; + return e2eRuntimeProviders(support, gatewayRuntimes).map((runtimeProvider) => ({ + ...row, + execution_id: runtimeExecutionId(row.id, "", runtimeProvider), + runtime_provider: runtimeProvider, + coverage_variant: runtimeCoverageVariant("", runtimeProvider), + })); + }); +} + const invokedFile = process.argv[1] ? path.resolve(process.argv[1]) : ""; if (invokedFile === fileURLToPath(import.meta.url)) { try { diff --git a/tools/e2e/execution-coverage.mts b/tools/e2e/execution-coverage.mts index 54ce8141698..2a38912c50a 100644 --- a/tools/e2e/execution-coverage.mts +++ b/tools/e2e/execution-coverage.mts @@ -101,6 +101,7 @@ export function validateE2eExecutionRows( row.agentRuntime, row.observableOutcome, row.environmentOrInferenceEndpoint, + row.variant, ].join("\u0000"); const previous = coverageEvidence.get(evidenceKey); if (previous) { diff --git a/tools/e2e/gateway-runtime.mts b/tools/e2e/gateway-runtime.mts new file mode 100644 index 00000000000..c447a6750f2 --- /dev/null +++ b/tools/e2e/gateway-runtime.mts @@ -0,0 +1,59 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export const E2E_GATEWAY_RUNTIMES = ["docker", "podman"] as const; +export type E2eGatewayRuntime = (typeof E2E_GATEWAY_RUNTIMES)[number]; +export const E2E_RUNTIME_AGNOSTIC = "agnostic" as const; +export type E2eGatewayRuntimeSupport = readonly E2eGatewayRuntime[] | typeof E2E_RUNTIME_AGNOSTIC; +export type E2eRuntimeProvider = E2eGatewayRuntime | "none"; + +export function e2eGatewayRuntime(value: string | undefined): E2eGatewayRuntime { + const runtime = value ?? "docker"; + if (!E2E_GATEWAY_RUNTIMES.includes(runtime as E2eGatewayRuntime)) { + throw new Error(`Invalid gateway runtime: ${runtime}`); + } + return runtime as E2eGatewayRuntime; +} + +export function e2eGatewayRuntimes(value: string | undefined): E2eGatewayRuntime[] { + const requested = (value ?? "docker").split(","); + if ( + requested.length === 0 || + new Set(requested).size !== requested.length || + requested.some((runtime) => !E2E_GATEWAY_RUNTIMES.includes(runtime as E2eGatewayRuntime)) + ) { + throw new Error(`Invalid gateway runtimes: ${value ?? ""}`); + } + return requested as E2eGatewayRuntime[]; +} + +export function supportsE2eGatewayRuntime( + supported: E2eGatewayRuntimeSupport, + runtime: E2eGatewayRuntime, +): boolean { + return supported === E2E_RUNTIME_AGNOSTIC || supported.includes(runtime); +} + +export function e2eRuntimeProviders( + supported: E2eGatewayRuntimeSupport, + requested: readonly E2eGatewayRuntime[], +): E2eRuntimeProvider[] { + if (supported === E2E_RUNTIME_AGNOSTIC) return ["none"]; + return requested.filter((runtime) => supported.includes(runtime)); +} + +export function runtimeCoverageVariant( + variant: string, + runtimeProvider: E2eRuntimeProvider, +): string { + const runtimeVariant = runtimeProvider === "none" ? "runtime-agnostic" : runtimeProvider; + return variant === "" ? runtimeVariant : `${variant}-${runtimeVariant}`; +} + +export function runtimeExecutionId( + id: string, + variant: string, + runtimeProvider: E2eRuntimeProvider, +): string { + return `${id}-${runtimeCoverageVariant(variant, runtimeProvider)}`; +} diff --git a/tools/e2e/hermes-gpu-startup-workflow-boundary.mts b/tools/e2e/hermes-gpu-startup-workflow-boundary.mts index f7767b77148..da1c67e0e69 100644 --- a/tools/e2e/hermes-gpu-startup-workflow-boundary.mts +++ b/tools/e2e/hermes-gpu-startup-workflow-boundary.mts @@ -8,6 +8,7 @@ import { fileURLToPath } from "node:url"; import YAML from "yaml"; import { CLI_ARTIFACT_RESTORE_STEP } from "./cli-artifact-workflow-boundary.mts"; +import { E2E_ACTION_PROVENANCE } from "./workflow-boundary-policy.mts"; /** * SOURCE_OF_TRUTH_REVIEW @@ -44,6 +45,8 @@ type WorkflowRecord = Record; type WorkflowStep = WorkflowRecord & { name?: string; run?: string; + uses?: string; + with?: WorkflowRecord; }; function asRecord(value: unknown): WorkflowRecord { @@ -136,44 +139,36 @@ export function validateHermesGpuStartupWorkflow( const matrix = asRecord(strategy.matrix); if ( strategy["fail-fast"] !== false || - strategy["max-parallel"] !== 1 || - JSON.stringify(matrix.include) !== + strategy["max-parallel"] !== 2 || + JSON.stringify(matrix.scenario) !== + JSON.stringify(["native", "fallback", "compatibility-only"]) || + matrix.runtime_provider !== + "${{ fromJSON(needs.generate-matrix.outputs.runtime_providers_by_job)['hermes-gpu-startup'] }}" || + matrix.include !== undefined || + JSON.stringify(matrix.exclude) !== JSON.stringify([ - { - scenario: "native", - sandbox_name: "e2e-hgpu-native", - observable_outcome: "Native GPU startup reaches the stable Ready route", - coverage_variant: "native", - }, - { - scenario: "fallback", - sandbox_name: "e2e-hgpu-fallback", - observable_outcome: "Fallback GPU startup reaches the stable Ready route", - coverage_variant: "fallback", - }, - { - scenario: "compatibility-only", - sandbox_name: "e2e-hgpu-compat", - observable_outcome: "Compatibility-only GPU startup reaches the stable Ready route", - coverage_variant: "compatibility-only", - }, + { scenario: "fallback", runtime_provider: "podman" }, + { scenario: "compatibility-only", runtime_provider: "podman" }, ]) ) { - errors.push(`${JOB_NAME} must serialize GPU scenarios`); + errors.push(`${JOB_NAME} must expand reviewed GPU scenarios by supported runtime`); } const jobEnv = asRecord(job.env); const requiredEnv = { E2E_ARTIFACT_DIR: - "${{ github.workspace }}/e2e-artifacts/live/hermes-gpu-startup/${{ matrix.scenario }}", + "${{ github.workspace }}/e2e-artifacts/live/hermes-gpu-startup/${{ matrix.scenario }}/${{ matrix.runtime_provider }}", + E2E_GATEWAY_RUNTIMES: "docker,podman", E2E_HERMES_GPU_STARTUP_SCENARIO: "${{ matrix.scenario }}", E2E_JOB: "1", + E2E_OBSERVABLE_OUTCOME: "Hermes GPU startup reaches the stable Ready route", E2E_TARGET_ID: JOB_NAME, NEMOCLAW_AGENT: "hermes", NEMOCLAW_E2E_SHARD: "${{ matrix.scenario }}", + NEMOCLAW_GATEWAY_RUNTIME: "${{ matrix.runtime_provider }}", NEMOCLAW_RUN_LIVE_E2E: "1", NEMOCLAW_SANDBOX_GPU: "1", - NEMOCLAW_SANDBOX_NAME: "${{ matrix.sandbox_name }}", + NEMOCLAW_SANDBOX_NAME: "${{ matrix.scenario }}", } as const; for (const [name, expected] of Object.entries(requiredEnv)) { if (jobEnv[name] !== expected) { @@ -279,17 +274,22 @@ if ! @run restore`; const restoreI = steps.findIndex((step) => step.name === CLI_ARTIFACT_RESTORE_STEP); const ni = steps.findIndex((step) => step.name === "Reassert trusted Node runtime"); const node = steps[ni]; + const nativePodmanRuntime = steps[ni + 1]; if ( runStep.shell !== BASH || !trustedEnv(runStep) || pi < 0 || restoreI <= pi || ni !== restoreI + 1 || - ni + 1 !== steps.indexOf(runStep) || + ni + 2 !== steps.indexOf(runStep) || node?.uses !== "actions/setup-node@820762786026740c76f36085b0efc47a31fe5020" || asRecord(node?.with)["node-version"] !== "22" || !trustedEnv(node) || asRecord(node?.env).NODE_OPTIONS !== "" || + nativePodmanRuntime?.name !== "Prepare native Podman E2E runtime" || + nativePodmanRuntime?.uses !== E2E_ACTION_PROVENANCE.nativePodmanRuntime.reference || + asRecord(nativePodmanRuntime?.with).enabled !== + "${{ matrix.runtime_provider == 'podman' && 'true' || 'false' }}" || run.includes(SOURCE) || !hasProof( runStep.run, @@ -440,8 +440,10 @@ rm -rf -- @state`, steps.find((step) => step.name === "Upload Hermes GPU startup artifacts")?.with, ); if ( - upload.name !== "e2e-hermes-gpu-startup-${{ matrix.scenario }}" || - upload.path !== "e2e-artifacts/live/hermes-gpu-startup/${{ matrix.scenario }}/" + upload.name !== + "e2e-hermes-gpu-startup-${{ matrix.scenario }}-${{ matrix.runtime_provider }}" || + upload.path !== + "e2e-artifacts/live/hermes-gpu-startup/${{ matrix.scenario }}/${{ matrix.runtime_provider }}/" ) { errors.push(`${JOB_NAME} upload needs a scenario artifact path`); } diff --git a/tools/e2e/mcp-dev-workflow-boundary-digests.mts b/tools/e2e/mcp-dev-workflow-boundary-digests.mts index ed9672a7c6c..8c65825f2d0 100644 --- a/tools/e2e/mcp-dev-workflow-boundary-digests.mts +++ b/tools/e2e/mcp-dev-workflow-boundary-digests.mts @@ -4,15 +4,15 @@ import { createHash } from "node:crypto"; export const MCP_DEV_WORKFLOW_EXECUTION_CONTEXT_SHA256 = - "052c49d5e8688266dbf38fa911733132d33e4470a29a61deb6e7a11067737559"; + "d1415509251931c82ad6c48960cc7801078c8f523d977e0eadf27296338bc6e0"; export const MCP_DEV_JOB_EXECUTION_CONTEXT_SHA256 = - "aeabd0776df3f02174594194272ef4ad6870b3b91928be7f90f96c55c4095cf4"; + "9b70d22accbd7b413932e73b7e865097291af95eb3bacd4f862ff3f574325ab4"; export const MCP_DEV_TRUSTED_NODE_SETUP_CONTENT_SHA256 = "504821ad93c57971d0281ef1130ed6008fadd331bd56acb1a6b5e6a3358f3e49"; export const MCP_DEV_TRUSTED_PREFIX_CONTENT_SHA256 = "4c03445d26a30aabef34ab12879c95c01d5bc48ee1d192f588d47782935104cf"; export const MCP_DEV_POST_INSTALL_TRANSITION_CONTENT_SHA256 = - "5b517388f3f47f92452e038a591cdea00501e76bec22144f1b8264e5c21b963f"; + "9fae24e2a586143abeb36916b556e924d268300d03375bfe6936c5ad97aab9d5"; export function contentSha256(value: unknown): string { return createHash("sha256") diff --git a/tools/e2e/mcp-workflow-boundary.mts b/tools/e2e/mcp-workflow-boundary.mts index 008d34ab36d..3f549ca23f4 100644 --- a/tools/e2e/mcp-workflow-boundary.mts +++ b/tools/e2e/mcp-workflow-boundary.mts @@ -24,6 +24,13 @@ const DEV_ARTIFACT_JOB = "openshell-dev-artifact"; const CREDENTIAL_WINDOW_JOB = "openshell-credential-generation-window"; const MCP_AGENT_SHARDS = ["openclaw", "hermes", "deepagents"] as const; const MATRIX_AGENT_EXPRESSION = "${{ matrix.agent }}"; +const MATRIX_RUNTIME_PROVIDER_EXPRESSION = "${{ matrix.runtime_provider }}"; +const DOCKER_EXACT_MAIN_PROOF_EXPRESSION = + "${{ matrix.runtime_provider == 'docker' && '1' || '0' }}"; +const MANAGED_IMAGE_REVISION_EXPRESSION = + "${{ needs.base-image-publication.outputs.managed_image_revision }}"; +const MANAGED_IMAGE_RECEIPT_EXPRESSION = + "${{ needs.base-image-publication.outputs.managed_image_receipt }}"; const TERMINAL_JOBS = [ "release-qualification", "relevant-e2e", @@ -69,7 +76,7 @@ const DEV_COMPATIBILITY_STEP_ID = "mcp_runtime_compatibility"; const DEV_COMPATIBILITY_TOOL = "tools/e2e/mcp-bridge-runtime-compatibility.mts"; const CREDENTIAL_WINDOW_ID = "openshell-credential-generation-window"; const CREDENTIAL_WINDOW_FILE = `test/e2e/live/${CREDENTIAL_WINDOW_ID}.test.ts`; -const CREDENTIAL_WINDOW_ARTIFACT_DIR = "e2e-artifacts/live/openshell-credential-generation-window"; +const CREDENTIAL_WINDOW_ARTIFACT_DIR = `e2e-artifacts/live/openshell-credential-generation-window/${MATRIX_RUNTIME_PROVIDER_EXPRESSION}`; const CREDENTIAL_WINDOW_RUN_STEP = "Run OpenShell credential generation-window live test"; const CREDENTIAL_WINDOW_JOB_CONDITION = "${{ contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'openshell-credential-generation-window') }}"; @@ -172,13 +179,13 @@ function validateJobIdentity( requireEqual( errors, env.E2E_MANAGED_IMAGE_REVISION, - "${{ needs.base-image-publication.outputs.managed_image_revision }}", + MANAGED_IMAGE_REVISION_EXPRESSION, `${jobName} must receive the selected managed-image cohort revision`, ); requireEqual( errors, env.E2E_MANAGED_IMAGE_COHORT_RECEIPT, - "${{ needs.base-image-publication.outputs.managed_image_receipt }}", + MANAGED_IMAGE_RECEIPT_EXPRESSION, `${jobName} must receive the complete selected managed-image cohort receipt`, ); requireEqual( @@ -238,8 +245,8 @@ function validateJobIdentity( requireEqual( errors, env.NEMOCLAW_OPENSHELL_EXACT_MAIN_PROOF, - "1", - "mcp-bridge must enable the exact stable release proof", + DOCKER_EXACT_MAIN_PROOF_EXPRESSION, + "mcp-bridge must enable the exact stable release proof only for its Docker rows", ); requireEqual( errors, @@ -681,7 +688,7 @@ function validateJobExecution( ); for (const required of [ "tools/e2e/assert-mcp-artifact-secrets-absent.mts", - `e2e-artifacts/live/${jobName}/${MATRIX_AGENT_EXPRESSION}`, + `e2e-artifacts/live/${jobName}/${MATRIX_AGENT_EXPRESSION}/${MATRIX_RUNTIME_PROVIDER_EXPRESSION}`, ]) { requireContains(errors, scan.run, required, `${jobName} artifact secret scan is incomplete`); } @@ -701,13 +708,13 @@ function validateJobExecution( requireEqual( errors, uploadOptions.path, - `e2e-artifacts/live/${jobName}/${MATRIX_AGENT_EXPRESSION}/`, + `e2e-artifacts/live/${jobName}/${MATRIX_AGENT_EXPRESSION}/${MATRIX_RUNTIME_PROVIDER_EXPRESSION}/`, `${jobName} artifact upload must use exactly the scanned directory`, ); requireEqual( errors, uploadOptions.name, - `e2e-${jobName}-${MATRIX_AGENT_EXPRESSION}`, + `e2e-${jobName}-${MATRIX_AGENT_EXPRESSION}-${MATRIX_RUNTIME_PROVIDER_EXPRESSION}`, `${jobName} artifact upload must use its isolated artifact name`, ); if (Object.keys(uploadOptions).sort().join(",") !== "name,path") { @@ -878,25 +885,25 @@ function validateCredentialWindowJob( const env = asRecord(job.env); const expectedEnv = { - E2E_MANAGED_IMAGE_REVISION: - "${{ needs.base-image-publication.outputs.managed_image_revision }}", - E2E_MANAGED_IMAGE_COHORT_RECEIPT: - "${{ needs.base-image-publication.outputs.managed_image_receipt }}", + E2E_MANAGED_IMAGE_REVISION: MANAGED_IMAGE_REVISION_EXPRESSION, + E2E_MANAGED_IMAGE_COHORT_RECEIPT: MANAGED_IMAGE_RECEIPT_EXPRESSION, E2E_WORKLOAD_SOURCE: "${{ needs.generate-matrix.outputs.workload_source }}", NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG_JSON: "${{ needs.base-image-publication.outputs.managed_image_catalog }}", E2E_JOB: "1", + E2E_GATEWAY_RUNTIMES: "docker,podman", E2E_TARGET_ID: CREDENTIAL_WINDOW_JOB, E2E_AGENT_RUNTIME: "openclaw", E2E_OBSERVABLE_OUTCOME: "Credential expiry rotation detach and rebuild preserve the intended access window", E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: - "Ubuntu Docker host; local compatible inference and MCP endpoint", + "Ubuntu managed runtime host; local compatible inference and MCP endpoint", E2E_ARTIFACT_DIR: `\${{ github.workspace }}/${CREDENTIAL_WINDOW_ARTIFACT_DIR}`, NEMOCLAW_CLI_BIN: "${{ github.workspace }}/bin/nemoclaw.js", NEMOCLAW_OPENSHELL_CHANNEL: "stable", NEMOCLAW_OPENSHELL_EXACT_MAIN_PROOF: "1", NEMOCLAW_RUN_LIVE_E2E: "1", + NEMOCLAW_GATEWAY_RUNTIME: MATRIX_RUNTIME_PROVIDER_EXPRESSION, OPENSHELL_DOCKER_SUPERVISOR_IMAGE: `ghcr.io/nvidia/openshell/supervisor@sha256:${STABLE_RELEASE_SUPERVISOR_INDEX}`, }; if (!hasExactEntries(env, expectedEnv)) { @@ -1026,7 +1033,7 @@ function validateCredentialWindowJob( const uploadOptions = asRecord(upload.with); if ( !hasExactEntries(uploadOptions, { - name: `e2e-${CREDENTIAL_WINDOW_JOB}`, + name: `e2e-${CREDENTIAL_WINDOW_JOB}-${MATRIX_RUNTIME_PROVIDER_EXPRESSION}`, path: `${CREDENTIAL_WINDOW_ARTIFACT_DIR}/`, }) ) { diff --git a/tools/e2e/operations-workflow-boundary.mts b/tools/e2e/operations-workflow-boundary.mts index 41061ae76b0..64f906369fd 100644 --- a/tools/e2e/operations-workflow-boundary.mts +++ b/tools/e2e/operations-workflow-boundary.mts @@ -391,7 +391,7 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow } for (const fragment of [ '"$WORKFLOW_EVENT" == "workflow_dispatch"', - '"$WORKFLOW_REF" == "refs/heads/main"', + '"$WORKFLOW_REF" == refs/heads/*', '"$PR_NUMBER" =~ ^[1-9][0-9]*$', '"$CHECKOUT_REPOSITORY" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$', '"$CHECKOUT_SHA" =~ ^[a-f0-9]{40}$', @@ -399,7 +399,6 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow '"$EXPECTED_WORKFLOW_SHA" == "$WORKFLOW_SHA"', "https://api.github.com/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}", `[[ "$(jq -r '.base.repo.full_name // ""' <<< "$pull_json")" == "NVIDIA/NemoClaw" ]]`, - `[[ "$(jq -r '.base.ref // ""' <<< "$pull_json")" == "main" ]]`, `[[ "$(jq -r '.head.repo.full_name // ""' <<< "$pull_json")" == "$CHECKOUT_REPOSITORY" ]]`, `[[ "$(jq -r '.head.sha' <<< "$pull_json")" == "$CHECKOUT_SHA" ]]`, `[[ "$(jq -r '.base.sha' <<< "$pull_json")" == "$BASE_SHA" ]]`, @@ -420,6 +419,12 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow if (!authSource.includes(fragment)) errors.push(`Manual PR authentication must retain ${fragment}`); } + if ( + authSource.includes("Authorization: Bearer") || + Object.hasOwn(authentication.env ?? {}, "GITHUB_TOKEN") + ) { + errors.push("Manual PR authentication must use public PR metadata without a job token"); + } const qualificationPlanName = "native-runtime-qualification-producer-plan"; const qualificationPlan = workflow.jobs[qualificationPlanName] ?? {}; @@ -459,7 +464,6 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow "https://api.github.com/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}", "pull request must still be open", "pull request base repository changed before execution", - "pull request base branch changed before execution", "checkout_repository changed before execution", "checkout_sha changed before execution", "base_sha changed before execution", @@ -470,6 +474,12 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow errors.push(`Manual PR checkout validation must retain ${fragment}`); } } + if ( + validationSource.includes("Authorization: Bearer") || + Object.hasOwn(validation.env ?? {}, "GITHUB_TOKEN") + ) { + errors.push("Manual PR checkout validation must use public PR metadata without a job token"); + } const credentialAuthorization = credentialAuthorizationIndex >= 0 ? steps[credentialAuthorizationIndex] : {}; @@ -502,7 +512,7 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow '"$WORKFLOW_REPOSITORY" == "NVIDIA/NemoClaw"', '"$NVIDIA_OWNED" == "true"', '"$EVENT_NAME" == "workflow_dispatch"', - '"$REF" == "refs/heads/main"', + '"$REF" == refs/heads/*', '"$CHECKOUT_SHA" =~ ^[a-f0-9]{40}$', '"$WORKFLOW_SHA" =~ ^[a-f0-9]{40}$', '"$EXPECTED_WORKFLOW_SHA" == "$WORKFLOW_SHA"', @@ -825,6 +835,13 @@ export function validateBaseImagePublicationGate(workflow: OperationsWorkflow): ) { errors.push("cloud-onboard must use the exact PR managed-image catalog"); } + if ( + (cloudOnboard.steps ?? []).some( + (step) => step.name === "Materialize cloud-onboard managed-image catalog", + ) + ) { + errors.push("cloud-onboard must not duplicate the exact inline managed-image catalog"); + } if ( live.env?.E2E_MANAGED_IMAGE_REVISION !== "${{ needs.base-image-publication.outputs.managed_image_revision }}" @@ -867,7 +884,9 @@ export function validateBaseImagePublicationGate(workflow: OperationsWorkflow): } if ( catalogue.with?.managed_image_revision !== - "${{ needs.base-image-publication.outputs.managed_image_revision }}" + "${{ needs.base-image-publication.outputs.managed_image_revision }}" || + catalogue.with?.managed_image_catalog !== + "${{ needs.base-image-publication.outputs.managed_image_catalog }}" ) { errors.push(`${jobName} must use the selected managed-image revision`); } diff --git a/tools/e2e/standard-profile-workflow-boundary.mts b/tools/e2e/standard-profile-workflow-boundary.mts index d5fbfa1edf9..51d7813142b 100644 --- a/tools/e2e/standard-profile-workflow-boundary.mts +++ b/tools/e2e/standard-profile-workflow-boundary.mts @@ -132,7 +132,7 @@ function validateProfileCallers(errors: string[], workflow: WorkflowRecord): voi `${contract.job} must call the standard E2E profile after matrix generation and base-image publication`, ); } - if (job.name !== "${{ matrix.display_name }}") { + if (job.name !== "${{ matrix.display_name }} (${{ matrix.runtime_provider }})") { errors.push(`${contract.job} must use the planned outcome-first display name`); } const matrixOutput = `needs.generate-matrix.outputs.${contract.matrix}`; @@ -153,6 +153,9 @@ function validateProfileCallers(errors: string[], workflow: WorkflowRecord): voi for (const [name, expected] of Object.entries({ candidate_repository: "${{ inputs.checkout_repository || github.repository }}", candidate_sha: "${{ inputs.checkout_sha || github.sha }}", + runtime_provider: "${{ matrix.runtime_provider }}", + execution_id: "${{ matrix.execution_id }}", + coverage_variant: "${{ matrix.coverage_variant }}", risk_signal_expected_sha: "${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.checkout_sha || '' }}", risk_signal_correlation_id: @@ -206,6 +209,9 @@ function validateProfileWorkflow(errors: string[], profile: WorkflowRecord): voi const requiredInputs = { candidate_repository: "string", candidate_sha: "string", + runtime_provider: "string", + execution_id: "string", + coverage_variant: "string", risk_signal_expected_sha: "string", risk_signal_correlation_id: "string", cli_artifact_provenance: "string", @@ -280,6 +286,7 @@ function validateProfileWorkflow(errors: string[], profile: WorkflowRecord): voi const jobEnv = record(runJob.env); const expectedJobEnv = { E2E_JOB: "1", + E2E_EXECUTION_ID: "${{ inputs.execution_id }}", NEMOCLAW_E2E_MANAGED_IMAGE_CATALOG_JSON: "${{ inputs.managed_image_catalog }}", E2E_MANAGED_IMAGE_REVISION: "${{ inputs.managed_image_revision }}", E2E_TARGET_ID: "${{ inputs.target_id }}", @@ -307,6 +314,8 @@ function validateProfileWorkflow(errors: string[], profile: WorkflowRecord): voi "Install target host dependencies", "Prepare E2E workspace", "Restore exact-commit CLI artifact", + "Prepare native Podman E2E runtime", + "Stage immutable stopped-state cleanup helper", "Install reviewed cloudflared", "Add swap for Hermes image rebuild", "Initialize runner comparison telemetry", @@ -329,6 +338,9 @@ function validateProfileWorkflow(errors: string[], profile: WorkflowRecord): voi const executionPlanRun = String(executionPlan?.run ?? ""); const executionPlanFragments = [ '[[ "$CATALOGUE_ID" =~ ^[a-z0-9]+(-[a-z0-9]+)*$ ]]', + '[[ "$COVERAGE_VARIANT" =~ ^[a-z0-9]+(-[a-z0-9]+)*$ ]]', + '[[ "$EXECUTION_ID" =~ ^[a-z0-9]+(-[a-z0-9]+)*$ ]]', + '[[ "$EXECUTION_ID" == "${CATALOGUE_ID}-${COVERAGE_VARIANT}" ]]', '[[ "$TARGET_ID" =~ ^[a-z0-9]+(-[a-z0-9]+)*$ ]]', '[[ "$SHARD" =~ ^[a-z0-9]+(-[a-z0-9]+)*$ ]]', '[[ "$CANDIDATE_REPOSITORY" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]]', @@ -342,6 +354,7 @@ function validateProfileWorkflow(errors: string[], profile: WorkflowRecord): voi 'printf \'upload_name=%s\\n\' "$upload_name" >>"$GITHUB_OUTPUT"', 'printf \'E2E_ARTIFACT_DIR=%s/%s\\n\' "$GITHUB_WORKSPACE_VALUE" "$artifact_directory" >>"$GITHUB_ENV"', 'printf \'NEMOCLAW_E2E_SHARD=%s\\n\' "$SHARD" >>"$GITHUB_ENV"', + 'printf \'NEMOCLAW_GATEWAY_RUNTIME=%s\\n\' "$RUNTIME_PROVIDER" >>"$GITHUB_ENV"', ]; if ( executionPlan?.id !== "execution_plan" || @@ -353,12 +366,15 @@ function validateProfileWorkflow(errors: string[], profile: WorkflowRecord): voi CANDIDATE_REPOSITORY: "${{ inputs.candidate_repository }}", CANDIDATE_SHA: "${{ inputs.candidate_sha }}", CATALOGUE_ID: "${{ inputs.catalogue_id }}", + COVERAGE_VARIANT: "${{ inputs.coverage_variant }}", ENV: "/dev/null", + EXECUTION_ID: "${{ inputs.execution_id }}", GITHUB_WORKSPACE_VALUE: "${{ github.workspace }}", HOST_PACKAGES: "${{ inputs.host_packages }}", HOST_PREPARATION: "${{ inputs.host_preparation }}", INSTALL_MODE: "${{ inputs.install_mode }}", LC_ALL: "C", + RUNTIME_PROVIDER: "${{ inputs.runtime_provider }}", SHARD: "${{ inputs.shard }}", TARGET_ID: "${{ inputs.target_id }}", TEST_FILE: "${{ inputs.test_file }}", @@ -457,6 +473,43 @@ function validateProfileWorkflow(errors: string[], profile: WorkflowRecord): voi ) { errors.push("standard E2E profile must restore the planned exact-commit CLI artifact"); } + const nativePodmanRuntime = requireStep( + errors, + workflowSteps, + "Prepare native Podman E2E runtime", + ); + if ( + nativePodmanRuntime?.uses !== E2E_ACTION_PROVENANCE.nativePodmanRuntime.reference || + record(nativePodmanRuntime?.with).enabled !== + "${{ inputs.runtime_provider == 'podman' && 'true' || 'false' }}" || + !restore || + workflowSteps.indexOf(nativePodmanRuntime ?? {}) !== workflowSteps.indexOf(restore) + 1 + ) { + errors.push("standard E2E profile must prepare the selected native Podman runtime"); + } + const stoppedStateHelper = requireStep( + errors, + workflowSteps, + "Stage immutable stopped-state cleanup helper", + ); + const stoppedStateHelperRun = String(stoppedStateHelper?.run ?? ""); + if ( + stoppedStateHelper?.if !== + "${{ inputs.target_id == 'channels-stop-start' && (inputs.runtime_provider == 'docker' || inputs.runtime_provider == 'podman') }}" || + stoppedStateHelper.shell !== EXECUTION_PLAN_SHELL || + !isDeepStrictEqual(record(stoppedStateHelper.env), { + CLEANUP_IMAGE: + "node:22-trixie-slim@sha256:db8a96a63e5264607ada2d206758876ebbed6a12be2ada7517793cbfb0c2a29c", + RUNTIME_PROVIDER: "${{ inputs.runtime_provider }}", + }) || + !stoppedStateHelperRun.includes('docker pull "$CLEANUP_IMAGE"') || + !stoppedStateHelperRun.includes('podman --url "unix://$OPENSHELL_PODMAN_SOCKET" pull') || + !stoppedStateHelperRun.includes('--authfile "$DOCKER_CONFIG/config.json" "$CLEANUP_IMAGE"') || + workflowSteps.indexOf(stoppedStateHelper ?? {}) !== + workflowSteps.indexOf(nativePodmanRuntime ?? {}) + 1 + ) { + errors.push("standard E2E profile must stage the immutable stopped-state helper once"); + } const cloudflared = requireStep(errors, workflowSteps, "Install reviewed cloudflared"); const cloudflaredRun = String(cloudflared?.run ?? ""); if ( @@ -473,7 +526,7 @@ function validateProfileWorkflow(errors: string[], profile: WorkflowRecord): voi !cloudflaredRun.includes('dpkg-deb -f "${cloudflared_deb}" Package') || !cloudflaredRun.includes('"${architecture}" != "amd64"') || cloudflaredRun.includes("command -v cloudflared") || - workflowSteps.indexOf(cloudflared ?? {}) !== workflowSteps.indexOf(restore ?? {}) + 1 + workflowSteps.indexOf(cloudflared ?? {}) !== workflowSteps.indexOf(stoppedStateHelper ?? {}) + 1 ) { errors.push("standard E2E profile must install only the reviewed cloudflared package"); } @@ -628,9 +681,15 @@ function validateProfileWorkflow(errors: string[], profile: WorkflowRecord): voi evidence?.if !== "${{ always() && steps.execution_plan.outcome == 'success' }}" || evidenceEnv.ARTIFACT_DIRECTORY !== "${{ steps.execution_plan.outputs.artifact_directory }}" || evidenceEnv.CANDIDATE_SHA !== "${{ inputs.candidate_sha }}" || + evidenceEnv.COVERAGE_VARIANT !== "${{ inputs.coverage_variant }}" || + evidenceEnv.EXECUTION_ID !== "${{ inputs.execution_id }}" || + evidenceEnv.RUNTIME_PROVIDER !== "${{ inputs.runtime_provider }}" || evidenceEnv.WORKFLOW_SHA !== "${{ github.workflow_sha }}" || evidenceEnv.JOB_STATUS !== "${{ job.status }}" || !evidenceRun.includes('kind: "nemoclaw-e2e-evidence-v1"') || + !evidenceRun.includes("executionId: $executionId") || + !evidenceRun.includes("coverageVariant: $coverageVariant") || + !evidenceRun.includes("runtimeProvider: $runtimeProvider") || !evidenceRun.includes("successful E2E target produced no product evidence") || !evidenceRun.includes('>"$ARTIFACT_DIRECTORY/evidence-manifest.json"') || workflowSteps.indexOf(evidence ?? {}) >= workflowSteps.indexOf(upload ?? {}) diff --git a/tools/e2e/target-catalogue.mts b/tools/e2e/target-catalogue.mts index 8068b5d56c5..fd4e8185265 100644 --- a/tools/e2e/target-catalogue.mts +++ b/tools/e2e/target-catalogue.mts @@ -8,6 +8,16 @@ import { pathToFileURL } from "node:url"; import { isDeepStrictEqual } from "node:util"; import { type E2eAgentRuntime, validateE2eExecutionMetadata } from "./execution-coverage.mts"; +import { + E2E_GATEWAY_RUNTIMES, + type E2eGatewayRuntime, + type E2eGatewayRuntimeSupport, + type E2eRuntimeProvider, + E2E_RUNTIME_AGNOSTIC, + e2eRuntimeProviders, + runtimeCoverageVariant, + runtimeExecutionId, +} from "./gateway-runtime.mts"; import { ONBOARD_RESUME_TARGET_TIMEOUT_MINUTES, ONBOARD_SINGLE_FINAL_HANDOFF_TARGET_TIMEOUT_MINUTES, @@ -79,10 +89,14 @@ export interface E2eCatalogueTarget { artifactLayout: E2eArtifactLayout; selector?: string; environment: Readonly>; + gatewayRuntimes: E2eGatewayRuntimeSupport; } export interface E2eCatalogueMatrixRow { id: string; + execution_id: string; + runtime_provider: E2eRuntimeProvider; + coverage_variant: string; target_id: string; display_name: string; agent_runtime: E2eAgentRuntime; @@ -130,6 +144,7 @@ type TargetOptions = Omit< | "prAdvisorSelectable" | "shard" | "artifactLayout" + | "gatewayRuntimes" > & { agentRuntime: E2eAgentRuntime; environmentOrInferenceEndpoint: string; @@ -151,6 +166,7 @@ type TargetOptions = Omit< shard?: string; artifactLayout?: E2eArtifactLayout; testFile?: string; + gatewayRuntimes: E2eGatewayRuntimeSupport; }; function target(id: string, options: TargetOptions): E2eCatalogueTarget { @@ -176,6 +192,7 @@ function target(id: string, options: TargetOptions): E2eCatalogueTarget { shard = "default", artifactLayout = "target-shard", testFile = `test/e2e/live/${id}.test.ts`, + gatewayRuntimes, ...execution } = options; return { @@ -202,10 +219,32 @@ function target(id: string, options: TargetOptions): E2eCatalogueTarget { shard, artifactLayout, installNonInteractive, + gatewayRuntimes, ...execution, }; } +function managedRuntimeTarget( + id: string, + options: Omit, +): E2eCatalogueTarget { + return target(id, { ...options, gatewayRuntimes: E2E_GATEWAY_RUNTIMES }); +} + +function dockerOnlyTarget( + id: string, + options: Omit, +): E2eCatalogueTarget { + return target(id, { ...options, gatewayRuntimes: ["docker"] }); +} + +function runtimeAgnosticTarget( + id: string, + options: Omit, +): E2eCatalogueTarget { + return target(id, { ...options, gatewayRuntimes: E2E_RUNTIME_AGNOSTIC }); +} + const hostedInference = { NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1", } as const; @@ -253,7 +292,7 @@ function commonEgressTarget(options: { selector: string; shard: string; }): E2eCatalogueTarget { - return target(`common-egress-agent-${options.shard}`, { + return managedRuntimeTarget(`common-egress-agent-${options.shard}`, { targetId: "common-egress-agent", displayName: options.displayName, agentRuntime: options.hermes ? "hermes" : "openclaw", @@ -301,7 +340,7 @@ interface GatewayUpgradeTargetOptions { } function gatewayUpgradeTarget(options: GatewayUpgradeTargetOptions): E2eCatalogueTarget { - return target(`openshell-gateway-upgrade-${options.shard}`, { + return dockerOnlyTarget(`openshell-gateway-upgrade-${options.shard}`, { targetId: "openshell-gateway-upgrade", displayName: options.displayName, agentRuntime: "openclaw", @@ -417,7 +456,7 @@ export function catalogueExclusionReason(id: string): string | undefined { } export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ - target("agent-turn-latency", { + managedRuntimeTarget("agent-turn-latency", { displayName: "Performance: bounds hosted inference turns for OpenClaw and Hermes", agentRuntime: "openclaw + hermes", environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference", @@ -443,7 +482,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ OPENSHELL_GATEWAY: "nemoclaw", }, }), - target("bedrock-runtime-compatible-anthropic-openclaw", { + managedRuntimeTarget("bedrock-runtime-compatible-anthropic-openclaw", { targetId: "bedrock-runtime-compatible-anthropic", displayName: "Inference: OpenClaw routes an Anthropic request through Amazon Bedrock", agentRuntime: "openclaw", @@ -463,7 +502,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ OPENSHELL_GATEWAY: "nemoclaw", }, }), - target("bedrock-runtime-compatible-anthropic-hermes", { + managedRuntimeTarget("bedrock-runtime-compatible-anthropic-hermes", { targetId: "bedrock-runtime-compatible-anthropic", displayName: "Inference: Hermes routes an Anthropic request through Amazon Bedrock", agentRuntime: "hermes", @@ -484,7 +523,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ OPENSHELL_GATEWAY: "nemoclaw", }, }), - target("bootstrap-install-smoke", { + dockerOnlyTarget("bootstrap-install-smoke", { displayName: "Install: bootstraps NemoClaw and completes hosted inference", agentRuntime: "openclaw", environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference", @@ -507,7 +546,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ SKIP_DOCKER_PULL: "1", }, }), - target("brave-search", { + managedRuntimeTarget("brave-search", { displayName: "Search: OpenClaw returns a Brave result without exposing its key", agentRuntime: "openclaw", environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference and Brave Search", @@ -525,7 +564,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ OPENSHELL_GATEWAY: "nemoclaw", }, }), - target("channels-add-remove", { + managedRuntimeTarget("channels-add-remove", { displayName: "Messaging: adds and removes Telegram configuration", agentRuntime: "openclaw", environmentOrInferenceEndpoint: "Ubuntu; no inference endpoint", @@ -543,7 +582,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ TELEGRAM_REQUIRE_MENTION: "0", }, }), - target("channels-stop-start-openclaw", { + managedRuntimeTarget("channels-stop-start-openclaw", { targetId: "channels-stop-start", displayName: "Messaging: OpenClaw preserves channels across stop and start", agentRuntime: "openclaw", @@ -571,7 +610,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ WECHAT_BOT_TOKEN: "test-fake-wechat-token-stop-start-openclaw", }, }), - target("channels-stop-start-hermes", { + managedRuntimeTarget("channels-stop-start-hermes", { targetId: "channels-stop-start", displayName: "Messaging: Hermes preserves channels across stop and start", agentRuntime: "hermes", @@ -602,7 +641,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ WECHAT_BOT_TOKEN: "test-fake-wechat-token-stop-start-hermes", }, }), - target("cloud-inference", { + managedRuntimeTarget("cloud-inference", { displayName: "Inference: OpenClaw uses hosted inference", agentRuntime: "openclaw", environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference", @@ -658,7 +697,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ TAVILY_API_KEY: "", }, }), - target("concurrent-gateway-ports", { + dockerOnlyTarget("concurrent-gateway-ports", { displayName: "Gateway: isolates ports for concurrent sandboxes", agentRuntime: "openclaw", environmentOrInferenceEndpoint: "Ubuntu Docker host; local gateway; no inference endpoint", @@ -669,7 +708,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ exposeCliBin: true, environment: nonInteractive, }), - target("cron-preflight-inference-local", { + managedRuntimeTarget("cron-preflight-inference-local", { displayName: "Preflight: reaches managed inference without DNS failure", agentRuntime: "openclaw", environmentOrInferenceEndpoint: "Ubuntu Docker host; local managed inference", @@ -686,7 +725,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ OPENSHELL_GATEWAY: "nemoclaw", }, }), - target("dashboard-remote-bind", { + managedRuntimeTarget("dashboard-remote-bind", { displayName: "Dashboard: retains audit findings when bound remotely", agentRuntime: "openclaw", environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference", @@ -703,7 +742,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ OPENSHELL_GATEWAY: "nemoclaw", }, }), - target("device-auth-health", { + managedRuntimeTarget("device-auth-health", { displayName: "Health: treats a 401 authentication response as reachable", agentRuntime: "openclaw", environmentOrInferenceEndpoint: "Ubuntu; local authentication fixture; no inference endpoint", @@ -719,7 +758,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ OPENSHELL_GATEWAY: "nemoclaw", }, }), - target("double-onboard", { + managedRuntimeTarget("double-onboard", { displayName: "Onboarding: reuses the gateway and preserves sibling sandboxes", agentRuntime: "openclaw", environmentOrInferenceEndpoint: "Ubuntu Docker host; local gateway fixtures", @@ -730,7 +769,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ exposeCliBin: true, environment: nonInteractive, }), - target("gpu-double-onboard", { + managedRuntimeTarget("gpu-double-onboard", { displayName: "Onboarding: preserves Ollama authentication after GPU re-onboarding", agentRuntime: "openclaw", environmentOrInferenceEndpoint: "NVIDIA GPU runner; local Ollama", @@ -748,7 +787,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ NEMOCLAW_OLLAMA_PROXY_PORT: "11435", }, }), - target("gpu-e2e", { + managedRuntimeTarget("gpu-e2e", { displayName: "Inference: validates OpenClaw and Hermes turns through GPU Ollama", agentRuntime: "openclaw", environmentOrInferenceEndpoint: "NVIDIA GPU runner; local Ollama", @@ -768,7 +807,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ OPENSHELL_GATEWAY: "nemoclaw", }, }), - target("full-e2e", { + managedRuntimeTarget("full-e2e", { displayName: "OpenClaw: installs, onboards, and completes an agent turn", agentRuntime: "openclaw", environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference", @@ -788,7 +827,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ NEMOCLAW_SANDBOX_NAME: "e2e-full", }, }), - target("gateway-guard-recovery", { + dockerOnlyTarget("gateway-guard-recovery", { displayName: "Gateway: restores the guard chain after recreation", agentRuntime: "openclaw", environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference", @@ -805,7 +844,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ OPENSHELL_GATEWAY: "nemoclaw", }, }), - target("hermes-discord", { + managedRuntimeTarget("hermes-discord", { displayName: "Messaging: Hermes preserves Discord configuration across rebuild", agentRuntime: "hermes", environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference and Discord", @@ -830,7 +869,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ DISCORD_REQUIRE_MENTION: "0", }, }), - target("hermes-inference-switch", { + managedRuntimeTarget("hermes-inference-switch", { displayName: "Inference: Hermes switches to an Anthropic-compatible endpoint", agentRuntime: "hermes", environmentOrInferenceEndpoint: "Ubuntu; Anthropic-compatible inference fixture", @@ -855,7 +894,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ OPENSHELL_GATEWAY: "nemoclaw", }, }), - target("hermes-shields-config", { + managedRuntimeTarget("hermes-shields-config", { displayName: "Shields: restores stopped Hermes across posture changes", agentRuntime: "hermes", environmentOrInferenceEndpoint: "Ubuntu Docker host; no inference endpoint", @@ -874,7 +913,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ OPENSHELL_GATEWAY: "nemoclaw", }, }), - target("hermes-slack", { + managedRuntimeTarget("hermes-slack", { displayName: "Messaging: isolates Hermes Slack credentials and reaches Slack APIs", agentRuntime: "hermes", environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference and Slack", @@ -898,7 +937,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ SLACK_BOT_TOKEN: "xoxb-test-hermes-slack-token", }, }), - target("issue-2478-crash-loop-recovery", { + managedRuntimeTarget("issue-2478-crash-loop-recovery", { displayName: "Gateway: recovers after process termination and remains stable", agentRuntime: "openclaw", environmentOrInferenceEndpoint: "Ubuntu Docker host; local gateway; no inference endpoint", @@ -913,7 +952,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ OPENSHELL_GATEWAY: "nemoclaw", }, }), - target("issue-4462-scope-upgrade-approval", { + managedRuntimeTarget("issue-4462-scope-upgrade-approval", { displayName: "Authorization: approves a write-scope upgrade without operator.admin", agentRuntime: "openclaw", environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference", @@ -928,7 +967,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ NEMOCLAW_SANDBOX_NAME: "e2e-issue-4462", }, }), - target("inference-routing", { + managedRuntimeTarget("inference-routing", { displayName: "Inference: rejects unsafe routes and proves runtime identities", agentRuntime: "openclaw + langchain-deepagents-code", environmentOrInferenceEndpoint: "Ubuntu; local compatible and HTTPS inference fixtures", @@ -940,7 +979,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ cloudflared: true, owningPaths: ["tools/e2e/onboard-timeout-contract.mts"], }), - target("kimi-inference-compat", { + managedRuntimeTarget("kimi-inference-compat", { displayName: "Inference: configures a Kimi-compatible endpoint", agentRuntime: "openclaw", environmentOrInferenceEndpoint: "Ubuntu; Kimi-compatible inference fixture", @@ -956,7 +995,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ OPENSHELL_GATEWAY: "nemoclaw", }, }), - target("llama-cpp-generic-gpu", { + dockerOnlyTarget("llama-cpp-generic-gpu", { displayName: "Inference: completes an agent turn with llama.cpp on a generic NVIDIA GPU", agentRuntime: "openclaw", environmentOrInferenceEndpoint: "NVIDIA GPU runner; local llama.cpp", @@ -974,7 +1013,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ OPENSHELL_GATEWAY: "nemoclaw", }, }), - target("messaging-compatible-endpoint", { + managedRuntimeTarget("messaging-compatible-endpoint", { displayName: "Messaging: routes Telegram through a compatible endpoint", agentRuntime: "openclaw", environmentOrInferenceEndpoint: "Ubuntu; compatible inference and Telegram fixtures", @@ -991,7 +1030,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ TELEGRAM_BOT_TOKEN: "test-fake-telegram-token-e2e", }, }), - target("model-router-provider-routed-inference", { + managedRuntimeTarget("model-router-provider-routed-inference", { displayName: "Inference: Model Router returns a provider-routed response", agentRuntime: "openclaw", environmentOrInferenceEndpoint: "Ubuntu; NVIDIA API and Model Router", @@ -1002,7 +1041,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ exposeCliBin: true, environment: { OPENSHELL_GATEWAY: "nemoclaw" }, }), - target("network-policy", { + managedRuntimeTarget("network-policy", { displayName: "Network policy: enforces restricted allow and deny rules", agentRuntime: "openclaw", environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference and network probes", @@ -1030,7 +1069,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ OPENSHELL_GATEWAY: "nemoclaw", }, }), - target("ollama-auth-proxy", { + dockerOnlyTarget("ollama-auth-proxy", { displayName: "Inference: Ollama proxy enforces and preserves authentication", agentRuntime: "none", environmentOrInferenceEndpoint: "Ubuntu Docker host; local Ollama proxy", @@ -1044,7 +1083,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ NEMOCLAW_E2E_OLLAMA_PROXY_PORT: "11435", }, }), - target("onboard-repair", { + managedRuntimeTarget("onboard-repair", { displayName: "Onboarding: repairs a missing sandbox and rejects conflicting resume input", agentRuntime: "openclaw", environmentOrInferenceEndpoint: "Ubuntu Docker host; local onboarding fixtures", @@ -1055,7 +1094,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ exposeCliBin: true, environment: { ...nonInteractive, NEMOCLAW_SANDBOX_NAME: "e2e-repair" }, }), - target("onboard-policy-preset-sequencing", { + managedRuntimeTarget("onboard-policy-preset-sequencing", { displayName: "Onboarding: preserves policy preset step order", agentRuntime: "openclaw", environmentOrInferenceEndpoint: "Ubuntu; no inference endpoint", @@ -1067,7 +1106,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ owningPaths: ["test/e2e/live/onboard-interactive-pty.ts"], environment: { NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" }, }), - target("onboard-resume", { + managedRuntimeTarget("onboard-resume", { displayName: "Onboarding: resumes interrupted setup from recorded progress", agentRuntime: "openclaw", environmentOrInferenceEndpoint: "Ubuntu Docker host; local onboarding fixtures", @@ -1079,7 +1118,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ owningPaths: ["tools/e2e/onboard-timeout-contract.mts"], environment: { ...nonInteractive, NEMOCLAW_SANDBOX_NAME: "e2e-resume" }, }), - target("openclaw-discord-pairing", { + managedRuntimeTarget("openclaw-discord-pairing", { displayName: "Messaging: shares OpenClaw Discord pairing approval", agentRuntime: "openclaw", environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference and Discord", @@ -1096,7 +1135,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ DISCORD_BOT_TOKEN: "test-fake-discord-pairing-e2e", }, }), - target("openclaw-skill-cli", { + managedRuntimeTarget("openclaw-skill-cli", { displayName: "Skills: OpenClaw installs and inspects workspace skills", agentRuntime: "openclaw", environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference", @@ -1111,7 +1150,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ OPENSHELL_GATEWAY: "nemoclaw", }, }), - target("openclaw-inference-switch", { + managedRuntimeTarget("openclaw-inference-switch", { displayName: "Inference: OpenClaw switches providers and remains responsive", agentRuntime: "openclaw", environmentOrInferenceEndpoint: "Ubuntu; compatible inference fixtures", @@ -1133,7 +1172,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ OPENSHELL_GATEWAY: "nemoclaw", }, }), - target("openclaw-tui-chat-correlation", { + managedRuntimeTarget("openclaw-tui-chat-correlation", { displayName: "TUI: keeps rapid OpenClaw turns correlated", agentRuntime: "openclaw", environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference", @@ -1158,7 +1197,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ NEMOCLAW_PREFERRED_API: "openai-completions", }, }), - target("openclaw-slack-pairing", { + managedRuntimeTarget("openclaw-slack-pairing", { displayName: "Messaging: shares OpenClaw Slack pairing approval", agentRuntime: "openclaw", environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference and Slack", @@ -1176,7 +1215,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ SLACK_APP_TOKEN: "xapp-fake-slack-pairing-e2e", }, }), - target("pi-agent-qualification-amd64", { + dockerOnlyTarget("pi-agent-qualification-amd64", { targetId: "pi-agent-qualification", displayName: "Pi: qualifies managed runtime on Linux AMD64", agentRuntime: "pi", @@ -1210,7 +1249,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ OPENSHELL_GATEWAY: "nemoclaw", }, }), - target("pi-agent-qualification-arm64", { + dockerOnlyTarget("pi-agent-qualification-arm64", { targetId: "pi-agent-qualification", displayName: "Pi: qualifies managed runtime on Linux ARM64", agentRuntime: "pi", @@ -1245,7 +1284,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ }, }), ...GATEWAY_UPGRADE_TARGETS, - target("rebuild-openclaw", { + dockerOnlyTarget("rebuild-openclaw", { displayName: "Rebuild: preserves OpenClaw state and rotates the gateway token", agentRuntime: "openclaw", environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference", @@ -1260,7 +1299,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ ], environment: hostedInference, }), - target("rebuild-hermes", { + dockerOnlyTarget("rebuild-hermes", { displayName: "Rebuild: preserves Hermes state and recovers cron dispatch", agentRuntime: "hermes", environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference", @@ -1289,7 +1328,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ OPENSHELL_GATEWAY: "nemoclaw", }, }), - target("rebuild-hermes-stale-base", { + dockerOnlyTarget("rebuild-hermes-stale-base", { displayName: "Rebuild: refreshes a stale Hermes base and restores state", agentRuntime: "hermes", environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference", @@ -1320,7 +1359,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ OPENSHELL_GATEWAY: "nemoclaw", }, }), - target("sandbox-survival", { + managedRuntimeTarget("sandbox-survival", { displayName: "Lifecycle: preserves sandbox state after an OpenShell gateway restart", agentRuntime: "openclaw", environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference", @@ -1336,7 +1375,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ OPENSHELL_GATEWAY: "nemoclaw", }, }), - target("sandbox-operations", { + managedRuntimeTarget("sandbox-operations", { displayName: "Sandbox: preserves lifecycle and multi-sandbox operations", agentRuntime: "openclaw", environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference", @@ -1355,7 +1394,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ OPENSHELL_GATEWAY: "nemoclaw", }, }), - target("security-posture-openclaw", { + managedRuntimeTarget("security-posture-openclaw", { targetId: "security-posture", displayName: "Security: OpenClaw retains the required sandbox posture", agentRuntime: "openclaw", @@ -1383,7 +1422,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ OPENSHELL_GATEWAY: "nemoclaw", }, }), - target("security-posture-hermes", { + managedRuntimeTarget("security-posture-hermes", { targetId: "security-posture", displayName: "Security: Hermes retains the required sandbox posture", agentRuntime: "hermes", @@ -1414,7 +1453,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ OPENSHELL_GATEWAY: "nemoclaw", }, }), - target("sessions-agents-cli", { + managedRuntimeTarget("sessions-agents-cli", { displayName: "CLI: routes sessions and agents to OpenClaw", agentRuntime: "openclaw", environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference", @@ -1430,7 +1469,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ OPENSHELL_GATEWAY: "nemoclaw", }, }), - target("shields-config", { + managedRuntimeTarget("shields-config", { displayName: "Shields: restores stopped OpenClaw across posture changes", agentRuntime: "openclaw", environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference", @@ -1447,7 +1486,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ OPENSHELL_GATEWAY: "nemoclaw", }, }), - target("snapshot-commands", { + managedRuntimeTarget("snapshot-commands", { displayName: "Snapshot: restores selected sandbox state without credential leaks", agentRuntime: "openclaw", environmentOrInferenceEndpoint: "Ubuntu Docker host; no inference endpoint", @@ -1468,7 +1507,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ OPENSHELL_GATEWAY: "nemoclaw", }, }), - target("spark-install", { + runtimeAgnosticTarget("spark-install", { displayName: "Install: leaves NemoClaw and OpenShell usable after standard installation", agentRuntime: "unresolved", environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference", @@ -1487,7 +1526,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ OPENSHELL_GATEWAY: "nemoclaw", }, }), - target("skill-agent", { + managedRuntimeTarget("skill-agent", { displayName: "Skills: OpenClaw reads an injected sandbox skill", agentRuntime: "openclaw", environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference", @@ -1499,7 +1538,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ exposeCliBin: true, environment: hostedInference, }), - target("state-backup-restore", { + managedRuntimeTarget("state-backup-restore", { displayName: "Backup: restores workspace files and memory", agentRuntime: "openclaw", environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference", @@ -1515,7 +1554,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ OPENSHELL_GATEWAY: "nemoclaw", }, }), - target("telegram-injection", { + managedRuntimeTarget("telegram-injection", { displayName: "Messaging: treats Telegram shell metacharacters as data", agentRuntime: "openclaw", environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference and Telegram fixture", @@ -1531,7 +1570,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ OPENSHELL_GATEWAY: "nemoclaw", }, }), - target("token-rotation", { + managedRuntimeTarget("token-rotation", { displayName: "Messaging: rotates one provider token without rebuilding siblings", agentRuntime: "openclaw", environmentOrInferenceEndpoint: "Ubuntu; no inference endpoint", @@ -1551,7 +1590,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ SLACK_APP_TOKEN_B: "xapp-fake-B-rotation-e2e", }, }), - target("tunnel-lifecycle", { + managedRuntimeTarget("tunnel-lifecycle", { displayName: "Tunnel: starts, probes, and stops a public dashboard tunnel", agentRuntime: "openclaw", environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference and Cloudflare tunnel", @@ -1568,7 +1607,7 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ OPENSHELL_GATEWAY: "nemoclaw", }, }), - target("whatsapp-qr-compact", { + runtimeAgnosticTarget("whatsapp-qr-compact", { displayName: "Messaging: renders a compact WhatsApp pairing QR code", agentRuntime: "none", environmentOrInferenceEndpoint: "Ubuntu; no sandbox or inference endpoint", @@ -1586,6 +1625,7 @@ export const E2E_CATALOGUE_SHARED_PATHS = [ ".github/workflows/e2e-standard-profile.yaml", "scripts/install-openshell.sh", "tools/e2e/target-catalogue.mts", + "tools/e2e/gateway-runtime.mts", ] as const; const ID_PATTERN = /^[a-z0-9]+(?:-[a-z0-9]+)*$/u; @@ -1639,6 +1679,14 @@ export function validateE2eTargetCatalogue( if (!E2E_EXECUTION_PROFILES.includes(entry.profile)) { throw new Error(`E2E target ${entry.id} has an invalid execution profile`); } + if ( + entry.gatewayRuntimes !== E2E_RUNTIME_AGNOSTIC && + (entry.gatewayRuntimes.length === 0 || + new Set(entry.gatewayRuntimes).size !== entry.gatewayRuntimes.length || + entry.gatewayRuntimes.some((runtime) => !E2E_GATEWAY_RUNTIMES.includes(runtime))) + ) { + throw new Error(`E2E target ${entry.id} has invalid gateway runtime support`); + } if (!/^[A-Za-z0-9._-]+$/u.test(entry.runner)) { throw new Error(`E2E target ${entry.id} has an invalid runner`); } @@ -1786,33 +1834,39 @@ export function catalogueTargetsForChangedFiles( export function catalogueMatrix( profile: E2eExecutionProfile, targets: readonly E2eCatalogueTarget[], + gatewayRuntimes: readonly E2eGatewayRuntime[] = ["docker"], ): E2eCatalogueMatrixRow[] { return targets .filter((entry) => entry.profile === profile) - .map((entry) => ({ - id: entry.id, - target_id: entry.targetId, - display_name: entry.displayName, - agent_runtime: entry.agentRuntime, - observable_outcome: entry.displayName, - environment_or_inference_endpoint: entry.environmentOrInferenceEndpoint, - unresolved_reason: entry.unresolvedReason, - runner: entry.runner, - runner_key: entry.runnerKey, - test_file: entry.testFile, - timeout_minutes: entry.timeoutMinutes, - install_mode: entry.installMode, - install_non_interactive: entry.installNonInteractive, - restore_cli: entry.restoreCli, - cloudflared: entry.cloudflared, - host_packages: entry.hostPackages.join(" "), - host_preparation: entry.hostPreparation, - runner_comparison: entry.runnerComparison, - runner_pressure: entry.runnerPressure, - compatible_api_key: entry.compatibleApiKey, - shard: entry.shard, - artifact_layout: entry.artifactLayout, - })); + .flatMap((entry) => + e2eRuntimeProviders(entry.gatewayRuntimes, gatewayRuntimes).map((runtimeProvider) => ({ + id: entry.id, + execution_id: runtimeExecutionId(entry.id, entry.shard, runtimeProvider), + runtime_provider: runtimeProvider, + coverage_variant: runtimeCoverageVariant(entry.shard, runtimeProvider), + target_id: entry.targetId, + display_name: entry.displayName, + agent_runtime: entry.agentRuntime, + observable_outcome: entry.displayName, + environment_or_inference_endpoint: entry.environmentOrInferenceEndpoint, + unresolved_reason: entry.unresolvedReason, + runner: entry.runner, + runner_key: entry.runnerKey, + test_file: entry.testFile, + timeout_minutes: entry.timeoutMinutes, + install_mode: entry.installMode, + install_non_interactive: entry.installNonInteractive, + restore_cli: entry.restoreCli, + cloudflared: entry.cloudflared, + host_packages: entry.hostPackages.join(" "), + host_preparation: entry.hostPreparation, + runner_comparison: entry.runnerComparison, + runner_pressure: entry.runnerPressure, + compatible_api_key: entry.compatibleApiKey, + shard: entry.shard, + artifact_layout: entry.artifactLayout, + })), + ); } export async function runCatalogueTarget(id: string, testFile: string): Promise { diff --git a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts index d661e36644c..da29895fb8d 100644 --- a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts +++ b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts @@ -179,7 +179,7 @@ const EXPLICIT_UPLOAD_CONTRACTS = new Map([ [ "live", { - name: "e2e-${{ matrix.id }}", + name: "e2e-${{ matrix.execution_id }}", path: [ "e2e-artifacts/live/${{ matrix.id }}/run-plan.json", "e2e-artifacts/live/${{ matrix.id }}/target.json", @@ -237,8 +237,8 @@ const EXPLICIT_UPLOAD_CONTRACTS = new Map([ [ "hermes-gpu-startup", { - name: "e2e-hermes-gpu-startup-${{ matrix.scenario }}", - path: "e2e-artifacts/live/hermes-gpu-startup/${{ matrix.scenario }}/", + name: "e2e-hermes-gpu-startup-${{ matrix.scenario }}-${{ matrix.runtime_provider }}", + path: "e2e-artifacts/live/hermes-gpu-startup/${{ matrix.scenario }}/${{ matrix.runtime_provider }}/", }, ], [ @@ -251,15 +251,15 @@ const EXPLICIT_UPLOAD_CONTRACTS = new Map([ [ "mcp-bridge", { - name: "e2e-mcp-bridge-${{ matrix.agent }}", - path: "e2e-artifacts/live/mcp-bridge/${{ matrix.agent }}/", + name: "e2e-mcp-bridge-${{ matrix.agent }}-${{ matrix.runtime_provider }}", + path: "e2e-artifacts/live/mcp-bridge/${{ matrix.agent }}/${{ matrix.runtime_provider }}/", }, ], [ "mcp-bridge-dev", { - name: "e2e-mcp-bridge-dev-${{ matrix.agent }}", - path: "e2e-artifacts/live/mcp-bridge-dev/${{ matrix.agent }}/", + name: "e2e-mcp-bridge-dev-${{ matrix.agent }}-${{ matrix.runtime_provider }}", + path: "e2e-artifacts/live/mcp-bridge-dev/${{ matrix.agent }}/${{ matrix.runtime_provider }}/", }, ], [ @@ -272,8 +272,36 @@ const EXPLICIT_UPLOAD_CONTRACTS = new Map([ [ "openshell-credential-generation-window", { - name: "e2e-openshell-credential-generation-window", - path: "e2e-artifacts/live/openshell-credential-generation-window/", + name: "e2e-openshell-credential-generation-window-${{ matrix.runtime_provider }}", + path: "e2e-artifacts/live/openshell-credential-generation-window/${{ matrix.runtime_provider }}/", + }, + ], + [ + SHARED_E2E_JOB_ID, + { + name: "e2e-${{ matrix.execution_id }}", + path: "e2e-artifacts/live/${{ matrix.execution_id }}/", + }, + ], + [ + "hermes-e2e", + { + name: "e2e-hermes-e2e-${{ matrix.runtime_provider }}", + path: "e2e-artifacts/live/hermes-e2e/${{ matrix.runtime_provider }}/", + }, + ], + [ + "cloud-onboard", + { + name: "e2e-cloud-onboard-${{ matrix.runtime_provider }}", + path: "e2e-artifacts/live/cloud-onboard/${{ matrix.runtime_provider }}/", + }, + ], + [ + "messaging-providers", + { + name: "e2e-messaging-providers-${{ matrix.runtime_provider }}", + path: "e2e-artifacts/live/messaging-providers/${{ matrix.runtime_provider }}/", }, ], ]); diff --git a/tools/e2e/workflow-boundary-policy.mts b/tools/e2e/workflow-boundary-policy.mts index b7dfac2eeed..045fec6a3fc 100644 --- a/tools/e2e/workflow-boundary-policy.mts +++ b/tools/e2e/workflow-boundary-policy.mts @@ -7,10 +7,20 @@ export const E2E_ACTION_PROVENANCE = { "NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75", contentSha256: "1283c2eadfbc38ccb3b795684ba5ced9c89ae2040fffbb6b81854a9d1926802b", }, + nativePodmanRuntime: { + reference: + "NVIDIA/NemoClaw/.github/actions/setup-native-podman-e2e@c87144de2c8e2d90b14cf11b31718846e32c65de", + contentSha256: "ea633b602a0c44f19cdb4c4e4ca28c9b22732e848c34edd871c148675da83349", + }, + stageNativePodmanToolchains: { + reference: + "NVIDIA/NemoClaw/.github/actions/stage-native-podman-e2e-toolchains@1a0f53d5d7e5420556be72b50d79ed5a333d637d", + contentSha256: "e6be7f926407795a2575a6dac8dc8b61738c9f19f7dd09ff6e52dff50ec2140f", + }, restoreCliArtifact: { reference: - "NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@813ca162c2867a34ba3692ad60dba73f3282baea", - contentSha256: "6f8d0138589b7c48a977d004f41cc60f7328d193d85ea5365f0a48b65f3e7485", + "NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@17759906bd7f80319c58af759dd60cfb893109bf", + contentSha256: "4a6a6b21993e579855916dfb897995a3f35dc4461d04666094af7eddb8676077", }, uploadArtifacts: { reference: diff --git a/tools/e2e/workflow-boundary.mts b/tools/e2e/workflow-boundary.mts index c9f250ea2fa..c7b0d099659 100644 --- a/tools/e2e/workflow-boundary.mts +++ b/tools/e2e/workflow-boundary.mts @@ -44,6 +44,12 @@ import { validateE2eExecutionRows, validateE2eExecutionMetadata, } from "./execution-coverage.mts"; +import { + E2E_RUNTIME_AGNOSTIC, + E2E_GATEWAY_RUNTIMES as SUPPORTED_E2E_GATEWAY_RUNTIMES, + type E2eGatewayRuntime, + type E2eGatewayRuntimeSupport, +} from "./gateway-runtime.mts"; import { validateStandardProfileWorkflowBoundary } from "./standard-profile-workflow-boundary.mts"; import { validateTrustedHermesSwapHelperSource, @@ -97,6 +103,13 @@ const DEFAULT_HOST_DEPENDENCY_ACTION_PATH = join( "host-dependency-setup", "action.yaml", ); +const DEFAULT_NATIVE_PODMAN_SETUP_ACTION_PATH = join( + REPO_ROOT, + ".github", + "actions", + "setup-native-podman-e2e", + "action.yaml", +); const DEFAULT_HOST_DEPENDENCY_SCRIPT_PATH = join( REPO_ROOT, ".github", @@ -123,6 +136,8 @@ export interface FreeStandingJobsInventory { targetToJob: Map; liveTestToJobs: Map; coverageRows: E2eExecutionRow[]; + gatewayRuntimesByJob: Map; + gatewayRuntimesByCoverageRow: Map; } export interface FocusedE2eJob { @@ -163,6 +178,7 @@ const LIVE_TEST_FILE_PATTERN = /test\/e2e\/live\/(?:[A-Za-z0-9._-]+\/)*[A-Za-z0- const FREE_STANDING_JOB_MARKER = "E2E_JOB"; const FREE_STANDING_TARGET_MARKER = "E2E_TARGET_ID"; const FREE_STANDING_DEFAULT_ENABLED_MARKER = "E2E_DEFAULT_ENABLED"; +const GATEWAY_RUNTIMES_MARKER = "E2E_GATEWAY_RUNTIMES"; const AGENT_RUNTIME_MARKER = "E2E_AGENT_RUNTIME"; const OUTCOME_MARKER = "E2E_OBSERVABLE_OUTCOME"; const ENVIRONMENT_MARKER = "E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT"; @@ -174,6 +190,7 @@ const COVERAGE_MATRIX_KEYS = [ "unresolved_reason", "coverage_variant", ] as const; +const COVERAGE_GATEWAY_RUNTIMES_KEY = "gateway_runtimes"; const STAGING_BREV_JOB_ID = "staging-brev-launchable"; const STAGING_BREV_IDENTITY_JOB_ID = "staging-brev-launchable-identity"; const STAGING_BREV_JOB_IDS = new Set([STAGING_BREV_JOB_ID, STAGING_BREV_IDENTITY_JOB_ID]); @@ -479,7 +496,47 @@ function findDuplicates(values: readonly string[]): string[] { return [...duplicates].sort(); } -function workflowCoverageRows(jobId: string, job: WorkflowRecord): E2eExecutionRow[] { +function gatewayRuntimeSupport(value: unknown): E2eGatewayRuntimeSupport | undefined { + const declaration = stringValue(value); + if (declaration === E2E_RUNTIME_AGNOSTIC) return E2E_RUNTIME_AGNOSTIC; + const runtimes = declaration.split(","); + return runtimes.length > 0 && + new Set(runtimes).size === runtimes.length && + runtimes.every((runtime) => + SUPPORTED_E2E_GATEWAY_RUNTIMES.includes(runtime as E2eGatewayRuntime), + ) + ? (runtimes as E2eGatewayRuntime[]) + : undefined; +} + +function scenarioCoverageCandidates( + matrix: WorkflowRecord, + jobGatewayRuntimes: E2eGatewayRuntimeSupport, +): WorkflowRecord[] { + if (!Array.isArray(matrix.scenario) || jobGatewayRuntimes === E2E_RUNTIME_AGNOSTIC) return []; + const scenarios = matrix.scenario.map(stringValue).filter(Boolean); + if (scenarios.length !== matrix.scenario.length || new Set(scenarios).size !== scenarios.length) { + return []; + } + const exclusions = Array.isArray(matrix.exclude) ? matrix.exclude.map(asRecord) : []; + return scenarios.map((scenario) => ({ + coverage_variant: scenario, + gateway_runtimes: jobGatewayRuntimes + .filter( + (runtime) => + !exclusions.some( + (entry) => entry.scenario === scenario && entry.runtime_provider === runtime, + ), + ) + .join(","), + })); +} + +function workflowCoverageRows( + jobId: string, + job: WorkflowRecord, + jobGatewayRuntimes: E2eGatewayRuntimeSupport, +): Array<{ row: E2eExecutionRow; gatewayRuntimes: E2eGatewayRuntimeSupport }> { const env = asRecord(job.env); const matrix = asRecord(asRecord(job.strategy).matrix); const includes = Array.isArray(matrix.include) @@ -495,7 +552,9 @@ function workflowCoverageRows(jobId: string, job: WorkflowRecord): E2eExecutionR ].some((key) => Object.hasOwn(env, key)); if (!hasEnvironmentMetadata && includes.length === 0) return []; - const candidates = includes.length > 0 ? includes : [{}]; + const scenarioCandidates = scenarioCoverageCandidates(matrix, jobGatewayRuntimes); + const candidates = + includes.length > 0 ? includes : scenarioCandidates.length > 0 ? scenarioCandidates : [{}]; return candidates.map((entry) => { const metadata = validateE2eExecutionMetadata( { @@ -509,10 +568,14 @@ function workflowCoverageRows(jobId: string, job: WorkflowRecord): E2eExecutionR `E2E workflow job ${jobId}`, ); return { - id: jobId, - variant: stringValue(entry.coverage_variant), - source: STAGING_BREV_JOB_IDS.has(jobId) ? "staging" : "retained-workflow", - ...metadata, + row: { + id: jobId, + variant: stringValue(entry.coverage_variant), + source: STAGING_BREV_JOB_IDS.has(jobId) ? "staging" : "retained-workflow", + ...metadata, + }, + gatewayRuntimes: + gatewayRuntimeSupport(entry[COVERAGE_GATEWAY_RUNTIMES_KEY]) ?? jobGatewayRuntimes, }; }); } @@ -529,15 +592,12 @@ function deriveFreeStandingJobsInventoryFromJobs(jobs: WorkflowRecord): { const targetToJob = new Map(); const liveTestToJobs = new Map(); const coverageRows: E2eExecutionRow[] = []; + const gatewayRuntimesByJob = new Map(); + const gatewayRuntimesByCoverageRow = new Map(); for (const [jobId, rawJob] of Object.entries(jobs)) { const job = asRecord(rawJob); const env = asRecord(job.env); - try { - coverageRows.push(...workflowCoverageRows(jobId, job)); - } catch (error) { - errors.push(error instanceof Error ? error.message : String(error)); - } if (jobId === SHARED_E2E_JOB_ID) continue; const hasJobMarker = Object.hasOwn(env, FREE_STANDING_JOB_MARKER); const hasTargetMarker = Object.hasOwn(env, FREE_STANDING_TARGET_MARKER); @@ -559,6 +619,24 @@ function deriveFreeStandingJobsInventoryFromJobs(jobs: WorkflowRecord): { allowedJobs.push(jobId); workflowJobs.push(jobId); + const gatewayRuntimes = + gatewayRuntimeSupport(env[GATEWAY_RUNTIMES_MARKER]) ?? E2E_RUNTIME_AGNOSTIC; + if (gatewayRuntimes === undefined) { + errors.push(`${jobId} job ${GATEWAY_RUNTIMES_MARKER} is invalid`); + } else { + gatewayRuntimesByJob.set(jobId, gatewayRuntimes); + try { + for (const declaration of workflowCoverageRows(jobId, job, gatewayRuntimes)) { + coverageRows.push(declaration.row); + gatewayRuntimesByCoverageRow.set( + `${declaration.row.id}:${declaration.row.variant}`, + declaration.gatewayRuntimes, + ); + } + } catch (error) { + errors.push(error instanceof Error ? error.message : String(error)); + } + } for (const file of collectLiveTestFiles(rawJob)) addMapValue(liveTestToJobs, file, jobId); if (Object.hasOwn(env, FREE_STANDING_DEFAULT_ENABLED_MARKER)) { if (env[FREE_STANDING_DEFAULT_ENABLED_MARKER] !== "0") { @@ -632,6 +710,8 @@ function deriveFreeStandingJobsInventoryFromJobs(jobs: WorkflowRecord): { freeStandingTargets, targetToJob, coverageRows, + gatewayRuntimesByJob, + gatewayRuntimesByCoverageRow, liveTestToJobs: new Map( [...liveTestToJobs] .sort(([left], [right]) => left.localeCompare(right)) @@ -660,6 +740,18 @@ function cloneFreeStandingJobsInventory( freeStandingTargets: [...inventory.freeStandingTargets], targetToJob: new Map(inventory.targetToJob), coverageRows: inventory.coverageRows.map((row) => ({ ...row })), + gatewayRuntimesByJob: new Map( + [...inventory.gatewayRuntimesByJob].map(([job, runtimes]) => [ + job, + runtimes === E2E_RUNTIME_AGNOSTIC ? runtimes : [...runtimes], + ]), + ), + gatewayRuntimesByCoverageRow: new Map( + [...inventory.gatewayRuntimesByCoverageRow].map(([key, runtimes]) => [ + key, + runtimes === E2E_RUNTIME_AGNOSTIC ? runtimes : [...runtimes], + ]), + ), liveTestToJobs: cloneStringArrayMap(inventory.liveTestToJobs), }; } @@ -1318,7 +1410,7 @@ function validateSharedE2eJob(errors: string[], jobs: WorkflowRecord): void { return; } - if (job.name !== "Shared E2E (${{ matrix.id }})") { + if (job.name !== "Shared E2E (${{ matrix.execution_id }})") { errors.push("shared E2E job name must expose the test ID"); } if (job.needs !== "generate-matrix") { @@ -1348,12 +1440,14 @@ function validateSharedE2eJob(errors: string[], jobs: WorkflowRecord): void { const jobEnv = asRecord(job.env); const expectedEnv = { CHECK_DOC_LINKS_REMOTE: "0", - E2E_ARTIFACT_DIR: "${{ github.workspace }}/e2e-artifacts/live/${{ matrix.id }}", + E2E_ARTIFACT_DIR: "${{ github.workspace }}/e2e-artifacts/live/${{ matrix.execution_id }}", + E2E_EXECUTION_ID: "${{ matrix.execution_id }}", E2E_TARGET_ID: "${{ matrix.id }}", NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", NEMOCLAW_CLI_BIN: "${{ github.workspace }}/bin/nemoclaw.js", NEMOCLAW_NON_INTERACTIVE: "1", NEMOCLAW_RUN_LIVE_E2E: "1", + NEMOCLAW_GATEWAY_RUNTIME: "${{ matrix.runtime_provider }}", }; for (const [name, expected] of Object.entries(expectedEnv)) { if (jobEnv[name] !== expected) { @@ -1662,7 +1756,10 @@ function validateHermesE2EJob(errors: string[], jobs: WorkflowRecord): void { if (jobEnv.NEMOCLAW_CLI_BIN !== "${{ github.workspace }}/bin/nemoclaw.js") { errors.push("hermes-e2e job must point NEMOCLAW_CLI_BIN at the repo CLI"); } - if (jobEnv.E2E_ARTIFACT_DIR !== "${{ github.workspace }}/e2e-artifacts/live/hermes-e2e") { + if ( + jobEnv.E2E_ARTIFACT_DIR !== + "${{ github.workspace }}/e2e-artifacts/live/hermes-e2e/${{ matrix.runtime_provider }}" + ) { errors.push("hermes-e2e job must write artifacts under e2e-artifacts/live/hermes-e2e"); } if (jobEnv.NEMOCLAW_AGENT !== "hermes") { @@ -2101,6 +2198,7 @@ function validateStagingBrevLaunchableIdentityJob(errors: string[], jobs: Workfl const expectedJobEnv = { CANDIDATE_SHA: "${{ github.sha }}", E2E_DEFAULT_ENABLED: "0", + E2E_GATEWAY_RUNTIMES: "agnostic", E2E_JOB: "1", INSTANCE_NAME: "nclaw-identity-${{ github.run_id }}-${{ github.run_attempt }}", E2E_AGENT_RUNTIME: "none", @@ -3117,8 +3215,8 @@ export function validateE2eWorkflow(workflowValue: unknown): string[] { const upload = requireStep(errors, steps, "Upload E2E artifacts"); const uploadWith = asRecord(upload?.with); - if (uploadWith.name !== "e2e-${{ matrix.id }}") { - errors.push("artifact upload name must include matrix.id"); + if (uploadWith.name !== "e2e-${{ matrix.execution_id }}") { + errors.push("artifact upload name must include matrix.execution_id"); } const uploadPath = stringValue(uploadWith.path); requireUploadPathContains( @@ -3350,9 +3448,57 @@ export function validateE2eWorkflowBoundary(workflowPath = DEFAULT_E2E_WORKFLOW_ ...validateDockerHubAuthAction(), ...validateDockerHubCleanupAction(), ...validateHostDependencyAction(), + ...validateNativePodmanSetupAction(), ...validateE2eWorkflow(workflow), ...validateTrustedHermesSwapHelperSource( readFileSync(DEFAULT_LIVE_VITEST_INVOCATION_PATH, "utf8"), ), ]; } + +export function validateNativePodmanSetupAction( + actionPath = DEFAULT_NATIVE_PODMAN_SETUP_ACTION_PATH, +): string[] { + const action = asRecord(YAML.parse(readFileSync(actionPath, "utf8"))); + const steps = asSteps(asRecord(action.runs).steps); + const start = steps.find((step) => step.name === "Start native Podman runtime"); + const run = stringValue(start?.run); + const errors: string[] = []; + + if (!start) return ["native Podman setup action must start the runtime"]; + if (!run.includes('systemctl start "user-runtime-dir@${uid}.service" "user@${uid}.service"')) { + errors.push("native Podman setup must start the runner user manager"); + } + if (!run.includes("/usr/bin/systemctl --user start dbus.socket")) { + errors.push("native Podman setup must start the runner user D-Bus socket"); + } + if (!run.includes('[[ -S "$runtime_directory/bus" && ! -L "$runtime_directory/bus" ]]')) { + errors.push("native Podman setup must verify the runner user D-Bus authority"); + } + if (run.includes("printf 'DOCKER_HOST=")) { + errors.push("native Podman setup must not expose its API socket as Docker"); + } + if ( + !run.includes("systemctl stop docker.service docker.socket") || + !run.includes("systemctl mask --runtime docker.service docker.socket") || + !run.includes("! pgrep -x dockerd >/dev/null") || + !run.includes("docker info >/dev/null 2>&1") + ) { + errors.push("native Podman setup must make Docker unavailable before qualification"); + } + if ( + !run.includes('export DBUS_SESSION_BUS_ADDRESS="unix:path=$runtime_directory/bus"') || + !run.includes('systemctl --user start "$service_name.socket"') || + run.indexOf('export DBUS_SESSION_BUS_ADDRESS="unix:path=$runtime_directory/bus"') > + run.indexOf('systemctl --user start "$service_name.socket"') + ) { + errors.push("native Podman setup must bind user D-Bus before starting the API service"); + } + if (!run.includes("printf 'OPENSHELL_PODMAN_SOCKET=%s\\n'")) { + errors.push("native Podman setup must expose the provider-owned socket authority"); + } + if (!run.includes('printf \'PATH=%s:%s\\n\' "$toolchain_install_root/bin" "$PATH"')) { + errors.push("native Podman setup must preserve the reviewed executable authority on PATH"); + } + return errors; +} diff --git a/tools/e2e/workflow-plan.mts b/tools/e2e/workflow-plan.mts index 549fef3c7dd..0e306b581b7 100644 --- a/tools/e2e/workflow-plan.mts +++ b/tools/e2e/workflow-plan.mts @@ -9,12 +9,18 @@ import { isDeepStrictEqual } from "node:util"; import { buildLiveTargetInventory, buildLiveTargetMatrix, + liveTargetGatewayRuntimes, type LiveTargetMatrixEntry, } from "../../test/e2e/registry/run.ts"; +import { listTargets } from "../../test/e2e/registry/registry.ts"; import { buildRiskPlan } from "../advisors/risk-plan.mts"; import { + type CredentialFreeTestDefinitionRow, type CredentialFreeTestMatrixRow, credentialFreeTestCoverage, + credentialFreeTestGatewayRuntimes, + credentialFreeTestMatrix, + credentialFreeTestSupportsGatewayRuntime, discoverCredentialFreeTests, SHARED_E2E_JOB_ID, } from "./credential-free-tests.mts"; @@ -46,6 +52,16 @@ import { validateE2eExecutionRows, validateE2eExecutionMetadata, } from "./execution-coverage.mts"; +import { + E2E_RUNTIME_AGNOSTIC, + type E2eGatewayRuntime, + type E2eGatewayRuntimeSupport, + type E2eRuntimeProvider, + e2eGatewayRuntimes, + e2eRuntimeProviders, + runtimeCoverageVariant, + supportsE2eGatewayRuntime, +} from "./gateway-runtime.mts"; export type WorkflowPlanSelectors = { jobs?: string; @@ -53,17 +69,20 @@ export type WorkflowPlanSelectors = { }; export type E2eWorkflowPlan = { + gatewayRuntimes: E2eGatewayRuntime[]; matrix: LiveTargetMatrixEntry[]; testMatrix: CredentialFreeTestMatrixRow[]; catalogueMatrices: Record; coverageMatrix: E2eExecutionRow[]; selectedJobs: string[]; + runtimeProvidersByJob: Record; hermesSelected: boolean; explicitOnlyJobs: string[]; }; type WorkflowPlanOptions = { changedFiles?: readonly string[]; + gatewayRuntimes?: readonly E2eGatewayRuntime[]; }; type WorkflowPlanCliOptions = WorkflowPlanSelectors & { @@ -152,6 +171,7 @@ function isLiveTargetMatrixEntry(value: unknown): value is LiveTargetMatrixEntry !hasExactKeys(value, [ "agentRuntime", "environmentOrInferenceEndpoint", + "execution_id", "expectedStateId", "id", "install", @@ -161,6 +181,8 @@ function isLiveTargetMatrixEntry(value: unknown): value is LiveTargetMatrixEntry "pendingRuntimeSuites", "platform", "requiredSecrets", + "runtime_provider", + "coverage_variant", "runner", "runtime", "suites", @@ -175,6 +197,13 @@ function isLiveTargetMatrixEntry(value: unknown): value is LiveTargetMatrixEntry return ( typeof value.id === "string" && /^[a-z0-9]+(?:-[a-z0-9]+)*$/u.test(value.id) && + typeof value.execution_id === "string" && + /^[a-z0-9]+(?:-[a-z0-9]+)*$/u.test(value.execution_id) && + typeof value.coverage_variant === "string" && + /^[a-z0-9]+(?:-[a-z0-9]+)*$/u.test(value.coverage_variant) && + (value.runtime_provider === "docker" || + value.runtime_provider === "podman" || + value.runtime_provider === "none") && typeof value.runner === "string" && /^[A-Za-z0-9_-]+$/u.test(value.runner) && typeof value.label === "string" && @@ -200,14 +229,32 @@ function isLiveTargetMatrixEntry(value: unknown): value is LiveTargetMatrixEntry } function isCredentialFreeTestMatrixRow(value: unknown): value is CredentialFreeTestMatrixRow { - if (!isRecord(value) || !hasExactKeys(value, ["file", "id", "project"])) return false; + if ( + !isRecord(value) || + !hasExactKeys(value, [ + "coverage_variant", + "execution_id", + "file", + "id", + "project", + "runtime_provider", + ]) + ) + return false; if ( typeof value.id !== "string" || !/^[a-z0-9]+(?:-[a-z0-9]+)*$/u.test(value.id) || typeof value.file !== "string" || value.file.split("/").some((segment) => segment === "." || segment === "..") || !/^test\/(?:[A-Za-z0-9._-]+\/)*[A-Za-z0-9._-]+[.]test[.](?:js|ts)$/u.test(value.file) || - typeof value.project !== "string" + typeof value.project !== "string" || + typeof value.execution_id !== "string" || + !/^[a-z0-9]+(?:-[a-z0-9]+)*$/u.test(value.execution_id) || + typeof value.coverage_variant !== "string" || + !/^[a-z0-9]+(?:-[a-z0-9]+)*$/u.test(value.coverage_variant) || + (value.runtime_provider !== "docker" && + value.runtime_provider !== "podman" && + value.runtime_provider !== "none") ) { return false; } @@ -219,8 +266,8 @@ function isCredentialFreeTestMatrixRow(value: unknown): value is CredentialFreeT ); } -function hasUniqueIds(rows: readonly { id: string }[]): boolean { - return new Set(rows.map((row) => row.id)).size === rows.length; +function hasUniqueValues(rows: readonly T[], value: (row: T) => string): boolean { + return new Set(rows.map(value)).size === rows.length; } function isCatalogueMatrixRow(value: unknown): value is E2eCatalogueMatrixRow { @@ -231,7 +278,9 @@ function isCatalogueMatrixRow(value: unknown): value is E2eCatalogueMatrixRow { "agent_runtime", "cloudflared", "compatible_api_key", + "coverage_variant", "id", + "execution_id", "display_name", "environment_or_inference_endpoint", "host_preparation", @@ -244,6 +293,7 @@ function isCatalogueMatrixRow(value: unknown): value is E2eCatalogueMatrixRow { "runner_comparison", "runner_key", "runner_pressure", + "runtime_provider", "shard", "target_id", "test_file", @@ -252,6 +302,13 @@ function isCatalogueMatrixRow(value: unknown): value is E2eCatalogueMatrixRow { ]) && typeof value.id === "string" && /^[a-z0-9]+(?:-[a-z0-9]+)*$/u.test(value.id) && + typeof value.execution_id === "string" && + /^[a-z0-9]+(?:-[a-z0-9]+)*$/u.test(value.execution_id) && + typeof value.coverage_variant === "string" && + /^[a-z0-9]+(?:-[a-z0-9]+)*$/u.test(value.coverage_variant) && + (value.runtime_provider === "docker" || + value.runtime_provider === "podman" || + value.runtime_provider === "none") && typeof value.display_name === "string" && /^[A-Z][A-Za-z0-9 .'+()-]+: [^/\r\n]{1,72}$/u.test(value.display_name) && typeof value.agent_runtime === "string" && @@ -360,7 +417,12 @@ function isCatalogueMatrixRowForProfile( target.runnerPressure === value.runner_pressure && target.compatibleApiKey === value.compatible_api_key && target.shard === value.shard && - target.artifactLayout === value.artifact_layout + target.artifactLayout === value.artifact_layout && + value.coverage_variant === runtimeCoverageVariant(target.shard, value.runtime_provider) && + value.execution_id === `${target.id}-${value.coverage_variant}` && + e2eRuntimeProviders(target.gatewayRuntimes, ["docker", "podman"]).includes( + value.runtime_provider, + ) ); } @@ -390,20 +452,54 @@ function emptyCatalogueMatrices(): Record { return Object.fromEntries( - E2E_EXECUTION_PROFILES.map((profile) => [profile, catalogueMatrix(profile, targets)]), + E2E_EXECUTION_PROFILES.map((profile) => [ + profile, + catalogueMatrix(profile, targets, gatewayRuntimes), + ]), ) as Record; } -function registryTargetsForChangedFiles(changedFiles: readonly string[]): LiveTargetMatrixEntry[] { +function registryTargetsForChangedFiles( + changedFiles: readonly string[], + gatewayRuntimes: readonly E2eGatewayRuntime[], +): LiveTargetMatrixEntry[] { return changedFiles.some((file) => REGISTRY_OWNING_PATHS.some((owner) => pathMatches(file, owner)), ) - ? buildLiveTargetMatrix() + ? buildLiveTargetMatrix([], gatewayRuntimes) : []; } +function workflowJobRuntimeProviders( + inventory: ReturnType, + job: string, + gatewayRuntimes: readonly E2eGatewayRuntime[], +): E2eRuntimeProvider[] { + return e2eRuntimeProviders( + inventory.gatewayRuntimesByJob.get(job) ?? E2E_RUNTIME_AGNOSTIC, + gatewayRuntimes, + ); +} + +function runtimeProvidersByJob( + inventory: ReturnType, + jobs: readonly string[], + gatewayRuntimes: readonly E2eGatewayRuntime[], + sharedRows: readonly CredentialFreeTestMatrixRow[] = [], +): Record { + return Object.fromEntries( + jobs.map((job) => [ + job, + job === SHARED_E2E_JOB_ID && sharedRows.length > 0 + ? [...new Set(sharedRows.map((row) => row.runtime_provider))] + : workflowJobRuntimeProviders(inventory, job, gatewayRuntimes), + ]), + ); +} + function changedFilesFromEnvironment(environment: NodeJS.ProcessEnv): string[] | undefined { if (environment.EVENT_NAME !== "push") return undefined; const declared = environment.CHANGED_FILES; @@ -424,9 +520,9 @@ function selectorIds(value: string | undefined, label: "jobs" | "targets"): stri } function selectTestRows( - rows: readonly CredentialFreeTestMatrixRow[], + rows: readonly CredentialFreeTestDefinitionRow[], ids: readonly string[], -): CredentialFreeTestMatrixRow[] { +): CredentialFreeTestDefinitionRow[] { if (ids.length === 0) return [...rows]; const selected = new Set(ids); return rows.filter((row) => selected.has(row.id)); @@ -478,13 +574,15 @@ function mapTrustedControllerJobs( }; } -function emptyE2eWorkflowPlan(): E2eWorkflowPlan { +function emptyE2eWorkflowPlan(gatewayRuntimes: readonly E2eGatewayRuntime[]): E2eWorkflowPlan { return { + gatewayRuntimes: [...gatewayRuntimes], matrix: [], testMatrix: [], catalogueMatrices: emptyCatalogueMatrices(), coverageMatrix: [], selectedJobs: [], + runtimeProvidersByJob: {}, hermesSelected: false, explicitOnlyJobs: readFreeStandingJobsInventory().explicitOnlyJobs, }; @@ -499,7 +597,7 @@ function coverageMatrixForPlan( const catalogueRows = E2E_EXECUTION_PROFILES.flatMap((profile) => plan.catalogueMatrices[profile].map((row) => ({ id: row.id, - variant: "", + variant: row.coverage_variant, source: "catalogue" as const, agentRuntime: row.agent_runtime, observableOutcome: row.observable_outcome, @@ -509,7 +607,7 @@ function coverageMatrixForPlan( ); const registryRows = plan.matrix.map((row) => ({ id: row.id, - variant: "", + variant: row.coverage_variant, source: "typed-registry" as const, agentRuntime: row.agentRuntime, observableOutcome: row.observableOutcome, @@ -518,12 +616,23 @@ function coverageMatrixForPlan( })); const sharedRows = plan.testMatrix.map((row) => ({ id: row.id, - variant: "", + variant: row.coverage_variant, source: "shared-e2e" as const, ...credentialFreeTestCoverage(row.id), })); const selectedJobs = new Set(plan.selectedJobs); - const workflowRows = inventory.coverageRows.filter((row) => selectedJobs.has(row.id)); + const workflowRows = inventory.coverageRows + .filter((row) => selectedJobs.has(row.id)) + .flatMap((row) => { + const support = + inventory.gatewayRuntimesByCoverageRow.get(`${row.id}:${row.variant}`) ?? + inventory.gatewayRuntimesByJob.get(row.id) ?? + E2E_RUNTIME_AGNOSTIC; + return e2eRuntimeProviders(support, plan.gatewayRuntimes).map((runtimeProvider) => ({ + ...row, + variant: runtimeCoverageVariant(row.variant, runtimeProvider), + })); + }); const rows = [...catalogueRows, ...registryRows, ...sharedRows, ...workflowRows]; validateE2eExecutionRows(rows); return rows; @@ -575,6 +684,7 @@ export function buildE2eWorkflowPlan( selectors: WorkflowPlanSelectors = {}, options: WorkflowPlanOptions = {}, ): E2eWorkflowPlan { + const gatewayRuntimes = e2eGatewayRuntimes((options.gatewayRuntimes ?? ["docker"]).join(",")); const jobs = selectorIds(selectors.jobs, "jobs"); const targets = selectorIds(selectors.targets, "targets"); @@ -596,10 +706,12 @@ export function buildE2eWorkflowPlan( if (jetsonDispatchSelected) { return withCoverageMatrix( { + gatewayRuntimes, matrix: [], testMatrix: [], catalogueMatrices: emptyCatalogueMatrices(), selectedJobs: [JETSON_DISPATCH_TARGET], + runtimeProvidersByJob: { [JETSON_DISPATCH_TARGET]: ["none"] }, hermesSelected: false, explicitOnlyJobs: [...inventory.explicitOnlyJobs], }, @@ -627,6 +739,26 @@ export function buildE2eWorkflowPlan( const selectedCatalogueTargets = E2E_TARGET_CATALOGUE.filter( (target) => selectedIds.has(target.id) || selectedIds.has(target.targetId), ); + const unsupportedCatalogueTarget = selectedCatalogueTargets.find( + (target) => e2eRuntimeProviders(target.gatewayRuntimes, gatewayRuntimes).length === 0, + ); + if (unsupportedCatalogueTarget) { + throw new Error( + `E2E target ${unsupportedCatalogueTarget.id} does not support requested gateway runtimes ${gatewayRuntimes.join(",")}`, + ); + } + const unsupportedSharedTest = discoverCredentialFreeTests().find( + (row) => + selectedIds.has(row.id) && + !gatewayRuntimes.some((runtime) => + credentialFreeTestSupportsGatewayRuntime(row.id, runtime), + ), + ); + if (unsupportedSharedTest) { + throw new Error( + `E2E target ${unsupportedSharedTest.id} does not support requested gateway runtimes ${gatewayRuntimes.join(",")}`, + ); + } const registryTargets = targets.filter( (target) => !inventory.targetToJob.has(target) && !catalogueIds.has(target), ); @@ -639,12 +771,38 @@ export function buildE2eWorkflowPlan( selectedJobSet.add("openshell-credential-generation-window"); } const selectedJobs = [...selectedJobSet]; + const unsupportedWorkflowJob = selectedJobs.find( + (job) => workflowJobRuntimeProviders(inventory, job, gatewayRuntimes).length === 0, + ); + if (unsupportedWorkflowJob) { + throw new Error( + `E2E job ${unsupportedWorkflowJob} does not support requested gateway runtimes ${gatewayRuntimes.join(",")}`, + ); + } + const registryMatrix = + registryTargets.length > 0 ? buildLiveTargetMatrix(registryTargets, gatewayRuntimes) : []; + if (registryTargets.some((target) => !registryMatrix.some((row) => row.id === target))) { + throw new Error( + `Selected typed E2E target does not support requested gateway runtimes ${gatewayRuntimes.join(",")}`, + ); + } + const selectedTestDefinitions = selectedIds.has(SHARED_E2E_JOB_ID) + ? credentialFreeTests + : selectTestRows(credentialFreeTests, [...jobs, ...targets]); + const testMatrix = credentialFreeTestMatrix(selectedTestDefinitions, gatewayRuntimes); return withCoverageMatrix( { - matrix: registryTargets.length > 0 ? buildLiveTargetMatrix(registryTargets) : [], - testMatrix: selectTestRows(credentialFreeTests, [...jobs, ...targets]), - catalogueMatrices: catalogueMatrices(selectedCatalogueTargets), + gatewayRuntimes, + matrix: registryMatrix, + testMatrix, + catalogueMatrices: catalogueMatrices(selectedCatalogueTargets, gatewayRuntimes), selectedJobs, + runtimeProvidersByJob: runtimeProvidersByJob( + inventory, + selectedJobs, + gatewayRuntimes, + testMatrix, + ), hermesSelected: selectedJobs.includes(HERMES_JOB_ID), explicitOnlyJobs: [...inventory.explicitOnlyJobs], }, @@ -657,11 +815,20 @@ export function buildE2eWorkflowPlan( if ( changedFiles.some((file) => FULL_SUITE_OWNING_PATHS.some((owner) => pathMatches(file, owner))) ) { - const plan = buildE2eWorkflowPlan(selectors); - return { - ...plan, - selectedJobs: [...new Set([...plan.selectedJobs, JETSON_DISPATCH_TARGET])], - }; + const plan = buildE2eWorkflowPlan(selectors, { gatewayRuntimes }); + const { coverageMatrix: _coverageMatrix, ...planWithoutCoverage } = plan; + const selectedJobs = [...new Set([...plan.selectedJobs, JETSON_DISPATCH_TARGET])]; + return withCoverageMatrix( + { + ...planWithoutCoverage, + selectedJobs, + runtimeProvidersByJob: { + ...plan.runtimeProvidersByJob, + [JETSON_DISPATCH_TARGET]: ["none"], + }, + }, + inventory, + ); } const focusedLegacyJobs = focusedE2eJobsForChangedFiles(changedFiles, inventory); const directlySelectedCatalogueTargets = catalogueTargetsForChangedFiles(changedFiles); @@ -692,7 +859,13 @@ export function buildE2eWorkflowPlan( } selectedJobSet.add(JETSON_DISPATCH_TARGET); const selectedJobs = [...selectedJobSet]; - const selectedTests = credentialFreeTests.filter((row) => changedFiles.includes(row.file)); + const runtimeSelectedJobs = selectedJobs.filter( + (job) => workflowJobRuntimeProviders(inventory, job, gatewayRuntimes).length > 0, + ); + const selectedTests = credentialFreeTestMatrix( + credentialFreeTests.filter((row) => changedFiles.includes(row.file)), + gatewayRuntimes, + ); const selectedCatalogueIds = new Set([ ...directlySelectedCatalogueTargets.map((target) => target.id), ...riskJobIds, @@ -702,31 +875,54 @@ export function buildE2eWorkflowPlan( ); const riskTargetIds = riskPlan.requiredTargets.map((target) => target.id); const registryMatrix = [ - ...registryTargetsForChangedFiles(changedFiles), - ...(riskTargetIds.length > 0 ? buildLiveTargetMatrix(riskTargetIds) : []), - ].filter((entry, index, rows) => rows.findIndex((row) => row.id === entry.id) === index); + ...registryTargetsForChangedFiles(changedFiles, gatewayRuntimes), + ...(riskTargetIds.length > 0 ? buildLiveTargetMatrix(riskTargetIds, gatewayRuntimes) : []), + ].filter( + (entry, index, rows) => + rows.findIndex((row) => row.execution_id === entry.execution_id) === index, + ); return withCoverageMatrix( { + gatewayRuntimes, matrix: registryMatrix, testMatrix: selectedTests, - catalogueMatrices: catalogueMatrices(selectedCatalogueTargets), - selectedJobs, - hermesSelected: selectedJobs.includes(HERMES_JOB_ID), + catalogueMatrices: catalogueMatrices(selectedCatalogueTargets, gatewayRuntimes), + selectedJobs: runtimeSelectedJobs, + runtimeProvidersByJob: runtimeProvidersByJob( + inventory, + runtimeSelectedJobs, + gatewayRuntimes, + selectedTests, + ), + hermesSelected: runtimeSelectedJobs.includes(HERMES_JOB_ID), explicitOnlyJobs: [...inventory.explicitOnlyJobs], }, inventory, ); } + const testMatrix = credentialFreeTestMatrix(credentialFreeTests, gatewayRuntimes); + const selectedJobs = inventory.workflowJobs.filter( + (job) => + !inventory.explicitOnlyJobs.includes(job) && + (job !== SHARED_E2E_JOB_ID || testMatrix.length > 0) && + workflowJobRuntimeProviders(inventory, job, gatewayRuntimes).length > 0, + ); return withCoverageMatrix( { - matrix: buildLiveTargetMatrix(), - testMatrix: credentialFreeTests, - catalogueMatrices: catalogueMatrices(E2E_TARGET_CATALOGUE), - selectedJobs: inventory.workflowJobs.filter( - (job) => !inventory.explicitOnlyJobs.includes(job), + gatewayRuntimes, + matrix: buildLiveTargetMatrix([], gatewayRuntimes), + testMatrix, + catalogueMatrices: catalogueMatrices(E2E_TARGET_CATALOGUE, gatewayRuntimes), + selectedJobs, + runtimeProvidersByJob: runtimeProvidersByJob( + inventory, + selectedJobs, + gatewayRuntimes, + testMatrix, ), - hermesSelected: true, + hermesSelected: + workflowJobRuntimeProviders(inventory, HERMES_JOB_ID, gatewayRuntimes).length > 0, explicitOnlyJobs: [...inventory.explicitOnlyJobs], }, inventory, @@ -739,9 +935,11 @@ export function validateE2eWorkflowPlan(plan: unknown): E2eWorkflowPlan { !hasExactKeys(plan, [ "catalogueMatrices", "explicitOnlyJobs", + "gatewayRuntimes", "hermesSelected", "matrix", "coverageMatrix", + "runtimeProvidersByJob", "selectedJobs", "testMatrix", ]) @@ -755,16 +953,64 @@ export function validateE2eWorkflowPlan(plan: unknown): E2eWorkflowPlan { const catalogueMatrixRows = E2E_EXECUTION_PROFILES.flatMap( (profile) => catalogueMatricesValue[profile], ); + const credentialFreeDefinitions = new Map( + discoverCredentialFreeTests().map((row) => [row.id, row]), + ); + const validCredentialFreeTestRows = + Array.isArray(plan.testMatrix) && + plan.testMatrix.every((row) => { + if (!isCredentialFreeTestMatrixRow(row)) return false; + const definition = credentialFreeDefinitions.get(row.id); + return ( + definition?.file === row.file && + definition.project === row.project && + row.runtime_provider !== "none" && + credentialFreeTestSupportsGatewayRuntime(row.id, row.runtime_provider) + ); + }); + const validLiveTargetRows = + Array.isArray(plan.matrix) && plan.matrix.every(isLiveTargetMatrixEntry); + const uniqueExecutionIds = + validLiveTargetRows && + validCredentialFreeTestRows && + hasUniqueValues( + [ + ...(plan.matrix as LiveTargetMatrixEntry[]), + ...(plan.testMatrix as CredentialFreeTestMatrixRow[]), + ...catalogueMatrixRows, + ], + (row) => row.execution_id, + ); + const validGatewayRuntimes = + Array.isArray(plan.gatewayRuntimes) && + plan.gatewayRuntimes.length > 0 && + plan.gatewayRuntimes.every((runtime) => runtime === "docker" || runtime === "podman") && + new Set(plan.gatewayRuntimes).size === plan.gatewayRuntimes.length; + const selectedJobsValue = plan.selectedJobs; + const validRuntimeProvidersByJob = + isRecord(plan.runtimeProvidersByJob) && + isStringArray(selectedJobsValue) && + Object.keys(plan.runtimeProvidersByJob).length === selectedJobsValue.length && + Object.keys(plan.runtimeProvidersByJob).every((job) => selectedJobsValue.includes(job)) && + Object.values(plan.runtimeProvidersByJob).every( + (providers) => + Array.isArray(providers) && + providers.length > 0 && + providers.every( + (provider) => provider === "docker" || provider === "podman" || provider === "none", + ) && + new Set(providers).size === providers.length, + ); if ( - !Array.isArray(plan.matrix) || - !plan.matrix.every(isLiveTargetMatrixEntry) || - !Array.isArray(plan.testMatrix) || - !plan.testMatrix.every(isCredentialFreeTestMatrixRow) || + !validGatewayRuntimes || + !validLiveTargetRows || + !validCredentialFreeTestRows || !isE2eExecutionRows(plan.coverageMatrix) || - !hasUniqueIds([...plan.matrix, ...plan.testMatrix, ...catalogueMatrixRows]) || - !isStringArray(plan.selectedJobs) || - !plan.selectedJobs.every((job) => /^[A-Za-z0-9_-]+$/u.test(job)) || - !hasUniqueIds(plan.selectedJobs.map((id) => ({ id }))) || + !uniqueExecutionIds || + !isStringArray(selectedJobsValue) || + !selectedJobsValue.every((job) => /^[A-Za-z0-9_-]+$/u.test(job)) || + !hasUniqueValues(selectedJobsValue, (id) => id) || + !validRuntimeProvidersByJob || typeof plan.hermesSelected !== "boolean" || !isStringArray(plan.explicitOnlyJobs) || !plan.explicitOnlyJobs.every((job) => /^[A-Za-z0-9_-]+$/u.test(job)) || @@ -844,16 +1090,79 @@ function restrictUnauthorizedCandidatePlan( ): E2eWorkflowPlan { const candidatePlan = withoutCredentialedCatalogueProfiles(plan); const { coverageMatrix: _coverageMatrix, ...planWithoutCoverage } = candidatePlan; + const selectedJobs = hasPlannerSelectors ? plan.selectedJobs : []; return withCoverageMatrix( { ...planWithoutCoverage, - selectedJobs: hasPlannerSelectors ? plan.selectedJobs : [], + selectedJobs, + runtimeProvidersByJob: Object.fromEntries( + selectedJobs.map((job) => [job, plan.runtimeProvidersByJob[job]]), + ), hermesSelected: hasPlannerSelectors && plan.hermesSelected, }, readFreeStandingJobsInventory(), ); } +type RuntimeExclusion = { + id: string; + excluded: E2eGatewayRuntime[]; + supported: readonly E2eGatewayRuntime[]; +}; + +function runtimeExclusion( + id: string, + support: E2eGatewayRuntimeSupport, + requested: readonly E2eGatewayRuntime[], +): RuntimeExclusion | undefined { + if (support === E2E_RUNTIME_AGNOSTIC) return undefined; + const excluded = requested.filter((runtime) => !supportsE2eGatewayRuntime(support, runtime)); + return excluded.length > 0 ? { id, excluded, supported: support } : undefined; +} + +function runtimeExclusionsForPlan( + plan: E2eWorkflowPlan, + inventory: ReturnType, +): RuntimeExclusion[] { + const catalogueIds = new Set( + Object.values(plan.catalogueMatrices) + .flat() + .map((row) => row.id), + ); + const liveIds = new Set(plan.matrix.map((row) => row.id)); + const sharedIds = new Set(plan.testMatrix.map((row) => row.id)); + const selectedJobs = new Set(plan.selectedJobs); + const candidates = [ + ...E2E_TARGET_CATALOGUE.filter((target) => catalogueIds.has(target.id)).map((target) => + runtimeExclusion(target.id, target.gatewayRuntimes, plan.gatewayRuntimes), + ), + ...listTargets() + .filter((target) => liveIds.has(target.id)) + .map((target) => + runtimeExclusion(target.id, liveTargetGatewayRuntimes(target), plan.gatewayRuntimes), + ), + ...discoverCredentialFreeTests() + .filter((row) => sharedIds.has(row.id)) + .map((row) => + runtimeExclusion(row.id, credentialFreeTestGatewayRuntimes(row.id), plan.gatewayRuntimes), + ), + ...inventory.coverageRows + .filter((row) => selectedJobs.has(row.id)) + .map((row) => + runtimeExclusion( + e2eExecutionLabel(row), + inventory.gatewayRuntimesByCoverageRow.get(`${row.id}:${row.variant}`) ?? + inventory.gatewayRuntimesByJob.get(row.id) ?? + E2E_RUNTIME_AGNOSTIC, + plan.gatewayRuntimes, + ), + ), + ].filter((row): row is RuntimeExclusion => row !== undefined); + return [...new Map(candidates.map((row) => [row.id, row])).values()].sort((a, b) => + a.id.localeCompare(b.id), + ); +} + export function renderE2eWorkflowPlanSummary( plan: E2eWorkflowPlan, options: { includeCoverageAudit?: boolean } = {}, @@ -876,6 +1185,7 @@ export function renderE2eWorkflowPlanSummary( const explicitOnlyRows = inventory.coverageRows.filter((row) => plan.explicitOnlyJobs.includes(row.id), ); + const runtimeExclusions = runtimeExclusionsForPlan(plan, inventory); const unsupportedDeclarations = buildLiveTargetInventory().filter((row) => !row.supported); const outcomeRows = new Map(); for (const row of plan.coverageMatrix) { @@ -897,11 +1207,22 @@ export function renderE2eWorkflowPlanSummary( new Set(rows.map((row) => row.environmentOrInferenceEndpoint)).size > 1 ? "environment or inference endpoint" : "", + new Set(rows.map((row) => row.variant)).size > 1 ? "coverage variant" : "", ].filter(Boolean); lines.push( `| ${outcome} | ${rows.map((row) => `\`${e2eExecutionLabel(row)}\``).join(", ")} | ${dimensions.join(" and ")} |`, ); } + lines.push( + "", + "### Intentional runtime exclusions", + "", + "| Target or job | Requested runtime not scheduled | Declared runtime support |", + "| --- | --- | --- |", + ); + for (const row of runtimeExclusions) { + lines.push(`| \`${row.id}\` | ${row.excluded.join(", ")} | ${row.supported.join(", ")} |`); + } lines.push( "", "### Intentional exclusions", @@ -945,12 +1266,15 @@ export function writeE2eWorkflowPlanCiOutput( } const controllerMap = mapTrustedControllerJobs(selectors, environment); const plannerSelectors = controllerMap.selectors; + const gatewayRuntimes = e2eGatewayRuntimes( + environment.NEMOCLAW_GATEWAY_RUNTIMES ?? environment.NEMOCLAW_GATEWAY_RUNTIME, + ); const hasPlannerSelectors = Boolean(plannerSelectors.jobs || plannerSelectors.targets); const changedFiles = hasPlannerSelectors ? undefined : changedFilesFromEnvironment(environment); const planned = controllerMap.retiredSelectorSelected && !hasPlannerSelectors - ? emptyE2eWorkflowPlan() - : buildE2eWorkflowPlan(plannerSelectors, { changedFiles }); + ? emptyE2eWorkflowPlan(gatewayRuntimes) + : buildE2eWorkflowPlan(plannerSelectors, { changedFiles, gatewayRuntimes }); const availableOptionalCredentials = new Set( E2E_OPTIONAL_CREDENTIALS.filter( (credential) => environment[`NEMOCLAW_E2E_${credential}_AVAILABLE`] !== "false", @@ -986,6 +1310,8 @@ export function writeE2eWorkflowPlanCiOutput( `catalogue_nvidia_inference_matrix=${JSON.stringify(plan.catalogueMatrices["nvidia-inference"])}`, `catalogue_github_read_matrix=${JSON.stringify(plan.catalogueMatrices["github-read"])}`, `catalogue_brave_nvidia_inference_matrix=${JSON.stringify(plan.catalogueMatrices["brave-nvidia-inference"])}`, + `gateway_runtimes=${JSON.stringify(plan.gatewayRuntimes)}`, + `runtime_providers_by_job=${JSON.stringify(plan.runtimeProvidersByJob)}`, `selected_jobs=${JSON.stringify(plan.selectedJobs)}`, `selected_workflow_jobs=${JSON.stringify(selectedWorkflowJobs(plan))}`, `hermes_selected=${plan.hermesSelected}`, diff --git a/tools/mcp-tool-discovery-runtime/reviewed-runtime-bundle/managed-startup-image-runtime.bundle b/tools/mcp-tool-discovery-runtime/reviewed-runtime-bundle/managed-startup-image-runtime.bundle index 11e46caa608..7635ad15736 100644 --- a/tools/mcp-tool-discovery-runtime/reviewed-runtime-bundle/managed-startup-image-runtime.bundle +++ b/tools/mcp-tool-discovery-runtime/reviewed-runtime-bundle/managed-startup-image-runtime.bundle @@ -1,6 +1,6 @@ var __create=Object.create;var __defProp=Object.defineProperty;var __getOwnPropDesc=Object.getOwnPropertyDescriptor;var __getOwnPropNames=Object.getOwnPropertyNames;var __getProtoOf=Object.getPrototypeOf;var __hasOwnProp=Object.prototype.hasOwnProperty;var __export=(target,all)=>{for(var name in all)__defProp(target,name,{get:all[name],enumerable:true})};var __copyProps=(to,from,except,desc)=>{if(from&&typeof from==="object"||typeof from==="function"){for(let key of __getOwnPropNames(from))if(!__hasOwnProp.call(to,key)&&key!==except)__defProp(to,key,{get:()=>from[key],enumerable:!(desc=__getOwnPropDesc(from,key))||desc.enumerable})}return to};var __toESM=(mod,isNodeMode,target)=>(target=mod!=null?__create(__getProtoOf(mod)):{},__copyProps(isNodeMode||!mod||!mod.__esModule?__defProp(target,"default",{value:mod,enumerable:true}):target,mod));var __toCommonJS=mod=>__copyProps(__defProp({},"__esModule",{value:true}),mod);var image_runtime_exports={};__export(image_runtime_exports,{applyManagedBootstrapEnvelope:()=>applyManagedBootstrapEnvelope,main:()=>main2,managedBootstrapEnvelopeClaimPaths:()=>managedBootstrapEnvelopeClaimPaths,readManagedBootstrapEnvelope:()=>readManagedBootstrapEnvelope,recoverManagedBootstrapEnvelopeClaim:()=>recoverManagedBootstrapEnvelopeClaim,verifyManagedBootstrapImageCompletion:()=>verifyManagedBootstrapImageCompletion,waitForManagedBootstrapImageCompletion:()=>waitForManagedBootstrapImageCompletion});module.exports=__toCommonJS(image_runtime_exports);var import_node_fs4=__toESM(require("node:fs"));var import_node_path4=__toESM(require("node:path"));var import_node_child_process=require("node:child_process");var import_node_crypto6=require("node:crypto");var import_node_fs3=__toESM(require("node:fs"));var import_node_path3=__toESM(require("node:path"));var MAX_CORPORATE_CA_BYTES=128*1024;var PEM_CERTIFICATE_RE_GLOBAL=/-----BEGIN CERTIFICATE-----[\s\S]+?-----END CERTIFICATE-----/g;var import_node_buffer2=require("node:buffer");function isObjectRecord(value){return typeof value==="object"&&value!==null&&!Array.isArray(value)}var ChannelManifestRegistry=class{manifests=new Map;constructor(manifests=[]){for(const manifest of manifests){this.register(manifest)}}register(manifest){if(this.manifests.has(manifest.id)){throw new Error(`Duplicate channel manifest id '${manifest.id}'`)}this.manifests.set(manifest.id,manifest);return this}get(channelId){return this.manifests.get(channelId)}list(){return Array.from(this.manifests.values())}listAvailable(ctx={}){const supportedChannelIds=Array.isArray(ctx.supportedChannelIds)?new Set(ctx.supportedChannelIds):null;return this.list().filter(manifest=>{if(ctx.agent&&!manifest.supportedAgents.includes(ctx.agent)){return false}if(supportedChannelIds&&!supportedChannelIds.has(manifest.id)){return false}return true})}};function createChannelManifestRegistry(manifests=[]){return new ChannelManifestRegistry(manifests)}var discordManifest={schemaVersion:1,id:"discord",displayName:"Discord",description:"Discord bot messaging",supportedAgents:["openclaw","hermes"],auth:{mode:"token-paste"},inputs:[{id:"botToken",kind:"secret",required:true,envKey:"DISCORD_BOT_TOKEN",prompt:{label:"Discord Bot Token",help:"Discord Developer Portal \u2192 Applications \u2192 Bot \u2192 Reset/Copy Token."}},{id:"serverId",kind:"config",required:false,envKey:"DISCORD_SERVER_ID",statePath:"discordGuilds.serverId",prompt:{label:"Discord Server ID (for guild workspace access)",help:"Enable Developer Mode in Discord, then right-click your server and copy the Server ID.",emptyValueMessage:"guild channels stay disabled"}},{id:"requireMention",kind:"config",required:false,envKey:"DISCORD_REQUIRE_MENTION",statePath:"discordGuilds.requireMention",promptWhenInput:"serverId",validValues:["0","1"],defaultValue:"1",prompt:{label:"Discord mention mode",help:"Choose whether the bot should reply only when @mentioned or to all messages in this server."}},{id:"userId",kind:"config",required:false,envKey:"DISCORD_USER_ID",statePath:"discordGuilds.userIds",promptWhenInput:"serverId",prompt:{label:"Discord User ID (optional guild allowlist)",help:"Optional: enable Developer Mode in Discord, then right-click your user/avatar and copy the User ID. Leave blank to allow any member of the configured server to message the bot.",emptyValueMessage:"any member in the configured server can message the bot"}}],credentials:[{id:"discordBotToken",sourceInput:"botToken",providerName:"{sandboxName}-discord-bridge",providerEnvKey:"DISCORD_BOT_TOKEN",placeholder:"openshell:resolve:env:DISCORD_BOT_TOKEN"}],policyPresets:[{name:"discord",requiredAtCreate:true,validationWarningLines:["For Discord preset validation, do not use curl as the success signal:","curl is not in the preset binary allowlist, so curl probes can fail even","when the policy is working. Use Node HTTPS against","https://discord.com/api/v10/gateway or validate the configured",'messaging bridge/gateway path. DNS-only checks such as dns.resolve("gateway.discord.gg")',"can also be inconclusive behind a proxy."]}],render:[{id:"discord-openclaw-channel",kind:"json-fragment",agent:"openclaw",target:"openclaw.json",fragment:{path:"channels.discord",value:{enabled:true,accounts:{default:{enabled:true,healthMonitor:{enabled:false},proxy:"{{discordProxyUrl}}",dmPolicy:"{{discord.allowedUsers.dmPolicy}}",allowFrom:"{{discord.allowedUsers.values}}"}}}}},{id:"discord-openclaw-guilds",kind:"json-fragment",agent:"openclaw",target:"openclaw.json",when:"{{discord.hasGuilds}}",fragment:{path:"channels.discord",value:{groupPolicy:"allowlist",guilds:"{{discord.guilds}}"}}},{id:"discord-openclaw-plugin",kind:"json-fragment",agent:"openclaw",target:"openclaw.json",fragment:{path:"plugins.entries.discord",value:{enabled:true}}},{id:"discord-hermes-env",kind:"env-lines",agent:"hermes",target:"~/.hermes/.env",lines:["NEMOCLAW_DISCORD_GUILD_IDS={{discord.guildIds.csv}}","DISCORD_ALLOWED_USERS={{discord.allowedUsers.csv}}","DISCORD_ALLOW_ALL_USERS={{discord.allowAllUsers}}"]},{id:"discord-hermes-config",kind:"json-fragment",agent:"hermes",target:"~/.hermes/config.yaml",fragment:{path:"discord",value:{require_mention:"{{discord.requireMention}}",free_response_channels:"",allowed_channels:"",auto_thread:true,reactions:true,channel_prompts:{}}}},{id:"discord-hermes-platform",kind:"json-fragment",agent:"hermes",target:"~/.hermes/config.yaml",fragment:{path:"platforms.discord",value:{enabled:true}}}],runtime:{openclaw:{channelName:"discord",visibility:{configKeys:["discord"],logPatterns:["discord"]}}},agentPackages:[{id:"openclawPluginPackage",agent:"openclaw",manager:"openclaw-plugin",spec:"npm:@openclaw/discord@{{openclaw.version}}",pin:true,integrityByVersion:{"2026.7.1":"sha512-tZfdC1YA8oVLvc2BK1w0F6rUljS5ugCOp2uWe0vPsbG1fbzVVIO4V32RoqZznGHe5u2R9u4n1aV5Z/qa1m2oFg=="},tarballUrlByVersion:{"2026.7.1":"https://registry.npmjs.org/@openclaw/discord/-/discord-2026.7.1.tgz"},required:true}],hooks:[{id:"discord-openclaw-bridge-health",phase:"health-check",handler:"discord.openclawBridgeHealth",agents:["openclaw"],onFailure:"abort"},{id:"discord-token-paste",phase:"enroll",handler:"common.tokenPaste",outputs:[{id:"botToken",kind:"secret",required:true}],onFailure:"skip-channel"},{id:"discord-config-prompt",phase:"enroll",handler:"common.configPrompt",outputs:[{id:"serverId",kind:"config"},{id:"requireMention",kind:"config"},{id:"userId",kind:"config"}]}]};var googlechatManifest={schemaVersion:1,id:"googlechat",displayName:"Google Chat",description:"Google Chat (Chat API) bot messaging (experimental)",supportedAgents:["openclaw","hermes"],auth:{mode:"token-paste"},inputs:[{id:"serviceAccount",kind:"secret",required:true,envKey:"GOOGLECHAT_SERVICE_ACCOUNT",maskCap:40,formatHint:"Paste the entire service-account JSON key on one line (minified) \u2014 the whole downloaded JSON file.",maxTokenAttempts:3,prompt:{label:"Google Chat service account JSON",help:["\u2503 GOOGLE CHAT \u2014 service account key","\u2503","\u2503 Google Cloud Console \u2192 IAM & Admin \u2192 Service Accounts","\u2503 \u2192 your bot's SA \u2192 Keys \u2192 Add key \u2192 Create new key \u2192 JSON","\u2503","\u2503 A .json file downloads. Paste its contents below as ONE line (minified).",""].join("\n")}},{id:"audienceType",kind:"config",required:false,envKey:"GOOGLECHAT_AUDIENCE_TYPE",statePath:"googlechatConfig.audienceType",validValues:["app-url","project-number"],defaultValue:"app-url"},{id:"audience",kind:"config",required:false,envKey:"GOOGLECHAT_AUDIENCE",statePath:"googlechatConfig.audience",prompt:{label:"Google Chat webhook audience",help:"Usually filled automatically from the public tunnel URL. For audienceType 'project-number', enter your GCP project number instead.",emptyValueMessage:"inbound webhook verification will be unconfigured"}},{id:"appPrincipal",kind:"config",required:false,envKey:"GOOGLECHAT_APP_PRINCIPAL",statePath:"googlechatConfig.appPrincipal",formatPattern:"^[0-9]{6,32}$",formatHint:"appPrincipal is the add-on's numeric OAuth client ID (uniqueId, ~21 digits), not an email.",prompt:{label:"Google Chat appPrincipal",help:[" Workspace account \u2192 leave blank, done."," Personal Gmail \u2192 needs the add-on's ~21-digit ID (not an email), stable across rebuilds.",""," If you already know it, paste it at the prompt and you're done."," If not, leave it blank \u2014 the first DM reveals it once the sandbox is live:",""," 1. Watch the gateway log:",' nemoclaw logs --follow | grep "unexpected add-on principal"'," 2. DM the bot once \u2014 it won't reply yet, that's expected. The log prints:"," unexpected add-on principal: "," 3. Save that and rebuild:"," GOOGLECHAT_APP_PRINCIPAL= nemoclaw channels add googlechat"," nemoclaw rebuild --yes"].join("\n"),emptyValueMessage:"Workspace accounts do not need it; personal accounts must set it later"}},{id:"allowFrom",kind:"config",required:false,envKey:"GOOGLECHAT_ALLOWED_USERS",statePath:"allowedIds.googlechat",prompt:{label:"Google Chat DM allowlist (comma-separated)",help:["Optional: restrict who can DM the bot."," OpenClaw: users/NNN (emails ignored)"," Hermes: email (users/NNN ignored)"," Blank: pairing mode (recommended) \u2014 OpenClaw's pairing reply shows your users/NNN"," Filling this switches DM policy to allowlist \u2014 a wrong-form entry is dropped silently, with no pairing code."].join("\n"),emptyValueMessage:"bot will require manual pairing"}},{id:"projectId",kind:"config",required:false,envKey:"GOOGLE_CHAT_PROJECT_ID",statePath:"googlechatConfig.projectId",prompt:{label:"Google Chat GCP project ID (Hermes Pub/Sub pull)",help:"The Google Cloud project that owns the Pub/Sub subscription Hermes pulls Chat events from. OpenClaw ignores this.",emptyValueMessage:"required for the Hermes Google Chat channel"}},{id:"subscriptionName",kind:"config",required:false,envKey:"GOOGLE_CHAT_SUBSCRIPTION_NAME",statePath:"googlechatConfig.subscriptionName",prompt:{label:"Google Chat Pub/Sub subscription (projects/

/subscriptions/)",help:["The pull subscription bound to the Chat events topic. Hermes pulls from it over the Pub/Sub REST API; the gateway-minted token is scoped to both chat.bot and pubsub."," Its topic must grant roles/pubsub.publisher to the app's push account:"," Interactive features service-@gcp-sa-gsuiteaddons.iam.gserviceaccount.com"," Classic bot chat-api-push@system.gserviceaccount.com"," Shown at Chat API \u2192 Configuration \u2192 Connection settings"," Missing it channel connects, no event arrives, Chat says the bot is not responding"].join("\n"),emptyValueMessage:"required for the Hermes Google Chat channel"}}],credentials:[],policyPresets:[{name:"googlechat",policyKeys:["googlechat"],agentPolicyKeys:{hermes:["googlechat_hermes"]}}],render:[{id:"googlechat-openclaw-channel",kind:"json-fragment",agent:"openclaw",target:"openclaw.json",fragment:{path:"channels.googlechat",value:{enabled:true,serviceAccountFile:"/nonexistent/googlechat-gateway-minted-no-service-account-file",audienceType:"{{googlechatConfig.audienceType}}",audience:"{{googlechatConfig.audience}}",appPrincipal:"{{googlechatConfig.appPrincipal}}",webhookPath:"/googlechat",healthMonitor:{enabled:false},dm:{policy:"{{allowedIds.googlechat.dmPolicy}}",allowFrom:"{{allowedIds.googlechat.values}}"}}}},{id:"googlechat-openclaw-plugin",kind:"json-fragment",agent:"openclaw",target:"openclaw.json",fragment:{path:"plugins.entries.googlechat",value:{enabled:true}}},{id:"googlechat-openclaw-gateway-reload-off",kind:"json-fragment",agent:"openclaw",target:"openclaw.json",fragment:{path:"gateway.reload",value:{mode:"off"}}},{id:"googlechat-hermes-env",kind:"env-lines",agent:"hermes",target:"~/.hermes/.env",lines:["GOOGLE_CHAT_PROJECT_ID={{googlechatConfig.projectId}}","GOOGLE_CHAT_SUBSCRIPTION_NAME={{googlechatConfig.subscriptionName}}","GOOGLE_CHAT_ALLOWED_USERS={{allowedIds.googlechat.csv}}"]},{id:"googlechat-hermes-platform",kind:"json-fragment",agent:"hermes",target:"~/.hermes/config.yaml",fragment:{path:"platforms.google_chat",value:{enabled:true}}}],runtime:{openclaw:{channelName:"googlechat",visibility:{configKeys:["googlechat"],logPatterns:["googlechat"]},nodePreloads:[{module:"googlechat-trusted-proxy-fetch",injectInto:["boot"],optional:false,installMessage:"[channels] Installing Google Chat trusted-proxy-fetch patch (route googleapis via trusted env proxy)",installedMessage:"[channels] Google Chat trusted-proxy-fetch patch installed (NODE_OPTIONS updated)"},{module:"googlechat-outbound-auth",injectInto:["boot"],optional:false,installMessage:"[channels] Installing Google Chat outbound-auth patch (gateway-minted bearer)",installedMessage:"[channels] Google Chat outbound-auth patch installed (NODE_OPTIONS updated)"}],secretScans:[{path:"/sandbox/.openclaw/openclaw.json",pattern:"-----BEGIN (?:RSA )?PRIVATE KEY-----",message:"[SECURITY] Google Chat service account private key leaked into {path} - refusing to serve",exitCode:78}]}},agentPackages:[{id:"openclawPluginPackage",agent:"openclaw",manager:"openclaw-plugin",spec:"npm:@openclaw/googlechat@{{openclaw.version}}",pin:true,integrityByVersion:{"2026.7.1":"sha512-Dv0xOmcxAThEr6hoK+ioofHNu18hfbIceQrEHX3AHZPpOUiTJvToVpA5eX87NQINewwfSJf0gVhE6kSbSk2Aew=="},tarballUrlByVersion:{"2026.7.1":"https://registry.npmjs.org/@openclaw/googlechat/-/googlechat-2026.7.1.tgz"},required:true},{id:"hermesGooglePubsubPackage",agent:"hermes",manager:"hermes-uv-pip",spec:"google-cloud-pubsub==2.39.0",required:true},{id:"hermesGoogleApiClientPackage",agent:"hermes",manager:"hermes-uv-pip",spec:"google-api-python-client==2.194.0",required:true},{id:"hermesGoogleAuthPackage",agent:"hermes",manager:"hermes-uv-pip",spec:"google-auth==2.55.1",required:true}],hooks:[{id:"googlechat-tunnel-audience-gate",phase:"enroll",handler:"googlechat.tunnelAudienceGate",agents:["openclaw"],inputs:["audienceType","audience"],outputs:[{id:"audience",kind:"config"}],onFailure:"skip-channel"},{id:"googlechat-service-account",phase:"enroll",handler:"googlechat.tokenPaste",outputs:[{id:"serviceAccount",kind:"secret",required:true}],onFailure:"skip-channel"},{id:"googlechat-config-prompt",phase:"enroll",handler:"common.configPrompt",outputs:[{id:"allowFrom",kind:"config"}]},{id:"googlechat-openclaw-config-prompt",phase:"enroll",handler:"common.configPrompt",agents:["openclaw"],outputs:[{id:"appPrincipal",kind:"config"}]},{id:"googlechat-hermes-config-prompt",phase:"enroll",handler:"common.configPrompt",agents:["hermes"],outputs:[{id:"projectId",kind:"config"},{id:"subscriptionName",kind:"config"}]}]};var slackManifest={schemaVersion:1,id:"slack",displayName:"Slack",description:"Slack bot messaging",supportedAgents:["openclaw","hermes"],auth:{mode:"token-paste"},inputs:[{id:"botToken",kind:"secret",required:true,envKey:"SLACK_BOT_TOKEN",formatPattern:"^xoxb-[A-Za-z0-9_-]+$",formatHint:"Slack bot tokens start with 'xoxb-' (e.g. xoxb---).",prompt:{label:"Slack Bot Token",help:"Slack API \u2192 Your Apps \u2192 OAuth & Permissions \u2192 Bot User OAuth Token (xoxb-...)."}},{id:"appToken",kind:"secret",required:true,envKey:"SLACK_APP_TOKEN",formatPattern:"^xapp-[A-Za-z0-9_-]+$",formatHint:"Slack app tokens start with 'xapp-' (e.g. xapp----).",prompt:{label:"Slack App Token (Socket Mode)",help:"Slack API \u2192 Your Apps \u2192 Basic Information \u2192 App-Level Tokens (xapp-...)."}},{id:"allowedUsers",kind:"config",required:false,envKey:"SLACK_ALLOWED_USERS",statePath:"allowedIds.slack",prompt:{label:"Slack Member IDs (comma-separated allowlist)",help:"In Slack, open each allowed human user's profile -> More -> Copy member ID. Enter one or more comma-separated member IDs, not the app or bot user ID. Member IDs look like U01ABC2DEF3.",emptyValueMessage:"bot will require manual pairing"}},{id:"allowedChannels",kind:"config",required:false,envKey:"SLACK_ALLOWED_CHANNELS",statePath:"slackConfig.allowedChannels",prompt:{label:"Slack Channel IDs (comma-separated allowlist)",help:"Optional: enter comma-separated Slack channel IDs where the bot may answer @mentions. Channel IDs look like C012AB3CD.",emptyValueMessage:"channel @mentions stay unrestricted by channel ID"}}],credentials:[{id:"slackBotToken",sourceInput:"botToken",providerName:"{sandboxName}-slack-bridge",providerEnvKey:"SLACK_BOT_TOKEN",placeholder:"xoxb-OPENSHELL-RESOLVE-ENV-SLACK_BOT_TOKEN",primary:true},{id:"slackAppToken",sourceInput:"appToken",providerName:"{sandboxName}-slack-app",providerEnvKey:"SLACK_APP_TOKEN",placeholder:"xapp-OPENSHELL-RESOLVE-ENV-SLACK_APP_TOKEN"}],policyPresets:[{name:"slack",requiredAtCreate:true}],render:[{id:"slack-openclaw-channel",kind:"json-fragment",agent:"openclaw",target:"openclaw.json",fragment:{path:"channels.slack",value:{enabled:true,accounts:{default:{enabled:true,healthMonitor:{enabled:false},dmPolicy:"{{allowedIds.slack.dmPolicy}}",allowFrom:"{{allowedIds.slack.values}}",groupPolicy:"{{allowedIds.slack.groupPolicy}}",channels:"{{allowedIds.slack.channels}}"}}}}},{id:"slack-openclaw-plugin",kind:"json-fragment",agent:"openclaw",target:"openclaw.json",fragment:{path:"plugins.entries.slack",value:{enabled:true}}},{id:"slack-hermes-env",kind:"env-lines",agent:"hermes",target:"~/.hermes/.env",lines:["SLACK_ALLOWED_USERS={{allowedIds.slack.csv}}","SLACK_ALLOWED_CHANNELS={{slackConfig.allowedChannels.csv}}"]},{id:"slack-hermes-platform",kind:"json-fragment",agent:"hermes",target:"~/.hermes/config.yaml",fragment:{path:"platforms.slack",value:{enabled:true,extra:{rich_blocks:true}}}}],runtime:{openclaw:{channelName:"slack",visibility:{configKeys:["slack"],logPatterns:["slack"]},nodePreloads:[{module:"slack-channel-guard",injectInto:["boot","connect"],optional:false,installMessage:"[channels] Installing Slack channel guard (unhandled-rejection safety net)",installedMessage:"[channels] Slack channel guard installed (NODE_OPTIONS updated)"}],secretScans:[{path:"/sandbox/.openclaw/openclaw.json",pattern:"(?:xoxb|xapp)-(?!OPENSHELL-RESOLVE-ENV-)",message:"[SECURITY] Slack token leaked into {path} - refusing to serve",exitCode:78}]},hermes:{}},agentPackages:[{id:"openclawPluginPackage",agent:"openclaw",manager:"openclaw-plugin",spec:"npm:@openclaw/slack@{{openclaw.version}}",pin:true,integrityByVersion:{"2026.7.1":"sha512-dwVGEVCmoTQrOIeZaSCIOPg8pT7hB883QQEXdp9EZUDzTGuvSc+KxH2iERSOV/59hROQctYdcobGn/vdB1H4XA=="},tarballUrlByVersion:{"2026.7.1":"https://registry.npmjs.org/@openclaw/slack/-/slack-2026.7.1.tgz"},required:true}],hooks:[{id:"slack-socket-mode-gateway-conflict",phase:"pre-enable",handler:"slack.socketModeGatewayConflict",onFailure:"abort"},{id:"slack-openclaw-bridge-health",phase:"health-check",handler:"slack.openclawBridgeHealth",agents:["openclaw"],onFailure:"abort"},{id:"slack-socket-mode-gateway-status",phase:"status",handler:"slack.socketModeGatewayStatus",outputs:[{id:"gatewayOverlaps",kind:"status"}]},{id:"slack-status-health",phase:"status",handler:"slack.statusHealth",providesReadiness:true,agents:["openclaw"],outputs:[{id:"channelHealth",kind:"status"}]},{id:"slack-token-paste",phase:"enroll",handler:"common.tokenPaste",outputs:[{id:"botToken",kind:"secret",required:true},{id:"appToken",kind:"secret",required:true}],onFailure:"skip-channel"},{id:"slack-config-prompt",phase:"enroll",handler:"common.configPrompt",outputs:[{id:"allowedUsers",kind:"config"},{id:"allowedChannels",kind:"config"}]},{id:"slack-credential-validation",phase:"reachability-check",handler:"slack.validateCredentials",inputs:["botToken","appToken"],onFailure:"skip-channel"}]};var TEAMS_OPENCLAW_WEBHOOK_RENDER_CONTRACT={channelId:"teams",renderId:"teams-openclaw-channel",hookId:"teams-openclaw-channel",handlerId:"common.staticOutputs",kind:"json-fragment",agent:"openclaw",target:"openclaw.json",configPath:"channels.msteams",webhookPath:"/api/messages"};function authorizeTeamsOpenClawWebhookField(entry){if(!isPlainDataObject(entry))return[];const contract=TEAMS_OPENCLAW_WEBHOOK_RENDER_CONTRACT;if(ownDataPropertyValue(entry,"channelId")!==contract.channelId||ownDataPropertyValue(entry,"renderId")!==contract.renderId||ownDataPropertyValue(entry,"hookId")!==contract.hookId||ownDataPropertyValue(entry,"handler")!==contract.handlerId||ownDataPropertyValue(entry,"kind")!==contract.kind||ownDataPropertyValue(entry,"agent")!==contract.agent||ownDataPropertyValue(entry,"target")!==contract.target||ownDataPropertyValue(entry,"path")!==contract.configPath){return[]}const value=ownDataPropertyValue(entry,"value");if(!isPlainDataObject(value))return[];const webhook=ownDataPropertyValue(value,"webhook");if(!isPlainDataObject(webhook)||!hasExactlyOwnDataProperties(webhook,["path","port"])||!isTcpPort(ownDataPropertyValue(webhook,"port"))||ownDataPropertyValue(webhook,"path")!==contract.webhookPath){return[]}return[{path:["value","webhook"],value:webhook}]}function isPlainDataObject(value){if(value===null||typeof value!=="object"||Array.isArray(value))return false;const prototype=Object.getPrototypeOf(value);return prototype===Object.prototype||prototype===null}function ownDataPropertyValue(value,key){const descriptor=Object.getOwnPropertyDescriptor(value,key);return descriptor&&"value"in descriptor?descriptor.value:void 0}function hasExactlyOwnDataProperties(value,expected){const actual=Object.getOwnPropertyNames(value).sort();return actual.length===expected.length&&actual.every((key,index)=>key===expected[index])}function isTcpPort(value){return Number.isInteger(value)&&value>=1&&value<=65535}var teamsManifest={schemaVersion:1,id:"teams",displayName:"Microsoft Teams",description:"Microsoft Teams bot messaging (experimental)",enrollmentNotes:["Microsoft Teams requires a public HTTPS webhook endpoint at /api/messages; expose the configured Teams webhook port before installing the Teams app.","Use Azure AD object IDs in TEAMS_ALLOWED_USERS so only authorized users can interact with the bot."],supportedAgents:["openclaw","hermes"],auth:{mode:"token-paste"},inputs:[{id:"appId",kind:"config",required:true,envKey:"MSTEAMS_APP_ID",statePath:"teamsConfig.appId",prompt:{label:"Microsoft Teams Client ID",help:"Run `teams app create --endpoint https:///api/messages`, then copy CLIENT_ID."}},{id:"clientSecret",kind:"secret",required:true,envKey:"MSTEAMS_APP_PASSWORD",prompt:{label:"Microsoft Teams Client Secret",help:"Use the CLIENT_SECRET printed by `teams app create`. It is shown once; rotate it in Entra ID if it was lost."}},{id:"tenantId",kind:"config",required:true,envKey:"MSTEAMS_TENANT_ID",statePath:"teamsConfig.tenantId",prompt:{label:"Microsoft Teams Tenant ID",help:"Use the TENANT_ID printed by `teams app create` or shown by `teams status --verbose`."}},{id:"allowedUsers",kind:"config",required:false,envKey:"TEAMS_ALLOWED_USERS",statePath:"allowedIds.teams",prompt:{label:"Microsoft Teams AAD Object IDs (comma-separated allowlist)",help:"Recommended: run `teams status --verbose` and enter the Azure AD object IDs allowed to use the bot."}},{id:"webhookPort",kind:"config",required:false,envKey:"MSTEAMS_PORT",statePath:"teamsConfig.webhookPort",defaultValue:"3978",prompt:{label:"Microsoft Teams webhook port",help:"Local bot webhook port to expose publicly. Defaults to 3978 and serves /api/messages."}},{id:"requireMention",kind:"config",required:false,envKey:"TEAMS_REQUIRE_MENTION",statePath:"teamsConfig.requireMention",validValues:["0","1"],defaultValue:"1",prompt:{label:"Microsoft Teams mention mode",help:"Controls OpenClaw group and channel behavior only. Direct messages are unaffected."}}],credentials:[{id:"teamsClientSecret",sourceInput:"clientSecret",providerName:"{sandboxName}-teams-bridge",providerEnvKey:"MSTEAMS_APP_PASSWORD",placeholder:"openshell:resolve:env:MSTEAMS_APP_PASSWORD",primary:true}],policyPresets:[{name:"teams",policyKeys:["teams"],requiredAtCreate:true}],hostForward:{port:"{{teamsConfig.webhookPort}}",label:"Microsoft Teams webhook"},render:[{id:TEAMS_OPENCLAW_WEBHOOK_RENDER_CONTRACT.renderId,kind:TEAMS_OPENCLAW_WEBHOOK_RENDER_CONTRACT.kind,agent:TEAMS_OPENCLAW_WEBHOOK_RENDER_CONTRACT.agent,target:TEAMS_OPENCLAW_WEBHOOK_RENDER_CONTRACT.target,fragment:{path:TEAMS_OPENCLAW_WEBHOOK_RENDER_CONTRACT.configPath,value:{enabled:true,appId:"{{teamsConfig.appId}}",tenantId:"{{teamsConfig.tenantId}}",webhook:{port:"{{teamsConfig.webhookPort}}",path:TEAMS_OPENCLAW_WEBHOOK_RENDER_CONTRACT.webhookPath},healthMonitor:{enabled:false},streaming:{mode:"off"},dmPolicy:"{{allowedIds.teams.dmPolicy}}",allowFrom:"{{allowedIds.teams.values}}",groupPolicy:"open",requireMention:"{{teamsConfig.requireMention}}"}}},{id:"teams-openclaw-plugin",kind:"json-fragment",agent:"openclaw",target:"openclaw.json",fragment:{path:"plugins.entries.msteams",value:{enabled:true}}},{id:"teams-hermes-env",kind:"env-lines",agent:"hermes",target:"~/.hermes/.env",lines:["TEAMS_CLIENT_ID={{teamsConfig.appId}}","TEAMS_TENANT_ID={{teamsConfig.tenantId}}","TEAMS_ALLOWED_USERS={{allowedIds.teams.csv}}","TEAMS_PORT={{teamsConfig.webhookPort}}"]},{id:"teams-hermes-platform",kind:"json-fragment",agent:"hermes",target:"~/.hermes/config.yaml",fragment:{path:"platforms.teams",value:{enabled:true}}}],runtime:{openclaw:{channelName:"msteams",visibility:{configKeys:["msteams"],logPatterns:["msteams","teams"]},nodePreloads:[{module:"msteams-message-hints",injectInto:["boot","connect"],optional:false,installMessage:"[channels] Installing Microsoft Teams message hint patch (native mentions)",installedMessage:"[channels] Microsoft Teams message hint patch installed (NODE_OPTIONS updated)"}]},hermes:{envAliases:[{envKey:"MSTEAMS_APP_PASSWORD",targetEnvKey:"TEAMS_CLIENT_SECRET",match:"^openshell:resolve:env:v[0-9]+_MSTEAMS_APP_PASSWORD$",value:"openshell:resolve:env:MSTEAMS_APP_PASSWORD"}]}},agentPackages:[{id:"openclawPluginPackage",agent:"openclaw",manager:"openclaw-plugin",spec:"npm:@openclaw/msteams@{{openclaw.version}}",pin:true,integrityByVersion:{"2026.7.1":"sha512-gG/Yk6HZAguHwrmKjsqdONbFz5WNy126PEAXQWNW/TulO1kIifQ6tktM16BQPNLnkmWqLbj+TrrO55Cjas1aFg=="},tarballUrlByVersion:{"2026.7.1":"https://registry.npmjs.org/@openclaw/msteams/-/msteams-2026.7.1.tgz"},required:true},{id:"hermesTeamsAppsPackage",agent:"hermes",manager:"hermes-uv-pip",spec:"microsoft-teams-apps==2.0.13.4",required:true}],hooks:[{id:"teams-host-forward-port-conflict",phase:"pre-enable",handler:"teams.hostForwardPortConflict",inputs:["webhookPort"],onFailure:"abort"},{id:"teams-host-forward-port-status",phase:"status",handler:"teams.hostForwardPortStatus",outputs:[{id:"hostForwardPortOverlaps",kind:"status"}]},{id:"teams-token-paste",phase:"enroll",handler:"common.tokenPaste",outputs:[{id:"clientSecret",kind:"secret",required:true}],onFailure:"skip-channel"},{id:"teams-config-prompt",phase:"enroll",handler:"common.configPrompt",outputs:[{id:"appId",kind:"config",required:true},{id:"tenantId",kind:"config",required:true},{id:"allowedUsers",kind:"config"},{id:"webhookPort",kind:"config"},{id:"requireMention",kind:"config"}]}]};var telegramManifest={schemaVersion:1,id:"telegram",displayName:"Telegram",description:"Telegram bot messaging",diagnosticsProbe:"log-tail",enrollmentNotes:["For Telegram group chats, disable privacy mode in @BotFather (/setprivacy -> your bot -> Disable).","After changing privacy mode, remove and re-add the bot to each group before testing @mentions."],supportedAgents:["openclaw","hermes"],auth:{mode:"token-paste"},inputs:[{id:"botToken",kind:"secret",required:true,envKey:"TELEGRAM_BOT_TOKEN",prompt:{label:"Telegram Bot Token",help:"Create a bot via @BotFather on Telegram, then copy the token."}},{id:"allowedIds",kind:"config",required:false,envKey:"TELEGRAM_ALLOWED_IDS",statePath:"allowedIds.telegram",prompt:{label:"Telegram User ID (for DM access)",help:"Send /start to @userinfobot on Telegram to get your numeric user ID.",emptyValueMessage:"bot will require manual pairing"}},{id:"requireMention",kind:"config",required:false,envKey:"TELEGRAM_REQUIRE_MENTION",statePath:"telegramConfig.requireMention",validValues:["0","1"],defaultValue:"1",prompt:{label:"Telegram group mention mode",help:"Controls Telegram group-chat behavior only \u2014 reply only when @mentioned vs. to all group messages. Direct messages are unaffected by this setting and remain subject to pairing and TELEGRAM_ALLOWED_IDS."}},{id:"groupPolicy",kind:"config",required:false,envKey:"TELEGRAM_GROUP_POLICY",statePath:"telegramConfig.groupPolicy",validValues:["open","allowlist","disabled"],defaultValue:"open",prompt:{label:"Telegram group policy",help:"Controls OpenClaw Telegram group access. Hermes does not expose an equivalent disable-groups policy."}}],credentials:[{id:"telegramBotToken",sourceInput:"botToken",providerName:"{sandboxName}-telegram-bridge",providerEnvKey:"TELEGRAM_BOT_TOKEN",placeholder:"openshell:resolve:env:TELEGRAM_BOT_TOKEN"}],policyPresets:[{name:"telegram",requiredAtCreate:true,policyKeys:["telegram_bot"],agentPolicyKeys:{hermes:["telegram"]}}],render:[{id:"telegram-openclaw-channel",kind:"json-fragment",agent:"openclaw",target:"openclaw.json",fragment:{path:"channels.telegram",value:{enabled:true,accounts:{default:{enabled:true,healthMonitor:{enabled:false},proxy:"{{proxyUrl}}",groupPolicy:"{{telegramConfig.groupPolicy}}",dmPolicy:"{{allowedIds.telegram.dmPolicy}}",allowFrom:"{{allowedIds.telegram.values}}"}}}}},{id:"telegram-openclaw-groups",kind:"json-fragment",agent:"openclaw",target:"openclaw.json",when:"{{telegramConfig.openclawGroups}}",fragment:{path:"channels.telegram.groups",value:"{{telegramConfig.openclawGroups}}"}},{id:"telegram-openclaw-plugin",kind:"json-fragment",agent:"openclaw",target:"openclaw.json",fragment:{path:"plugins.entries.telegram",value:{enabled:true}}},{id:"telegram-hermes-env",kind:"env-lines",agent:"hermes",target:"~/.hermes/.env",lines:["TELEGRAM_ALLOWED_USERS={{allowedIds.telegram.csv}}"]},{id:"telegram-hermes-config",kind:"json-fragment",agent:"hermes",target:"~/.hermes/config.yaml",fragment:{path:"telegram",value:{require_mention:"{{telegramConfig.requireMention}}"}}},{id:"telegram-hermes-platform",kind:"json-fragment",agent:"hermes",target:"~/.hermes/config.yaml",fragment:{path:"platforms.telegram",value:{enabled:true}}}],runtime:{openclaw:{channelName:"telegram",visibility:{configKeys:["telegram"],logPatterns:["telegram"]},nodePreloads:[{module:"telegram-diagnostics",injectInto:["boot","connect"],optional:false,installMessage:"[channels] Installing Telegram diagnostics (provider readiness + inference errors)",installedMessage:"[channels] Telegram diagnostics installed (NODE_OPTIONS updated)"}]}},hooks:[{id:"telegram-token-paste",phase:"enroll",handler:"common.tokenPaste",outputs:[{id:"botToken",kind:"secret",required:true}],onFailure:"skip-channel"},{id:"telegram-allowlist-aliases",phase:"enroll",handler:"telegram.allowlistAliases",outputs:[{id:"allowedIds",kind:"config"}]},{id:"telegram-config-prompt",phase:"enroll",handler:"common.configPrompt",outputs:[{id:"requireMention",kind:"config"},{id:"allowedIds",kind:"config"}]},{id:"telegram-openclaw-config-prompt",phase:"enroll",handler:"common.configPrompt",agents:["openclaw"],outputs:[{id:"groupPolicy",kind:"config"}]},{id:"telegram-get-me-reachability",phase:"reachability-check",handler:"telegram.getMeReachability",inputs:["botToken"],onFailure:"skip-channel"},{id:"telegram-openclaw-bridge-health",phase:"health-check",handler:"telegram.openclawBridgeHealth",agents:["openclaw"],onFailure:"abort"},{id:"telegram-gateway-conflict-status",phase:"status",handler:"telegram.gatewayConflictStatus",outputs:[{id:"bridgeHealth",kind:"status"}]},{id:"telegram-status-health",phase:"status",handler:"telegram.statusHealth",agents:["openclaw"],outputs:[{id:"channelHealth",kind:"status"}]}]};var WECHAT_OPENCLAW_ACCOUNT_FILE_CONTRACT={channelId:"wechat",planHookId:"wechat-seed-openclaw-account",handlerId:"wechat.seedOpenClawAccount",outputId:"openclawWeixinAccountFile",kind:"build-file",required:true,mode:"0600"};var WECHAT_SEED_OPENCLAW_ACCOUNT_HOOK_ID=WECHAT_OPENCLAW_ACCOUNT_FILE_CONTRACT.handlerId;var WECHAT_SEED_OPENCLAW_ACCOUNT_PLAN_HOOK_ID=WECHAT_OPENCLAW_ACCOUNT_FILE_CONTRACT.planHookId;var WECHAT_OPENCLAW_ACCOUNT_FILE_OUTPUT_ID=WECHAT_OPENCLAW_ACCOUNT_FILE_CONTRACT.outputId;var WECHAT_TOKEN_PLACEHOLDER="openshell:resolve:env:WECHAT_BOT_TOKEN";function authorizeWechatAccountFilePlaceholders(value){const content=isPlainDataObject2(value)?ownDataPropertyValue2(value,"content"):void 0;if(!isPlainDataObject2(value)||!hasExactlyOwnDataProperties2(value,["content","mode","path"])||!isWechatAccountFilePath(ownDataPropertyValue2(value,"path"))||ownDataPropertyValue2(value,"mode")!==WECHAT_OPENCLAW_ACCOUNT_FILE_CONTRACT.mode||!isPlainDataObject2(content)||!hasOnlyOwnDataProperties(content,["baseUrl","savedAt","token","userId"])||!hasOwnDataProperty(content,"savedAt")||!hasOwnDataProperty(content,"token")||ownDataPropertyValue2(content,"token")!==WECHAT_TOKEN_PLACEHOLDER||!isNonEmptyString(ownDataPropertyValue2(content,"savedAt"))||!isOptionalNonEmptyString(content,"baseUrl")||!isOptionalNonEmptyString(content,"userId")){return[]}return[{path:["content","token"],value:WECHAT_TOKEN_PLACEHOLDER}]}function isWechatAccountFilePath(value){if(typeof value!=="string")return false;const prefix="openclaw-weixin/accounts/";const suffix=".json";if(!value.startsWith(prefix)||!value.endsWith(suffix))return false;const accountId=value.slice(prefix.length,-suffix.length);return accountId===accountId.trim()&&isSafeWechatAccountId(accountId)}function isSafeWechatAccountId(accountId){return accountId.length>0&&accountId!=="."&&accountId!==".."&&!/[\\/\0-\x1F\x7F]/.test(accountId)&&!accountId.includes("..")}function isPlainDataObject2(value){if(value===null||typeof value!=="object"||Array.isArray(value))return false;const prototype=Object.getPrototypeOf(value);return prototype===Object.prototype||prototype===null}function ownDataPropertyValue2(value,key){const descriptor=Object.getOwnPropertyDescriptor(value,key);return descriptor&&"value"in descriptor?descriptor.value:void 0}function hasOwnDataProperty(value,key){const descriptor=Object.getOwnPropertyDescriptor(value,key);return descriptor!==void 0&&"value"in descriptor}function hasExactlyOwnDataProperties2(value,expected){const actual=Object.getOwnPropertyNames(value).sort();return actual.length===expected.length&&actual.every((key,index)=>key===expected[index])}function hasOnlyOwnDataProperties(value,allowed){return Object.getOwnPropertyNames(value).every(key=>allowed.includes(key))}function isNonEmptyString(value){return typeof value==="string"&&value.length>0}function isOptionalNonEmptyString(value,key){return!hasOwnDataProperty(value,key)||isNonEmptyString(ownDataPropertyValue2(value,key))}var wechatManifest={schemaVersion:1,id:"wechat",displayName:"WeChat",description:"WeChat (personal) bot messaging",enrollmentHelp:"Captured automatically via a host-side QR scan during onboard \u2014 pair the bot by scanning the QR with WeChat on your phone (Discover \u2192 Scan). DM-only.",supportedAgents:["openclaw","hermes"],auth:{mode:"host-qr"},inputs:[{id:"botToken",kind:"secret",required:true,envKey:"WECHAT_BOT_TOKEN",prompt:{label:"WeChat Bot Token",help:"Captured automatically via a host-side QR scan during onboard \u2014 pair the bot by scanning the QR with WeChat on your phone (Discover \u2192 Scan). DM-only."}},{id:"accountId",kind:"config",required:true,envKey:"WECHAT_ACCOUNT_ID",statePath:"wechatConfig.accountId"},{id:"baseUrl",kind:"config",required:false,envKey:"WECHAT_BASE_URL",statePath:"wechatConfig.baseUrl"},{id:"userId",kind:"config",required:false,envKey:"WECHAT_USER_ID",statePath:"wechatConfig.userId"},{id:"allowedIds",kind:"config",required:false,envKey:"WECHAT_ALLOWED_IDS",statePath:"allowedIds.wechat",prompt:{label:"WeChat User ID(s) (DM allowlist)",help:"Optional: restrict who can DM the bot. The WeChat user id of the operator who scanned is added automatically; supply additional ids as a comma-separated list.",emptyValueMessage:"bot will require manual pairing"}}],credentials:[{id:"wechatBotToken",sourceInput:"botToken",providerName:"{sandboxName}-wechat-bridge",providerEnvKey:"WECHAT_BOT_TOKEN",placeholder:"openshell:resolve:env:WECHAT_BOT_TOKEN"}],state:{openclaw:["wechat","openclaw-weixin"]},policyPresets:[{name:"wechat",policyKeys:["wechat_bridge"],requiredAtCreate:true}],render:[{id:"wechat-openclaw-plugin",kind:"json-fragment",agent:"openclaw",target:"openclaw.json",fragment:{path:"plugins.entries.openclaw-weixin",value:{enabled:true}}},{id:"wechat-openclaw-channel",kind:"json-fragment",agent:"openclaw",target:"openclaw.json",fragment:{path:"channels.openclaw-weixin",value:{enabled:true}}},{id:"wechat-hermes-env",kind:"env-lines",agent:"hermes",target:"~/.hermes/.env",lines:["WEIXIN_ACCOUNT_ID={{wechatConfig.accountId}}","WEIXIN_BASE_URL={{wechatConfig.baseUrl}}","WEIXIN_ALLOWED_USERS={{allowedIds.wechat.csv}}"]},{id:"wechat-hermes-platform",kind:"json-fragment",agent:"hermes",target:"~/.hermes/config.yaml",fragment:{path:"platforms.weixin",value:{enabled:true}}}],runtime:{openclaw:{channelName:"openclaw-weixin",visibility:{configKeys:["openclaw-weixin"],logPatterns:["wechat","openclaw-weixin"]},nodePreloads:[{module:"wechat-account-placeholder",injectInto:["boot"],optional:false},{module:"wechat-diagnostics",injectInto:["boot","connect"],optional:false,installMessage:"[channels] Installing WeChat diagnostics (provider readiness + inference errors)",installedMessage:"[channels] WeChat diagnostics installed (NODE_OPTIONS updated)"}]},hermes:{envAliases:[{envKey:"WECHAT_BOT_TOKEN",targetEnvKey:"WEIXIN_TOKEN",match:"^openshell:resolve:env:v[0-9]+_WECHAT_BOT_TOKEN$",value:"openshell:resolve:env:WECHAT_BOT_TOKEN"}]}},agentPackages:[{id:"openclawPluginPackage",agent:"openclaw",manager:"openclaw-plugin",spec:"npm:@tencent-weixin/openclaw-weixin@2.4.3",pin:true,integrity:"sha512-dPQbidUNWigC6V10vGW4i+GLH09x+6zUhafZRjuxkJ9GDu8o62WBsnUTojp4KqUH756hz+t2v9khiCRSi0dBDw==",tarballUrl:"https://registry.npmjs.org/@tencent-weixin/openclaw-weixin/-/openclaw-weixin-2.4.3.tgz",runtimeLock:{cachePath:"/usr/local/share/nemoclaw/wechat-npm-cache",installCacheEnvKey:"NEMOCLAW_WECHAT_NPM_INSTALL_CACHE",lockFile:"/usr/local/lib/nemoclaw/wechat-runtime/package-lock.json",projectsRoot:"/sandbox/.openclaw/npm/projects",verifierPath:"/usr/local/lib/nemoclaw/verify-wechat-runtime-lock.mts",offline:true,legacyPeerDeps:true},required:true}],hooks:[{id:"wechat-host-qr",phase:"enroll",handler:"wechat.ilinkLogin",inputs:["allowedIds"],outputs:[{id:"botToken",kind:"secret",required:true},{id:"accountId",kind:"config",required:true},{id:"baseUrl",kind:"config"},{id:"userId",kind:"config"},{id:"allowedIds",kind:"config"}],onFailure:"skip-channel"},{id:"wechat-config-prompt",phase:"enroll",handler:"common.configPrompt",outputs:[{id:"allowedIds",kind:"config"}]},{id:WECHAT_OPENCLAW_ACCOUNT_FILE_CONTRACT.planHookId,phase:"post-agent-install",handler:WECHAT_OPENCLAW_ACCOUNT_FILE_CONTRACT.handlerId,agents:["openclaw"],inputs:["wechatConfig.accountId","wechatConfig.baseUrl","wechatConfig.userId","credential.wechatBotToken.placeholder"],outputs:[{id:"openclawWeixinAccountsIndex",kind:"build-file",required:true},{id:WECHAT_OPENCLAW_ACCOUNT_FILE_CONTRACT.outputId,kind:WECHAT_OPENCLAW_ACCOUNT_FILE_CONTRACT.kind,required:WECHAT_OPENCLAW_ACCOUNT_FILE_CONTRACT.required},{id:"openclawConfigPatch",kind:"build-file",required:true}],onFailure:"abort"},{id:"wechat-health-check",phase:"health-check",handler:"wechat.healthCheck",inputs:["wechatConfig.accountId"],onFailure:"abort"}]};var whatsappManifest={schemaVersion:1,id:"whatsapp",displayName:"WhatsApp",description:"WhatsApp Web messaging (QR pairing)",enrollmentHelp:"WhatsApp Web pairs via QR code scanned with your phone \u2014 no host-side token. After the sandbox is running, run `openshell term` and then use `openclaw channels login --channel whatsapp` for OpenClaw or `hermes whatsapp` for Hermes to display the QR.",enrollmentNotes:["After pairing, run `nemoclaw channels status --channel whatsapp`. OpenClaw reports inbound delivery evidence; Hermes reports gateway and dashboard session-path diagnostics."],supportedAgents:["openclaw","hermes"],auth:{mode:"in-sandbox-qr"},inputs:[{id:"mode",kind:"config",required:false,envKey:"WHATSAPP_MODE",statePath:"whatsappConfig.mode",validValues:["self-chat","bot"],defaultValue:"self-chat",prompt:{label:"WhatsApp reply mode",help:"self-chat replies only to messages the paired account sends to itself. bot replies to other senders and stops replying to that self-chat: an unknown sender receives a pairing code you approve with `hermes pairing approve whatsapp `, unless you set WHATSAPP_ALLOWED_IDS to a fixed sender list before this command.",emptyValueMessage:"the sandbox replies only in your own self-chat"}},{id:"allowedIds",kind:"config",required:false,envKey:"WHATSAPP_ALLOWED_IDS",statePath:"allowedIds.whatsapp"}],credentials:[],policyPresets:["whatsapp"],render:[{id:"whatsapp-openclaw-channel",kind:"json-fragment",agent:"openclaw",target:"openclaw.json",fragment:{path:"channels.whatsapp",value:{enabled:true,accounts:{default:{enabled:true,healthMonitor:{enabled:false}}}}}},{id:"whatsapp-openclaw-plugin",kind:"json-fragment",agent:"openclaw",target:"openclaw.json",fragment:{path:"plugins.entries.whatsapp",value:{enabled:true}}},{id:"whatsapp-hermes-env",kind:"env-lines",agent:"hermes",target:"~/.hermes/.env",lines:["WHATSAPP_ENABLED=true","WHATSAPP_MODE={{whatsappConfig.mode}}","WHATSAPP_DM_POLICY={{whatsappConfig.dmPolicy}}","WHATSAPP_ALLOWED_USERS={{allowedIds.whatsapp.csv}}"]},{id:"whatsapp-hermes-platform",kind:"json-fragment",agent:"hermes",target:"~/.hermes/config.yaml",fragment:{path:"platforms.whatsapp",value:{enabled:true}}}],runtime:{openclaw:{channelName:"whatsapp",visibility:{configKeys:["whatsapp"],logPatterns:["whatsapp"]},nodePreloads:[{module:"whatsapp-qr-compact",injectInto:["connect"],optional:true,installMessage:"[channels] Installing WhatsApp compact-QR renderer (scan-friendly pairing)"}]}},agentPackages:[{id:"openclawPluginPackage",agent:"openclaw",manager:"openclaw-plugin",spec:"npm:@openclaw/whatsapp@{{openclaw.version}}",pin:true,integrityByVersion:{"2026.7.1":"sha512-wLY/Omc5fleRpl2lKGN8sxt/8hYfHGwLRezmWsk8oCbea5pRKUPE6ZX+wJO1O52NOJkAGCuiXvS7x0qIeKxXbQ=="},tarballUrlByVersion:{"2026.7.1":"https://registry.npmjs.org/@openclaw/whatsapp/-/whatsapp-2026.7.1.tgz"},required:true}],hooks:[{id:"whatsapp-config-prompt",phase:"enroll",handler:"common.configPrompt",agents:["hermes"],outputs:[{id:"mode",kind:"config"}]},{id:"whatsapp-status-health",phase:"status",handler:"whatsapp.statusHealth",agents:["openclaw","hermes"],outputs:[{id:"channelHealth",kind:"status"}]}]};var BUILT_IN_CHANNEL_MANIFESTS=[telegramManifest,discordManifest,wechatManifest,slackManifest,whatsappManifest,teamsManifest,googlechatManifest];function createBuiltInChannelManifestRegistry(){return createChannelManifestRegistry(BUILT_IN_CHANNEL_MANIFESTS)}var EXACT_TEMPLATE_PATTERN=/^\{\{\s*([^}]+?)\s*\}\}$/;var TEMPLATE_REFERENCE_PATTERN=/\{\{\s*([^}]+?)\s*\}\}/g;function resolvedRenderTemplateReference(value){return{matched:true,value}}function resolveSandboxNameTemplate(value,sandboxName){return value.replaceAll("{sandboxName}",sandboxName)}function resolveRenderTemplatesInValue(value,context){if(typeof value==="string")return resolveRenderTemplatesInString(value,context);if(Array.isArray(value)){if(value.length===0)return value;const resolved=value.map(entry=>resolveRenderTemplatesInValue(entry,context)).filter(entry=>entry!==void 0);return resolved.length>0?resolved:void 0}if(value&&typeof value==="object"){const sourceEntries=Object.entries(value);if(sourceEntries.length===0)return value;const entries=sourceEntries.map(([key,entry])=>[key,resolveRenderTemplatesInValue(entry,context)]).filter(entry=>entry[1]!==void 0);return entries.length>0?Object.fromEntries(entries):void 0}return value}function isTruthyRenderTemplate(value,context){if(!value)return true;const resolved=resolveRenderTemplatesInString(value,context);if(resolved===void 0||resolved===null||resolved===false)return false;if(Array.isArray(resolved))return resolved.length>0;if(typeof resolved==="object")return Object.keys(resolved).length>0;if(typeof resolved==="string")return resolved.trim().length>0;return true}function resolveRenderTemplatesInString(value,context){const exact=value.match(EXACT_TEMPLATE_PATTERN);if(exact?.[1])return resolveTemplateReference(exact[1].trim(),context);let omitted=false;const resolved=value.replace(TEMPLATE_REFERENCE_PATTERN,(match,reference)=>{const replacement=resolveTemplateReference(reference.trim(),context);if(replacement===void 0||replacement===null){omitted=true;return""}if(Array.isArray(replacement))return replacement.map(String).join(",");if(typeof replacement==="object")return JSON.stringify(replacement);return String(replacement)});return omitted?void 0:resolved}function resolveTemplateReference(reference,context){const resolved=context.referenceResolver?.(reference,context);return resolved?.matched?resolved.value:"{{"+reference+"}}"}function allowedIds(context,channel){return parseList(stateValue(context,`allowedIds.${channel}`))}function stateValue(context,path5){const stateInput=context.inputs.find(input=>input.statePath===path5);if(stateInput?.value!==void 0)return stateInput.value;const inputId=path5.split(".").at(-1);return context.inputs.find(input=>input.inputId===inputId)?.value}function parseList(value){if(Array.isArray(value))return unique(value.map(String).map(cleanString).filter(Boolean));const text=cleanString(value);if(!text)return[];return unique(text.split(",").map(cleanString).filter(Boolean))}function parseBoolean(value){if(typeof value==="boolean")return value;const text=cleanString(value)?.toLowerCase();if(text==="1"||text==="true"||text==="yes"||text==="on")return true;if(text==="0"||text==="false"||text==="no"||text==="off")return false;return void 0}function nonEmptyString(value){return cleanString(value)||void 0}function cleanString(value){const text=String(value??"");if(/[\r\n]/.test(text)){throw new Error("Messaging template values must not contain line breaks.")}return text.trim()}function nonEmptyArray(values){return values.length>0?[...values]:void 0}function nonEmptyCsv(values){return values.length>0?values.join(","):void 0}function nonEmptyObject(value){return Object.keys(value).length>0?value:void 0}function unique(values){return[...new Set(values)]}var resolveDiscordTemplateReference=(reference,context)=>{if(reference==="discordProxyUrl")return resolvedRenderTemplateReference(void 0);switch(reference){case"discord.guilds":return resolvedRenderTemplateReference(nonEmptyObject(discordGuilds(context)));case"discord.hasGuilds":return resolvedRenderTemplateReference(Object.keys(discordGuilds(context)).length>0);case"discord.guildIds.csv":return resolvedRenderTemplateReference(nonEmptyCsv(Object.keys(discordGuilds(context))));case"discord.allowedUsers.values":return resolvedRenderTemplateReference(nonEmptyArray(discordAllowedUsers(context)));case"discord.allowedUsers.csv":return resolvedRenderTemplateReference(nonEmptyCsv(discordAllowedUsers(context)));case"discord.allowedUsers.dmPolicy":return resolvedRenderTemplateReference(discordAllowedUsers(context).length>0?"allowlist":void 0);case"discord.allowAllUsers":return resolvedRenderTemplateReference(Object.keys(discordGuilds(context)).length>0&&discordAllowedUsers(context).length===0?true:void 0);case"discord.requireMention":return resolvedRenderTemplateReference(discordRequireMention(context));default:return void 0}};function discordGuilds(context){const serverIds=parseList(stateValue(context,"discordGuilds.serverId"));if(serverIds.length===0)return{};const users=parseList(stateValue(context,"discordGuilds.userIds"));const requireMention=parseBoolean(stateValue(context,"discordGuilds.requireMention"))??true;return Object.fromEntries(serverIds.map(serverId=>[serverId,{requireMention,...users.length>0?{users}:{}}]))}function discordAllowedUsers(context){const users=new Set(allowedIds(context,"discord"));for(const guild of Object.values(discordGuilds(context))){for(const user of guild.users??[])users.add(String(user))}return[...users]}function discordRequireMention(context){for(const guild of Object.values(discordGuilds(context))){if(typeof guild.requireMention==="boolean")return guild.requireMention}return true}var DEFAULT_AUDIENCE_TYPE="app-url";var APP_PRINCIPAL_DISCOVERY_SENTINEL="000000000000000000000";var resolveGooglechatTemplateReference=(reference,context)=>{switch(reference){case"googlechatConfig.audienceType":return resolvedRenderTemplateReference(nonEmptyString(stateValue(context,"googlechatConfig.audienceType"))??DEFAULT_AUDIENCE_TYPE);case"googlechatConfig.audience":return resolvedRenderTemplateReference(nonEmptyString(stateValue(context,"googlechatConfig.audience")));case"googlechatConfig.appPrincipal":return resolvedRenderTemplateReference(nonEmptyString(stateValue(context,"googlechatConfig.appPrincipal"))??APP_PRINCIPAL_DISCOVERY_SENTINEL);case"googlechatConfig.projectId":return resolvedRenderTemplateReference(nonEmptyString(stateValue(context,"googlechatConfig.projectId")));case"googlechatConfig.subscriptionName":return resolvedRenderTemplateReference(nonEmptyString(stateValue(context,"googlechatConfig.subscriptionName")));default:break}const allowReference=reference.match(/^allowedIds[.]googlechat[.](values|dmPolicy|csv)$/);if(!allowReference?.[1])return void 0;const ids=allowedIds(context,"googlechat");switch(allowReference[1]){case"values":return resolvedRenderTemplateReference(nonEmptyArray(ids));case"dmPolicy":return resolvedRenderTemplateReference(ids.length>0?"allowlist":void 0);case"csv":return resolvedRenderTemplateReference(ids.length>0?ids.join(","):void 0);default:return void 0}};var resolveSlackTemplateReference=(reference,context)=>{if(reference==="slackConfig.allowedChannels.csv"){return resolvedRenderTemplateReference(nonEmptyCsv(slackAllowedChannels(context)))}const allowedIdsReference=reference.match(/^allowedIds[.]slack[.](values|csv|dmPolicy|groupPolicy|channels)$/);if(!allowedIdsReference?.[1])return void 0;const ids=allowedIds(context,"slack");switch(allowedIdsReference[1]){case"values":return resolvedRenderTemplateReference(nonEmptyArray(ids));case"csv":return resolvedRenderTemplateReference(nonEmptyCsv(ids));case"dmPolicy":return resolvedRenderTemplateReference(ids.length>0?"allowlist":void 0);case"groupPolicy":return resolvedRenderTemplateReference(ids.length>0||slackAllowedChannels(context).length>0?"allowlist":void 0);case"channels":return resolvedRenderTemplateReference(slackChannelConfig(context,ids));default:return void 0}};function slackChannelConfig(context,users){const allowedChannels=slackAllowedChannels(context);const entry={enabled:true,requireMention:true,...users.length>0?{users:[...users]}:{}};if(allowedChannels.length>0){return Object.fromEntries(allowedChannels.map(channelId=>[channelId,{...entry}]))}return users.length>0?{"*":entry}:void 0}function slackAllowedChannels(context){return parseList(stateValue(context,"slackConfig.allowedChannels"))}var DEFAULT_TEAMS_WEBHOOK_PORT=3978;var resolveTeamsTemplateReference=(reference,context)=>{switch(reference){case"teamsConfig.appId":return resolvedRenderTemplateReference(nonEmptyString(stateValue(context,"teamsConfig.appId")));case"teamsConfig.tenantId":return resolvedRenderTemplateReference(nonEmptyString(stateValue(context,"teamsConfig.tenantId")));case"teamsConfig.webhookPort":return resolvedRenderTemplateReference(teamsWebhookPort(context));case"teamsConfig.requireMention":return resolvedRenderTemplateReference(parseBoolean(stateValue(context,"teamsConfig.requireMention")));default:break}const allowedIdsReference=reference.match(/^allowedIds[.]teams[.](values|csv|dmPolicy)$/);if(!allowedIdsReference?.[1])return void 0;const ids=allowedIds(context,"teams");switch(allowedIdsReference[1]){case"values":return resolvedRenderTemplateReference(nonEmptyArray(ids));case"csv":return resolvedRenderTemplateReference(nonEmptyCsv(ids));case"dmPolicy":return resolvedRenderTemplateReference(ids.length>0?"allowlist":void 0);default:return void 0}};function teamsWebhookPort(context){const raw=nonEmptyString(stateValue(context,"teamsConfig.webhookPort"));if(!raw)return DEFAULT_TEAMS_WEBHOOK_PORT;const port=Number(raw);if(!Number.isInteger(port)||port<1||port>65535){throw new Error("Microsoft Teams webhook port must be an integer TCP port between 1 and 65535.")}return port}var DEFAULT_PROXY_HOST="10.200.0.1";var DEFAULT_PROXY_PORT="3128";var DEFAULT_TELEGRAM_GROUP_POLICY="open";var TELEGRAM_GROUP_POLICIES=new Set(["open","allowlist","disabled"]);var resolveTelegramTemplateReference=(reference,context)=>{if(reference==="proxyUrl")return resolvedRenderTemplateReference(proxyUrl(context.env));if(reference==="telegramConfig.groupPolicy"){return resolvedRenderTemplateReference(telegramGroupPolicy(context))}if(reference==="telegramConfig.openclawGroups"){return resolvedRenderTemplateReference(telegramOpenClawGroups(context))}if(reference==="telegramConfig.requireMention"){return resolvedRenderTemplateReference(parseBoolean(stateValue(context,"telegramConfig.requireMention")))}const allowedIdsReference=reference.match(/^allowedIds[.]telegram[.](values|csv|dmPolicy)$/);if(!allowedIdsReference?.[1])return void 0;const ids=allowedIds(context,"telegram");switch(allowedIdsReference[1]){case"values":return resolvedRenderTemplateReference(nonEmptyArray(ids));case"csv":return resolvedRenderTemplateReference(nonEmptyCsv(ids));case"dmPolicy":return resolvedRenderTemplateReference(ids.length>0?"allowlist":void 0);default:return void 0}};function proxyUrl(env){const host=nonEmptyString(env?.NEMOCLAW_PROXY_HOST)??DEFAULT_PROXY_HOST;const port=nonEmptyString(env?.NEMOCLAW_PROXY_PORT)??DEFAULT_PROXY_PORT;return`http://${host}:${port}`}function telegramGroupPolicy(context){const value=nonEmptyString(stateValue(context,"telegramConfig.groupPolicy"));return value&&TELEGRAM_GROUP_POLICIES.has(value)?value:DEFAULT_TELEGRAM_GROUP_POLICY}function telegramOpenClawGroups(context){if(telegramGroupPolicy(context)!=="open")return void 0;const requireMention=parseBoolean(stateValue(context,"telegramConfig.requireMention"));return requireMention===true?{"*":{requireMention:true}}:void 0}var WECHAT_ILINK_HOSTS=new Set(["ilinkai.weixin.qq.com","ilinkai.wechat.com"]);var WECHAT_ILINK_IDC_HOST_PATTERN=/^idc-[0-9]+[.]weixin[.]qq[.]com$/u;function normalizeWechatIlinkBaseUrl(value){const raw=String(value??"");if(/[\r\n]/.test(raw)){throw new Error("WeChat baseUrl must not contain line breaks.")}const text=raw.trim();if(!text)return void 0;let url;try{url=new URL(text)}catch{throw new Error("WeChat baseUrl must be a valid URL.")}if(url.protocol!=="https:"){throw new Error("WeChat baseUrl must use HTTPS.")}if(url.username||url.password){throw new Error("WeChat baseUrl must not include credentials.")}const authority=text.match(/^[a-z][a-z0-9+.-]*:\/\/([^/?#]*)/iu)?.[1]??"";if(authority.includes(":")){throw new Error("WeChat baseUrl must not include an explicit port.")}if(!isWechatIlinkHost(url.hostname)){throw new Error("WeChat baseUrl must use an expected iLink host.")}if(url.pathname&&url.pathname!=="/"||url.search||url.hash){throw new Error("WeChat baseUrl must be an iLink origin URL.")}return url.origin}function isWechatIlinkHost(hostname){const normalized=hostname.toLowerCase();return WECHAT_ILINK_HOSTS.has(normalized)||isWechatIlinkIdcHost(normalized)}function isWechatIlinkIdcHost(hostname){return WECHAT_ILINK_IDC_HOST_PATTERN.test(hostname.toLowerCase())}var resolveWechatTemplateReference=(reference,context)=>{const wechatConfig=reference.match(/^wechatConfig[.](accountId|baseUrl|userId)$/);if(wechatConfig?.[1]){if(wechatConfig[1]==="baseUrl"){return resolvedRenderTemplateReference(normalizeWechatIlinkBaseUrl(stateValue(context,"wechatConfig.baseUrl")))}return resolvedRenderTemplateReference(nonEmptyString(stateValue(context,"wechatConfig."+wechatConfig[1])))}const allowedIdsReference=reference.match(/^allowedIds[.]wechat[.](values|csv|dmPolicy)$/);if(!allowedIdsReference?.[1])return void 0;const ids=wechatAllowedIds(context);switch(allowedIdsReference[1]){case"values":return resolvedRenderTemplateReference(nonEmptyArray(ids));case"csv":return resolvedRenderTemplateReference(nonEmptyCsv(ids));case"dmPolicy":return resolvedRenderTemplateReference(ids.length>0?"allowlist":void 0);default:return void 0}};function wechatAllowedIds(context){const ids=allowedIds(context,"wechat");const userId=nonEmptyString(stateValue(context,"wechatConfig.userId"));return userId&&!ids.includes(userId)?[userId,...ids]:ids}var DEFAULT_WHATSAPP_MODE="self-chat";var BOT_WHATSAPP_MODE="bot";var WHATSAPP_MODES=new Set([DEFAULT_WHATSAPP_MODE,BOT_WHATSAPP_MODE]);var resolveWhatsappTemplateReference=(reference,context)=>{if(reference==="whatsappConfig.mode"){return resolvedRenderTemplateReference(whatsappMode(context))}if(reference==="whatsappConfig.dmPolicy"){return resolvedRenderTemplateReference(whatsappDmPolicy(context))}const allowedIdsReference=reference.match(/^allowedIds[.]whatsapp[.](values|csv)$/);if(!allowedIdsReference?.[1])return void 0;const ids=allowedIds(context,"whatsapp");switch(allowedIdsReference[1]){case"values":return resolvedRenderTemplateReference(nonEmptyArray(ids));case"csv":return resolvedRenderTemplateReference(nonEmptyCsv(ids));default:return void 0}};function whatsappMode(context){const value=nonEmptyString(stateValue(context,"whatsappConfig.mode"));return value&&WHATSAPP_MODES.has(value)?value:DEFAULT_WHATSAPP_MODE}function whatsappDmPolicy(context){if(whatsappMode(context)!==BOT_WHATSAPP_MODE)return void 0;return allowedIds(context,"whatsapp").length>0?"allowlist":"pairing"}var BUILT_IN_TEMPLATE_REFERENCE_RESOLVERS=[resolveTelegramTemplateReference,resolveDiscordTemplateReference,resolveWechatTemplateReference,resolveSlackTemplateReference,resolveWhatsappTemplateReference,resolveTeamsTemplateReference,resolveGooglechatTemplateReference];function createBuiltInRenderTemplateResolver(){return(reference,context)=>{for(const resolver of BUILT_IN_TEMPLATE_REFERENCE_RESOLVERS){const resolved=resolver(reference,context);if(resolved)return resolved}return void 0}}var import_node_crypto=__toESM(require("node:crypto"));function hashCredential(value){const normalized=String(value??"").trim();if(!normalized)return null;return import_node_crypto.default.createHash("sha256").update(normalized).digest("hex")}function planCredentialBindings(manifest,context,inputs,environment=process.env){return manifest.credentials.map(credential=>{const sourceInput=inputs.find(input=>input.inputId===credential.sourceInput);const credentialAvailable=sourceInput?.credentialAvailable===true||context.credentialAvailability?.[credential.id]===true||context.credentialAvailability?.[`${manifest.id}.${credential.id}`]===true;const envKey=sourceInput?.sourceEnv??credential.providerEnvKey;const credentialHash=credentialAvailable?hashCredential(environment[envKey])??void 0:void 0;return{channelId:manifest.id,credentialId:credential.id,sourceInput:credential.sourceInput,providerName:resolveSandboxNameTemplate(credential.providerName,context.sandboxName),providerEnvKey:credential.providerEnvKey,placeholder:credential.placeholder,credentialAvailable,...credentialHash!==void 0?{credentialHash}:{}}})}function planHostForward(manifest,inputs,active,referenceResolver,environment=process.env){if(!active||!manifest.hostForward)return void 0;const context={inputs,env:environment,referenceResolver};if(!isTruthyRenderTemplate(manifest.hostForward.when,context))return void 0;const portValue=resolveRenderTemplatesInValue(manifest.hostForward.port,context);const port=normalizeForwardPort(manifest.id,portValue);return{channelId:manifest.id,port,label:manifest.hostForward.label}}function normalizeForwardPort(channelId,value){const port=typeof value==="number"?value:Number(String(value??"").trim());if(!Number.isInteger(port)||port<1||port>65535){throw new Error(`Channel manifest '${channelId}' declares invalid host forward port '${String(value)}'.`)}return port}var OPENSHELL_ENV_PLACEHOLDER_PREFIX="openshell:resolve:env:";var OPENSHELL_ALIAS_PLACEHOLDER_RE=/^[A-Za-z0-9]+-OPENSHELL-RESOLVE-ENV-(.+)$/;function normalizeProviderPlaceholderForEnvKey(value,envKey){if(value.startsWith(OPENSHELL_ENV_PLACEHOLDER_PREFIX)){return placeholderSuffixMatchesEnvKey(value.slice(OPENSHELL_ENV_PLACEHOLDER_PREFIX.length),envKey)?`${OPENSHELL_ENV_PLACEHOLDER_PREFIX}${envKey}`:null}const aliasMatch=value.match(OPENSHELL_ALIAS_PLACEHOLDER_RE);if(!aliasMatch||!placeholderSuffixMatchesEnvKey(aliasMatch[1],envKey)){return null}return value.replace(/-OPENSHELL-RESOLVE-ENV-.+$/,`-OPENSHELL-RESOLVE-ENV-${envKey}`)}function placeholderSuffixMatchesEnvKey(suffix,envKey){if(suffix===envKey)return true;const revisionMatch=suffix.match(/^v[0-9]+_(.+)$/);return revisionMatch?.[1]===envKey}function hasFullPersistedCredentialBindingShape(binding){return typeof binding.channelId==="string"&&typeof binding.credentialId==="string"&&typeof binding.sourceInput==="string"&&typeof binding.providerName==="string"&&typeof binding.providerEnvKey==="string"&&typeof binding.placeholder==="string"&&typeof binding.credentialAvailable==="boolean"}function normalizeFullPersistedCredentialBindings(bindings){return bindings.map(binding=>({channelId:binding.channelId,credentialId:binding.credentialId,sourceInput:binding.sourceInput,providerName:binding.providerName,providerEnvKey:binding.providerEnvKey,placeholder:normalizeProviderPlaceholderForEnvKey(binding.placeholder,binding.providerEnvKey)??binding.placeholder,credentialAvailable:binding.credentialAvailable===true,...typeof binding.credentialHash==="string"?{credentialHash:binding.credentialHash}:{}}))}function normalizePersistedAgentCredentialPlaceholders(render,credentialBindings){const credentialEnvKeys=new Set(credentialBindings.map(binding=>binding.providerEnvKey).filter(Boolean));if(credentialEnvKeys.size===0)return[...render];return render.map(entry=>{if(entry.kind!=="env-lines")return entry;return{...entry,lines:entry.lines.map(line=>normalizeCredentialEnvLine(line,credentialEnvKeys))}})}function normalizeCredentialEnvLine(line,credentialEnvKeys){const index=line.indexOf("=");if(index<=0)return line;const envKey=line.slice(0,index).trim();if(!credentialEnvKeys.has(envKey))return line;const value=line.slice(index+1);const normalized=normalizeProviderPlaceholderForEnvKey(value,envKey);return normalized?`${envKey}=${normalized}`:line}function normalizePersistedSandboxMessagingPlanShape(plan,environment=process.env){const manifestRegistry=createBuiltInChannelManifestRegistry();const disabledChannels=plan.disabledChannels.filter(channelId=>typeof channelId==="string");const disabledSet=new Set(disabledChannels);const channels=plan.channels.map(channel=>normalizePersistedChannel(channel,disabledSet,manifestRegistry.get(channel.channelId),environment));const credentialBindings=normalizePersistedCredentialBindings(plan,channels,manifestRegistry,environment);const normalizedPlan={...plan,channels,disabledChannels,credentialBindings,networkPolicy:plan.networkPolicy&&Array.isArray(plan.networkPolicy.entries)?plan.networkPolicy:{presets:[],entries:[]},agentRender:normalizePersistedAgentCredentialPlaceholders(Array.isArray(plan.agentRender)?[...plan.agentRender]:[],credentialBindings),buildSteps:Array.isArray(plan.buildSteps)?[...plan.buildSteps]:[],...plan.runtimeSetup!==void 0?{runtimeSetup:normalizeRuntimeSetup(plan.runtimeSetup)}:{},stateUpdates:Array.isArray(plan.stateUpdates)?[...plan.stateUpdates]:[],healthChecks:Array.isArray(plan.healthChecks)?[...plan.healthChecks]:[]};return normalizedPlan}function normalizePersistedChannel(channel,disabledSet,manifest,environment){const disabled=channel.disabled??disabledSet.has(channel.channelId);const configured=channel.configured??true;const hasFullShape=hasFullChannelShape(channel);const inputs=hasFullShape?normalizeFullInputs(channel.channelId,channel.inputs??[]):normalizePersistedInputs(channel,manifest);const active=channel.active??(configured&&!disabled&&requiredInputsAvailable(manifest,inputs));const hostForward=manifest?planHostForward(manifest,inputs,active&&!disabled,createBuiltInRenderTemplateResolver(),environment):void 0;return{channelId:channel.channelId,displayName:channel.displayName??manifest?.displayName??channel.channelId,authMode:channel.authMode??manifest?.auth.mode??"none",active,selected:channel.selected??configured,configured,disabled,...channel.pendingRemoval===true?{pendingRemoval:true}:{},inputs,...hostForward?{hostForward}:{},hooks:Array.isArray(channel.hooks)?[...channel.hooks]:[]}}function normalizePersistedInputs(channel,manifest){const persistedById=new Map((channel.inputs??[]).filter(input=>typeof input.inputId==="string").map(input=>[input.inputId,input]));const fromManifest=(manifest?.inputs??[]).map(input=>inputReferenceFromManifest(channel.channelId,input,persistedById.get(input.id)));const manifestInputIds=new Set((manifest?.inputs??[]).map(input=>input.id));const unknownInputs=[...persistedById.values()].flatMap(input=>{if(!input.inputId||manifestInputIds.has(input.inputId))return[];return[normalizeUnknownInput(channel.channelId,input)]});return[...fromManifest,...unknownInputs]}function normalizeFullInputs(channelId,inputs){return inputs.filter(input=>typeof input.inputId==="string").map(input=>({channelId:typeof input.channelId==="string"?input.channelId:channelId,inputId:input.inputId,kind:input.kind==="secret"||input.kind==="config"?input.kind:"config",required:typeof input.required==="boolean"?input.required:false,...typeof input.sourceEnv==="string"?{sourceEnv:input.sourceEnv}:{},...typeof input.statePath==="string"?{statePath:input.statePath}:{},...input.credentialAvailable!==void 0?{credentialAvailable:input.credentialAvailable}:{},...input.value!==void 0?{value:input.value}:{}}))}function inputReferenceFromManifest(channelId,input,persisted){return{channelId,inputId:input.id,kind:input.kind,required:input.required,...input.envKey?{sourceEnv:input.envKey}:{},...input.kind==="config"&&input.statePath?{statePath:input.statePath}:{},...persisted?.credentialAvailable!==void 0?{credentialAvailable:persisted.credentialAvailable}:{},...persisted?.value!==void 0?{value:persisted.value}:{}}}function normalizeUnknownInput(channelId,input){const kind=input.kind==="secret"||input.kind==="config"?input.kind:"config";return{channelId,inputId:input.inputId,kind,required:input.required===true,...typeof input.sourceEnv==="string"?{sourceEnv:input.sourceEnv}:{},...typeof input.statePath==="string"?{statePath:input.statePath}:{},...input.credentialAvailable!==void 0?{credentialAvailable:input.credentialAvailable}:{},...input.value!==void 0?{value:input.value}:{}}}function requiredInputsAvailable(manifest,inputs){if(!manifest)return true;return manifest.inputs.every(manifestInput=>{if(!manifestInput.required)return true;const input=inputs.find(entry=>entry.inputId===manifestInput.id);if(!input)return false;if(input.kind==="secret")return input.credentialAvailable===true;if(input.value===void 0)return false;return typeof input.value==="string"?input.value.trim().length>0:true})}function normalizePersistedCredentialBindings(plan,channels,manifestRegistry,environment){const persisted=plan.credentialBindings??[];if(Array.isArray(plan.credentialBindings)&&plan.channels.every(hasFullChannelShape)&&persisted.every(hasFullPersistedCredentialBindingShape)){return normalizeFullPersistedCredentialBindings(persisted)}const manifests=channels.flatMap(channel=>{const manifest=manifestRegistry.get(channel.channelId);return manifest?[manifest]:[]});const planForBindings={...plan,channels,credentialBindings:[],networkPolicy:{presets:[],entries:[]},agentRender:[],buildSteps:[],runtimeSetup:{nodePreloads:[],envAliases:[],secretScans:[]},stateUpdates:[],healthChecks:[]};const generated=credentialBindingsFromManifests(planForBindings,manifests,new Map(channels.map(channel=>[channel.channelId,channel.inputs])),environment);return generated.map(binding=>overlayPersistedCredentialBinding(binding,persisted))}function credentialBindingsFromManifests(plan,manifests,inputRegistry,environment){const context=compilerContext(plan);return manifests.flatMap(manifest=>planCredentialBindings(manifest,context,inputRegistry.get(manifest.id)??[],environment).map(binding=>overlayPersistedCredentialBinding(binding,plan.credentialBindings)))}function overlayPersistedCredentialBinding(binding,persisted){const match=persisted.find(candidate=>credentialBindingMatches(binding,candidate));if(!match)return binding;return{...binding,credentialAvailable:typeof match.credentialAvailable==="boolean"?match.credentialAvailable:binding.credentialAvailable,...typeof match.credentialHash==="string"&&match.credentialHash.length>0?{credentialHash:match.credentialHash}:binding.credentialHash?{credentialHash:binding.credentialHash}:{}}}function credentialBindingMatches(binding,candidate){if(candidate.channelId&&candidate.channelId!==binding.channelId)return false;if(candidate.providerEnvKey&&candidate.providerEnvKey===binding.providerEnvKey)return true;if(candidate.credentialId&&candidate.credentialId===binding.credentialId)return true;if(candidate.sourceInput&&candidate.sourceInput===binding.sourceInput)return true;return false}function hasFullChannelShape(channel){return typeof channel.displayName==="string"&&typeof channel.authMode==="string"&&typeof channel.active==="boolean"&&typeof channel.selected==="boolean"&&typeof channel.configured==="boolean"&&typeof channel.disabled==="boolean"&&Array.isArray(channel.inputs)}function normalizeRuntimeSetup(setup){return{nodePreloads:Array.isArray(setup?.nodePreloads)?[...setup.nodePreloads]:[],envAliases:Array.isArray(setup?.envAliases)?[...setup.envAliases]:[],secretScans:Array.isArray(setup?.secretScans)?[...setup.secretScans]:[]}}function compilerContext(plan){return{sandboxName:plan.sandboxName,agent:plan.agent,workflow:plan.workflow,isInteractive:false,configuredChannels:plan.channels.map(channel=>channel.channelId),disabledChannels:plan.disabledChannels,credentialAvailability:credentialAvailabilityFromPlan(plan)}}function credentialAvailabilityFromPlan(plan){const availability={};for(const channel of plan.channels){for(const input of channel.inputs){if(input.kind!=="secret"||input.credentialAvailable!==true)continue;availability[`${channel.channelId}.${input.inputId}`]=true;if(input.sourceEnv)availability[input.sourceEnv]=true}}for(const credential of plan.credentialBindings){if(!credential.credentialAvailable)continue;availability[credential.credentialId]=true;availability[`${credential.channelId}.${credential.credentialId}`]=true;availability[`${credential.channelId}.${credential.sourceInput}`]=true;availability[credential.providerEnvKey]=true}return availability}function normalizeMessagingChannelId(channelId){return channelId.trim().toLowerCase()}function enabledPlanChannels(plan){const disabled=new Set((plan.disabledChannels??[]).map(normalizeMessagingChannelId).filter(Boolean));return plan.channels.filter(channel=>{const channelId=normalizeMessagingChannelId(channel.channelId);return channelId.length>0&&channel.active&&!channel.disabled&&!disabled.has(channelId)})}function selectActiveMessagingChannelIds(plan){const seen=new Set;const channels=[];for(const item of enabledPlanChannels(plan)){const channel=normalizeMessagingChannelId(item.channelId);if(!channel||seen.has(channel))continue;seen.add(channel);channels.push(channel)}return channels}function selectEnabledMessagingAgentRender(plan){const active=new Set(selectActiveMessagingChannelIds(plan));return plan.agentRender.filter(render=>render.agent===plan.agent&&active.has(normalizeMessagingChannelId(render.channelId)))}function selectEnabledPostAgentInstallBuildFiles(plan){const active=new Set(selectActiveMessagingChannelIds(plan));const channels=enabledPlanChannels(plan);return plan.buildSteps.filter(step=>{const channelId=normalizeMessagingChannelId(step.channelId);if(!active.has(channelId)||step.kind!=="build-file")return false;if(!step.hookId)return true;const matchingChannels=channels.filter(channel=>normalizeMessagingChannelId(channel.channelId)===channelId);if(matchingChannels.length!==1)return false;const matchedHook=matchingChannels[0]?.hooks?.find(hook=>hook.id===step.hookId);return matchedHook!==void 0&&matchedHook.phase==="post-agent-install"})}function parseSandboxMessagingPlan(value,options={}){if(!isObjectRecord(value)||value.schemaVersion!==1||typeof value.sandboxName!=="string"||typeof value.agent!=="string"||typeof value.workflow!=="string"||!Array.isArray(value.channels)||!Array.isArray(value.disabledChannels)||!isOptionalObjectArray(value,"credentialBindings")||Object.hasOwn(value,"networkPolicy")&&!isObjectRecord(value.networkPolicy)||!isOptionalObjectArray(value,"agentRender")||!isOptionalObjectArray(value,"buildSteps")||!isRuntimeSetup(value.runtimeSetup)||!isOptionalObjectArray(value,"stateUpdates")||!isOptionalObjectArray(value,"healthChecks")){return null}if(options.sandboxName&&value.sandboxName!==options.sandboxName)return null;if(options.agent&&value.agent!==options.agent)return null;const supported=Array.isArray(options.supportedChannelIds)?new Set(options.supportedChannelIds):null;const normalizedChannelIds=new Set;for(const channel of value.channels){if(!isObjectRecord(channel)||typeof channel.channelId!=="string")return null;const normalizedChannelId=normalizeMessagingChannelId(channel.channelId);if(!normalizedChannelId||normalizedChannelId!==channel.channelId||normalizedChannelIds.has(normalizedChannelId)){return null}if(Object.hasOwn(channel,"configured")&&typeof channel.configured!=="boolean"){return null}if(Object.hasOwn(channel,"active")&&typeof channel.active!=="boolean")return null;if(Object.hasOwn(channel,"disabled")&&typeof channel.disabled!=="boolean")return null;if(Object.hasOwn(channel,"pendingRemoval")&&typeof channel.pendingRemoval!=="boolean"){return null}if(Object.hasOwn(channel,"inputs")&&!Array.isArray(channel.inputs))return null;if(Object.hasOwn(channel,"hostForward")&&!isHostForward(channel.hostForward))return null;if(Object.hasOwn(channel,"hooks")&&!Array.isArray(channel.hooks))return null;if(Array.isArray(channel.inputs)&&channel.inputs.some(input=>!isObjectRecord(input)||typeof input.inputId!=="string"||Object.hasOwn(input,"channelId")&&input.channelId!==normalizedChannelId)){return null}if(Array.isArray(channel.hooks)&&channel.hooks.some(hook=>!isObjectRecord(hook)||Object.hasOwn(hook,"channelId")&&hook.channelId!==normalizedChannelId)){return null}if(Object.hasOwn(channel,"hostForward")&&isObjectRecord(channel.hostForward)&&channel.hostForward.channelId!==normalizedChannelId){return null}if(supported&&!supported.has(channel.channelId))return null;normalizedChannelIds.add(normalizedChannelId)}if(!value.disabledChannels.every(isCanonicalMessagingChannelId))return null;const disabledChannelIds=new Set(value.disabledChannels);if(disabledChannelIds.size!==value.disabledChannels.length||[...disabledChannelIds].some(channelId=>!normalizedChannelIds.has(channelId))||value.channels.some(channel=>isObjectRecord(channel)&&channel.disabled===true!==disabledChannelIds.has(String(channel.channelId)))){return null}if(!hasCanonicalChannelReferences(value.credentialBindings)||!hasMatchingAgentRenderEntries(value.agentRender,value.agent)||!hasCanonicalChannelReferences(value.agentRender)||!hasCanonicalChannelReferences(value.buildSteps)||!hasCanonicalChannelReferences(value.stateUpdates)||!hasCanonicalChannelReferences(value.healthChecks)||!hasCanonicalNetworkPolicyReferences(value.networkPolicy)||!hasCanonicalRuntimeSetupReferences(value.runtimeSetup)){return null}return cloneSandboxMessagingPlan(normalizePersistedSandboxMessagingPlanShape(value,options.environment))}function hasMatchingAgentRenderEntries(value,agent){return!Array.isArray(value)||value.every(render=>isObjectRecord(render)&&render.agent===agent)}function hasCanonicalNetworkPolicyReferences(value){if(!isObjectRecord(value)||!Object.hasOwn(value,"entries"))return true;return hasCanonicalChannelReferences(value.entries)}function cloneSandboxMessagingPlan(plan){return JSON.parse(JSON.stringify(plan))}function isOptionalObjectArray(value,key){if(!Object.hasOwn(value,key))return true;const entries=value[key];return Array.isArray(entries)&&entries.every(isObjectRecord)}function isHostForward(value){return isObjectRecord(value)&&typeof value.channelId==="string"&&typeof value.port==="number"&&Number.isInteger(value.port)&&value.port>=1&&value.port<=65535&&typeof value.label==="string"}function isRuntimeSetup(value){if(value===void 0)return true;return isObjectRecord(value)&&Array.isArray(value.nodePreloads)&&Array.isArray(value.envAliases)&&Array.isArray(value.secretScans)&&value.nodePreloads.every(isObjectRecord)&&value.envAliases.every(isObjectRecord)&&value.secretScans.every(isObjectRecord)}function isCanonicalMessagingChannelId(value){return typeof value==="string"&&value.length>0&&normalizeMessagingChannelId(value)===value}function hasCanonicalChannelReferences(value){return value===void 0||Array.isArray(value)&&value.every(entry=>isObjectRecord(entry)&&isCanonicalMessagingChannelId(entry.channelId))}function hasCanonicalRuntimeSetupReferences(value){if(value===void 0)return true;if(!isObjectRecord(value))return false;return["nodePreloads","envAliases","secretScans"].every(field=>hasCanonicalChannelReferences(value[field]))}var import_node_buffer=require("node:buffer");var import_node_crypto2=require("node:crypto");var import_node_util=require("node:util");function isLoopbackHostname(hostname=""){const normalized=String(hostname||"").trim().toLowerCase().replace(/^\[|\]$/g,"");return normalized==="localhost"||normalized==="::1"||/^127(?:\.\d{1,3}){3}$/.test(normalized)}function isLoopbackDashboardUrl(value){return isLoopbackHostname(new URL(value).hostname)}function listMessagingCredentialEnvAssignments(options={}){return selectManifests(options).flatMap(manifest=>{const credentialsByTemplate=new Map(manifest.credentials.map(credential=>[`{{credential.${credential.id}.placeholder}}`,credential]));const renderedAssignments=manifest.render.flatMap(render=>{if(options.agent&&render.agent!==options.agent)return[];if(render.kind!=="env-lines")return[];return render.lines.flatMap(line=>{const separator=line.indexOf("=");if(separator<=0)return[];const credential=credentialsByTemplate.get(line.slice(separator+1));if(!credential)return[];return[{channelId:manifest.id,agent:render.agent,sourceEnvKey:credential.providerEnvKey,targetEnvKey:line.slice(0,separator),placeholder:credential.placeholder}]})});const runtimeAssignments=["openclaw","hermes"].flatMap(agent=>{if(options.agent&&agent!==options.agent)return[];if(!manifest.supportedAgents.includes(agent))return[];return(manifest.runtime?.[agent]?.envAliases??[]).flatMap(alias=>{if(!alias.targetEnvKey)return[];const credential=manifest.credentials.find(candidate=>candidate.providerEnvKey===alias.envKey);if(!credential)return[];return[{channelId:manifest.id,agent,sourceEnvKey:alias.envKey,targetEnvKey:alias.targetEnvKey,placeholder:credential.placeholder}]})});return[...renderedAssignments,...runtimeAssignments]})}function selectManifests(options){const manifests=options.manifests??BUILT_IN_CHANNEL_MANIFESTS;const agent=options.agent;const selected=agent?manifests.filter(manifest=>manifest.supportedAgents.includes(agent)):manifests;return[...selected]}function authorizeMessagingManagedStartupFields(entry,section){if(section==="agentRender")return authorizeTeamsOpenClawWebhookField(entry);if(!isPlainDataObject3(entry))return[];const contract=WECHAT_OPENCLAW_ACCOUNT_FILE_CONTRACT;if(ownDataPropertyValue3(entry,"channelId")!==contract.channelId||ownDataPropertyValue3(entry,"hookId")!==contract.planHookId||ownDataPropertyValue3(entry,"handler")!==contract.handlerId||ownDataPropertyValue3(entry,"outputId")!==contract.outputId||ownDataPropertyValue3(entry,"kind")!==contract.kind||ownDataPropertyValue3(entry,"required")!==contract.required){return[]}return authorizeWechatAccountFilePlaceholders(ownDataPropertyValue3(entry,"value")).map(authorization=>({...authorization,path:["value",...authorization.path]}))}function isPlainDataObject3(value){if(value===null||typeof value!=="object"||Array.isArray(value))return false;const prototype=Object.getPrototypeOf(value);return prototype===Object.prototype||prototype===null}function ownDataPropertyValue3(value,key){const descriptor=Object.getOwnPropertyDescriptor(value,key);return descriptor&&"value"in descriptor?descriptor.value:void 0}var DCODE_UPSTREAM_PROVIDER_RE=/^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/u;function isValidDcodeUpstreamProvider(value){return DCODE_UPSTREAM_PROVIDER_RE.test(value)}var MANAGED_STARTUP_PROFILE_SCHEMA_VERSION=1;var MANAGED_STARTUP_PROFILE_MAX_BYTES=64*1024;var MANAGED_STARTUP_PROFILE_MAX_ENCODED_BYTES=Math.ceil(MANAGED_STARTUP_PROFILE_MAX_BYTES/3)*4;var MAX_IDENTIFIER_BYTES=256;var MAX_MODEL_BYTES=1024;var MAX_URL_BYTES=2048;var MAX_LIST_ITEMS=128;var MAX_JSON_NODES=4096;var MAX_JSON_DEPTH=32;var MAX_TUNING_INTEGER=1e9;var MIN_HERMES_CONTEXT_WINDOW=64e3;var SHA256_RE=/^[a-f0-9]{64}$/;var CONTROL_CHARACTER_RE=/[\u0000-\u001f\u007f-\u009f]/u;var BASE64URL_RE=/^[A-Za-z0-9_-]+$/;var RAW_CA_PEM_RE=/-----BEGIN (?:TRUSTED )?CERTIFICATE-----/iu;var RAW_CA_PEM_BASE64_RE=/^LS0tLS1CRUdJTi(?:BDRVJUSUZJQ0FURS0tLS0t|BUlVTVEVEIENFUlRJRklDQVRFLS0tLS0)/u;var RAW_CA_DER_BASE64_RE=/^MII[A-Za-z0-9+/=\r\n]{253,}$/u;var RAW_CA_DATA_URI_RE=/data:application\/(?:pkix-cert|x-x509-ca-cert);base64,MII[A-Za-z0-9+/=]{253,}/iu;var URL_CANDIDATE_RE=/[A-Za-z][A-Za-z0-9+.-]*:\/\/[^\s"'<>]+/gu;var UTF8_DECODER=new import_node_util.TextDecoder("utf-8",{fatal:true});var CREDENTIAL_SHAPED_NAME_PATTERN=/(?:^|[_-])(?:api[_-]?key|access[_-]?key|secret[_-]?key|auth[_-]?token|refresh[_-]?token|access[_-]?token|client[_-]?secret|private[_-]?key|pass[_-]?code|personal[_-]?access[_-]?token|connection[_-]?string|webhook(?:[_-]?url)?|key|secret|token|password|passwd|passcode|auth|authorization|credential|credentials|bearer|bearer[_-]?token|cookie|cookies|pat|private|privatekey|pin|webhookurl|dsn|connectionstring)(?:$|[_-])/iu;var CREDENTIAL_COMPOUND_NAME_PATTERN=/^(?:access|refresh|client|bearer|auth|api|private|signing|session|bot|app|resolved)(?:token|key|secret|password)$/iu;var CREDENTIAL_CAMEL_SUFFIX_PATTERN=/(?:apiKey|accessKey|secretKey|authToken|refreshToken|accessToken|clientSecret|privateKey|passcode|password|passwd|passphrase|bearerToken|botToken|appToken|sessionToken|signingKey|secretPublicKey|personalAccessToken|connectionString|webhookUrl)$/iu;var CREDENTIAL_CAMEL_BOUNDARY_PATTERN=/[a-z0-9](?:Token|Key|Secret|Password|Passphrase|Pat)$/u;var CREDENTIAL_ENV_NAME_PATTERN=/^(?:[A-Z0-9]+_)*(?:TOKEN|KEY|SECRET|PASSWORD|PASSWD|PASS|PASSPHRASE|CREDENTIAL)S?$/u;var CREDENTIAL_HEADER_NAME_PATTERN=/^(?:authorization|proxy-authorization|cookie|set-cookie|.+-(?:key|token|secret|password|passphrase|credential|auth)s?)$/iu;var PUBLIC_KEY_NAME_PATTERN=/^public[-_]?keys?$/iu;var PASS_CREDENTIAL_NAME_PATTERN=/(?:^|[-_])pass(?:wd)?$/iu;var NON_SECRET_KEY_METADATA_NAMES=new Set(["envKey","installCacheEnvKey","providerEnvKey","stateKey","targetEnvKey"]);var MESSAGING_CREDENTIAL_PLACEHOLDER_RE=/^(?:openshell:resolve:env:|[A-Za-z0-9]+-OPENSHELL-RESOLVE-ENV-)(?:v[0-9]+_)?[A-Z][A-Z0-9_]*$/u;var MESSAGING_CREDENTIAL_ENV_ALIASES=new Set(listMessagingCredentialEnvAssignments().filter(({sourceEnvKey,targetEnvKey})=>sourceEnvKey!==targetEnvKey).map(({agent,sourceEnvKey,targetEnvKey})=>`${agent}\0${sourceEnvKey}\0${targetEnvKey}`));var MESSAGING_CREDENTIAL_RUNTIME_ENV_ALIASES=new Set(listMessagingCredentialEnvAssignments().filter(({sourceEnvKey,targetEnvKey})=>sourceEnvKey!==targetEnvKey).map(({agent,channelId,sourceEnvKey,targetEnvKey})=>`${agent}\0${channelId}\0${sourceEnvKey}\0${targetEnvKey}`));var JSON_ARRAY_INDEX_SEGMENT_RE=/^\[(?:0|[1-9][0-9]*)\]$/u;var SECRET_VALUE_PATTERNS=[/nvapi-[A-Za-z0-9_-]{10,}/u,/nvcf-[A-Za-z0-9_-]{10,}/u,/ghp_[A-Za-z0-9_-]{10,}/u,/github_pat_[A-Za-z0-9_]{30,}/u,/sk-(?:proj-|ant-)?[A-Za-z0-9_-]{10,}/u,/(?:xox[bpas]|xapp)-[A-Za-z0-9-]{10,}/u,/A(?:K|S)IA[A-Z0-9]{16}/u,/hf_[A-Za-z0-9]{10,}/u,/glpat-[A-Za-z0-9_-]{10,}/u,/gsk_[A-Za-z0-9]{10,}/u,/pypi-[A-Za-z0-9_-]{10,}/u,/tvly-[A-Za-z0-9_-]{10,}/u,/lsv2_(?:pt|sk)_[A-Za-z0-9]{10,}(?:_[A-Za-z0-9]+)*/u,/\bbot\d{8,10}:[A-Za-z0-9_-]{35}\b/u,/\b\d{8,10}:[A-Za-z0-9_-]{35}\b/u,/\b[A-Za-z0-9]{24}\.[A-Za-z0-9_-]{6}\.[A-Za-z0-9_-]{27,}\b/u,/\beyJ[A-Za-z0-9_-]{5,}\.[A-Za-z0-9_-]{2,}\.[A-Za-z0-9_-]{10,}\b/u,/\bBearer\s+[A-Za-z0-9_.+/=-]{10,}/iu,/-----BEGIN (?:[A-Z0-9]+ )?PRIVATE KEY-----/u];var MANAGED_STARTUP_INFERENCE_APIS=["openai-completions","openai-responses","anthropic-messages"];var MANAGED_STARTUP_REASONING_EFFORTS=["default","low","medium","high"];var MANAGED_STARTUP_DCODE_AUTO_APPROVAL_MODES=["disabled","thread-opt-in"];var MANAGED_STARTUP_HERMES_TOOL_GATEWAYS=["nous-web","nous-image","nous-audio","nous-browser","nous-code"];var MANAGED_STARTUP_AGENTS=["openclaw","hermes","langchain-deepagents-code","pi"];var MANAGED_STARTUP_MESSAGING_AGENTS=["openclaw","hermes"];function freezeAgentCapabilities(capabilities){return Object.freeze({...capabilities,inferenceApis:Object.freeze([...capabilities.inferenceApis]),dashboardModes:Object.freeze([...capabilities.dashboardModes]),inputModalities:Object.freeze([...capabilities.inputModalities]),webSearchProviders:Object.freeze([...capabilities.webSearchProviders]),toolGateways:Object.freeze([...capabilities.toolGateways]),tuningFields:Object.freeze([...capabilities.tuningFields])})}var PROFILE_CAPABILITIES={openclaw:{inferenceApis:[...MANAGED_STARTUP_INFERENCE_APIS],dashboardModes:["loopback","remote"],inputModalities:["text","image"],webSearchProviders:["brave","tavily"],toolGateways:[],tuningFields:["contextWindow","maxTokens","reasoning","reasoningEffort"],supportsMessaging:true,supportsInferenceCompatibility:true,supportsUpstreamEndpoint:false,supportsHostProxyIntent:true,supportsPrimaryModelRef:true,supportsAgentTimeout:true,supportsHeartbeat:true,supportsExtraAgents:true,supportsDeviceAuth:true,observability:"openclaw-otel",supportsMinimalBootstrap:true},hermes:{inferenceApis:[...MANAGED_STARTUP_INFERENCE_APIS],dashboardModes:["disabled","loopback-forwarded"],inputModalities:[],webSearchProviders:["tavily"],toolGateways:[...MANAGED_STARTUP_HERMES_TOOL_GATEWAYS],tuningFields:["contextWindow"],supportsMessaging:true,supportsInferenceCompatibility:false,supportsUpstreamEndpoint:false,supportsHostProxyIntent:true,supportsPrimaryModelRef:false,supportsAgentTimeout:false,supportsHeartbeat:false,supportsExtraAgents:false,supportsDeviceAuth:false,observability:"none",supportsMinimalBootstrap:false},"langchain-deepagents-code":{inferenceApis:["openai-completions"],dashboardModes:["disabled"],inputModalities:[],webSearchProviders:[],toolGateways:[],tuningFields:["reasoningEffort"],supportsMessaging:false,supportsInferenceCompatibility:false,supportsUpstreamEndpoint:true,supportsHostProxyIntent:true,supportsPrimaryModelRef:false,supportsAgentTimeout:false,supportsHeartbeat:false,supportsExtraAgents:false,supportsDeviceAuth:false,observability:"dcode-marker",supportsMinimalBootstrap:false},pi:{inferenceApis:["openai-completions"],dashboardModes:["disabled"],inputModalities:[],webSearchProviders:[],toolGateways:[],tuningFields:["contextWindow","maxTokens","reasoning"],supportsMessaging:false,supportsInferenceCompatibility:false,supportsUpstreamEndpoint:false,supportsHostProxyIntent:true,supportsPrimaryModelRef:false,supportsAgentTimeout:false,supportsHeartbeat:false,supportsExtraAgents:false,supportsDeviceAuth:false,observability:"none",supportsMinimalBootstrap:false}};for(const agent of MANAGED_STARTUP_AGENTS){Object.defineProperty(PROFILE_CAPABILITIES,agent,{configurable:false,enumerable:true,value:freezeAgentCapabilities(PROFILE_CAPABILITIES[agent]),writable:false})}var MANAGED_STARTUP_PROFILE_CAPABILITIES=Object.freeze(PROFILE_CAPABILITIES);function affordance(input,profilePath,source="docker-arg",representation="value"){return{input,profilePath,source,representation}}var HOST_PROXY_AFFORDANCES=[affordance("HTTP_PROXY","proxy.hostHttpUrl","runtime-env"),affordance("http_proxy","proxy.hostHttpUrl","runtime-env","derived"),affordance("HTTPS_PROXY","proxy.hostHttpsUrl","runtime-env"),affordance("https_proxy","proxy.hostHttpsUrl","runtime-env","derived"),affordance("NO_PROXY","proxy.hostNoProxy","runtime-env"),affordance("no_proxy","proxy.hostNoProxy","runtime-env","derived")];var MANAGED_STARTUP_PROFILE_AFFORDANCE_INVENTORY={openclaw:[affordance("NEMOCLAW_MODEL","inference.model"),affordance("NEMOCLAW_INFERENCE_PROVIDER_ID","inference.routeProvider"),affordance("NEMOCLAW_UPSTREAM_PROVIDER","inference.upstreamProvider"),affordance("NEMOCLAW_PRIMARY_MODEL_REF","inference.primaryModelRef"),affordance("NEMOCLAW_INFERENCE_BASE_URL","inference.routedBaseUrl"),affordance("NEMOCLAW_INFERENCE_API","inference.api"),affordance("NEMOCLAW_INFERENCE_COMPAT_B64","inference.compatibility"),affordance("NEMOCLAW_INFERENCE_INPUTS","inference.inputModalities"),affordance("NEMOCLAW_CONTEXT_WINDOW","tuning.contextWindow"),affordance("NEMOCLAW_MAX_TOKENS","tuning.maxTokens"),affordance("NEMOCLAW_REASONING","tuning.reasoning"),affordance("NEMOCLAW_REASONING_EFFORT","tuning.reasoningEffort"),affordance("NEMOCLAW_TOOL_DISCLOSURE","tools.disclosure"),affordance("NEMOCLAW_AGENT_TIMEOUT","agentConfig.agentTimeoutSeconds"),affordance("NEMOCLAW_AGENT_HEARTBEAT_EVERY","agentConfig.heartbeatEvery"),affordance("NEMOCLAW_EXTRA_AGENTS_JSON_B64","agentConfig.extraAgents"),affordance("NEMOCLAW_DISABLE_DEVICE_AUTH","agentConfig.deviceAuth.disabled"),affordance("NEMOCLAW_DEVICE_AUTH_OPT_OUT_SOURCE","agentConfig.deviceAuth.optOutSource"),affordance("NEMOCLAW_WEB_SEARCH_ENABLED","agentConfig.webSearch.enabled"),affordance("NEMOCLAW_WEB_SEARCH_PROVIDER","agentConfig.webSearch.provider"),affordance("NEMOCLAW_OPENCLAW_OTEL","agentConfig.otel.enabled"),affordance("NEMOCLAW_OPENCLAW_OTEL_ENDPOINT","agentConfig.otel.endpointUrl"),affordance("NEMOCLAW_OPENCLAW_OTEL_SERVICE_NAME","agentConfig.otel.serviceName"),affordance("NEMOCLAW_OPENCLAW_OTEL_SAMPLE_RATE","agentConfig.otel.sampleRate"),affordance("CHAT_UI_URL","dashboard.url"),affordance("NEMOCLAW_DASHBOARD_BIND","dashboard.bindAddress"),affordance("NEMOCLAW_WSL_DASHBOARD_EXPOSURE","dashboard.wslExposure"),affordance("NEMOCLAW_DASHBOARD_PORT","dashboard.port","runtime-env"),affordance("NEMOCLAW_PROXY_HOST","proxy.managedHost"),affordance("NEMOCLAW_PROXY_PORT","proxy.managedPort"),affordance("NEMOCLAW_MESSAGING_PLAN_B64","messaging.plan"),affordance("NEMOCLAW_MINIMAL_BOOTSTRAP","agentConfig.minimalBootstrap","runtime-env"),affordance("NEMOCLAW_CORPORATE_CA_B64","corporateCa.bundleSha256","host-material","digest-handoff"),...HOST_PROXY_AFFORDANCES],hermes:[affordance("NEMOCLAW_MODEL","inference.model"),affordance("NEMOCLAW_INFERENCE_PROVIDER_ID","inference.routeProvider"),affordance("NEMOCLAW_UPSTREAM_PROVIDER","inference.upstreamProvider"),affordance("NEMOCLAW_INFERENCE_BASE_URL","inference.routedBaseUrl"),affordance("NEMOCLAW_INFERENCE_API","inference.api"),affordance("NEMOCLAW_CONTEXT_WINDOW","tuning.contextWindow"),affordance("NEMOCLAW_TOOL_DISCLOSURE","tools.disclosure"),affordance("NEMOCLAW_HERMES_TOOL_GATEWAY_BROKER","tools.enabledGateways","docker-arg","derived"),affordance("NEMOCLAW_HERMES_TOOL_GATEWAY_PRESETS_B64","tools.enabledGateways"),affordance("NEMOCLAW_WEB_SEARCH_ENABLED","agentConfig.webSearch.enabled"),affordance("NEMOCLAW_WEB_SEARCH_PROVIDER","agentConfig.webSearch.provider"),affordance("NEMOCLAW_MESSAGING_PLAN_B64","messaging.plan"),affordance("CHAT_UI_URL","dashboard.browserUrl"),affordance("NEMOCLAW_DASHBOARD_PORT","dashboard.publicPort","runtime-env"),affordance("NEMOCLAW_HERMES_DASHBOARD","dashboard.mode","runtime-env"),affordance("NEMOCLAW_HERMES_DASHBOARD_PORT","dashboard.publicPort","runtime-env"),affordance("NEMOCLAW_HERMES_DASHBOARD_INTERNAL_PORT","dashboard.internalPort","runtime-env"),affordance("NEMOCLAW_HERMES_DASHBOARD_TUI","dashboard.tuiEnabled","runtime-env"),affordance("NEMOCLAW_PROXY_HOST","proxy.managedHost","runtime-env"),affordance("NEMOCLAW_PROXY_PORT","proxy.managedPort","runtime-env"),affordance("NEMOCLAW_CORPORATE_CA_B64","corporateCa.bundleSha256","host-material","digest-handoff"),...HOST_PROXY_AFFORDANCES],"langchain-deepagents-code":[affordance("NEMOCLAW_MODEL","inference.model"),affordance("NEMOCLAW_INFERENCE_PROVIDER_ID","inference.routeProvider"),affordance("NEMOCLAW_UPSTREAM_PROVIDER","inference.upstreamProvider"),affordance("NEMOCLAW_UPSTREAM_ENDPOINT_URL","inference.upstreamEndpointUrl"),affordance("NEMOCLAW_INFERENCE_BASE_URL","inference.routedBaseUrl"),affordance("NEMOCLAW_INFERENCE_API","inference.api"),affordance("NEMOCLAW_REASONING_EFFORT","tuning.reasoningEffort"),affordance("NEMOCLAW_TOOL_DISCLOSURE","tools.disclosure"),affordance("NEMOCLAW_DCODE_AUTO_APPROVAL","agentConfig.autoApprovalMode"),affordance("NEMOCLAW_PROXY_HOST","proxy.managedHost"),affordance("NEMOCLAW_PROXY_PORT","proxy.managedPort"),affordance("NEMOCLAW_OBSERVABILITY","agentConfig.observabilityEnabled","runtime-env"),affordance("NEMOCLAW_CORPORATE_CA_B64","corporateCa.bundleSha256","host-material","digest-handoff"),...HOST_PROXY_AFFORDANCES],pi:[affordance("NEMOCLAW_MODEL","inference.model"),affordance("NEMOCLAW_INFERENCE_PROVIDER_ID","inference.routeProvider"),affordance("NEMOCLAW_UPSTREAM_PROVIDER","inference.upstreamProvider"),affordance("NEMOCLAW_INFERENCE_BASE_URL","inference.routedBaseUrl"),affordance("NEMOCLAW_INFERENCE_API","inference.api"),affordance("NEMOCLAW_CONTEXT_WINDOW","tuning.contextWindow"),affordance("NEMOCLAW_MAX_TOKENS","tuning.maxTokens"),affordance("NEMOCLAW_REASONING","tuning.reasoning"),affordance("NEMOCLAW_TOOL_DISCLOSURE","tools.disclosure"),affordance("NEMOCLAW_PROXY_HOST","proxy.managedHost"),affordance("NEMOCLAW_PROXY_PORT","proxy.managedPort"),affordance("NEMOCLAW_CORPORATE_CA_B64","corporateCa.bundleSha256","host-material","digest-handoff"),...HOST_PROXY_AFFORDANCES]};function deferredRuntimeInput(input,owner,reason,admission="managed-launch-forwarded"){return Object.freeze({input,owner,admission,reason})}var MANAGED_STARTUP_PROFILE_DEFERRED_RUNTIME_INPUTS=Object.freeze({openclaw:Object.freeze([deferredRuntimeInput("NEMOCLAW_AUTO_PAIR_DEADLINE_SECS","application-environment","operator scheduler tuning is applied by the application environment transaction"),deferredRuntimeInput("NEMOCLAW_AUTO_PAIR_FAST_DEADLINE_SECS","application-environment","operator scheduler tuning is applied by the application environment transaction"),deferredRuntimeInput("NEMOCLAW_AUTO_PAIR_FAST_REENTRY_INTERVAL_SECS","application-environment","operator scheduler tuning is applied by the application environment transaction"),deferredRuntimeInput("NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS","application-environment","operator scheduler tuning is applied by the application environment transaction"),deferredRuntimeInput("NEMOCLAW_AUTO_PAIR_RUN_TIMEOUT_SECS","application-environment","operator scheduler tuning is applied by the application environment transaction"),deferredRuntimeInput("NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS","application-environment","operator scheduler tuning is applied by the application environment transaction"),deferredRuntimeInput("NEMOCLAW_MCP_SHADOW_DIAGNOSTICS","application-environment","operator shadow-diagnostics tuning is applied by the application environment transaction"),deferredRuntimeInput("NEMOCLAW_MCP_TOOLS_LIST_TIMEOUT_MS","application-environment","operator MCP discovery timeout tuning is applied by the application environment transaction"),deferredRuntimeInput("OPENCLAW_HOME","fixed-image-contract","the managed image and agent definition own this fixed runtime layout path"),deferredRuntimeInput("OPENCLAW_STATE_DIR","fixed-image-contract","the managed image and agent definition own this fixed runtime layout path"),deferredRuntimeInput("OPENCLAW_WORKSPACE_DIR","fixed-image-contract","the managed image and agent definition own this fixed runtime layout path"),deferredRuntimeInput("NEMOCLAW_EXTRA_PLACEHOLDER_KEYS","credential-plumbing","credential provider construction owns key metadata outside the secret-free profile")]),hermes:Object.freeze([deferredRuntimeInput("NEMOCLAW_EXTRA_PLACEHOLDER_KEYS","credential-plumbing","credential provider construction owns key metadata outside the secret-free profile")]),"langchain-deepagents-code":Object.freeze([deferredRuntimeInput("NEMOCLAW_SANDBOX_NAME","engine-identity","the lifecycle engine owns instance identity outside reusable startup intent"),deferredRuntimeInput("NEMOCLAW_EXTRA_PLACEHOLDER_KEYS","credential-plumbing","credential provider construction owns key metadata outside the secret-free profile")]),pi:Object.freeze([deferredRuntimeInput("NEMOCLAW_EXTRA_PLACEHOLDER_KEYS","credential-plumbing","credential provider construction owns key metadata outside the secret-free profile")])});function runtimeCleanupObligation(input,emittedFor,supportedFor,reason){return Object.freeze({input,emittedFor:Object.freeze([...emittedFor]),supportedFor:Object.freeze([...supportedFor]),owner:"application-environment",reason})}var MANAGED_STARTUP_RUNTIME_CLEANUP_OBLIGATIONS=Object.freeze([runtimeCleanupObligation("NEMOCLAW_DASHBOARD_BIND",["hermes"],["openclaw"],"generic managed-dashboard construction currently emits the OpenClaw-only bind control for Hermes"),runtimeCleanupObligation("NEMOCLAW_MINIMAL_BOOTSTRAP",["hermes","langchain-deepagents-code"],["openclaw"],"generic host-proxy construction currently emits the OpenClaw-only bootstrap control for other agents")]);var ManagedStartupProfileError=class extends Error{constructor(message){super(`Invalid managed startup profile: ${message}`);this.name="ManagedStartupProfileError"}};var PROFILE_KEYS=new Set(["schemaVersion","agent","agentConfig","inference","proxy","dashboard","tools","messaging","tuning","corporateCa"]);var INFERENCE_KEYS=new Set(["routeProvider","upstreamProvider","model","routedBaseUrl","upstreamEndpointUrl","api","primaryModelRef","compatibility","inputModalities"]);var PROXY_KEYS=new Set(["managedHost","managedPort","hostHttpUrl","hostHttpsUrl","hostNoProxy"]);var OPENCLAW_DASHBOARD_KEYS=new Set(["agent","mode","url","port","bindAddress","wslExposure"]);var HERMES_DASHBOARD_KEYS=new Set(["agent","mode","url","browserUrl","publicPort","internalPort","tuiEnabled"]);var DCODE_DASHBOARD_KEYS=new Set(["agent","mode"]);var TOOLS_KEYS=new Set(["disclosure","enabledGateways"]);var MESSAGING_KEYS=new Set(["plan"]);var TUNING_FIELD_ORDER=["contextWindow","maxTokens","reasoning","reasoningEffort"];var TUNING_KEYS=new Set(TUNING_FIELD_ORDER);var CORPORATE_CA_KEYS=new Set(["bundleSha256"]);var OPENCLAW_CONFIG_KEYS=new Set(["agent","webSearch","otel","agentTimeoutSeconds","heartbeatEvery","extraAgents","deviceAuth","minimalBootstrap"]);var HERMES_CONFIG_KEYS=new Set(["agent","webSearch"]);var DCODE_CONFIG_KEYS=new Set(["agent","autoApprovalMode","observabilityEnabled"]);var PI_CONFIG_KEYS=new Set(["agent"]);var PI_DASHBOARD_KEYS=new Set(["agent","mode"]);var WEB_SEARCH_KEYS=new Set(["enabled","provider"]);var OTEL_KEYS=new Set(["enabled","endpointUrl","serviceName","sampleRate"]);var DEVICE_AUTH_KEYS=new Set(["disabled","optOutSource"]);var EXTRA_AGENTS_KEYS=new Set(["agents","defaults","main"]);var MANAGED_STARTUP_AGENT_SET=new Set(MANAGED_STARTUP_AGENTS);var DCODE_AUTO_APPROVAL_MODE_SET=new Set(MANAGED_STARTUP_DCODE_AUTO_APPROVAL_MODES);var REASONING_EFFORT_SET=new Set(MANAGED_STARTUP_REASONING_EFFORTS);var HERMES_INTERNAL_API_PORT=18642;var HERMES_API_PORT_RANGE_START=8642;var HERMES_API_PORT_RANGE_END=8652;function isHermesApiPort(port){return port>=HERMES_API_PORT_RANGE_START&&port<=HERMES_API_PORT_RANGE_END}function isHermesReservedApiPort(port){return port===HERMES_INTERNAL_API_PORT||isHermesApiPort(port)}var HERMES_RESERVED_API_PORT_LABEL=`${HERMES_API_PORT_RANGE_START}-${HERMES_API_PORT_RANGE_END} or ${HERMES_INTERNAL_API_PORT}`;function isPlainObject(value){if(typeof value!=="object"||value===null||Array.isArray(value))return false;const prototype=Object.getPrototypeOf(value);return prototype===Object.prototype||prototype===null}function isCredentialShapedName(name){if(PUBLIC_KEY_NAME_PATTERN.test(name)||NON_SECRET_KEY_METADATA_NAMES.has(name))return false;return CREDENTIAL_SHAPED_NAME_PATTERN.test(name)||CREDENTIAL_COMPOUND_NAME_PATTERN.test(name)||CREDENTIAL_CAMEL_SUFFIX_PATTERN.test(name)||CREDENTIAL_CAMEL_BOUNDARY_PATTERN.test(name)||CREDENTIAL_ENV_NAME_PATTERN.test(name)||CREDENTIAL_HEADER_NAME_PATTERN.test(name)||PASS_CREDENTIAL_NAME_PATTERN.test(name)}function valueLooksLikeSecret(value){for(let index=0;index=5&&path5[0]==="messaging"&&path5[1]==="plan"&&path5[2]==="agentRender"&&JSON_ARRAY_INDEX_SEGMENT_RE.test(path5[3]??"")&&path5[4]==="value";const isAuthorizedBuildStepPlaceholder=allowedBuildStepPlaceholders.has(buildStepPlaceholderKey(path5,value));return isCredentialBindingPlaceholder||isAgentRenderValuePlaceholder||isAuthorizedBuildStepPlaceholder}function requiresMessagingSchemaFieldAuthorization(path5){const fieldName=path5[path5.length-1];return fieldName==="webhook"}function messagingAuthorizedFieldKey(path5){return JSON.stringify(path5)}function buildStepPlaceholderKey(path5,value){return JSON.stringify([path5,value])}function messagingCredentialPlaceholderEnvKey(value){if(!MESSAGING_CREDENTIAL_PLACEHOLDER_RE.test(value))return null;const marker=value.startsWith("openshell:resolve:env:")?"openshell:resolve:env:":"-OPENSHELL-RESOLVE-ENV-";const key=value.slice(value.indexOf(marker)+marker.length);return key.replace(/^v[0-9]+_/u,"")}function containsMessagingCredentialPlaceholder(value){return value.includes("openshell:resolve:env:")||value.includes("-OPENSHELL-RESOLVE-ENV-")}function isMessagingCredentialPlaceholderAssignment(selectedAgent,path5,value){if(path5.length!==6||path5[0]!=="messaging"||path5[1]!=="plan"||path5[2]!=="agentRender"||!JSON_ARRAY_INDEX_SEGMENT_RE.test(path5[3]??"")||path5[4]!=="lines"||!JSON_ARRAY_INDEX_SEGMENT_RE.test(path5[5]??"")){return false}const separator=value.indexOf("=");if(separator<=0||value.indexOf("=",separator+1)!==-1)return false;const envKey=value.slice(0,separator);const placeholder=value.slice(separator+1);const placeholderEnvKey=messagingCredentialPlaceholderEnvKey(placeholder);return CREDENTIAL_ENV_NAME_PATTERN.test(envKey)&&placeholderEnvKey!==null&&(envKey===placeholderEnvKey||typeof selectedAgent==="string"&&MESSAGING_CREDENTIAL_ENV_ALIASES.has(`${selectedAgent}\0${placeholderEnvKey}\0${envKey}`))}function isMessagingRuntimeEnvAliasPath(path5){return path5.length===5&&path5[0]==="messaging"&&path5[1]==="plan"&&path5[2]==="runtimeSetup"&&path5[3]==="envAliases"&&JSON_ARRAY_INDEX_SEGMENT_RE.test(path5[4]??"")}function ownDataPropertyValue4(value,key){const descriptor=Object.getOwnPropertyDescriptor(value,key);return descriptor&&"value"in descriptor?descriptor.value:void 0}function isCanonicalMessagingRuntimeEnvAlias(selectedAgent,path5,value){if(!isMessagingRuntimeEnvAliasPath(path5))return false;const channelId=ownDataPropertyValue4(value,"channelId");const envKey=ownDataPropertyValue4(value,"envKey");const targetEnvKey=ownDataPropertyValue4(value,"targetEnvKey");const match=ownDataPropertyValue4(value,"match");const placeholder=ownDataPropertyValue4(value,"value");const expectedMatch=targetEnvKey===void 0?`^openshell:resolve:env:(v[0-9]+_)?${envKey}$`:`^openshell:resolve:env:v[0-9]+_${envKey}$`;return typeof envKey==="string"&&CREDENTIAL_ENV_NAME_PATTERN.test(envKey)&&match===expectedMatch&&typeof placeholder==="string"&&messagingCredentialPlaceholderEnvKey(placeholder)===envKey&&(targetEnvKey===void 0||typeof selectedAgent==="string"&&typeof channelId==="string"&&typeof targetEnvKey==="string"&&CREDENTIAL_ENV_NAME_PATTERN.test(targetEnvKey)&&MESSAGING_CREDENTIAL_RUNTIME_ENV_ALIASES.has(`${selectedAgent}\0${channelId}\0${envKey}\0${targetEnvKey}`))}function isAllowedMessagingRuntimeAliasStringPath(path5,allowedAliasIndexes){return path5.length===6&&path5[0]==="messaging"&&path5[1]==="plan"&&path5[2]==="runtimeSetup"&&path5[3]==="envAliases"&&allowedAliasIndexes.has(path5[4]??"")&&(path5[5]==="match"||path5[5]==="value"||path5[5]==="targetEnvKey")}function isMessagingPackagePin(path5,value){return path5.length===6&&path5[0]==="messaging"&&path5[1]==="plan"&&path5[2]==="buildSteps"&&JSON_ARRAY_INDEX_SEGMENT_RE.test(path5[3]??"")&&path5[4]==="value"&&path5[5]==="pin"&&typeof value==="boolean"}function containsUrlWithCredentialMaterial(value){const candidates=value.match(URL_CANDIDATE_RE)??[];for(let index=0;index{if(isCredentialShapedName(key))credentialQuery=true});const fragment=url.hash.startsWith("#")?url.hash.slice(1):url.hash;const queryStart=fragment.indexOf("?");const fragmentParameters=new URLSearchParams(queryStart>=0?fragment.slice(queryStart+1):fragment);let credentialFragment=false;fragmentParameters.forEach((_fragmentValue,key)=>{if(isCredentialShapedName(key))credentialFragment=true});if(url.username||url.password||credentialQuery||credentialFragment)return true}catch{}}return false}function invalid(reason){throw new ManagedStartupProfileError(reason)}function payloadPath(path5){return path5.reduce((result,segment)=>segment.startsWith("[")?`${result}${segment}`:`${result}${result?".":""}${segment}`,"")}function mapArrayByIndex(values,mapper){const mapped=[];for(let index=0;index0&&values[insertion-1]>selected){Object.defineProperty(values,String(insertion),{configurable:true,enumerable:true,value:values[insertion-1],writable:true});insertion-=1}Object.defineProperty(values,String(insertion),{configurable:true,enumerable:true,value:selected,writable:true})}return values}function requireRecord(value,where){if(!isPlainObject(value))invalid(`${where} must be an object`);return value}function rejectUnknownKeys(value,allowed,where){const keys=Object.keys(value);for(let index=0;indexmaxBytes||CONTROL_CHARACTER_RE.test(value)){invalid(`${where} must be a bounded, non-empty string without control characters`)}return value}function requireStringEnum(value,allowed,where){const normalized=requireBoundedString(value,where);if(!allowed.has(normalized))invalid(`${where} is not supported`);return normalized}function requireNullablePositiveInteger(value,where){if(value===null)return null;if(typeof value!=="number"||!Number.isSafeInteger(value)||value<1||value>MAX_TUNING_INTEGER){invalid(`${where} must be null or a bounded positive integer`)}return value}function requirePositiveInteger(value,where,maximum=MAX_TUNING_INTEGER){if(typeof value!=="number"||!Number.isSafeInteger(value)||value<1||value>maximum){invalid(`${where} must be a bounded positive integer`)}return value}function requirePort(value,where,minimum=1){if(typeof value!=="number"||!Number.isInteger(value)||value<1||value>65535){invalid(`${where} must be a valid TCP port`)}if(valueMAX_LIST_ITEMS){invalid(`${where} must be a bounded string list`)}const items=mapArrayByIndex(value,item=>requireBoundedString(item,`${where} item`));const unique2=new Set;for(let index=0;index{if(depth>MAX_JSON_DEPTH)invalid(`${where} exceeds the JSON depth limit`);if(current===null||typeof current==="string"||typeof current==="boolean"){return current}if(typeof current==="number"){if(!Number.isFinite(current))invalid(`${where} contains a non-finite number`);return current}if(Array.isArray(current)){return mapArrayByIndex(current,item=>clone(item,depth+1))}if(!isPlainObject(current))invalid(`${where} contains a non-JSON value`);const result=options.nullPrototypeObjects?Object.create(null):{};const keys=Object.getOwnPropertyNames(current);for(let index=0;indexMAX_IDENTIFIER_BYTES||CONTROL_CHARACTER_RE.test(key)){invalid(`${where} contains an invalid object key`)}const descriptor=Object.getOwnPropertyDescriptor(current,key);if(!descriptor||!("value"in descriptor)){invalid(`${where} contains a non-JSON value`)}Object.defineProperty(result,key,{configurable:true,enumerable:true,value:clone(descriptor.value,depth+1),writable:true})}return result};return clone(value,0)}function requireJsonObjectOrNull(value,where){if(value===null)return null;if(!isPlainObject(value))invalid(`${where} must be null or a plain JSON object`);return cloneJsonValue(value,where,{nullPrototypeObjects:true})}function requireJsonObject(value,where){const object=requireJsonObjectOrNull(value,where);if(object===null)invalid(`${where} must be a plain JSON object`);return object}function requireHttpUrl(value,where){const raw=requireBoundedString(value,where,MAX_URL_BYTES);let parsed;try{parsed=new URL(raw)}catch{invalid(`${where} must be a valid HTTP(S) URL`)}if(parsed.protocol!=="http:"&&parsed.protocol!=="https:"||parsed.username||parsed.password||parsed.search||parsed.hash){invalid(`${where} must be a credential-free HTTP(S) URL without query or fragment data`)}const pathname=parsed.pathname.replace(/\/+$/u,"");return pathname===""?parsed.origin:`${parsed.origin}${pathname}`}function requireProxyUrl(value,allowedSchemes,where){if(value===null)return null;const raw=requireBoundedString(value,where,MAX_URL_BYTES);let parsed;try{parsed=new URL(raw)}catch{invalid(`${where} must be a valid HTTP(S) proxy URL`)}if(!allowedSchemes.has(parsed.protocol)||parsed.username||parsed.password||parsed.pathname!=="/"||parsed.search||parsed.hash){invalid(`${where} must be a credential-free HTTP(S) proxy origin`)}return parsed.origin}function requireManagedProxyHost(value,where){const host=requireBoundedString(value,where);if(!/^[A-Za-z0-9._-]+$/u.test(host)){invalid(`${where} must be a hostname or IPv4 address without a scheme or separators`)}return host}function configuredDashboardPort(value){const explicit=new URL(value).port;return explicit===""?18789:Number(explicit)}function requireSampleRate(value,where){if(typeof value!=="number"||!Number.isFinite(value)||value<0||value>1){invalid(`${where} must be a number between 0 and 1`)}return value}function assertPayloadStructureAndCredentialShapes(root){const pending=[{value:root,depth:0,path:[]}];const allowedRuntimeAliasIndexes=new Set;const allowedMessagingCredentialFields=new Set;const allowedBuildStepPlaceholders=new Set;const selectedAgent=isPlainObject(root)?ownDataPropertyValue4(root,"agent"):void 0;let discoveredNodes=1;let observedBytes=0;const observeText=value=>{observedBytes+=import_node_buffer.Buffer.byteLength(value,"utf8");if(observedBytes>MANAGED_STARTUP_PROFILE_MAX_BYTES){invalid(`payload exceeds ${String(MANAGED_STARTUP_PROFILE_MAX_BYTES)} bytes`)}};const reserveNode=depth=>{discoveredNodes+=1;if(discoveredNodes>MAX_JSON_NODES||depth>MAX_JSON_DEPTH){invalid("payload structure exceeds the complexity limit")}observedBytes+=1};while(pending.length>0){const current=pending.pop();if(!current)break;if(current.depth>MAX_JSON_DEPTH){invalid("payload structure exceeds the complexity limit")}if(typeof current.value==="string"){observeText(current.value);if(!isAllowedMessagingRuntimeAliasStringPath(current.path,allowedRuntimeAliasIndexes)&&!isMessagingCredentialPlaceholder(current.path,current.value,allowedBuildStepPlaceholders,allowedMessagingCredentialFields)&&!isMessagingCredentialPlaceholderAssignment(selectedAgent,current.path,current.value)&&(valueLooksLikeSecret(current.value)||containsMessagingCredentialPlaceholder(current.value))){invalid(`payload field ${payloadPath(current.path)} contains credential-shaped string data`)}if(RAW_CA_PEM_RE.test(current.value)||RAW_CA_PEM_BASE64_RE.test(current.value)||RAW_CA_DER_BASE64_RE.test(current.value)||RAW_CA_DATA_URI_RE.test(current.value)){invalid(`payload field ${payloadPath(current.path)} contains raw certificate data; provide only the CA SHA-256 digest`)}if(containsUrlWithCredentialMaterial(current.value)){invalid(`payload field ${payloadPath(current.path)} contains a URL with embedded credentials`)}continue}if(Array.isArray(current.value)){if(Object.getPrototypeOf(current.value)!==Array.prototype){invalid("payload arrays must use the standard JSON prototype")}if("toJSON"in current.value){invalid("payload must not define a custom JSON serializer")}if(Object.getOwnPropertySymbols(current.value).length>0||Object.getOwnPropertyNames(current.value).length!==current.value.length+1){invalid("payload arrays must contain only indexed JSON values")}for(let index=0;index0||discoveredNodes+keys.length>MAX_JSON_NODES){invalid("payload structure exceeds the complexity limit")}for(let index=0;indexMANAGED_STARTUP_PROFILE_MAX_BYTES){invalid(`payload exceeds ${String(MANAGED_STARTUP_PROFILE_MAX_BYTES)} bytes`)}}function validateWebSearch(value,agent){const webSearch=requireRecord(value,"agentConfig.webSearch");rejectUnknownKeys(webSearch,WEB_SEARCH_KEYS,"agentConfig.webSearch");const provider=requireStringEnum(webSearch.provider,new Set(MANAGED_STARTUP_PROFILE_CAPABILITIES[agent].webSearchProviders),"agentConfig.webSearch.provider");return{enabled:requireBoolean(webSearch.enabled,"agentConfig.webSearch.enabled"),provider}}function validateOpenClawOtel(value){const otel=requireRecord(value,"agentConfig.otel");rejectUnknownKeys(otel,OTEL_KEYS,"agentConfig.otel");return{enabled:requireBoolean(otel.enabled,"agentConfig.otel.enabled"),endpointUrl:requireHttpUrl(otel.endpointUrl,"agentConfig.otel.endpointUrl"),serviceName:requireBoundedString(otel.serviceName,"agentConfig.otel.serviceName",MAX_IDENTIFIER_BYTES),sampleRate:requireSampleRate(otel.sampleRate,"agentConfig.otel.sampleRate")}}function validateExtraAgents(value){const extraAgents=requireRecord(value,"agentConfig.extraAgents");rejectUnknownKeys(extraAgents,EXTRA_AGENTS_KEYS,"agentConfig.extraAgents");if(!Array.isArray(extraAgents.agents)||extraAgents.agents.length>MAX_LIST_ITEMS){invalid("agentConfig.extraAgents.agents must be a bounded JSON object list")}return{agents:mapArrayByIndex(extraAgents.agents,(agent,index)=>requireJsonObject(agent,`agentConfig.extraAgents.agents[${String(index)}]`)),defaults:requireJsonObject(extraAgents.defaults,"agentConfig.extraAgents.defaults"),main:requireJsonObject(extraAgents.main,"agentConfig.extraAgents.main")}}function validateDeviceAuth(value){const deviceAuth=requireRecord(value,"agentConfig.deviceAuth");rejectUnknownKeys(deviceAuth,DEVICE_AUTH_KEYS,"agentConfig.deviceAuth");return{disabled:requireBoolean(deviceAuth.disabled,"agentConfig.deviceAuth.disabled"),optOutSource:requireStringEnum(deviceAuth.optOutSource,new Set(["operator","managed-onboard"]),"agentConfig.deviceAuth.optOutSource")}}function validateAgentConfig(value,expectedAgent){const config=requireRecord(value,"agentConfig");const agent=requireStringEnum(config.agent,MANAGED_STARTUP_AGENT_SET,"agentConfig.agent");if(agent!==expectedAgent)invalid("agentConfig.agent must match agent");if(agent==="openclaw"){rejectUnknownKeys(config,OPENCLAW_CONFIG_KEYS,"agentConfig");const heartbeatEvery=config.heartbeatEvery===null?null:requireBoundedString(config.heartbeatEvery,"agentConfig.heartbeatEvery",MAX_IDENTIFIER_BYTES);if(heartbeatEvery!==null&&!/^\d+(?:s|m|h)$/u.test(heartbeatEvery)){invalid("agentConfig.heartbeatEvery must be null or a duration ending in s, m, or h")}return{agent,webSearch:validateWebSearch(config.webSearch,agent),otel:validateOpenClawOtel(config.otel),agentTimeoutSeconds:requirePositiveInteger(config.agentTimeoutSeconds,"agentConfig.agentTimeoutSeconds"),heartbeatEvery,extraAgents:validateExtraAgents(config.extraAgents),deviceAuth:validateDeviceAuth(config.deviceAuth),minimalBootstrap:requireBoolean(config.minimalBootstrap,"agentConfig.minimalBootstrap")}}if(agent==="hermes"){rejectUnknownKeys(config,HERMES_CONFIG_KEYS,"agentConfig");return{agent,webSearch:validateWebSearch(config.webSearch,agent)}}if(agent==="pi"){rejectUnknownKeys(config,PI_CONFIG_KEYS,"agentConfig");return{agent}}rejectUnknownKeys(config,DCODE_CONFIG_KEYS,"agentConfig");return{agent,autoApprovalMode:requireStringEnum(config.autoApprovalMode,DCODE_AUTO_APPROVAL_MODE_SET,"agentConfig.autoApprovalMode"),observabilityEnabled:requireBoolean(config.observabilityEnabled,"agentConfig.observabilityEnabled")}}function validateDashboard(value,expectedAgent){const dashboard=requireRecord(value,"dashboard");const agent=requireStringEnum(dashboard.agent,MANAGED_STARTUP_AGENT_SET,"dashboard.agent");if(agent!==expectedAgent)invalid("dashboard.agent must match agent");if(agent==="openclaw"){rejectUnknownKeys(dashboard,OPENCLAW_DASHBOARD_KEYS,"dashboard");const mode=requireStringEnum(dashboard.mode,new Set(MANAGED_STARTUP_PROFILE_CAPABILITIES[agent].dashboardModes),"dashboard.mode");const url=requireHttpUrl(dashboard.url,"dashboard.url");const bindAddress=requireStringEnum(dashboard.bindAddress,new Set(["127.0.0.1","0.0.0.0"]),"dashboard.bindAddress");const wslExposure=requireBoolean(dashboard.wslExposure,"dashboard.wslExposure");const hasRemoteExposure=!isLoopbackDashboardUrl(url)||bindAddress==="0.0.0.0"||wslExposure;if(mode==="remote"!==hasRemoteExposure){invalid("OpenClaw dashboard.mode must reflect its URL, bind address, and WSL exposure")}const port=requirePort(dashboard.port,"dashboard.port",1024);if(isHermesApiPort(port))invalid(`OpenClaw dashboard.port must not use a reserved Hermes API port (${HERMES_API_PORT_RANGE_START}-${HERMES_API_PORT_RANGE_END})`);if(configuredDashboardPort(url)!==port){invalid("OpenClaw dashboard.port must match dashboard.url")}return{agent,mode,url,port,bindAddress,wslExposure}}if(agent==="hermes"){rejectUnknownKeys(dashboard,HERMES_DASHBOARD_KEYS,"dashboard");const mode=requireStringEnum(dashboard.mode,new Set(MANAGED_STARTUP_PROFILE_CAPABILITIES[agent].dashboardModes),"dashboard.mode");const url=requireHttpUrl(dashboard.url,"dashboard.url");if(!isLoopbackDashboardUrl(url)){invalid("Hermes dashboard.url must remain loopback; OpenShell owns the host forward")}const browserUrl=dashboard.browserUrl===void 0?void 0:requireHttpUrl(dashboard.browserUrl,"dashboard.browserUrl");if(browserUrl!==void 0&&!isLoopbackDashboardUrl(browserUrl)&&new URL(browserUrl).protocol!=="https:"){invalid("Hermes dashboard.browserUrl must use HTTPS unless it is loopback")}if(mode==="disabled"){if(dashboard.publicPort!==null||dashboard.internalPort!==null||dashboard.tuiEnabled!==false){invalid("disabled Hermes dashboard must not configure ports or TUI")}return{agent,mode,url,...browserUrl===void 0?{}:{browserUrl},publicPort:null,internalPort:null,tuiEnabled:false}}const publicPort=requirePort(dashboard.publicPort,"dashboard.publicPort",1024);const internalPort=requirePort(dashboard.internalPort,"dashboard.internalPort",1024);if(publicPort===internalPort){invalid("Hermes dashboard publicPort and internalPort must differ")}if(isHermesReservedApiPort(publicPort)||isHermesReservedApiPort(internalPort)){invalid(`Hermes dashboard ports must not use reserved API ports ${HERMES_RESERVED_API_PORT_LABEL}`)}if(configuredDashboardPort(url)!==publicPort){invalid("Hermes dashboard.publicPort must match dashboard.url")}if(browserUrl!==void 0&&isLoopbackDashboardUrl(browserUrl)&&configuredDashboardPort(browserUrl)!==publicPort){invalid("Hermes dashboard.publicPort must match dashboard.browserUrl")}return{agent,mode,url,...browserUrl===void 0?{}:{browserUrl},publicPort,internalPort,tuiEnabled:requireBoolean(dashboard.tuiEnabled,"dashboard.tuiEnabled")}}if(agent==="pi"){rejectUnknownKeys(dashboard,PI_DASHBOARD_KEYS,"dashboard");if(dashboard.mode!=="disabled")invalid("pi dashboard.mode must be disabled");return{agent,mode:"disabled"}}rejectUnknownKeys(dashboard,DCODE_DASHBOARD_KEYS,"dashboard");if(dashboard.mode!=="disabled"){invalid("langchain-deepagents-code dashboard.mode must be disabled")}return{agent,mode:"disabled"}}function validateInference(value,agent){const inference=requireRecord(value,"inference");rejectUnknownKeys(inference,INFERENCE_KEYS,"inference");const routeProvider=requireBoundedString(inference.routeProvider,"inference.routeProvider");const upstreamProvider=requireBoundedString(inference.upstreamProvider,"inference.upstreamProvider");const model=requireBoundedString(inference.model,"inference.model",MAX_MODEL_BYTES);const api=requireStringEnum(inference.api,new Set(MANAGED_STARTUP_PROFILE_CAPABILITIES[agent].inferenceApis),"inference.api");const upstreamEndpointUrl=inference.upstreamEndpointUrl===null?null:requireHttpUrl(inference.upstreamEndpointUrl,"inference.upstreamEndpointUrl");const primaryModelRef=inference.primaryModelRef===null?null:requireBoundedString(inference.primaryModelRef,"inference.primaryModelRef",MAX_MODEL_BYTES);const compatibility=requireJsonObjectOrNull(inference.compatibility,"inference.compatibility");const inputModalities=inference.inputModalities===null?null:requireEnumList(inference.inputModalities,new Set(MANAGED_STARTUP_PROFILE_CAPABILITIES[agent].inputModalities),"inference.inputModalities",{allowEmpty:false});if(upstreamEndpointUrl!==null&&!MANAGED_STARTUP_PROFILE_CAPABILITIES[agent].supportsUpstreamEndpoint){invalid(`inference.upstreamEndpointUrl must be null for ${agent}`)}if(agent==="openclaw"){if(primaryModelRef===null||inputModalities===null){invalid("openclaw requires primaryModelRef and inputModalities")}if(primaryModelRef!==`${routeProvider}/${model}`){invalid("openclaw primaryModelRef must match routeProvider and model")}}else{if(primaryModelRef!==null||compatibility!==null||inputModalities!==null){invalid(`${agent} does not support primaryModelRef, compatibility, or inputModalities`)}if(agent==="langchain-deepagents-code"&&!isValidDcodeUpstreamProvider(upstreamProvider)){invalid("inference.upstreamProvider must start with an ASCII letter or digit and contain 1-64 ASCII letters, digits, dots, underscores, or hyphens for DCode")}}return{routeProvider,upstreamProvider,model,routedBaseUrl:requireHttpUrl(inference.routedBaseUrl,"inference.routedBaseUrl"),upstreamEndpointUrl,api,primaryModelRef,compatibility,inputModalities}}function validateProxy(value,agent){const proxy=requireRecord(value,"proxy");rejectUnknownKeys(proxy,PROXY_KEYS,"proxy");const hostHttpUrl=requireProxyUrl(proxy.hostHttpUrl,new Set(["http:"]),"proxy.hostHttpUrl");const hostHttpsUrl=requireProxyUrl(proxy.hostHttpsUrl,new Set(["http:","https:"]),"proxy.hostHttpsUrl");const hostNoProxy=requireStringList(proxy.hostNoProxy,"proxy.hostNoProxy");if(!MANAGED_STARTUP_PROFILE_CAPABILITIES[agent].supportsHostProxyIntent&&(hostHttpUrl!==null||hostHttpsUrl!==null||hostNoProxy.length>0)){invalid(`${agent} rejects host proxy intent and accepts only its root-owned managed route`)}return{managedHost:requireManagedProxyHost(proxy.managedHost,"proxy.managedHost"),managedPort:requirePort(proxy.managedPort,"proxy.managedPort"),hostHttpUrl,hostHttpsUrl,hostNoProxy}}function validateTools(value,agent){const tools=requireRecord(value,"tools");rejectUnknownKeys(tools,TOOLS_KEYS,"tools");const enabledGateways=requireEnumList(tools.enabledGateways,new Set(MANAGED_STARTUP_PROFILE_CAPABILITIES[agent].toolGateways),"tools.enabledGateways",{allowEmpty:true});return{disclosure:requireStringEnum(tools.disclosure,new Set(["progressive","direct"]),"tools.disclosure"),enabledGateways}}function validateTuning(value,agent){const tuning=requireRecord(value,"tuning");rejectUnknownKeys(tuning,TUNING_KEYS,"tuning");const result={contextWindow:requireNullablePositiveInteger(tuning.contextWindow,"tuning.contextWindow"),maxTokens:requireNullablePositiveInteger(tuning.maxTokens,"tuning.maxTokens"),reasoning:requireNullableBoolean(tuning.reasoning,"tuning.reasoning"),reasoningEffort:tuning.reasoningEffort===null?null:requireStringEnum(tuning.reasoningEffort,REASONING_EFFORT_SET,"tuning.reasoningEffort")};const advertised=new Set(MANAGED_STARTUP_PROFILE_CAPABILITIES[agent].tuningFields);const unsupported=TUNING_FIELD_ORDER.filter(field=>result[field]!==null&&!advertised.has(field));if(unsupported.length>0){invalid(`${agent} does not support startup tuning fields: ${unsupported.join(", ")}`)}if(agent==="openclaw"){const missing=TUNING_FIELD_ORDER.filter(field=>advertised.has(field)&&result[field]===null);if(missing.length>0){invalid(`openclaw requires ${missing.join(", ")} tuning`)}}if(agent==="hermes"&&result.contextWindow!==null&&result.contextWindowcanonicalizeJson(item));if(!isPlainObject(value))return value;const result={};const keys=sortStrings(Object.keys(value));for(let index=0;indexMANAGED_STARTUP_PROFILE_MAX_BYTES){invalid(`canonical payload exceeds ${String(MANAGED_STARTUP_PROFILE_MAX_BYTES)} bytes`)}return serialized}function decodeManagedStartupProfile(encoded){if(typeof encoded!=="string"||encoded.length===0||import_node_buffer.Buffer.byteLength(encoded,"ascii")>MANAGED_STARTUP_PROFILE_MAX_ENCODED_BYTES||!BASE64URL_RE.test(encoded)||encoded.length%4===1){invalid("encoded payload is malformed or exceeds the size limit")}const bytes=import_node_buffer.Buffer.from(encoded,"base64url");if(bytes.length===0||bytes.length>MANAGED_STARTUP_PROFILE_MAX_BYTES||bytes.toString("base64url")!==encoded){invalid("encoded payload is malformed or exceeds the size limit")}let raw;try{raw=UTF8_DECODER.decode(bytes)}catch{invalid("payload is not valid UTF-8")}let parsed;try{parsed=JSON.parse(raw)}catch{invalid("payload is not valid JSON")}const profile=validateManagedStartupProfile(parsed);if(serializeManagedStartupProfile(profile)!==raw){invalid("payload is not in canonical form")}return profile}function fingerprintManagedStartupProfile(profile){return(0,import_node_crypto2.createHash)("sha256").update(serializeManagedStartupProfile(profile),"utf8").digest("hex")}var ManagedStartupAgentEnvironmentError=class extends Error{constructor(message){super(`Cannot map managed startup profile: ${message}`);this.name="ManagedStartupAgentEnvironmentError"}};var EMPTY_APPLICATION_ENVIRONMENT=Object.freeze({});var OPENCLAW_APPLICATION_RUNTIME_INPUTS=Object.freeze([["NEMOCLAW_AUTO_PAIR_DEADLINE_SECS","positive-finite-seconds"],["NEMOCLAW_AUTO_PAIR_FAST_DEADLINE_SECS","positive-finite-seconds"],["NEMOCLAW_AUTO_PAIR_FAST_REENTRY_INTERVAL_SECS","positive-finite-seconds"],["NEMOCLAW_AUTO_PAIR_FAST_REENTRY_POLLS","positive-safe-integer"],["NEMOCLAW_AUTO_PAIR_RUN_TIMEOUT_SECS","positive-finite-seconds"],["NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS","positive-finite-seconds"]]);function booleanFlag(value){return value?"1":"0"}function canonicalizeJson2(value){if(Array.isArray(value))return value.map(item=>canonicalizeJson2(item));if(value===null||typeof value!=="object")return value;const record=value;return Object.fromEntries(Object.keys(record).sort().map(key=>[key,canonicalizeJson2(record[key])]))}function encodeCanonicalJson(value){return import_node_buffer2.Buffer.from(JSON.stringify(canonicalizeJson2(value)),"utf8").toString("base64")}function sortedEnvironment(environment){return Object.freeze(Object.fromEntries(Object.entries(environment).sort(([left],[right])=>leftright?1:0)))}function canonicalApplicationRuntimeValue(name,raw,kind){if(raw.includes("\0")||/[\r\n]/u.test(raw)){throw new ManagedStartupAgentEnvironmentError(`${name} must be single-line text`)}const value=Number(raw.trim());const valid=kind==="positive-safe-integer"?Number.isSafeInteger(value)&&value>0:Number.isFinite(value)&&value>0;if(!valid){throw new ManagedStartupAgentEnvironmentError(`${name} must be ${kind==="positive-safe-integer"?"a positive safe integer":"finite positive seconds"}`)}return String(value)}function applicationRuntimePlan(profile,environment){const exportEnvironment={};if(profile.agent==="openclaw"){for(const[name,kind]of OPENCLAW_APPLICATION_RUNTIME_INPUTS){const raw=environment[name];if(raw!==void 0){exportEnvironment[name]=canonicalApplicationRuntimeValue(name,raw,kind)}}}const unsetEnvironment=new Set(MANAGED_STARTUP_RUNTIME_CLEANUP_OBLIGATIONS.filter(({supportedFor})=>!supportedFor.includes(profile.agent)).map(({input})=>input));if(profile.agent!=="openclaw"){for(const[name]of OPENCLAW_APPLICATION_RUNTIME_INPUTS){unsetEnvironment.add(name)}}return Object.freeze({exportEnvironment:sortedEnvironment(exportEnvironment),unsetEnvironment:Object.freeze([...unsetEnvironment].sort())})}function commonConfigurationEnvironment(profile){return{NEMOCLAW_INFERENCE_API:profile.inference.api,NEMOCLAW_INFERENCE_BASE_URL:profile.inference.routedBaseUrl,NEMOCLAW_INFERENCE_PROVIDER_ID:profile.inference.routeProvider,NEMOCLAW_MODEL:profile.inference.model,NEMOCLAW_TOOL_DISCLOSURE:profile.tools.disclosure,NEMOCLAW_UPSTREAM_PROVIDER:profile.inference.upstreamProvider}}function appendHostProxyEnvironment(environment,profile,options={}){if(options.preserveAmbientWhenAbsent===true&&profile.proxy.hostHttpUrl===null&&profile.proxy.hostHttpsUrl===null&&profile.proxy.hostNoProxy.length===0){return}const httpProxy=profile.proxy.hostHttpUrl??"";const httpsProxy=profile.proxy.hostHttpsUrl??"";const noProxy=profile.proxy.hostNoProxy.join(",");environment.HTTP_PROXY=httpProxy;environment.HTTPS_PROXY=httpsProxy;environment.NO_PROXY=noProxy;environment.http_proxy=httpProxy;environment.https_proxy=httpsProxy;environment.no_proxy=noProxy}function messagingEnvironment(profile,expectedAgent){if(profile.messaging.plan===null)return{};const plan=parseSandboxMessagingPlan(profile.messaging.plan,{agent:expectedAgent});if(!plan){throw new ManagedStartupAgentEnvironmentError(`messaging.plan must contain a validated ${expectedAgent} messaging plan`)}const{workflow:_workflow,...imageBuildPlan}=plan;return{NEMOCLAW_MESSAGING_PLAN_B64:encodeCanonicalJson(imageBuildPlan)}}function corporateCaMaterial(profile){return Object.freeze({kind:"corporate-ca-handoff",legacyInput:"NEMOCLAW_CORPORATE_CA_B64",expectedSha256:profile.corporateCa.bundleSha256})}function rootOwnedFile(legacyInput,path5,value){return Object.freeze({kind:"root-owned-file",legacyInput,path:path5,contents:`${value} `,owner:"root",group:"root",mode:292})}function dashboardAction(dashboard){return Object.freeze({kind:"configure-dashboard",dashboard:Object.freeze(structuredClone(dashboard))})}function applicationActions(profile,messagingAgent){const actions=[];if(messagingAgent!==null){actions.push(Object.freeze({kind:"apply-messaging-plan",agent:messagingAgent,mode:profile.messaging.plan===null?"clear":"apply",phase:"runtime-setup",runAs:"root"}))}actions.push(Object.freeze({kind:"generate-agent-config",agent:profile.agent,runAs:"sandbox"}));if(messagingAgent!==null){actions.push(Object.freeze({kind:"apply-messaging-plan",agent:messagingAgent,mode:profile.messaging.plan===null?"clear":"apply",phase:"post-agent-install",runAs:"sandbox"}))}actions.push(dashboardAction(profile.dashboard));return Object.freeze(actions)}function mapOpenClawProfile(profile,environment){if(profile.agent!=="openclaw"||profile.agentConfig.agent!=="openclaw"||profile.dashboard.agent!=="openclaw"||profile.inference.primaryModelRef===null||profile.inference.inputModalities===null||profile.tuning.contextWindow===null||profile.tuning.maxTokens===null||profile.tuning.reasoning===null||profile.tuning.reasoningEffort===null){throw new ManagedStartupAgentEnvironmentError("OpenClaw profile state is inconsistent")}const configurationEnvironment={...commonConfigurationEnvironment(profile),...messagingEnvironment(profile,"openclaw"),CHAT_UI_URL:profile.dashboard.url,NEMOCLAW_AGENT_HEARTBEAT_EVERY:profile.agentConfig.heartbeatEvery??"",NEMOCLAW_AGENT_TIMEOUT:String(profile.agentConfig.agentTimeoutSeconds),NEMOCLAW_CONTEXT_WINDOW:String(profile.tuning.contextWindow),NEMOCLAW_DASHBOARD_BIND:profile.dashboard.bindAddress==="0.0.0.0"?profile.dashboard.bindAddress:"",NEMOCLAW_DISABLE_DEVICE_AUTH:booleanFlag(profile.agentConfig.deviceAuth.disabled),NEMOCLAW_DEVICE_AUTH_OPT_OUT_SOURCE:profile.agentConfig.deviceAuth.optOutSource,NEMOCLAW_EXTRA_AGENTS_JSON_B64:encodeCanonicalJson(profile.agentConfig.extraAgents),NEMOCLAW_INFERENCE_COMPAT_B64:encodeCanonicalJson(profile.inference.compatibility),NEMOCLAW_INFERENCE_INPUTS:profile.inference.inputModalities.join(","),NEMOCLAW_MAX_TOKENS:String(profile.tuning.maxTokens),NEMOCLAW_OPENCLAW_OTEL:booleanFlag(profile.agentConfig.otel.enabled),NEMOCLAW_OPENCLAW_OTEL_ENDPOINT:profile.agentConfig.otel.endpointUrl,NEMOCLAW_OPENCLAW_OTEL_SAMPLE_RATE:String(profile.agentConfig.otel.sampleRate),NEMOCLAW_OPENCLAW_OTEL_SERVICE_NAME:profile.agentConfig.otel.serviceName,NEMOCLAW_PRIMARY_MODEL_REF:profile.inference.primaryModelRef,NEMOCLAW_PROXY_HOST:profile.proxy.managedHost,NEMOCLAW_PROXY_PORT:String(profile.proxy.managedPort),NEMOCLAW_REASONING:String(profile.tuning.reasoning),NEMOCLAW_REASONING_EFFORT:profile.tuning.reasoningEffort,NEMOCLAW_WEB_SEARCH_ENABLED:booleanFlag(profile.agentConfig.webSearch.enabled),NEMOCLAW_WEB_SEARCH_PROVIDER:profile.agentConfig.webSearch.provider,NEMOCLAW_WSL_DASHBOARD_EXPOSURE:booleanFlag(profile.dashboard.wslExposure)};const runtimeEnvironment={...configurationEnvironment};delete runtimeEnvironment.NEMOCLAW_MESSAGING_PLAN_B64;runtimeEnvironment.NEMOCLAW_DASHBOARD_PORT=String(profile.dashboard.port);runtimeEnvironment.NEMOCLAW_MINIMAL_BOOTSTRAP=booleanFlag(profile.agentConfig.minimalBootstrap);appendHostProxyEnvironment(runtimeEnvironment,profile,{preserveAmbientWhenAbsent:true});return Object.freeze({schemaVersion:profile.schemaVersion,agent:profile.agent,configurationEnvironment:sortedEnvironment(configurationEnvironment),runtimeEnvironment:sortedEnvironment(runtimeEnvironment),applicationRuntime:applicationRuntimePlan(profile,environment),materials:Object.freeze([corporateCaMaterial(profile)]),actions:applicationActions(profile,"openclaw")})}function mapHermesProfile(profile,environment){if(profile.agent!=="hermes"||profile.agentConfig.agent!=="hermes"||profile.dashboard.agent!=="hermes"){throw new ManagedStartupAgentEnvironmentError("Hermes profile state is inconsistent")}let chatUiUrl=profile.dashboard.browserUrl??profile.dashboard.url;if(profile.dashboard.mode==="loopback-forwarded"){if(profile.dashboard.browserUrl===void 0){throw new ManagedStartupAgentEnvironmentError("Cannot start the Hermes dashboard because its managed startup profile has no recorded browser URL. Rerun onboarding before starting the sandbox.")}chatUiUrl=profile.dashboard.browserUrl}const configurationEnvironment={...commonConfigurationEnvironment(profile),...messagingEnvironment(profile,"hermes"),CHAT_UI_URL:chatUiUrl,NEMOCLAW_CONTEXT_WINDOW:profile.tuning.contextWindow===null?"":String(profile.tuning.contextWindow),NEMOCLAW_HERMES_TOOL_GATEWAY_BROKER:booleanFlag(profile.tools.enabledGateways.length>0),NEMOCLAW_HERMES_TOOL_GATEWAY_PRESETS_B64:encodeCanonicalJson(profile.tools.enabledGateways),NEMOCLAW_WEB_SEARCH_ENABLED:booleanFlag(profile.agentConfig.webSearch.enabled),NEMOCLAW_WEB_SEARCH_PROVIDER:profile.agentConfig.webSearch.provider};const runtimeEnvironment={...configurationEnvironment};delete runtimeEnvironment.NEMOCLAW_MESSAGING_PLAN_B64;runtimeEnvironment.NEMOCLAW_DASHBOARD_PORT=profile.dashboard.publicPort===null?"":String(profile.dashboard.publicPort);runtimeEnvironment.NEMOCLAW_HERMES_DASHBOARD=profile.dashboard.mode==="loopback-forwarded"?"1":"0";runtimeEnvironment.NEMOCLAW_HERMES_DASHBOARD_INTERNAL_PORT=profile.dashboard.internalPort===null?"":String(profile.dashboard.internalPort);runtimeEnvironment.NEMOCLAW_HERMES_DASHBOARD_PORT=profile.dashboard.publicPort===null?"":String(profile.dashboard.publicPort);runtimeEnvironment.NEMOCLAW_HERMES_DASHBOARD_TUI=booleanFlag(profile.dashboard.tuiEnabled);runtimeEnvironment.NEMOCLAW_PROXY_HOST=profile.proxy.managedHost;runtimeEnvironment.NEMOCLAW_PROXY_PORT=String(profile.proxy.managedPort);appendHostProxyEnvironment(runtimeEnvironment,profile,{preserveAmbientWhenAbsent:true});return Object.freeze({schemaVersion:profile.schemaVersion,agent:profile.agent,configurationEnvironment:sortedEnvironment(configurationEnvironment),runtimeEnvironment:sortedEnvironment(runtimeEnvironment),applicationRuntime:applicationRuntimePlan(profile,environment),materials:Object.freeze([corporateCaMaterial(profile)]),actions:applicationActions(profile,"hermes")})}function mapDcodeProfile(profile,environment){if(profile.agent!=="langchain-deepagents-code"||profile.agentConfig.agent!=="langchain-deepagents-code"||profile.dashboard.agent!=="langchain-deepagents-code"||profile.messaging.plan!==null){throw new ManagedStartupAgentEnvironmentError("LangChain Deep Agents Code profile state is inconsistent")}const reasoningEffort=profile.tuning.reasoningEffort===null||profile.tuning.reasoningEffort==="default"?"":profile.tuning.reasoningEffort;const configurationEnvironment={...commonConfigurationEnvironment(profile),NEMOCLAW_REASONING_EFFORT:reasoningEffort,NEMOCLAW_UPSTREAM_ENDPOINT_URL:profile.inference.upstreamEndpointUrl??""};appendHostProxyEnvironment(configurationEnvironment,profile);const runtimeEnvironment={...configurationEnvironment,NEMOCLAW_OBSERVABILITY:booleanFlag(profile.agentConfig.observabilityEnabled)};delete runtimeEnvironment.NEMOCLAW_INFERENCE_BASE_URL;delete runtimeEnvironment.NEMOCLAW_REASONING_EFFORT;delete runtimeEnvironment.NEMOCLAW_UPSTREAM_PROVIDER;for(const name of["HTTP_PROXY","HTTPS_PROXY","NO_PROXY","http_proxy","https_proxy","no_proxy"]){delete runtimeEnvironment[name]}const materials=Object.freeze([corporateCaMaterial(profile),rootOwnedFile("NEMOCLAW_DCODE_AUTO_APPROVAL","/usr/local/share/nemoclaw/dcode-auto-approval",profile.agentConfig.autoApprovalMode),rootOwnedFile("NEMOCLAW_INFERENCE_BASE_URL","/usr/local/share/nemoclaw/dcode-inference-base-url",profile.inference.routedBaseUrl),rootOwnedFile("NEMOCLAW_UPSTREAM_PROVIDER","/usr/local/share/nemoclaw/dcode-upstream-provider",profile.inference.upstreamProvider),rootOwnedFile("NEMOCLAW_PROXY_HOST","/usr/local/share/nemoclaw/dcode-proxy-host",profile.proxy.managedHost),rootOwnedFile("NEMOCLAW_PROXY_PORT","/usr/local/share/nemoclaw/dcode-proxy-port",String(profile.proxy.managedPort)),rootOwnedFile("NEMOCLAW_REASONING_EFFORT","/usr/local/share/nemoclaw/dcode-reasoning-effort",reasoningEffort)]);return Object.freeze({schemaVersion:profile.schemaVersion,agent:profile.agent,configurationEnvironment:sortedEnvironment(configurationEnvironment),runtimeEnvironment:sortedEnvironment(runtimeEnvironment),applicationRuntime:applicationRuntimePlan(profile,environment),materials,actions:applicationActions(profile,null)})}function mapPiProfile(profile,environment){if(profile.agent!=="pi"||profile.agentConfig.agent!=="pi"||profile.dashboard.agent!=="pi"||profile.messaging.plan!==null){throw new ManagedStartupAgentEnvironmentError("Pi profile state is inconsistent")}const configurationEnvironment={...commonConfigurationEnvironment(profile),NEMOCLAW_CONTEXT_WINDOW:profile.tuning.contextWindow===null?"":String(profile.tuning.contextWindow),NEMOCLAW_MAX_TOKENS:profile.tuning.maxTokens===null?"":String(profile.tuning.maxTokens),NEMOCLAW_REASONING:profile.tuning.reasoning===null?"":String(profile.tuning.reasoning)};appendHostProxyEnvironment(configurationEnvironment,profile);const runtimeEnvironment={...configurationEnvironment};delete runtimeEnvironment.NEMOCLAW_INFERENCE_BASE_URL;delete runtimeEnvironment.NEMOCLAW_CONTEXT_WINDOW;delete runtimeEnvironment.NEMOCLAW_MAX_TOKENS;delete runtimeEnvironment.NEMOCLAW_REASONING;for(const name of["HTTP_PROXY","HTTPS_PROXY","NO_PROXY","http_proxy","https_proxy","no_proxy"]){delete runtimeEnvironment[name]}const materials=Object.freeze([corporateCaMaterial(profile),rootOwnedFile("NEMOCLAW_PROXY_HOST","/usr/local/share/nemoclaw/pi-proxy-host",profile.proxy.managedHost),rootOwnedFile("NEMOCLAW_PROXY_PORT","/usr/local/share/nemoclaw/pi-proxy-port",String(profile.proxy.managedPort))]);return Object.freeze({schemaVersion:profile.schemaVersion,agent:profile.agent,configurationEnvironment:sortedEnvironment(configurationEnvironment),runtimeEnvironment:sortedEnvironment(runtimeEnvironment),applicationRuntime:applicationRuntimePlan(profile,environment),materials,actions:applicationActions(profile,null)})}function mapManagedStartupProfileToAgentEnvironment(profile,environment=EMPTY_APPLICATION_ENVIRONMENT){const validated=validateManagedStartupProfile(profile);switch(validated.agent){case"openclaw":return mapOpenClawProfile(validated,environment);case"hermes":return mapHermesProfile(validated,environment);case"langchain-deepagents-code":return mapDcodeProfile(validated,environment);case"pi":return mapPiProfile(validated,environment)}}var import_node_buffer3=require("node:buffer");var import_node_crypto3=require("node:crypto");var import_node_fs=__toESM(require("node:fs"));var import_node_path=__toESM(require("node:path"));var import_node_util2=require("node:util");var MANAGED_STARTUP_APPLICATION_STATE_DIR="/var/lib/nemoclaw/startup-profile";var MANAGED_STARTUP_CA_MAX_BYTES=128*1024;var MANAGED_STARTUP_CA_MAX_CERTIFICATES=24;var STATE_SCHEMA_VERSION=1;var STATE_DIRECTORY_MODE=448;var STATE_FILE_MODE=384;var MAX_CONTROL_FILE_BYTES=512;var MAX_STATE_ENTRIES=32;var SHA256_RE2=/^[a-f0-9]{64}$/u;var GENERATION_RE=/^generation-([a-f0-9]{64})$/u;var PREPARE_TEMP_RE=/^\.prepare-[0-9]+-[a-f0-9]{24}$/u;var CONTROL_TEMP_RE=/^\.(?:committed|pending)\.json-[a-f0-9]{24}\.tmp$/u;var PEM_CERTIFICATE_RE=/-----BEGIN CERTIFICATE-----\r?\n[A-Za-z0-9+/=\r\n]+?-----END CERTIFICATE-----/gu;var UTF8_DECODER2=new import_node_util2.TextDecoder("utf-8",{fatal:true});var DEFAULT_RUNTIME={rootUid:0,rootGid:0};var ManagedStartupApplicationError=class extends Error{constructor(message){super(`Managed startup application failed: ${message}`);this.name="ManagedStartupApplicationError"}};function fail(message){throw new ManagedStartupApplicationError(message)}function runtimeFor(override){return override??DEFAULT_RUNTIME}function requireContainerRoot(){if(process.geteuid?.()!==0){fail("the image-side applicator must run with effective uid 0")}}function modeOf(stat){return stat.mode&511}function requireOwner(stat,target,runtime){if(stat.uid!==runtime.rootUid||stat.gid!==runtime.rootGid){fail(`${target} must be owned by root:root`)}}function requireSecureDirectory(target,runtime,exactMode){let stat;try{stat=import_node_fs.default.lstatSync(target)}catch{fail(`state directory component is missing or unreadable: ${target}`)}if(stat.isSymbolicLink()||!stat.isDirectory()){fail(`state directory component must be a real directory: ${target}`)}const runtimeOwned=stat.uid===runtime.rootUid&&stat.gid===runtime.rootGid;const systemRootOwned=stat.uid===0&&stat.gid===0;if(exactMode){requireOwner(stat,target,runtime)}else if(!runtimeOwned&&!systemRootOwned){fail(`state directory ancestor is not owned by a trusted identity: ${target}`)}const mode=modeOf(stat);const writableByUntrustedIdentity=(mode&18)!==0;const trustedStickyRoot=(stat.mode&512)!==0&&(runtimeOwned||systemRootOwned);if(exactMode&&mode!==STATE_DIRECTORY_MODE||!exactMode&&writableByUntrustedIdentity&&!trustedStickyRoot){fail(exactMode?`${target} must have mode 0700`:`${target} is a replaceable group- or world-writable ancestor`)}}function requireSecureAncestors(target,runtime){const root=import_node_path.default.parse(target).root;let current=root;requireSecureDirectory(current,runtime,false);for(const segment of import_node_path.default.relative(root,target).split(import_node_path.default.sep).filter(Boolean)){current=import_node_path.default.join(current,segment);let stat;try{stat=import_node_fs.default.lstatSync(current)}catch{fail(`state directory component is missing or unreadable: ${current}`)}if(stat.isSymbolicLink()){const runtimeOwned=stat.uid===runtime.rootUid&&stat.gid===runtime.rootGid;const systemRootOwned=stat.uid===0&&stat.gid===0;if(!runtimeOwned&&!systemRootOwned){fail(`state directory ancestor is a replaceable symlink: ${current}`)}let resolved;try{resolved=import_node_fs.default.realpathSync(current)}catch{fail(`state directory symlink is missing or unreadable: ${current}`)}requireSecureAncestors(resolved,runtime);continue}requireSecureDirectory(current,runtime,false)}}function ensureStateDirectory(rawStateDirectory,runtime){const stateDirectory=rawStateDirectory??MANAGED_STARTUP_APPLICATION_STATE_DIR;if(!import_node_path.default.isAbsolute(stateDirectory)||stateDirectory.includes("\0")){fail("stateDirectory must be an absolute path")}const normalized=import_node_path.default.resolve(stateDirectory);const parent=import_node_path.default.dirname(normalized);requireSecureAncestors(parent,runtime);try{import_node_fs.default.mkdirSync(normalized,{mode:STATE_DIRECTORY_MODE});import_node_fs.default.chownSync(normalized,runtime.rootUid,runtime.rootGid);import_node_fs.default.chmodSync(normalized,STATE_DIRECTORY_MODE)}catch(error){if(error.code!=="EEXIST"){fail(`could not create the managed startup state directory: ${normalized}`)}}requireSecureDirectory(normalized,runtime,true);return normalized}function requireSecureRegularFileStat(stat,target,runtime){if(!stat.isFile()||stat.isSymbolicLink()){fail(`${target} must be a regular file`)}if(stat.nlink!==1){fail(`${target} must not be hardlinked`)}requireOwner(stat,target,runtime);if(modeOf(stat)!==STATE_FILE_MODE){fail(`${target} must have mode 0600`)}}function readSecureFile(target,maxBytes,runtime){let descriptor;try{descriptor=import_node_fs.default.openSync(target,import_node_fs.default.constants.O_RDONLY|import_node_fs.default.constants.O_NOFOLLOW)}catch{fail(`state file is missing, unreadable, or a symlink: ${target}`)}try{const stat=import_node_fs.default.fstatSync(descriptor);requireSecureRegularFileStat(stat,target,runtime);if(stat.size<1||stat.size>maxBytes){fail(`${target} is empty or exceeds its size limit`)}const content=import_node_fs.default.readFileSync(descriptor);if(content.length!==stat.size){fail(`${target} changed while it was being read`)}return content}finally{import_node_fs.default.closeSync(descriptor)}}function writeSecureNewFile(target,content,runtime){let descriptor;try{descriptor=import_node_fs.default.openSync(target,import_node_fs.default.constants.O_CREAT|import_node_fs.default.constants.O_EXCL|import_node_fs.default.constants.O_WRONLY|import_node_fs.default.constants.O_NOFOLLOW,STATE_FILE_MODE)}catch{fail(`refused to replace an existing state file: ${target}`)}try{import_node_fs.default.fchownSync(descriptor,runtime.rootUid,runtime.rootGid);import_node_fs.default.fchmodSync(descriptor,STATE_FILE_MODE);import_node_fs.default.writeFileSync(descriptor,content);import_node_fs.default.fsyncSync(descriptor)}finally{import_node_fs.default.closeSync(descriptor)}}function syncDirectory(target){const descriptor=import_node_fs.default.openSync(target,import_node_fs.default.constants.O_RDONLY);try{import_node_fs.default.fsyncSync(descriptor)}finally{import_node_fs.default.closeSync(descriptor)}}function randomToken(){return(0,import_node_crypto3.randomBytes)(12).toString("hex")}function stateControl(fingerprint){return{schemaVersion:STATE_SCHEMA_VERSION,fingerprint,generation:`generation-${fingerprint}`}}function serializeStateControl(control){return JSON.stringify({fingerprint:control.fingerprint,generation:control.generation,schemaVersion:control.schemaVersion})}function parseStateControl(target,runtime){const bytes=readSecureFile(target,MAX_CONTROL_FILE_BYTES,runtime);let raw;try{raw=UTF8_DECODER2.decode(bytes)}catch{fail(`${target} is not valid UTF-8`)}let parsed;try{parsed=JSON.parse(raw)}catch{fail(`${target} is not valid JSON`)}if(typeof parsed!=="object"||parsed===null||Array.isArray(parsed)){fail(`${target} does not contain a valid state control`)}const record=parsed;if(Object.keys(record).sort().join(",")!=="fingerprint,generation,schemaVersion"||record.schemaVersion!==STATE_SCHEMA_VERSION||typeof record.fingerprint!=="string"||!SHA256_RE2.test(record.fingerprint)||record.generation!==`generation-${record.fingerprint}`){fail(`${target} does not contain a valid state control`)}const control=stateControl(record.fingerprint);if(serializeStateControl(control)!==raw){fail(`${target} is not in canonical form`)}return control}function publishStateControlIfAbsent(stateDirectory,basename,control,runtime){const target=import_node_path.default.join(stateDirectory,basename);const temporary=import_node_path.default.join(stateDirectory,`.${basename}-${randomToken()}.tmp`);writeSecureNewFile(temporary,serializeStateControl(control),runtime);try{import_node_fs.default.linkSync(temporary,target)}catch(error){try{unlinkSecureControlOrTemp(temporary,runtime)}catch{}if(error.code==="EEXIST"){return{control:parseStateControl(target,runtime),created:false}}fail(`could not atomically publish ${basename}`)}try{import_node_fs.default.unlinkSync(temporary)}catch(error){if(error.code!=="ENOENT"){fail(`could not finalize atomic publication of ${basename}`)}}syncDirectory(stateDirectory);return{control,created:true}}function validateCorporateCaBytes(bytes){if(bytes.length<1||bytes.length>MANAGED_STARTUP_CA_MAX_BYTES){fail(`corporate CA bundle must contain 1-${String(MANAGED_STARTUP_CA_MAX_BYTES)} bytes`)}let pem;try{pem=UTF8_DECODER2.decode(bytes)}catch{fail("corporate CA bundle must be valid UTF-8 PEM")}const matches=[...pem.matchAll(PEM_CERTIFICATE_RE)];if(matches.length<1||matches.length>MANAGED_STARTUP_CA_MAX_CERTIFICATES||matches[0]?.index!==0){fail(`corporate CA bundle must contain 1-${String(MANAGED_STARTUP_CA_MAX_CERTIFICATES)} PEM CA certificates`)}let cursor=0;for(const match of matches){const index=match.index;if(index===void 0||!/^(?:\r?\n)+$/u.test(pem.slice(cursor,index))&&index!==0){fail("corporate CA bundle contains non-PEM material between certificates")}const block=match[0];let certificate;try{certificate=new import_node_crypto3.X509Certificate(block)}catch{fail("corporate CA bundle contains an invalid X.509 certificate")}if(!certificate.ca){fail("corporate CA bundle contains a certificate without basicConstraints CA:TRUE")}cursor=index+block.length}if(!/^(?:\r?\n)?$/u.test(pem.slice(cursor))){fail("corporate CA bundle contains trailing non-PEM material")}}function validateManagedStartupCorporateCaTransport(encoded,profile){const expectedDigest=profile.corporateCa.bundleSha256;if(expectedDigest===null){if(encoded!==void 0){fail("corporate CA transport must be absent when the profile has no CA digest")}return null}if(typeof encoded!=="string"||encoded.length===0||encoded.length>Math.ceil(MANAGED_STARTUP_CA_MAX_BYTES/3)*4||!/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/u.test(encoded)){fail("corporate CA transport must be canonical standard base64")}const bytes=import_node_buffer3.Buffer.from(encoded,"base64");if(bytes.toString("base64")!==encoded){fail("corporate CA transport must be canonical standard base64")}validateCorporateCaBytes(bytes);const actualDigest=(0,import_node_crypto3.createHash)("sha256").update(bytes).digest("hex");if(actualDigest!==expectedDigest){fail("corporate CA bundle does not match the profile SHA-256 digest")}return bytes}function readCanonicalProfile(profilePath,runtime){const bytes=readSecureFile(profilePath,MANAGED_STARTUP_PROFILE_MAX_BYTES,runtime);let raw;try{raw=UTF8_DECODER2.decode(bytes)}catch{fail(`${profilePath} is not valid UTF-8`)}let parsed;try{parsed=JSON.parse(raw)}catch{fail(`${profilePath} is not valid JSON`)}let profile;try{profile=validateManagedStartupProfile(parsed)}catch(error){fail(`${profilePath} is invalid: ${error.message}`)}if(serializeManagedStartupProfile(profile)!==raw){fail(`${profilePath} is not a canonical managed startup profile`)}return{profile,fingerprint:fingerprintManagedStartupProfile(profile)}}function validateGeneration(stateDirectory,control,runtime,expectedAgent){if(!GENERATION_RE.test(control.generation)){fail("state control names an invalid generation")}const directory=import_node_path.default.join(stateDirectory,control.generation);requireSecureDirectory(directory,runtime,true);const entries=import_node_fs.default.readdirSync(directory).sort();if(entries.some(entry=>entry!=="profile.json"&&entry!=="corporate-ca.pem")||!entries.includes("profile.json")){fail(`${directory} contains missing or unsupported state files`)}const profilePath=import_node_path.default.join(directory,"profile.json");const{profile,fingerprint}=readCanonicalProfile(profilePath,runtime);if(fingerprint!==control.fingerprint){fail(`${directory} does not match its recorded profile fingerprint`)}if(expectedAgent!==void 0&&profile.agent!==expectedAgent){fail(`managed startup profile targets ${profile.agent}, expected ${expectedAgent}`)}const caPath=import_node_path.default.join(directory,"corporate-ca.pem");let corporateCaPath=null;if(profile.corporateCa.bundleSha256===null){if(entries.includes("corporate-ca.pem")){fail(`${directory} contains a CA bundle that is absent from the profile`)}}else{if(!entries.includes("corporate-ca.pem")){fail(`${directory} is missing the CA bundle recorded by the profile`)}const caBytes=readSecureFile(caPath,MANAGED_STARTUP_CA_MAX_BYTES,runtime);validateCorporateCaBytes(caBytes);if((0,import_node_crypto3.createHash)("sha256").update(caBytes).digest("hex")!==profile.corporateCa.bundleSha256){fail(`${directory} contains a CA bundle with the wrong SHA-256 digest`)}corporateCaPath=caPath}return{directory,profilePath,corporateCaPath,profile,fingerprint}}function validateDisposableDirectory(target,runtime){requireSecureDirectory(target,runtime,true);const entries=import_node_fs.default.readdirSync(target);if(entries.length>2||entries.some(entry=>entry!=="profile.json"&&entry!=="corporate-ca.pem")){fail(`${target} is not a recognized disposable generation`)}for(const entry of entries){const file=import_node_path.default.join(target,entry);const stat=import_node_fs.default.lstatSync(file);requireSecureRegularFileStat(stat,file,runtime)}}function discardDirectory(target,runtime){validateDisposableDirectory(target,runtime);import_node_fs.default.rmSync(target,{recursive:true})}function discardDirectoryIfPresent(target,runtime){try{import_node_fs.default.lstatSync(target)}catch(error){if(error.code==="ENOENT")return false;fail(`could not inspect disposable generation ${target}`)}discardDirectory(target,runtime);return true}function unlinkSecureControlOrTemp(target,runtime){const stat=import_node_fs.default.lstatSync(target);requireSecureRegularFileStat(stat,target,runtime);if(stat.size>MAX_CONTROL_FILE_BYTES){fail(`${target} exceeds the state-control size limit`)}import_node_fs.default.unlinkSync(target)}function listStateEntries(stateDirectory){const entries=import_node_fs.default.readdirSync(stateDirectory).sort();if(entries.length>MAX_STATE_ENTRIES){fail(`state directory exceeds ${String(MAX_STATE_ENTRIES)} entries`)}return entries}function unlinkRecoverableControlTemp(stateDirectory,entry,runtime){const temporary=import_node_path.default.join(stateDirectory,entry);const stat=import_node_fs.default.lstatSync(temporary);if(stat.nlink===1){unlinkSecureControlOrTemp(temporary,runtime);return}const basename=entry.startsWith(".committed.json-")?"committed.json":entry.startsWith(".pending.json-")?"pending.json":null;const target=basename===null?null:import_node_path.default.join(stateDirectory,basename);let targetStat=null;try{targetStat=target===null?null:import_node_fs.default.lstatSync(target)}catch{fail(`refused to remove an unpaired atomic-control temporary file: ${temporary}`)}if(stat.nlink!==2||targetStat===null||stat.dev!==targetStat.dev||stat.ino!==targetStat.ino||!stat.isFile()||stat.isSymbolicLink()||modeOf(stat)!==STATE_FILE_MODE||stat.size<1||stat.size>MAX_CONTROL_FILE_BYTES){fail(`refused to remove an unpaired atomic-control temporary file: ${temporary}`)}requireOwner(stat,temporary,runtime);requireOwner(targetStat,target,runtime);import_node_fs.default.unlinkSync(temporary)}function cleanAtomicTemps(stateDirectory,entries,runtime){let changed=false;for(const entry of entries){const target=import_node_path.default.join(stateDirectory,entry);if(PREPARE_TEMP_RE.test(entry)){discardDirectory(target,runtime);changed=true}else if(CONTROL_TEMP_RE.test(entry)){unlinkRecoverableControlTemp(stateDirectory,entry,runtime);changed=true}}if(changed)syncDirectory(stateDirectory)}function requireKnownStateEntries(stateDirectory,entries){for(const entry of entries){if(entry==="committed.json"||entry==="pending.json"||GENERATION_RE.test(entry)||PREPARE_TEMP_RE.test(entry)||CONTROL_TEMP_RE.test(entry)){continue}fail(`${stateDirectory} contains unsupported state component ${entry}`)}}function discardGenerationsExcept(stateDirectory,keepGeneration,runtime){for(const entry of listStateEntries(stateDirectory)){if(GENERATION_RE.test(entry)&&entry!==keepGeneration){discardDirectoryIfPresent(import_node_path.default.join(stateDirectory,entry),runtime)}}}function optionalStateControl(stateDirectory,basename,runtime){const target=import_node_path.default.join(stateDirectory,basename);try{import_node_fs.default.lstatSync(target)}catch(error){if(error.code==="ENOENT")return null;fail(`could not inspect ${target}`)}return parseStateControl(target,runtime)}function removePendingControl(stateDirectory,runtime){try{unlinkSecureControlOrTemp(import_node_path.default.join(stateDirectory,"pending.json"),runtime)}catch(error){if(error.code==="ENOENT")return;throw error}syncDirectory(stateDirectory)}function stateControlsMatch(left,right){return left.fingerprint===right.fingerprint&&left.generation===right.generation}function recoverCommittedState(stateDirectory,committedControl,pendingControl,requested,expectedAgent,runtime){const committed=validateGeneration(stateDirectory,committedControl,runtime,expectedAgent);if(pendingControl)removePendingControl(stateDirectory,runtime);discardGenerationsExcept(stateDirectory,committedControl.generation,runtime);syncDirectory(stateDirectory);if(!stateControlsMatch(committedControl,requested)){fail("a different startup profile is already committed; recreate the sandbox to change it")}return committed}function recoverState(stateDirectory,requested,expectedAgent,runtime){const initialEntries=listStateEntries(stateDirectory);requireKnownStateEntries(stateDirectory,initialEntries);cleanAtomicTemps(stateDirectory,initialEntries,runtime);const initiallyCommittedControl=optionalStateControl(stateDirectory,"committed.json",runtime);const pendingControl=optionalStateControl(stateDirectory,"pending.json",runtime);const committedAfterPendingRead=optionalStateControl(stateDirectory,"committed.json",runtime);const committedControl=committedAfterPendingRead??initiallyCommittedControl;if(committedControl){return{committed:recoverCommittedState(stateDirectory,committedControl,pendingControl,requested,expectedAgent,runtime),pending:null}}if(pendingControl){if(stateControlsMatch(pendingControl,requested)){const pending=validateGeneration(stateDirectory,pendingControl,runtime,expectedAgent);const committedAfterPendingValidation=optionalStateControl(stateDirectory,"committed.json",runtime);if(committedAfterPendingValidation){return{committed:recoverCommittedState(stateDirectory,committedAfterPendingValidation,pendingControl,requested,expectedAgent,runtime),pending:null}}discardGenerationsExcept(stateDirectory,pendingControl.generation,runtime);return{committed:null,pending}}fail("a different startup profile is already pending; wait for it to commit or recreate")}return{committed:null,pending:null}}function createGeneration(stateDirectory,control,profileJson,corporateCa,runtime){const temporaryName=`.prepare-${String(process.pid)}-${randomToken()}`;const temporary=import_node_path.default.join(stateDirectory,temporaryName);const generation=import_node_path.default.join(stateDirectory,control.generation);let renameAttempted=false;try{import_node_fs.default.mkdirSync(temporary,{mode:STATE_DIRECTORY_MODE});import_node_fs.default.chownSync(temporary,runtime.rootUid,runtime.rootGid);import_node_fs.default.chmodSync(temporary,STATE_DIRECTORY_MODE);writeSecureNewFile(import_node_path.default.join(temporary,"profile.json"),profileJson,runtime);if(corporateCa){writeSecureNewFile(import_node_path.default.join(temporary,"corporate-ca.pem"),corporateCa,runtime)}syncDirectory(temporary);renameAttempted=true;import_node_fs.default.renameSync(temporary,generation);syncDirectory(stateDirectory)}catch(error){try{import_node_fs.default.lstatSync(temporary);discardDirectory(temporary,runtime)}catch{}if(error instanceof ManagedStartupApplicationError)throw error;if(renameAttempted&&(error.code==="EEXIST"||error.code==="ENOTEMPTY")){return validateGeneration(stateDirectory,control,runtime)}fail(`could not atomically prepare generation ${control.generation}`)}return validateGeneration(stateDirectory,control,runtime)}function toPrepared(status,stateDirectory,generation,expectedAgent){return{status,stateDirectory,generationDirectory:generation.directory,profilePath:generation.profilePath,corporateCaPath:generation.corporateCaPath,fingerprint:generation.fingerprint,expectedAgent,profile:generation.profile}}function prepareManagedStartupApplication(input,testRuntime){const runtime=runtimeFor(testRuntime);requireContainerRoot();let profile;try{profile=decodeManagedStartupProfile(input.encodedProfile)}catch(error){fail(error.message)}if(profile.agent!==input.expectedAgent){fail(`managed startup profile targets ${profile.agent}, expected ${input.expectedAgent}`)}const corporateCa=validateManagedStartupCorporateCaTransport(input.corporateCaB64,profile);const profileJson=serializeManagedStartupProfile(profile);const control=stateControl(fingerprintManagedStartupProfile(profile));const stateDirectory=ensureStateDirectory(input.stateDirectory,runtime);const recovered=recoverState(stateDirectory,control,input.expectedAgent,runtime);if(recovered.committed){return toPrepared("already-committed",stateDirectory,recovered.committed,input.expectedAgent)}if(recovered.pending){return toPrepared("prepared",stateDirectory,recovered.pending,input.expectedAgent)}const generation=createGeneration(stateDirectory,control,profileJson,corporateCa,runtime);const publication=publishStateControlIfAbsent(stateDirectory,"pending.json",control,runtime);if(publication.control.fingerprint!==control.fingerprint||publication.control.generation!==control.generation){discardDirectoryIfPresent(generation.directory,runtime);syncDirectory(stateDirectory);fail("a different startup profile won the pending-state transaction")}const committedAfterPublication=optionalStateControl(stateDirectory,"committed.json",runtime);if(committedAfterPublication){if(committedAfterPublication.fingerprint!==control.fingerprint||committedAfterPublication.generation!==control.generation){if(publication.created){removePendingControl(stateDirectory,runtime);discardDirectoryIfPresent(generation.directory,runtime);syncDirectory(stateDirectory)}fail("a different startup profile committed during pending-state publication")}const committedGeneration=validateGeneration(stateDirectory,committedAfterPublication,runtime,input.expectedAgent);removePendingControl(stateDirectory,runtime);discardGenerationsExcept(stateDirectory,committedAfterPublication.generation,runtime);return toPrepared("already-committed",stateDirectory,committedGeneration,input.expectedAgent)}const activeGeneration=publication.created?generation:validateGeneration(stateDirectory,publication.control,runtime,input.expectedAgent);return toPrepared("prepared",stateDirectory,activeGeneration,input.expectedAgent)}function validatePreparedHandle(handle){if(!import_node_path.default.isAbsolute(handle.stateDirectory)||!SHA256_RE2.test(handle.fingerprint)||handle.generationDirectory!==import_node_path.default.join(handle.stateDirectory,`generation-${handle.fingerprint}`)||handle.profilePath!==import_node_path.default.join(handle.generationDirectory,"profile.json")||handle.corporateCaPath!==null&&handle.corporateCaPath!==import_node_path.default.join(handle.generationDirectory,"corporate-ca.pem")){fail("prepared startup handle is malformed")}return stateControl(handle.fingerprint)}function commitManagedStartupApplication(prepared,testRuntime){const runtime=runtimeFor(testRuntime);requireContainerRoot();const requested=validatePreparedHandle(prepared);const stateDirectory=ensureStateDirectory(prepared.stateDirectory,runtime);const committedControl=optionalStateControl(stateDirectory,"committed.json",runtime);if(committedControl){if(committedControl.fingerprint!==requested.fingerprint||committedControl.generation!==requested.generation){fail("a different startup profile is already committed")}const generation2=validateGeneration(stateDirectory,committedControl,runtime,prepared.expectedAgent);return{...toPrepared("already-committed",stateDirectory,generation2,prepared.expectedAgent),status:"committed"}}const pendingControl=optionalStateControl(stateDirectory,"pending.json",runtime);if(!pendingControl||pendingControl.fingerprint!==requested.fingerprint||pendingControl.generation!==requested.generation){fail("the prepared startup generation is not the active pending generation")}const generation=validateGeneration(stateDirectory,pendingControl,runtime,prepared.expectedAgent);const publication=publishStateControlIfAbsent(stateDirectory,"committed.json",pendingControl,runtime);if(publication.control.fingerprint!==requested.fingerprint||publication.control.generation!==requested.generation){fail("a different startup profile won the committed-state transaction")}removePendingControl(stateDirectory,runtime);discardGenerationsExcept(stateDirectory,publication.control.generation,runtime);syncDirectory(stateDirectory);return{...toPrepared("already-committed",stateDirectory,generation,prepared.expectedAgent),status:"committed"}}var SHIPPED_AGENT_SET=new Set(MANAGED_STARTUP_AGENTS);var DEFAULT_DEPENDENCIES={prepareApplication:input=>prepareManagedStartupApplication(input),commitApplication:prepared=>commitManagedStartupApplication(prepared)};var ManagedStartupCoordinatorError=class extends Error{constructor(message){super(`Managed startup coordination failed: ${message}`);this.name="ManagedStartupCoordinatorError"}};function fail2(message){throw new ManagedStartupCoordinatorError(message)}function createAdapterRegistry(adapters2){const byAgent=new Map;for(const adapter of adapters2){if(typeof adapter!=="object"||adapter===null||!SHIPPED_AGENT_SET.has(adapter.agent)||typeof adapter.apply!=="function"){fail2("every adapter must identify one shipped agent and provide an apply function")}if(byAgent.has(adapter.agent)){fail2(`duplicate adapter registered for ${adapter.agent}`)}byAgent.set(adapter.agent,adapter)}const missing=MANAGED_STARTUP_AGENTS.filter(agent=>!byAgent.has(agent));if(missing.length>0){fail2(`missing adapter for ${missing.join(", ")}`)}if(byAgent.size!==MANAGED_STARTUP_AGENTS.length){fail2("adapter registry must contain exactly the shipped agents")}return Object.freeze(Object.fromEntries(MANAGED_STARTUP_AGENTS.map(agent=>{const adapter=byAgent.get(agent);if(!adapter)fail2(`missing adapter for ${agent}`);return[agent,adapter]})))}function requirePreparedIdentity(prepared,requestedAgent){if(prepared.expectedAgent!==requestedAgent||prepared.profile.agent!==requestedAgent){fail2(`prepared profile targets ${prepared.profile.agent}, expected ${requestedAgent}`)}}function adapterContext(prepared){return Object.freeze({agent:prepared.profile.agent,profile:prepared.profile,fingerprint:prepared.fingerprint,generationDirectory:prepared.generationDirectory,profilePath:prepared.profilePath,corporateCaPath:prepared.corporateCaPath})}async function coordinateManagedStartupApplication(input,adapters2,dependencies=DEFAULT_DEPENDENCIES){const registry=createAdapterRegistry(adapters2);const prepared=await dependencies.prepareApplication(input);requirePreparedIdentity(prepared,input.expectedAgent);if(prepared.status==="already-committed"){return{adapterApplied:false,application:await dependencies.commitApplication(prepared)}}const adapter=registry[prepared.profile.agent];if(adapter.agent!==prepared.profile.agent){fail2(`adapter registry cross-dispatch detected for ${prepared.profile.agent}`)}await adapter.apply(adapterContext(prepared));return{adapterApplied:true,application:await dependencies.commitApplication(prepared)}}var import_node_crypto4=require("node:crypto");var MANAGED_STARTUP_ROOT_APPLY_SCHEMA_VERSION=1;var MANAGED_STARTUP_ROOT_APPLY_MAX_BYTES=320*1024;var MAX_CORPORATE_CA_ENCODED_BYTES=4*Math.ceil(128*1024/3);var SHA256_RE3=/^[a-f0-9]{64}$/u;var STANDARD_BASE64_RE=/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/u;var MCP_SHADOW_DIAGNOSTICS_ENV="NEMOCLAW_MCP_SHADOW_DIAGNOSTICS";var MANAGED_STARTUP_APPLICATION_RUNTIME_ENV_KEYS=Object.freeze(MANAGED_STARTUP_PROFILE_DEFERRED_RUNTIME_INPUTS.openclaw.filter(({admission,owner})=>admission==="managed-launch-forwarded"&&owner==="application-environment").map(({input})=>input));function selectManagedStartupApplicationRuntimeEnvironment(environment){const selected={};for(const name of MANAGED_STARTUP_APPLICATION_RUNTIME_ENV_KEYS){const value=environment[name];if(name===MCP_SHADOW_DIAGNOSTICS_ENV){if(value?.trim()==="1")selected[name]="1";continue}if(value!==void 0)selected[name]=value}return Object.freeze(selected)}function fail3(message){throw new Error(`Managed startup root application request is invalid: ${message}`)}function isManagedStartupRootApplyAgent(value){return typeof value==="string"&&MANAGED_STARTUP_AGENTS.includes(value)}function exactAgent(value){if(isManagedStartupRootApplyAgent(value))return value;return fail3("agent is unsupported")}function createManagedStartupRootApplyRequest(input){const agent=exactAgent(input.agent);if(input.encodedProfile.length===0||input.encodedProfile.length>MANAGED_STARTUP_PROFILE_MAX_ENCODED_BYTES){fail3("encoded profile exceeds its bounded transport")}const profile=decodeManagedStartupProfile(input.encodedProfile);if(profile.agent!==agent){fail3(`profile targets ${profile.agent}, expected ${agent}`)}const corporateCaB64=input.corporateCaB64??null;if(corporateCaB64!==null&&(corporateCaB64.length===0||corporateCaB64.length>MAX_CORPORATE_CA_ENCODED_BYTES||!STANDARD_BASE64_RE.test(corporateCaB64)||Buffer.from(corporateCaB64,"base64").toString("base64")!==corporateCaB64)){fail3("corporate CA is not canonical bounded base64")}if(profile.corporateCa.bundleSha256!==null!==(corporateCaB64!==null)){fail3("corporate CA transport does not match the profile")}if(corporateCaB64!==null&&(0,import_node_crypto4.createHash)("sha256").update(Buffer.from(corporateCaB64,"base64")).digest("hex")!==profile.corporateCa.bundleSha256){fail3("corporate CA does not match the profile digest")}return Object.freeze({schemaVersion:MANAGED_STARTUP_ROOT_APPLY_SCHEMA_VERSION,agent,encodedProfile:input.encodedProfile,profileFingerprint:fingerprintManagedStartupProfile(profile),corporateCaB64})}function serializeManagedStartupRootApplyRequest(request){const normalized=createManagedStartupRootApplyRequest({agent:request.agent,encodedProfile:request.encodedProfile,...request.corporateCaB64===null?{}:{corporateCaB64:request.corporateCaB64}});if(request.schemaVersion!==MANAGED_STARTUP_ROOT_APPLY_SCHEMA_VERSION||request.profileFingerprint!==normalized.profileFingerprint||!SHA256_RE3.test(request.profileFingerprint)){fail3("schema version or profile fingerprint is invalid")}const serialized=`${JSON.stringify({agent:normalized.agent,corporateCaB64:normalized.corporateCaB64,encodedProfile:normalized.encodedProfile,profileFingerprint:normalized.profileFingerprint,schemaVersion:normalized.schemaVersion})} -`;if(Buffer.byteLength(serialized,"utf8")>MANAGED_STARTUP_ROOT_APPLY_MAX_BYTES){fail3("serialized request exceeds its bounded transport")}return serialized}function parseManagedStartupRootApplyRequest(text){if(text.length===0||Buffer.byteLength(text,"utf8")>MANAGED_STARTUP_ROOT_APPLY_MAX_BYTES){fail3("serialized request is empty or too large")}let parsed;try{parsed=JSON.parse(text)}catch{fail3("serialized request is not valid JSON")}if(typeof parsed!=="object"||parsed===null||Array.isArray(parsed)){fail3("serialized request must be an object")}const record=parsed;const expectedKeys=["agent","corporateCaB64","encodedProfile","profileFingerprint","schemaVersion"];if(Object.keys(record).sort().join(",")!==expectedKeys.sort().join(",")||record.schemaVersion!==MANAGED_STARTUP_ROOT_APPLY_SCHEMA_VERSION||typeof record.encodedProfile!=="string"||typeof record.profileFingerprint!=="string"||record.corporateCaB64!==null&&typeof record.corporateCaB64!=="string"){fail3("serialized request has an invalid schema")}const request=createManagedStartupRootApplyRequest({agent:exactAgent(record.agent),encodedProfile:record.encodedProfile,...record.corporateCaB64===null?{}:{corporateCaB64:record.corporateCaB64}});if(record.profileFingerprint!==request.profileFingerprint||!SHA256_RE3.test(record.profileFingerprint)){fail3("profile fingerprint does not match the encoded profile")}if(serializeManagedStartupRootApplyRequest(request)!==text){fail3("serialized request is not canonical")}return request}var import_node_crypto5=require("node:crypto");var import_node_fs2=__toESM(require("node:fs"));var import_node_path2=__toESM(require("node:path"));var TRANSACTION_SCHEMA_VERSION=1;var MAX_TRANSACTION_FILES=128;var MAX_TRANSACTION_FILE_BYTES=8*1024*1024;var MAX_TRANSACTION_TOTAL_BYTES=32*1024*1024;var MAX_MANIFEST_BYTES=256*1024;var MAX_COMMIT_RECEIPT_BYTES=4096;var TRANSACTION_PARENT_DIRECTORY_MODE=493;var TRANSACTION_DIRECTORY_MODE=448;var TRANSACTION_FILE_MODE=256;var ATOMIC_TEMPORARY_FILE_MODE=384;var MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY="/var/lib/nemoclaw/managed-startup-shared-state-transaction-v1";var MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY="/run/nemoclaw/managed-startup-shared-rollback-receipt-v1";var MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY="/var/lib/nemoclaw/managed-startup-shared-state-commit-v1";var MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_FILE="receipt.json";function fail4(message){throw new Error(`Managed startup shared-state transaction failed: ${message}`)}function resolveOptions(options={}){const sandboxRoot=import_node_path2.default.resolve(options.sandboxRoot??"/sandbox");const transactionDirectory=import_node_path2.default.resolve(options.transactionDirectory??MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY);const commitReceiptDirectory=import_node_path2.default.resolve(options.commitReceiptDirectory??(options.transactionDirectory?import_node_path2.default.join(import_node_path2.default.dirname(transactionDirectory),import_node_path2.default.basename(MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY)):MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY));if(transactionDirectory===sandboxRoot||transactionDirectory.startsWith(`${sandboxRoot}${import_node_path2.default.sep}`)||commitReceiptDirectory===sandboxRoot||commitReceiptDirectory.startsWith(`${sandboxRoot}${import_node_path2.default.sep}`)||import_node_path2.default.dirname(commitReceiptDirectory)!==import_node_path2.default.dirname(transactionDirectory)||commitReceiptDirectory===transactionDirectory){fail4("transaction and commit receipts require distinct paths outside sandbox-shared state")}const bootstrapIdentity=options.bootstrapIdentity??null;if(bootstrapIdentity!==null&&!/^[a-f0-9]{64}$/u.test(bootstrapIdentity)){fail4("bootstrap identity must encode 32 lowercase-hex bytes")}return{sandboxRoot,transactionParentDirectory:import_node_path2.default.dirname(transactionDirectory),transactionDirectory,backupDirectory:import_node_path2.default.join(transactionDirectory,"backups"),manifestFile:import_node_path2.default.join(transactionDirectory,"manifest.json"),commitReceiptDirectory,commitReceiptFile:import_node_path2.default.join(commitReceiptDirectory,MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_FILE),trustedUid:options.trustedUid??0,trustedGid:options.trustedGid??0,readOnlyReceipt:options.readOnlyReceipt??false,bootstrapIdentity}}function modeOf2(stat){if(typeof stat.mode==="bigint"){return Number(stat.mode&0o7777n)}return stat.mode&4095}function requireTransactionIdentity(options){const expectedUid=options.readOnlyReceipt?0:options.trustedUid;const expectedGid=options.readOnlyReceipt?0:options.trustedGid;if(process.geteuid?.()!==expectedUid||process.getegid?.()!==expectedGid){fail4("transaction control requires the trusted effective identity")}}function pathExistsNoFollow(target){try{import_node_fs2.default.lstatSync(target);return true}catch(error){if(error.code==="ENOENT")return false;fail4(`could not inspect ${target}`)}}function requireDirectory(target,options,expectedMode=null){let stat;try{stat=import_node_fs2.default.lstatSync(target)}catch{fail4(`required directory is missing: ${target}`)}if(stat.isSymbolicLink()||!stat.isDirectory()){fail4(`required directory is unsafe: ${target}`)}if(expectedMode!==null&&(stat.uid!==options.trustedUid||stat.gid!==options.trustedGid||modeOf2(stat)!==expectedMode)){fail4(`${target} must be ${options.trustedUid}:${options.trustedGid} mode ${expectedMode.toString(8)}`)}return stat}function requireTransactionBoundaries(options){requireDirectory(options.sandboxRoot,options);requireDirectory(options.transactionParentDirectory,options,TRANSACTION_PARENT_DIRECTORY_MODE)}function sameStableMetadata(left,right){return left.dev===right.dev&&left.ino===right.ino&&left.mode===right.mode&&left.nlink===right.nlink&&left.uid===right.uid&&left.gid===right.gid&&left.size===right.size&&left.mtimeNs===right.mtimeNs&&left.ctimeNs===right.ctimeNs}function readStableFile(target,maxBytes){const noFollow=import_node_fs2.default.constants.O_NOFOLLOW;if(typeof noFollow!=="number")fail4("O_NOFOLLOW is unavailable");let descriptor;try{descriptor=import_node_fs2.default.openSync(target,import_node_fs2.default.constants.O_RDONLY|noFollow)}catch{fail4(`could not safely open ${target}`)}try{const before=import_node_fs2.default.fstatSync(descriptor,{bigint:true});if(!before.isFile()||before.nlink!==1n||before.size<0n||before.size>BigInt(maxBytes)){fail4(`refusing unsafe or oversized transaction file ${target}`)}const bytes=Buffer.alloc(Number(before.size));let offset=0;while(offset!segment||segment==="."||segment==="..")){fail4(`unsafe transaction path ${JSON.stringify(value)}`)}return segments.join("/")}function absoluteTarget(relativePath,options){const safe=safeRelativePath(relativePath);const target=import_node_path2.default.resolve(options.sandboxRoot,safe);if(!target.startsWith(`${options.sandboxRoot}${import_node_path2.default.sep}`)){fail4(`transaction target escapes the sandbox root: ${relativePath}`)}return target}function relativeTarget(target,options){return safeRelativePath(import_node_path2.default.relative(options.sandboxRoot,target))}function validateExistingAncestors(target,expectedAgent,options){const relative=relativeTarget(target,options);const sandboxStat=requireDirectory(options.sandboxRoot,options);const outputRoot=agentRoot(expectedAgent,options.sandboxRoot);if(target!==outputRoot&&!target.startsWith(`${outputRoot}${import_node_path2.default.sep}`)){fail4(`transaction target escapes the ${expectedAgent} state root: ${target}`)}let current=options.sandboxRoot;let expectedDevice=sandboxStat.dev;const segments=relative.split("/").slice(0,-1);for(const segment of segments){current=import_node_path2.default.join(current,segment);let stat;try{stat=import_node_fs2.default.lstatSync(current)}catch(error){if(error.code==="ENOENT")return;fail4(`could not inspect transaction path ancestor ${current}`)}if(stat.isSymbolicLink()||!stat.isDirectory()){fail4(`transaction path ancestor is unsafe: ${current}`)}if(current===outputRoot&&expectedAgent==="hermes"){expectedDevice=stat.dev}else if(stat.dev!==expectedDevice){fail4(`transaction path crosses a nested filesystem mount: ${current}`)}}}function managedOutputDevice(expectedAgent,options){const sandboxStat=requireDirectory(options.sandboxRoot,options);const outputRoot=agentRoot(expectedAgent,options.sandboxRoot);let stat;try{stat=import_node_fs2.default.lstatSync(outputRoot)}catch(error){if(error.code==="ENOENT")return sandboxStat.dev;fail4(`could not inspect managed output root ${outputRoot}`)}if(stat.isSymbolicLink()||!stat.isDirectory()){fail4(`managed output root is unsafe: ${outputRoot}`)}if(expectedAgent!=="hermes"&&stat.dev!==sandboxStat.dev){fail4(`managed output root crosses a nested filesystem mount: ${outputRoot}`)}return stat.dev}function agentRoot(agent,sandboxRoot){switch(agent){case"openclaw":return import_node_path2.default.join(sandboxRoot,".openclaw");case"hermes":return import_node_path2.default.join(sandboxRoot,".hermes");case"langchain-deepagents-code":return import_node_path2.default.join(sandboxRoot,".deepagents");case"pi":return import_node_path2.default.join(sandboxRoot,".pi")}}function resolveUnderAgentRoot(root,relativePath){const safe=safeRelativePath(relativePath);const target=import_node_path2.default.resolve(root,safe);if(!target.startsWith(`${root}${import_node_path2.default.sep}`)){fail4(`managed output escapes the agent root: ${relativePath}`)}return target}function renderTarget(root,agent,target){if(agent==="openclaw"&&target==="openclaw.json"){return import_node_path2.default.join(root,"openclaw.json")}const prefix=agent==="openclaw"?"~/.openclaw/":agent==="hermes"?"~/.hermes/":null;if(!prefix||!target.startsWith(prefix)){fail4(`unsupported managed messaging render target ${JSON.stringify(target)}`)}return resolveUnderAgentRoot(root,target.slice(prefix.length))}function managedOutputTargets(profile,options){const root=agentRoot(profile.agent,options.sandboxRoot);const files=new Set;const directories=new Set([root]);switch(profile.agent){case"openclaw":files.add(import_node_path2.default.join(root,"openclaw.json"));files.add(import_node_path2.default.join(root,".config-hash"));break;case"hermes":files.add(import_node_path2.default.join(root,"config.yaml"));files.add(import_node_path2.default.join(root,".env"));files.add(import_node_path2.default.join(root,".config-hash"));break;case"langchain-deepagents-code":files.add(import_node_path2.default.join(root,"config.toml"));directories.add(import_node_path2.default.join(root,".state"));directories.add(import_node_path2.default.join(root,"skills"));break;case"pi":directories.add(import_node_path2.default.join(root,"agent"));files.add(import_node_path2.default.join(root,"agent","models.json"));break}if(profile.messaging.plan!==null){const plan=parseSandboxMessagingPlan(profile.messaging.plan,{agent:profile.agent});if(!plan)fail4("managed messaging plan is invalid");for(const render of selectEnabledMessagingAgentRender(plan)){if(typeof render.target!=="string")continue;files.add(renderTarget(root,profile.agent,render.target))}for(const step of selectEnabledPostAgentInstallBuildFiles(plan)){if(typeof step.value!=="object"||step.value===null){continue}const outputPath=step.value.path;if(typeof outputPath==="string"){files.add(resolveUnderAgentRoot(root,outputPath))}}}for(const file of files){let parent=import_node_path2.default.dirname(file);while(parent!==options.sandboxRoot&&parent.startsWith(`${root}${import_node_path2.default.sep}`)){directories.add(parent);if(parent===root)break;parent=import_node_path2.default.dirname(parent)}}return{files:[...files].sort(),directories:[...directories].sort((left,right)=>left.split(import_node_path2.default.sep).length-right.split(import_node_path2.default.sep).length)}}function snapshotFile(target,index,expectedAgent,options){validateExistingAncestors(target,expectedAgent,options);let stat;try{stat=import_node_fs2.default.lstatSync(target)}catch(error){if(error.code==="ENOENT"){return{receipt:{path:relativeTarget(target,options),state:"absent"},bytes:null}}fail4(`could not inspect managed output ${target}`)}if(stat.isSymbolicLink()||!stat.isFile()||stat.nlink!==1){fail4(`managed output is not a safe regular file: ${target}`)}if(stat.dev!==managedOutputDevice(expectedAgent,options)){fail4(`managed output crosses a nested filesystem mount: ${target}`)}const stable=readStableFile(target,MAX_TRANSACTION_FILE_BYTES);const size=Number(stable.stat.size);const backup=`${String(index).padStart(3,"0")}.bin`;return{receipt:{path:relativeTarget(target,options),state:"file",backup,sha256:(0,import_node_crypto5.createHash)("sha256").update(stable.bytes).digest("hex"),size,uid:Number(stable.stat.uid),gid:Number(stable.stat.gid),mode:Number(stable.stat.mode&0o7777n)},bytes:stable.bytes}}function snapshotDirectory(target,expectedAgent,options){validateExistingAncestors(import_node_path2.default.join(target,".receipt"),expectedAgent,options);let stat;try{stat=import_node_fs2.default.lstatSync(target)}catch(error){if(error.code==="ENOENT"){return{path:relativeTarget(target,options),state:"absent"}}fail4(`could not inspect managed output directory ${target}`)}if(stat.isSymbolicLink()||!stat.isDirectory()){fail4(`managed output directory is unsafe: ${target}`)}if(stat.dev!==managedOutputDevice(expectedAgent,options)){fail4(`managed output directory crosses a nested filesystem mount: ${target}`)}return{path:relativeTarget(target,options),state:"directory",uid:stat.uid,gid:stat.gid,mode:modeOf2(stat)}}function atomicWriteTrustedFile(target,contents,mode,uid,gid){const parent=import_node_path2.default.dirname(target);const temporary=import_node_path2.default.join(parent,`.${import_node_path2.default.basename(target)}.${(0,import_node_crypto5.randomBytes)(12).toString("hex")}`);let descriptor;try{descriptor=import_node_fs2.default.openSync(temporary,import_node_fs2.default.constants.O_CREAT|import_node_fs2.default.constants.O_EXCL|import_node_fs2.default.constants.O_WRONLY|import_node_fs2.default.constants.O_NOFOLLOW,384);import_node_fs2.default.writeFileSync(descriptor,contents);import_node_fs2.default.fchownSync(descriptor,uid,gid);import_node_fs2.default.fchmodSync(descriptor,mode);import_node_fs2.default.fsyncSync(descriptor);import_node_fs2.default.closeSync(descriptor);descriptor=void 0;import_node_fs2.default.renameSync(temporary,target)}catch(error){if(descriptor!==void 0)import_node_fs2.default.closeSync(descriptor);try{import_node_fs2.default.unlinkSync(temporary)}catch{}fail4(`could not atomically write ${target}: ${error.message}`)}}function fsyncDirectory(directory){const descriptor=import_node_fs2.default.openSync(directory,import_node_fs2.default.constants.O_RDONLY);try{import_node_fs2.default.fsyncSync(descriptor)}finally{import_node_fs2.default.closeSync(descriptor)}}function canonicalManifest(manifest){return`${JSON.stringify(manifest,null,2)} +`;if(Buffer.byteLength(serialized,"utf8")>MANAGED_STARTUP_ROOT_APPLY_MAX_BYTES){fail3("serialized request exceeds its bounded transport")}return serialized}function parseManagedStartupRootApplyRequest(text){if(text.length===0||Buffer.byteLength(text,"utf8")>MANAGED_STARTUP_ROOT_APPLY_MAX_BYTES){fail3("serialized request is empty or too large")}let parsed;try{parsed=JSON.parse(text)}catch{fail3("serialized request is not valid JSON")}if(typeof parsed!=="object"||parsed===null||Array.isArray(parsed)){fail3("serialized request must be an object")}const record=parsed;const expectedKeys=["agent","corporateCaB64","encodedProfile","profileFingerprint","schemaVersion"];if(Object.keys(record).sort().join(",")!==expectedKeys.sort().join(",")||record.schemaVersion!==MANAGED_STARTUP_ROOT_APPLY_SCHEMA_VERSION||typeof record.encodedProfile!=="string"||typeof record.profileFingerprint!=="string"||record.corporateCaB64!==null&&typeof record.corporateCaB64!=="string"){fail3("serialized request has an invalid schema")}const request=createManagedStartupRootApplyRequest({agent:exactAgent(record.agent),encodedProfile:record.encodedProfile,...record.corporateCaB64===null?{}:{corporateCaB64:record.corporateCaB64}});if(record.profileFingerprint!==request.profileFingerprint||!SHA256_RE3.test(record.profileFingerprint)){fail3("profile fingerprint does not match the encoded profile")}if(serializeManagedStartupRootApplyRequest(request)!==text){fail3("serialized request is not canonical")}return request}var import_node_crypto5=require("node:crypto");var import_node_fs2=__toESM(require("node:fs"));var import_node_path2=__toESM(require("node:path"));var MANAGED_HERMES_STATE_ROOT="/sandbox/.hermes";var MANAGED_OPENCLAW_STATE_ROOT="/sandbox/.openclaw";var HERMES_STATE_VOLUME_NAME_PREFIX="nemoclaw-hermes-state-v1";var OPENCLAW_STATE_VOLUME_NAME_PREFIX="nemoclaw-openclaw-state-v1";var MANAGED_AGENT_STATE_ROOTS=Object.freeze({openclaw:Object.freeze([Object.freeze({mountTarget:MANAGED_OPENCLAW_STATE_ROOT,resourceIdentity:sandboxName=>`${OPENCLAW_STATE_VOLUME_NAME_PREFIX}-${sandboxName}`,ownershipLabels:(sandboxName,mountTarget)=>Object.freeze({"io.nvidia.nemoclaw.openclaw-state.managed":"true","io.nvidia.nemoclaw.openclaw-state.schema":"1","io.nvidia.nemoclaw.openclaw-state.sandbox":sandboxName,"io.nvidia.nemoclaw.openclaw-state.target":mountTarget}),uidAuthority:"agent",gidAuthority:"agent",mode:1528,readWrite:true})]),hermes:Object.freeze([Object.freeze({mountTarget:MANAGED_HERMES_STATE_ROOT,resourceIdentity:sandboxName=>`${HERMES_STATE_VOLUME_NAME_PREFIX}-${sandboxName}`,ownershipLabels:(sandboxName,mountTarget)=>Object.freeze({"io.nvidia.nemoclaw.hermes-state.managed":"true","io.nvidia.nemoclaw.hermes-state.schema":"1","io.nvidia.nemoclaw.hermes-state.sandbox":sandboxName,"io.nvidia.nemoclaw.hermes-state.target":mountTarget}),uidAuthority:"agent",gidAuthority:"agent",mode:2040,readWrite:true})]),"langchain-deepagents-code":Object.freeze([]),pi:Object.freeze([])});function managedStartupStateRootMountTargets(agent){return Object.freeze(MANAGED_AGENT_STATE_ROOTS[agent].map(({mountTarget})=>mountTarget))}var MANAGED_AGENT_WORKSPACE_ROOTS=Object.freeze({openclaw:Object.freeze({uidAuthority:"agent",gidAuthority:"agent",mode:493}),hermes:Object.freeze({uidAuthority:"agent",gidAuthority:"agent",mode:493}),"langchain-deepagents-code":Object.freeze({uidAuthority:"root",gidAuthority:"agent",mode:1021}),pi:Object.freeze({uidAuthority:"agent",gidAuthority:"agent",mode:493})});var TRANSACTION_SCHEMA_VERSION=1;var MAX_TRANSACTION_FILES=128;var MAX_TRANSACTION_FILE_BYTES=8*1024*1024;var MAX_TRANSACTION_TOTAL_BYTES=32*1024*1024;var MAX_MANIFEST_BYTES=256*1024;var MAX_COMMIT_RECEIPT_BYTES=4096;var TRANSACTION_PARENT_DIRECTORY_MODE=493;var TRANSACTION_DIRECTORY_MODE=448;var TRANSACTION_FILE_MODE=256;var ATOMIC_TEMPORARY_FILE_MODE=384;var MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY="/var/lib/nemoclaw/managed-startup-shared-state-transaction-v1";var MANAGED_STARTUP_SHARED_ROLLBACK_RECEIPT_DIRECTORY="/run/nemoclaw/managed-startup-shared-rollback-receipt-v1";var MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY="/var/lib/nemoclaw/managed-startup-shared-state-commit-v1";var MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_FILE="receipt.json";function fail4(message){throw new Error(`Managed startup shared-state transaction failed: ${message}`)}function resolveOptions(options={}){const sandboxRoot=import_node_path2.default.resolve(options.sandboxRoot??"/sandbox");const transactionDirectory=import_node_path2.default.resolve(options.transactionDirectory??MANAGED_STARTUP_SHARED_TRANSACTION_DIRECTORY);const commitReceiptDirectory=import_node_path2.default.resolve(options.commitReceiptDirectory??(options.transactionDirectory?import_node_path2.default.join(import_node_path2.default.dirname(transactionDirectory),import_node_path2.default.basename(MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY)):MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_DIRECTORY));if(transactionDirectory===sandboxRoot||transactionDirectory.startsWith(`${sandboxRoot}${import_node_path2.default.sep}`)||commitReceiptDirectory===sandboxRoot||commitReceiptDirectory.startsWith(`${sandboxRoot}${import_node_path2.default.sep}`)||import_node_path2.default.dirname(commitReceiptDirectory)!==import_node_path2.default.dirname(transactionDirectory)||commitReceiptDirectory===transactionDirectory){fail4("transaction and commit receipts require distinct paths outside sandbox-shared state")}const bootstrapIdentity=options.bootstrapIdentity??null;if(bootstrapIdentity!==null&&!/^[a-f0-9]{64}$/u.test(bootstrapIdentity)){fail4("bootstrap identity must encode 32 lowercase-hex bytes")}return{sandboxRoot,transactionParentDirectory:import_node_path2.default.dirname(transactionDirectory),transactionDirectory,backupDirectory:import_node_path2.default.join(transactionDirectory,"backups"),manifestFile:import_node_path2.default.join(transactionDirectory,"manifest.json"),commitReceiptDirectory,commitReceiptFile:import_node_path2.default.join(commitReceiptDirectory,MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_FILE),trustedUid:options.trustedUid??0,trustedGid:options.trustedGid??0,readOnlyReceipt:options.readOnlyReceipt??false,bootstrapIdentity}}function modeOf2(stat){if(typeof stat.mode==="bigint"){return Number(stat.mode&0o7777n)}return stat.mode&4095}function requireTransactionIdentity(options){const expectedUid=options.readOnlyReceipt?0:options.trustedUid;const expectedGid=options.readOnlyReceipt?0:options.trustedGid;if(process.geteuid?.()!==expectedUid||process.getegid?.()!==expectedGid){fail4("transaction control requires the trusted effective identity")}}function pathExistsNoFollow(target){try{import_node_fs2.default.lstatSync(target);return true}catch(error){if(error.code==="ENOENT")return false;fail4(`could not inspect ${target}`)}}function requireDirectory(target,options,expectedMode=null){let stat;try{stat=import_node_fs2.default.lstatSync(target)}catch{fail4(`required directory is missing: ${target}`)}if(stat.isSymbolicLink()||!stat.isDirectory()){fail4(`required directory is unsafe: ${target}`)}if(expectedMode!==null&&(stat.uid!==options.trustedUid||stat.gid!==options.trustedGid||modeOf2(stat)!==expectedMode)){fail4(`${target} must be ${options.trustedUid}:${options.trustedGid} mode ${expectedMode.toString(8)}`)}return stat}function requireTransactionBoundaries(options){requireDirectory(options.sandboxRoot,options);requireDirectory(options.transactionParentDirectory,options,TRANSACTION_PARENT_DIRECTORY_MODE)}function sameStableMetadata(left,right){return left.dev===right.dev&&left.ino===right.ino&&left.mode===right.mode&&left.nlink===right.nlink&&left.uid===right.uid&&left.gid===right.gid&&left.size===right.size&&left.mtimeNs===right.mtimeNs&&left.ctimeNs===right.ctimeNs}function readStableFile(target,maxBytes){const noFollow=import_node_fs2.default.constants.O_NOFOLLOW;if(typeof noFollow!=="number")fail4("O_NOFOLLOW is unavailable");let descriptor;try{descriptor=import_node_fs2.default.openSync(target,import_node_fs2.default.constants.O_RDONLY|noFollow)}catch{fail4(`could not safely open ${target}`)}try{const before=import_node_fs2.default.fstatSync(descriptor,{bigint:true});if(!before.isFile()||before.nlink!==1n||before.size<0n||before.size>BigInt(maxBytes)){fail4(`refusing unsafe or oversized transaction file ${target}`)}const bytes=Buffer.alloc(Number(before.size));let offset=0;while(offset!segment||segment==="."||segment==="..")){fail4(`unsafe transaction path ${JSON.stringify(value)}`)}return segments.join("/")}function absoluteTarget(relativePath,options){const safe=safeRelativePath(relativePath);const target=import_node_path2.default.resolve(options.sandboxRoot,safe);if(!target.startsWith(`${options.sandboxRoot}${import_node_path2.default.sep}`)){fail4(`transaction target escapes the sandbox root: ${relativePath}`)}return target}function relativeTarget(target,options){return safeRelativePath(import_node_path2.default.relative(options.sandboxRoot,target))}function isDeclaredAgentStateRoot(expectedAgent,outputRoot,options){const relative=import_node_path2.default.relative(options.sandboxRoot,outputRoot).split(import_node_path2.default.sep).join("/");const canonicalTarget=import_node_path2.default.posix.join("/sandbox",relative);return managedStartupStateRootMountTargets(expectedAgent).includes(canonicalTarget)}function validateExistingAncestors(target,expectedAgent,options){const relative=relativeTarget(target,options);const sandboxStat=requireDirectory(options.sandboxRoot,options);const outputRoot=agentRoot(expectedAgent,options.sandboxRoot);if(target!==outputRoot&&!target.startsWith(`${outputRoot}${import_node_path2.default.sep}`)){fail4(`transaction target escapes the ${expectedAgent} state root: ${target}`)}let current=options.sandboxRoot;let expectedDevice=sandboxStat.dev;const segments=relative.split("/").slice(0,-1);for(const segment of segments){current=import_node_path2.default.join(current,segment);let stat;try{stat=import_node_fs2.default.lstatSync(current)}catch(error){if(error.code==="ENOENT")return;fail4(`could not inspect transaction path ancestor ${current}`)}if(stat.isSymbolicLink()||!stat.isDirectory()){fail4(`transaction path ancestor is unsafe: ${current}`)}if(current===outputRoot&&isDeclaredAgentStateRoot(expectedAgent,outputRoot,options)){expectedDevice=stat.dev}else if(stat.dev!==expectedDevice){fail4(`transaction path crosses a nested filesystem mount: ${current}`)}}}function managedOutputDevice(expectedAgent,options){const sandboxStat=requireDirectory(options.sandboxRoot,options);const outputRoot=agentRoot(expectedAgent,options.sandboxRoot);let stat;try{stat=import_node_fs2.default.lstatSync(outputRoot)}catch(error){if(error.code==="ENOENT")return sandboxStat.dev;fail4(`could not inspect managed output root ${outputRoot}`)}if(stat.isSymbolicLink()||!stat.isDirectory()){fail4(`managed output root is unsafe: ${outputRoot}`)}if(!isDeclaredAgentStateRoot(expectedAgent,outputRoot,options)&&stat.dev!==sandboxStat.dev){fail4(`managed output root crosses a nested filesystem mount: ${outputRoot}`)}return stat.dev}function agentRoot(agent,sandboxRoot){switch(agent){case"openclaw":return import_node_path2.default.join(sandboxRoot,".openclaw");case"hermes":return import_node_path2.default.join(sandboxRoot,".hermes");case"langchain-deepagents-code":return import_node_path2.default.join(sandboxRoot,".deepagents");case"pi":return import_node_path2.default.join(sandboxRoot,".pi")}}function resolveUnderAgentRoot(root,relativePath){const safe=safeRelativePath(relativePath);const target=import_node_path2.default.resolve(root,safe);if(!target.startsWith(`${root}${import_node_path2.default.sep}`)){fail4(`managed output escapes the agent root: ${relativePath}`)}return target}function renderTarget(root,agent,target){if(agent==="openclaw"&&target==="openclaw.json"){return import_node_path2.default.join(root,"openclaw.json")}const prefix=agent==="openclaw"?"~/.openclaw/":agent==="hermes"?"~/.hermes/":null;if(!prefix||!target.startsWith(prefix)){fail4(`unsupported managed messaging render target ${JSON.stringify(target)}`)}return resolveUnderAgentRoot(root,target.slice(prefix.length))}function managedOutputTargets(profile,options){const root=agentRoot(profile.agent,options.sandboxRoot);const files=new Set;const directories=new Set([root]);switch(profile.agent){case"openclaw":files.add(import_node_path2.default.join(root,"openclaw.json"));files.add(import_node_path2.default.join(root,".config-hash"));break;case"hermes":files.add(import_node_path2.default.join(root,"config.yaml"));files.add(import_node_path2.default.join(root,".env"));files.add(import_node_path2.default.join(root,".config-hash"));break;case"langchain-deepagents-code":files.add(import_node_path2.default.join(root,"config.toml"));directories.add(import_node_path2.default.join(root,".state"));directories.add(import_node_path2.default.join(root,"skills"));break;case"pi":directories.add(import_node_path2.default.join(root,"agent"));files.add(import_node_path2.default.join(root,"agent","models.json"));break}if(profile.messaging.plan!==null){const plan=parseSandboxMessagingPlan(profile.messaging.plan,{agent:profile.agent});if(!plan)fail4("managed messaging plan is invalid");for(const render of selectEnabledMessagingAgentRender(plan)){if(typeof render.target!=="string")continue;files.add(renderTarget(root,profile.agent,render.target))}for(const step of selectEnabledPostAgentInstallBuildFiles(plan)){if(typeof step.value!=="object"||step.value===null){continue}const outputPath=step.value.path;if(typeof outputPath==="string"){files.add(resolveUnderAgentRoot(root,outputPath))}}}for(const file of files){let parent=import_node_path2.default.dirname(file);while(parent!==options.sandboxRoot&&parent.startsWith(`${root}${import_node_path2.default.sep}`)){directories.add(parent);if(parent===root)break;parent=import_node_path2.default.dirname(parent)}}return{files:[...files].sort(),directories:[...directories].sort((left,right)=>left.split(import_node_path2.default.sep).length-right.split(import_node_path2.default.sep).length)}}function snapshotFile(target,index,expectedAgent,options){validateExistingAncestors(target,expectedAgent,options);let stat;try{stat=import_node_fs2.default.lstatSync(target)}catch(error){if(error.code==="ENOENT"){return{receipt:{path:relativeTarget(target,options),state:"absent"},bytes:null}}fail4(`could not inspect managed output ${target}`)}if(stat.isSymbolicLink()||!stat.isFile()||stat.nlink!==1){fail4(`managed output is not a safe regular file: ${target}`)}if(stat.dev!==managedOutputDevice(expectedAgent,options)){fail4(`managed output crosses a nested filesystem mount: ${target}`)}const stable=readStableFile(target,MAX_TRANSACTION_FILE_BYTES);const size=Number(stable.stat.size);const backup=`${String(index).padStart(3,"0")}.bin`;return{receipt:{path:relativeTarget(target,options),state:"file",backup,sha256:(0,import_node_crypto5.createHash)("sha256").update(stable.bytes).digest("hex"),size,uid:Number(stable.stat.uid),gid:Number(stable.stat.gid),mode:Number(stable.stat.mode&0o7777n)},bytes:stable.bytes}}function snapshotDirectory(target,expectedAgent,options){validateExistingAncestors(import_node_path2.default.join(target,".receipt"),expectedAgent,options);let stat;try{stat=import_node_fs2.default.lstatSync(target)}catch(error){if(error.code==="ENOENT"){return{path:relativeTarget(target,options),state:"absent"}}fail4(`could not inspect managed output directory ${target}`)}if(stat.isSymbolicLink()||!stat.isDirectory()){fail4(`managed output directory is unsafe: ${target}`)}if(stat.dev!==managedOutputDevice(expectedAgent,options)){fail4(`managed output directory crosses a nested filesystem mount: ${target}`)}return{path:relativeTarget(target,options),state:"directory",uid:stat.uid,gid:stat.gid,mode:modeOf2(stat)}}function atomicWriteTrustedFile(target,contents,mode,uid,gid){const parent=import_node_path2.default.dirname(target);const temporary=import_node_path2.default.join(parent,`.${import_node_path2.default.basename(target)}.${(0,import_node_crypto5.randomBytes)(12).toString("hex")}`);let descriptor;try{descriptor=import_node_fs2.default.openSync(temporary,import_node_fs2.default.constants.O_CREAT|import_node_fs2.default.constants.O_EXCL|import_node_fs2.default.constants.O_WRONLY|import_node_fs2.default.constants.O_NOFOLLOW,384);import_node_fs2.default.writeFileSync(descriptor,contents);import_node_fs2.default.fchownSync(descriptor,uid,gid);import_node_fs2.default.fchmodSync(descriptor,mode);import_node_fs2.default.fsyncSync(descriptor);import_node_fs2.default.closeSync(descriptor);descriptor=void 0;import_node_fs2.default.renameSync(temporary,target)}catch(error){if(descriptor!==void 0)import_node_fs2.default.closeSync(descriptor);try{import_node_fs2.default.unlinkSync(temporary)}catch{}fail4(`could not atomically write ${target}: ${error.message}`)}}function fsyncDirectory(directory){const descriptor=import_node_fs2.default.openSync(directory,import_node_fs2.default.constants.O_RDONLY);try{import_node_fs2.default.fsyncSync(descriptor)}finally{import_node_fs2.default.closeSync(descriptor)}}function canonicalManifest(manifest){return`${JSON.stringify(manifest,null,2)} `}function canonicalLegacyManifest(manifest){return`${JSON.stringify({schemaVersion:manifest.schemaVersion,agent:manifest.agent,profileFingerprint:manifest.profileFingerprint,files:manifest.files,directories:manifest.directories},null,2)} `}function canonicalCommitReceipt(receipt){return`${JSON.stringify(receipt,null,2)} `}function requireExactKeys(record,keys){if(Object.keys(record).sort().join(",")!==[...keys].sort().join(",")){fail4("transaction manifest contains unexpected fields")}}function parseCommitReceipt(text){let parsed;try{parsed=JSON.parse(text)}catch{fail4("commit receipt is not valid JSON")}if(typeof parsed!=="object"||parsed===null||Array.isArray(parsed)){fail4("commit receipt must be an object")}const record=parsed;requireExactKeys(record,["agent","bootstrapIdentity","profileFingerprint","schemaVersion"]);if(record.schemaVersion!==TRANSACTION_SCHEMA_VERSION||!MANAGED_STARTUP_AGENTS.includes(String(record.agent))||typeof record.profileFingerprint!=="string"||!/^[a-f0-9]{64}$/u.test(record.profileFingerprint)||typeof record.bootstrapIdentity!=="string"||!/^[a-f0-9]{64}$/u.test(record.bootstrapIdentity)){fail4("commit receipt has an invalid envelope")}const receipt={schemaVersion:TRANSACTION_SCHEMA_VERSION,agent:record.agent,profileFingerprint:record.profileFingerprint,bootstrapIdentity:record.bootstrapIdentity};if(canonicalCommitReceipt(receipt)!==text){fail4("commit receipt is not canonical")}return receipt}function safeMetadata(value){return Number.isSafeInteger(value)&&value>=0}function parseManifest(text){let parsed;try{parsed=JSON.parse(text)}catch{fail4("transaction manifest is not valid JSON")}if(typeof parsed!=="object"||parsed===null||Array.isArray(parsed)){fail4("transaction manifest must be an object")}const record=parsed;const hasBootstrapIdentity=Object.hasOwn(record,"bootstrapIdentity");requireExactKeys(record,hasBootstrapIdentity?["agent","bootstrapIdentity","directories","files","profileFingerprint","schemaVersion"]:["agent","directories","files","profileFingerprint","schemaVersion"]);const bootstrapIdentity=hasBootstrapIdentity?record.bootstrapIdentity:null;if(record.schemaVersion!==TRANSACTION_SCHEMA_VERSION||!MANAGED_STARTUP_AGENTS.includes(String(record.agent))||typeof record.profileFingerprint!=="string"||!/^[a-f0-9]{64}$/u.test(record.profileFingerprint)||!(bootstrapIdentity===null||typeof bootstrapIdentity==="string"&&/^[a-f0-9]{64}$/u.test(bootstrapIdentity))||!Array.isArray(record.files)||!Array.isArray(record.directories)||record.files.length>MAX_TRANSACTION_FILES||record.directories.length>MAX_TRANSACTION_FILES*4){fail4("transaction manifest has an invalid envelope")}const files=record.files.map(value=>{if(typeof value!=="object"||value===null||Array.isArray(value)){return fail4("transaction file receipt must be an object")}const receipt=value;if(typeof receipt.path!=="string"){return fail4("transaction file receipt path must be a string")}const receiptPath=safeRelativePath(receipt.path);if(receipt.state==="absent"){requireExactKeys(receipt,["path","state"]);return{path:receiptPath,state:"absent"}}requireExactKeys(receipt,["backup","gid","mode","path","sha256","size","state","uid"]);if(receipt.state!=="file"||typeof receipt.backup!=="string"||!/^[0-9]{3}\.bin$/u.test(receipt.backup)||typeof receipt.sha256!=="string"||!/^[a-f0-9]{64}$/u.test(receipt.sha256)||!safeMetadata(receipt.size)||receipt.size>MAX_TRANSACTION_FILE_BYTES||!safeMetadata(receipt.uid)||!safeMetadata(receipt.gid)||!safeMetadata(receipt.mode)||receipt.mode>4095){return fail4("transaction file receipt is invalid")}return{path:receiptPath,state:"file",backup:receipt.backup,sha256:receipt.sha256,size:receipt.size,uid:receipt.uid,gid:receipt.gid,mode:receipt.mode}});const directories=record.directories.map(value=>{if(typeof value!=="object"||value===null||Array.isArray(value)){return fail4("transaction directory receipt must be an object")}const receipt=value;if(typeof receipt.path!=="string"){return fail4("transaction directory receipt path must be a string")}const receiptPath=safeRelativePath(receipt.path);if(receipt.state==="absent"){requireExactKeys(receipt,["path","state"]);return{path:receiptPath,state:"absent"}}requireExactKeys(receipt,["gid","mode","path","state","uid"]);if(receipt.state!=="directory"||!safeMetadata(receipt.uid)||!safeMetadata(receipt.gid)||!safeMetadata(receipt.mode)||receipt.mode>4095){return fail4("transaction directory receipt is invalid")}return{path:receiptPath,state:"directory",uid:receipt.uid,gid:receipt.gid,mode:receipt.mode}});const filePaths=files.map(receipt=>receipt.path);const directoryPaths=directories.map(receipt=>receipt.path);const backupNames=files.filter(receipt=>receipt.state==="file").map(receipt=>receipt.backup);if(new Set(filePaths).size!==filePaths.length||new Set(directoryPaths).size!==directoryPaths.length||new Set(backupNames).size!==backupNames.length){fail4("transaction manifest contains duplicate receipts")}const manifest={schemaVersion:TRANSACTION_SCHEMA_VERSION,agent:record.agent,profileFingerprint:record.profileFingerprint,bootstrapIdentity,files,directories};const canonical=hasBootstrapIdentity?canonicalManifest(manifest):canonicalLegacyManifest(manifest);if(canonical!==text){fail4("transaction manifest is not canonical")}return manifest}function requireTrustedTransactionPath(target,mode,options){const stat=import_node_fs2.default.lstatSync(target);if(stat.isSymbolicLink()||(mode===TRANSACTION_DIRECTORY_MODE?!stat.isDirectory():!stat.isFile())||!options.readOnlyReceipt&&(stat.uid!==options.trustedUid||stat.gid!==options.trustedGid)||modeOf2(stat)!==mode){fail4(`transaction artifact has unsafe metadata: ${target}`)}}function requireReadOnlyReceiptMount(target,options){if(!options.readOnlyReceipt)return;const probe=import_node_path2.default.join(target,".nemoclaw-write-probe");let descriptor;try{descriptor=import_node_fs2.default.openSync(probe,import_node_fs2.default.constants.O_CREAT|import_node_fs2.default.constants.O_EXCL|import_node_fs2.default.constants.O_WRONLY|import_node_fs2.default.constants.O_NOFOLLOW,384);import_node_fs2.default.closeSync(descriptor);descriptor=void 0;import_node_fs2.default.unlinkSync(probe)}catch(error){if(descriptor!==void 0)import_node_fs2.default.closeSync(descriptor);if(error.code==="EROFS")return;fail4("copied receipt must be mounted on a read-only filesystem")}fail4("copied receipt mount is writable")}function loadManifest(options){requireTransactionBoundaries(options);if(!pathExistsNoFollow(options.transactionDirectory))return null;requireTrustedTransactionPath(options.transactionDirectory,TRANSACTION_DIRECTORY_MODE,options);requireReadOnlyReceiptMount(options.transactionDirectory,options);requireTrustedTransactionPath(options.backupDirectory,TRANSACTION_DIRECTORY_MODE,options);requireTrustedTransactionPath(options.manifestFile,TRANSACTION_FILE_MODE,options);const stable=readStableFile(options.manifestFile,MAX_MANIFEST_BYTES);if(!options.readOnlyReceipt&&(Number(stable.stat.uid)!==options.trustedUid||Number(stable.stat.gid)!==options.trustedGid)||Number(stable.stat.mode&0o7777n)!==TRANSACTION_FILE_MODE){fail4("transaction manifest ownership changed while it was read")}return parseManifest(stable.bytes.toString("utf8"))}function transactionOptionsAt(options,transactionDirectory){return{...options,transactionDirectory,backupDirectory:import_node_path2.default.join(transactionDirectory,"backups"),manifestFile:import_node_path2.default.join(transactionDirectory,"manifest.json")}}function loadCommitReceipt(options){requireTransactionBoundaries(options);if(!pathExistsNoFollow(options.commitReceiptDirectory))return null;requireTrustedTransactionPath(options.commitReceiptDirectory,TRANSACTION_DIRECTORY_MODE,options);if(pathExistsNoFollow(options.commitReceiptFile)){requireReadOnlyReceiptMount(options.commitReceiptDirectory,options);requireTrustedTransactionPath(options.commitReceiptFile,TRANSACTION_FILE_MODE,options);const stable=readStableFile(options.commitReceiptFile,MAX_COMMIT_RECEIPT_BYTES);if(!options.readOnlyReceipt&&(Number(stable.stat.uid)!==options.trustedUid||Number(stable.stat.gid)!==options.trustedGid)||Number(stable.stat.mode&0o7777n)!==TRANSACTION_FILE_MODE){fail4("commit receipt ownership changed while it was read")}return{receipt:parseCommitReceipt(stable.bytes.toString("utf8")),compact:true}}const stagedOptions=transactionOptionsAt(options,options.commitReceiptDirectory);const staged=loadManifest(stagedOptions);if(!staged||staged.bootstrapIdentity===null){fail4("durable commit staging receipt is incomplete")}verifyAllBackups(staged.files,stagedOptions);return{receipt:{schemaVersion:TRANSACTION_SCHEMA_VERSION,agent:staged.agent,profileFingerprint:staged.profileFingerprint,bootstrapIdentity:staged.bootstrapIdentity},compact:false}}function verifyBackup(receipt,options){const backupPath=import_node_path2.default.join(options.backupDirectory,receipt.backup);requireTrustedTransactionPath(backupPath,TRANSACTION_FILE_MODE,options);const stable=readStableFile(backupPath,MAX_TRANSACTION_FILE_BYTES);const digest=(0,import_node_crypto5.createHash)("sha256").update(stable.bytes).digest("hex");if(stable.bytes.length!==receipt.size||digest!==receipt.sha256){fail4(`transaction backup does not match its receipt: ${receipt.path}`)}return stable.bytes}function verifyAllBackups(receipts,options){const backups=new Map;for(const receipt of receipts){if(receipt.state==="file"){backups.set(receipt.path,verifyBackup(receipt,options))}}return backups}function fileMatchesReceipt(target,receipt){let stat;try{stat=import_node_fs2.default.lstatSync(target)}catch(error){if(error.code==="ENOENT")return false;fail4(`could not inspect managed output ${target}`)}if(stat.isSymbolicLink()||!stat.isFile()||stat.nlink!==1)return false;const stable=readStableFile(target,MAX_TRANSACTION_FILE_BYTES);return stable.bytes.length===receipt.size&&(0,import_node_crypto5.createHash)("sha256").update(stable.bytes).digest("hex")===receipt.sha256&&Number(stable.stat.uid)===receipt.uid&&Number(stable.stat.gid)===receipt.gid&&Number(stable.stat.mode&0o7777n)===receipt.mode}function directoryMatchesReceipt(target,receipt){let stat;try{stat=import_node_fs2.default.lstatSync(target)}catch(error){if(error.code==="ENOENT")return false;fail4(`could not inspect managed output directory ${target}`)}return!stat.isSymbolicLink()&&stat.isDirectory()&&stat.uid===receipt.uid&&stat.gid===receipt.gid&&modeOf2(stat)===receipt.mode}function removeTransactionDirectory(options){requireTrustedTransactionPath(options.transactionDirectory,TRANSACTION_DIRECTORY_MODE,options);import_node_fs2.default.rmSync(options.transactionDirectory,{force:false,recursive:true});fsyncDirectory(options.transactionParentDirectory);if(pathExistsNoFollow(options.transactionDirectory)){fail4("transaction directory remained after cleanup")}}function assertCommitReceiptMatches(receipt,expected){if(receipt.agent!==expected.agent||expected.profileFingerprint!==void 0&&receipt.profileFingerprint!==expected.profileFingerprint||receipt.bootstrapIdentity!==expected.bootstrapIdentity){fail4("durable commit receipt belongs to a different bootstrap attempt")}}function loadCommitStagingManifest(options){if(!pathExistsNoFollow(options.manifestFile))return null;requireTrustedTransactionPath(options.manifestFile,TRANSACTION_FILE_MODE,options);const stable=readStableFile(options.manifestFile,MAX_MANIFEST_BYTES);if(Number(stable.stat.uid)!==options.trustedUid||Number(stable.stat.gid)!==options.trustedGid||Number(stable.stat.mode&0o7777n)!==TRANSACTION_FILE_MODE){fail4("durable commit staging manifest ownership changed while it was read")}return parseManifest(stable.bytes.toString("utf8"))}function retireInterruptedCommitReceiptWrites(receipt,options){const temporaryPattern=new RegExp(`^\\.${MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_FILE.replace(".","\\.")}\\.[a-f0-9]{24}$`,"u");for(const entry of import_node_fs2.default.readdirSync(options.commitReceiptDirectory)){if(!temporaryPattern.test(entry))continue;const target=import_node_path2.default.join(options.commitReceiptDirectory,entry);const stat=import_node_fs2.default.lstatSync(target);if(stat.isSymbolicLink()||!stat.isFile()||stat.nlink!==1||stat.uid!==options.trustedUid||stat.gid!==options.trustedGid||![ATOMIC_TEMPORARY_FILE_MODE,TRANSACTION_FILE_MODE].includes(modeOf2(stat))){fail4("interrupted durable commit receipt write has unsafe metadata")}const stable=readStableFile(target,MAX_COMMIT_RECEIPT_BYTES);const mode=Number(stable.stat.mode&0o7777n);if(Number(stable.stat.uid)!==options.trustedUid||Number(stable.stat.gid)!==options.trustedGid||![ATOMIC_TEMPORARY_FILE_MODE,TRANSACTION_FILE_MODE].includes(mode)){fail4("interrupted durable commit receipt write changed during verification")}if(stable.bytes.length>0){let interruptedReceipt=null;try{interruptedReceipt=parseCommitReceipt(stable.bytes.toString("utf8"))}catch{}if(interruptedReceipt)assertCommitReceiptMatches(interruptedReceipt,receipt)}import_node_fs2.default.unlinkSync(target);fsyncDirectory(options.commitReceiptDirectory)}}function compactDurableCommitReceipt(state,options){if(!state.compact){atomicWriteTrustedFile(options.commitReceiptFile,canonicalCommitReceipt(state.receipt),TRANSACTION_FILE_MODE,options.trustedUid,options.trustedGid);fsyncDirectory(options.commitReceiptDirectory)}retireInterruptedCommitReceiptWrites(state.receipt,options);const stagedOptions=transactionOptionsAt(options,options.commitReceiptDirectory);const manifestExists=pathExistsNoFollow(stagedOptions.manifestFile);const backupsExist=pathExistsNoFollow(stagedOptions.backupDirectory);const unexpectedBeforeCleanup=import_node_fs2.default.readdirSync(options.commitReceiptDirectory).filter(entry=>![MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_FILE,import_node_path2.default.basename(stagedOptions.backupDirectory),import_node_path2.default.basename(stagedOptions.manifestFile)].includes(entry));if(unexpectedBeforeCleanup.length!==0){fail4("durable commit receipt directory contains unexpected artifacts")}if(manifestExists){const staged=loadCommitStagingManifest(stagedOptions);if(!staged||staged.bootstrapIdentity===null){fail4("durable commit staging receipt disappeared during cleanup")}assertCommitReceiptMatches(state.receipt,{agent:staged.agent,profileFingerprint:staged.profileFingerprint,bootstrapIdentity:staged.bootstrapIdentity})}if(backupsExist){requireTrustedTransactionPath(stagedOptions.backupDirectory,TRANSACTION_DIRECTORY_MODE,options);import_node_fs2.default.rmSync(stagedOptions.backupDirectory,{force:false,recursive:true});fsyncDirectory(options.commitReceiptDirectory)}if(manifestExists){requireTrustedTransactionPath(stagedOptions.manifestFile,TRANSACTION_FILE_MODE,options);import_node_fs2.default.unlinkSync(stagedOptions.manifestFile);fsyncDirectory(options.commitReceiptDirectory)}const unexpected=import_node_fs2.default.readdirSync(options.commitReceiptDirectory).filter(entry=>entry!==MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_FILE);if(unexpected.length!==0){fail4("durable commit receipt directory contains unexpected artifacts")}const verified=loadCommitReceipt(options);if(!verified?.compact)fail4("durable commit receipt did not compact successfully");assertCommitReceiptMatches(verified.receipt,state.receipt)}function beginManagedStartupSharedStateTransaction(profile,inputOptions={}){const options=resolveOptions(inputOptions);requireTransactionIdentity(options);if(options.readOnlyReceipt){fail4("cannot begin a transaction from a read-only rollback receipt")}requireTransactionBoundaries(options);const profileFingerprint=fingerprintManagedStartupProfile(profile);const committed=loadCommitReceipt(options);if(committed){if(options.bootstrapIdentity===null){fail4("a durable managed bootstrap commit receipt already exists")}assertCommitReceiptMatches(committed.receipt,{agent:profile.agent,profileFingerprint,bootstrapIdentity:options.bootstrapIdentity});fail4("this managed bootstrap attempt is already durably committed")}const pending=loadManifest(options);if(pending){if(pending.agent!==profile.agent||pending.profileFingerprint!==profileFingerprint||pending.bootstrapIdentity!==options.bootstrapIdentity){fail4("a pending managed startup transaction belongs to a different agent, profile fingerprint, or bootstrap attempt")}verifyAllBackups(pending.files,options);return false}const targets=managedOutputTargets(profile,options);if(targets.files.length>MAX_TRANSACTION_FILES){fail4("managed startup transaction has too many file targets")}const snapshots=targets.files.map((target,index)=>snapshotFile(target,index,profile.agent,options));const totalBytes=snapshots.reduce((sum,snapshot)=>sum+(snapshot.bytes?.length??0),0);if(totalBytes>MAX_TRANSACTION_TOTAL_BYTES){fail4("managed startup transaction backup exceeds the total size limit")}const directories=targets.directories.map(target=>snapshotDirectory(target,profile.agent,options));const manifest={schemaVersion:TRANSACTION_SCHEMA_VERSION,agent:profile.agent,profileFingerprint,bootstrapIdentity:options.bootstrapIdentity,files:snapshots.map(({receipt})=>receipt),directories};let createdTransactionIdentity;try{import_node_fs2.default.mkdirSync(options.transactionDirectory,{mode:TRANSACTION_DIRECTORY_MODE});const created=import_node_fs2.default.lstatSync(options.transactionDirectory,{bigint:true});if(!created.isDirectory()||created.isSymbolicLink()){fail4("new transaction path is not a directory")}createdTransactionIdentity={dev:created.dev,ino:created.ino,uid:created.uid,gid:created.gid};import_node_fs2.default.chownSync(options.transactionDirectory,options.trustedUid,options.trustedGid);import_node_fs2.default.chmodSync(options.transactionDirectory,TRANSACTION_DIRECTORY_MODE);fsyncDirectory(options.transactionParentDirectory);import_node_fs2.default.mkdirSync(options.backupDirectory,{mode:TRANSACTION_DIRECTORY_MODE});import_node_fs2.default.chownSync(options.backupDirectory,options.trustedUid,options.trustedGid);import_node_fs2.default.chmodSync(options.backupDirectory,TRANSACTION_DIRECTORY_MODE);fsyncDirectory(options.transactionDirectory);for(const snapshot of snapshots){if(snapshot.receipt.state!=="file"||snapshot.bytes===null)continue;atomicWriteTrustedFile(import_node_path2.default.join(options.backupDirectory,snapshot.receipt.backup),snapshot.bytes,TRANSACTION_FILE_MODE,options.trustedUid,options.trustedGid)}fsyncDirectory(options.backupDirectory);atomicWriteTrustedFile(options.manifestFile,canonicalManifest(manifest),TRANSACTION_FILE_MODE,options.trustedUid,options.trustedGid);fsyncDirectory(options.transactionDirectory);loadManifest(options)}catch(error){try{if(createdTransactionIdentity&&pathExistsNoFollow(options.transactionDirectory)){const current=import_node_fs2.default.lstatSync(options.transactionDirectory,{bigint:true});if(!current.isSymbolicLink()&¤t.isDirectory()&¤t.dev===createdTransactionIdentity.dev&¤t.ino===createdTransactionIdentity.ino&¤t.uid===createdTransactionIdentity.uid&¤t.gid===createdTransactionIdentity.gid){import_node_fs2.default.chmodSync(options.transactionDirectory,TRANSACTION_DIRECTORY_MODE);import_node_fs2.default.chownSync(options.transactionDirectory,options.trustedUid,options.trustedGid)}requireTrustedTransactionPath(options.transactionDirectory,TRANSACTION_DIRECTORY_MODE,options);import_node_fs2.default.rmSync(options.transactionDirectory,{force:true,recursive:true})}}catch{}throw error}return true}function ensureOriginalDirectories(receipts,expectedAgent,options){for(const receipt of receipts){if(receipt.state!=="directory")continue;const target=absoluteTarget(receipt.path,options);validateExistingAncestors(import_node_path2.default.join(target,".restore"),expectedAgent,options);let stat=null;try{stat=import_node_fs2.default.lstatSync(target)}catch(error){if(error.code!=="ENOENT"){fail4(`could not inspect restore directory ${target}`)}}if(stat&&(stat.isSymbolicLink()||!stat.isDirectory())){fail4(`restore directory is unsafe: ${target}`)}if(stat&&directoryMatchesReceipt(target,receipt))continue;if(!stat)import_node_fs2.default.mkdirSync(target,{mode:receipt.mode});import_node_fs2.default.chownSync(target,receipt.uid,receipt.gid);import_node_fs2.default.chmodSync(target,receipt.mode)}}function restoreFiles(receipts,backups,expectedAgent,options){for(const receipt of receipts){const target=absoluteTarget(receipt.path,options);validateExistingAncestors(target,expectedAgent,options);if(receipt.state==="absent"){let stat;try{stat=import_node_fs2.default.lstatSync(target)}catch(error){if(error.code==="ENOENT")continue;fail4(`could not inspect new managed output ${target}`)}if(stat.isDirectory()){fail4(`new managed output unexpectedly became a directory: ${target}`)}import_node_fs2.default.unlinkSync(target);continue}if(fileMatchesReceipt(target,receipt))continue;const bytes=backups.get(receipt.path);if(!bytes)fail4(`verified transaction backup is missing: ${receipt.path}`);let current=null;try{current=import_node_fs2.default.lstatSync(target)}catch(error){if(error.code!=="ENOENT"){fail4(`could not inspect managed output before restore: ${target}`)}}if(current?.isDirectory()){fail4(`managed output unexpectedly became a directory: ${target}`)}atomicWriteTrustedFile(target,bytes,receipt.mode,receipt.uid,receipt.gid)}}function restoreDirectoryMetadata(receipts,options){for(const receipt of[...receipts].reverse()){const target=absoluteTarget(receipt.path,options);if(receipt.state==="absent"){try{import_node_fs2.default.rmdirSync(target)}catch(error){if(error.code==="ENOENT")continue;fail4(`could not remove newly created managed directory ${target}`)}continue}if(directoryMatchesReceipt(target,receipt))continue;const stat=import_node_fs2.default.lstatSync(target);if(stat.isSymbolicLink()||!stat.isDirectory()){fail4(`managed directory changed type during restore: ${target}`)}import_node_fs2.default.chownSync(target,receipt.uid,receipt.gid);import_node_fs2.default.chmodSync(target,receipt.mode)}}function verifyRestoration(manifest,options){for(const receipt of manifest.files){const target=absoluteTarget(receipt.path,options);if(receipt.state==="absent"){if(pathExistsNoFollow(target)){fail4(`new managed output remained after rollback: ${target}`)}continue}const stable=readStableFile(target,MAX_TRANSACTION_FILE_BYTES);if(stable.bytes.length!==receipt.size||(0,import_node_crypto5.createHash)("sha256").update(stable.bytes).digest("hex")!==receipt.sha256||Number(stable.stat.uid)!==receipt.uid||Number(stable.stat.gid)!==receipt.gid||Number(stable.stat.mode&0o7777n)!==receipt.mode){fail4(`managed output was not restored exactly: ${target}`)}}for(const receipt of manifest.directories){const target=absoluteTarget(receipt.path,options);if(receipt.state==="absent"){if(pathExistsNoFollow(target)){fail4(`new managed directory remained after rollback: ${target}`)}continue}const stat=import_node_fs2.default.lstatSync(target);if(stat.isSymbolicLink()||!stat.isDirectory()||stat.uid!==receipt.uid||stat.gid!==receipt.gid||modeOf2(stat)!==receipt.mode){fail4(`managed directory metadata was not restored exactly: ${target}`)}}}function rollbackManagedStartupSharedStateTransaction(expectedAgent,inputOptions={}){const options=resolveOptions(inputOptions);requireTransactionIdentity(options);const committed=loadCommitReceipt(options);if(committed){if(options.bootstrapIdentity===null){fail4("shared state is already durably committed")}assertCommitReceiptMatches(committed.receipt,{agent:expectedAgent,bootstrapIdentity:options.bootstrapIdentity});fail4("shared state is already durably committed and cannot be rolled back")}const manifest=loadManifest(options);if(!manifest)return false;if(manifest.agent!==expectedAgent){fail4(`pending transaction targets ${manifest.agent}, expected ${expectedAgent}`)}if(manifest.bootstrapIdentity!==options.bootstrapIdentity){fail4("pending transaction belongs to a different bootstrap attempt")}const backups=verifyAllBackups(manifest.files,options);ensureOriginalDirectories(manifest.directories,expectedAgent,options);restoreFiles(manifest.files,backups,expectedAgent,options);restoreDirectoryMetadata(manifest.directories,options);verifyRestoration(manifest,options);if(!options.readOnlyReceipt){removeTransactionDirectory(options)}return true}function commitManagedStartupSharedStateTransaction(expectedAgent,inputOptions={}){const options=resolveOptions(inputOptions);requireTransactionIdentity(options);if(options.readOnlyReceipt){fail4("cannot commit a read-only rollback receipt")}const committed=loadCommitReceipt(options);if(committed){if(options.bootstrapIdentity===null){fail4("durable commit receipt is missing its expected bootstrap identity")}assertCommitReceiptMatches(committed.receipt,{agent:expectedAgent,bootstrapIdentity:options.bootstrapIdentity});compactDurableCommitReceipt(committed,options);return true}const manifest=loadManifest(options);if(!manifest)return false;if(manifest.agent!==expectedAgent){fail4(`pending transaction targets ${manifest.agent}, expected ${expectedAgent}`)}if(manifest.bootstrapIdentity!==options.bootstrapIdentity){fail4("pending transaction belongs to a different bootstrap attempt")}if(manifest.bootstrapIdentity===null){removeTransactionDirectory(options);return true}verifyAllBackups(manifest.files,options);if(pathExistsNoFollow(options.commitReceiptDirectory)){fail4("durable commit receipt path appeared before transaction commit")}try{import_node_fs2.default.renameSync(options.transactionDirectory,options.commitReceiptDirectory);fsyncDirectory(options.transactionParentDirectory)}catch(error){fail4(`could not atomically establish durable commit state: ${error.message}`)}const renamed=loadCommitReceipt(options);if(!renamed)fail4("durable commit state disappeared after atomic rename");assertCommitReceiptMatches(renamed.receipt,{agent:expectedAgent,profileFingerprint:manifest.profileFingerprint,bootstrapIdentity:manifest.bootstrapIdentity});compactDurableCommitReceipt(renamed,options);return true}function clearManagedStartupSharedStateCommitReceipt(expectedAgent,inputOptions={}){const options=resolveOptions(inputOptions);requireTransactionIdentity(options);if(options.readOnlyReceipt){fail4("cannot clear a durable commit from a read-only receipt")}if(options.bootstrapIdentity===null){fail4("durable commit cleanup requires its bootstrap identity")}const committed=loadCommitReceipt(options);if(!committed)return false;assertCommitReceiptMatches(committed.receipt,{agent:expectedAgent,bootstrapIdentity:options.bootstrapIdentity});compactDurableCommitReceipt(committed,options);requireTrustedTransactionPath(options.commitReceiptDirectory,TRANSACTION_DIRECTORY_MODE,options);requireTrustedTransactionPath(options.commitReceiptFile,TRANSACTION_FILE_MODE,options);const entries=import_node_fs2.default.readdirSync(options.commitReceiptDirectory);if(entries.length!==1||entries[0]!==MANAGED_STARTUP_SHARED_COMMIT_RECEIPT_FILE){fail4("durable commit receipt directory contains unexpected artifacts")}import_node_fs2.default.rmSync(options.commitReceiptDirectory,{force:false,recursive:true});fsyncDirectory(options.transactionParentDirectory);if(pathExistsNoFollow(options.commitReceiptDirectory)){fail4("durable commit receipt remained after cleanup")}return true}function getManagedStartupSharedStateTransactionStatus(expected,inputOptions={}){const options=resolveOptions({...inputOptions,bootstrapIdentity:expected.bootstrapIdentity});requireTransactionIdentity(options);const manifest=loadManifest(options);if(manifest){if(manifest.agent!==expected.agent||manifest.profileFingerprint!==expected.profileFingerprint||manifest.bootstrapIdentity!==expected.bootstrapIdentity){fail4("pending transaction does not match the expected agent, profile fingerprint, or bootstrap identity")}verifyAllBackups(manifest.files,options);return"pending"}const committed=loadCommitReceipt(options);if(!committed)return"none";assertCommitReceiptMatches(committed.receipt,expected);return"committed"}var MANAGED_STARTUP_PROFILE_ENV="NEMOCLAW_STARTUP_PROFILE_B64";var MANAGED_STARTUP_CA_ENV="NEMOCLAW_CORPORATE_CA_B64";var MANAGED_STARTUP_RUNTIME_ENV_FILE="/run/nemoclaw/managed-startup-runtime.env";var MANAGED_STARTUP_RUNTIME_EXECUTABLE="/usr/local/lib/nemoclaw/managed-startup-image-runtime.cjs";var MANAGED_STARTUP_MERGED_CA_FILE="/run/nemoclaw/managed-startup-ca-bundle.pem";var MANAGED_STARTUP_COMPLETION_FILE="/run/nemoclaw/managed-startup-complete.json";var MANAGED_STARTUP_CORPORATE_CA_FILE="/usr/local/share/nemoclaw/corporate-ca.pem";var MANAGED_STARTUP_SYSTEM_CA_ANCHOR_DIRECTORY="/usr/local/share/ca-certificates";var MANAGED_STARTUP_SYSTEM_CA_ANCHOR_RE=/^nemoclaw-corporate-ca-[0-9]{2}\.crt$/u;var SYSTEM_CA_BUNDLE_FILE="/etc/ssl/certs/ca-certificates.crt";var UPDATE_CA_CERTIFICATES_EXECUTABLE="/usr/sbin/update-ca-certificates";var MANAGED_STARTUP_TLS_ENV_NAMES=new Set(["CURL_CA_BUNDLE","GIT_SSL_CAINFO","NODE_EXTRA_CA_CERTS","REQUESTS_CA_BUNDLE","SSL_CERT_FILE"]);var MESSAGING_RUNTIME_PLAN_FILE="/usr/local/share/nemoclaw/messaging-runtime-plan.json";var ROOT_STATE_PARENT="/var/lib/nemoclaw";var ROOT_RUNTIME_DIRECTORY="/run/nemoclaw";var ROOT_OWNED_DIRECTORY_MODE=493;var MAX_TRUST_BUNDLE_BYTES=4*1024*1024;var HERMES_MANAGED_CONFIG_FILES=["/sandbox/.hermes/config.yaml","/sandbox/.hermes/.env"];var HERMES_GENERATED_MANAGED_POLICY_FILE="/sandbox/.hermes/managed-policy.json";var HERMES_INSTALLED_MANAGED_POLICY_FILE="/usr/local/share/nemoclaw/hermes-managed-policy.json";var MAX_HERMES_MANAGED_POLICY_BYTES=4*1024*1024;var FIXED_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin";var SHA256_RE4=/^[a-f0-9]{64}$/u;var MANAGED_STARTUP_COMPLETION_SCHEMA_VERSION=1;var MAX_MANAGED_STARTUP_COMPLETION_BYTES=4096;var MAX_MANAGED_STARTUP_RUNTIME_ENVIRONMENT_BYTES=512*1024;var ManagedStartupImageActionPlanError=class extends Error{constructor(message){super(`Cannot build managed startup image action plan: ${message}`);this.name="ManagedStartupImageActionPlanError"}};var ManagedStartupImageRuntimeError=class extends Error{constructor(message){super(`Managed startup image application failed: ${message}`);this.name="ManagedStartupImageRuntimeError"}};function failActionPlan(message){throw new ManagedStartupImageActionPlanError(message)}function exactActionPlanAgent(value){if(MANAGED_STARTUP_AGENTS.includes(value)){return value}return failActionPlan(`unsupported agent ${JSON.stringify(value)}`)}function fail5(message){throw new ManagedStartupImageRuntimeError(message)}function validateManagedStartupApplicationRuntimePlan(plan){if(typeof plan!=="object"||plan===null){return fail5("application runtime plan must be an object")}const exportEnvironment=plan.exportEnvironment;const unsetEnvironment=plan.unsetEnvironment;if(typeof exportEnvironment!=="object"||exportEnvironment===null||Array.isArray(exportEnvironment)||!Array.isArray(unsetEnvironment)){return fail5("application runtime plan must contain exports and unsets")}const exports2={};for(const[name,value]of Object.entries(exportEnvironment)){if(!/^[A-Za-z_][A-Za-z0-9_]*$/u.test(name)){return fail5(`invalid application runtime environment key ${JSON.stringify(name)}`)}if(typeof value!=="string"||value.includes("\0")||/[\r\n]/u.test(value)){return fail5(`application runtime environment value for ${name} must be single-line text`)}exports2[name]=value}const unsets=new Set;for(const name of unsetEnvironment){if(typeof name!=="string"||!/^[A-Za-z_][A-Za-z0-9_]*$/u.test(name)){return fail5(`invalid application runtime unset ${JSON.stringify(name)}`)}if(unsets.has(name)){return fail5(`duplicate application runtime unset ${name}`)}if(Object.hasOwn(exports2,name)){return fail5(`application runtime cannot both export and unset ${name}`)}unsets.add(name)}return Object.freeze({exportEnvironment:Object.freeze(Object.fromEntries(Object.entries(exports2).sort(([left],[right])=>left.localeCompare(right)))),unsetEnvironment:Object.freeze([...unsets].sort())})}function applyManagedStartupCommandEnvironmentPlan(environment,plan){const validated=validateManagedStartupApplicationRuntimePlan(plan);const applied={...environment};for(const name of[...Object.keys(validated.exportEnvironment),...validated.unsetEnvironment]){delete applied[name]}return applied}function exactAgent2(value){if(MANAGED_STARTUP_AGENTS.includes(value)){return value}return fail5(`unsupported agent ${JSON.stringify(value)}`)}function managedTransactionProfile(expectedAgentInput,env=process.env){requireRoot();const expectedAgent=exactAgent2(expectedAgentInput);if(env.NEMOCLAW_MANAGED_IMAGE_CAPABILITY_UNION!=="1"){fail5("shared-state transactions require a complete managed image")}const encodedProfile=env[MANAGED_STARTUP_PROFILE_ENV];if(!encodedProfile)fail5(`${MANAGED_STARTUP_PROFILE_ENV} is required`);const profile=decodeManagedStartupProfile(encodedProfile);if(profile.agent!==expectedAgent){fail5(`shared-state transaction profile targets ${profile.agent}, expected ${expectedAgent}`)}return profile}function requireRoot(){if(process.geteuid?.()!==0){fail5("managed startup requires container effective uid 0")}}function modeOf3(stat){return stat.mode&511}function requireRootOwnedDirectory(target,mode){let stat;try{stat=import_node_fs3.default.lstatSync(target)}catch{fail5(`required root-owned directory is missing: ${target}`)}if(stat.isSymbolicLink()||!stat.isDirectory()||stat.uid!==0||stat.gid!==0||modeOf3(stat)!==mode){fail5(`${target} must be a root:root directory with mode ${mode.toString(8)}`)}}function ensureRootOwnedDirectory(target,mode=ROOT_OWNED_DIRECTORY_MODE){const parent=import_node_path3.default.dirname(target);const parentStat=import_node_fs3.default.lstatSync(parent);if(parentStat.isSymbolicLink()||!parentStat.isDirectory()||parentStat.uid!==0||parentStat.gid!==0||(modeOf3(parentStat)&18)!==0){fail5(`refusing unsafe parent directory for ${target}`)}try{import_node_fs3.default.mkdirSync(target,{mode});import_node_fs3.default.chownSync(target,0,0);import_node_fs3.default.chmodSync(target,mode)}catch(error){if(error.code!=="EEXIST"){fail5(`could not create ${target}`)}}requireRootOwnedDirectory(target,mode)}function requireSafeExistingRootTarget(target){let stat;try{stat=import_node_fs3.default.lstatSync(target)}catch(error){if(error.code==="ENOENT")return;fail5(`could not inspect ${target}`)}if(stat.isSymbolicLink()||!stat.isFile()||stat.nlink!==1||stat.uid!==0||stat.gid!==0){fail5(`refusing to replace unsafe root-owned file ${target}`)}}function atomicWriteRootFile(target,contents,mode){const parent=import_node_path3.default.dirname(target);const parentStat=import_node_fs3.default.lstatSync(parent);if(parentStat.isSymbolicLink()||!parentStat.isDirectory()||parentStat.uid!==0||parentStat.gid!==0||(modeOf3(parentStat)&18)!==0){fail5(`refusing unsafe root-owned file parent ${parent}`)}requireSafeExistingRootTarget(target);const temporary=import_node_path3.default.join(parent,`.${import_node_path3.default.basename(target)}.${(0,import_node_crypto6.randomBytes)(12).toString("hex")}`);let descriptor;try{descriptor=import_node_fs3.default.openSync(temporary,import_node_fs3.default.constants.O_CREAT|import_node_fs3.default.constants.O_EXCL|import_node_fs3.default.constants.O_WRONLY|import_node_fs3.default.constants.O_NOFOLLOW,384);import_node_fs3.default.fchownSync(descriptor,0,0);import_node_fs3.default.writeFileSync(descriptor,contents);import_node_fs3.default.fchmodSync(descriptor,mode);import_node_fs3.default.fsyncSync(descriptor);import_node_fs3.default.closeSync(descriptor);descriptor=void 0;import_node_fs3.default.renameSync(temporary,target)}catch(error){if(descriptor!==void 0)import_node_fs3.default.closeSync(descriptor);try{import_node_fs3.default.unlinkSync(temporary)}catch{}fail5(`could not atomically write ${target}: ${error.message}`)}const stat=import_node_fs3.default.lstatSync(target);if(stat.isSymbolicLink()||!stat.isFile()||stat.nlink!==1||stat.uid!==0||stat.gid!==0||modeOf3(stat)!==mode){fail5(`root-owned output failed metadata verification: ${target}`)}}function removeSafeRootFile(target){requireSafeExistingRootTarget(target);try{import_node_fs3.default.unlinkSync(target)}catch(error){if(error.code!=="ENOENT"){fail5(`could not remove ${target}`)}}}function trustedExecutable(target){try{const stat=import_node_fs3.default.lstatSync(target);return!stat.isSymbolicLink()&&stat.isFile()&&stat.uid===0&&stat.gid===0&&(modeOf3(stat)&18)===0&&(modeOf3(stat)&73)!==0}catch{return false}}function readSandboxIdentity(){const readId=flag=>{const result=(0,import_node_child_process.spawnSync)("/usr/bin/id",[flag,"sandbox"],{encoding:"utf8",env:{PATH:FIXED_PATH}});const value=result.stdout.trim();if(result.status!==0||!/^[1-9][0-9]*$/u.test(value)){fail5("could not resolve the sandbox account")}return value};return{uid:readId("-u"),gid:readId("-g")}}function managedStartupSandboxPrefix(){if(trustedExecutable("/usr/bin/setpriv")){const identity=readSandboxIdentity();return["/usr/bin/setpriv",`--reuid=${identity.uid}`,`--regid=${identity.gid}`,"--init-groups","--"]}return fail5("a trusted setpriv executable is required")}function commandEnvironment(configurationEnvironment,applicationRuntime){const env=applyManagedStartupCommandEnvironmentPlan({...process.env,...configurationEnvironment,HOME:"/sandbox",PATH:FIXED_PATH,NPM_CONFIG_OFFLINE:"true",npm_config_offline:"true",PIP_DISABLE_PIP_VERSION_CHECK:"1",PIP_NO_INDEX:"1",UV_OFFLINE:"1"},applicationRuntime);delete env[MANAGED_STARTUP_PROFILE_ENV];delete env[MANAGED_STARTUP_CA_ENV];return env}function execute(argv,runAs,configurationEnvironment,applicationRuntime,capture=false){if(argv.length===0)fail5("refusing an empty managed startup command");const command=runAs==="sandbox"?[...managedStartupSandboxPrefix(),...argv]:[...argv];const result=(0,import_node_child_process.spawnSync)(command[0],command.slice(1),{encoding:"utf8",env:commandEnvironment(configurationEnvironment,applicationRuntime),stdio:capture?"pipe":"inherit"});if(result.error){fail5(`could not execute ${argv[0]}: ${result.error.message}`)}if(result.status!==0){const detail=capture?`: ${(result.stderr||result.stdout).trim()}`:"";fail5(`${argv[0]} exited with status ${String(result.status??"unknown")}${detail}`)}return{status:result.status,stdout:result.stdout??"",stderr:result.stderr??""}}function generatorCommand(agent){switch(agent){case"openclaw":return["/usr/local/bin/node","--experimental-strip-types","/scripts/generate-openclaw-config.mts"];case"hermes":return["/usr/local/bin/node","--experimental-strip-types","/opt/nemoclaw-hermes-config/generate-config.ts"];case"langchain-deepagents-code":return["/usr/local/bin/node","--experimental-strip-types","/opt/nemoclaw-deepagents-code/generate-config.ts"];case"pi":return["/usr/local/bin/node","--experimental-strip-types","/opt/nemoclaw-pi/generate-config.ts"]}}function messagingCommand(agent,phase,mode){return["/usr/local/bin/node","--experimental-strip-types","/src/lib/messaging/applier/build/messaging-build-applier.mts","--agent",agent,"--phase",phase,"--mode",mode,...phase==="post-agent-install"?["--managed-startup-runtime"]:[]]}function assertActionAgent(inputAgent,actionAgent){if(inputAgent!==actionAgent){failActionPlan(`action for ${actionAgent} cannot be used by ${inputAgent}`)}}function buildManagedStartupImageActionPlan(input){const inputAgent=exactActionPlanAgent(input.agent);const commands=[];let dashboardActions=0;let generateActions=0;let runtimeMessagingActions=0;let postMessagingActions=0;for(const action of input.actions){switch(action.kind){case"configure-dashboard":{if(action.dashboard.agent!==input.agent){failActionPlan(`dashboard for ${action.dashboard.agent} cannot be used by ${input.agent}`)}dashboardActions+=1;break}case"generate-agent-config":{assertActionAgent(inputAgent,exactActionPlanAgent(action.agent));if(action.runAs!=="sandbox"){failActionPlan("agent configuration generation must run as sandbox")}generateActions+=1;commands.push({action:"generate-agent-config",runAs:action.runAs,argv:generatorCommand(action.agent)});break}case"apply-messaging-plan":{assertActionAgent(inputAgent,exactActionPlanAgent(action.agent));if(action.mode!=="apply"&&action.mode!=="clear"){failActionPlan("messaging intent must be apply or clear")}if(action.phase==="runtime-setup"){if(action.runAs!=="root"){failActionPlan("messaging runtime setup must run as root")}runtimeMessagingActions+=1;commands.push({action:"messaging-runtime-setup",runAs:action.runAs,argv:messagingCommand(action.agent,action.phase,action.mode)})}else if(action.phase==="post-agent-install"){if(action.runAs!=="sandbox"){failActionPlan("messaging post-agent configuration must run as sandbox")}postMessagingActions+=1;commands.push({action:"messaging-post-agent-install",runAs:action.runAs,argv:messagingCommand(action.agent,action.phase,action.mode)})}else{failActionPlan("unsupported messaging construction phase")}break}default:failActionPlan("unsupported managed startup construction action")}}if(dashboardActions!==1){failActionPlan("exactly one dashboard construction action is required")}if(generateActions!==1){failActionPlan("exactly one agent config construction action is required")}const supportsMessaging=MANAGED_STARTUP_MESSAGING_AGENTS.includes(inputAgent);const expectedMessagingActions=supportsMessaging?1:0;if(runtimeMessagingActions!==expectedMessagingActions||postMessagingActions!==expectedMessagingActions){failActionPlan(`${inputAgent} requires ${String(expectedMessagingActions)} action for each messaging phase`)}const expectedOrder=supportsMessaging?["messaging-runtime-setup","generate-agent-config","messaging-post-agent-install"]:["generate-agent-config"];if(commands.some((command,index)=>command.action!==expectedOrder[index])){failActionPlan(`${inputAgent} image actions are not in the required construction order`)}return Object.freeze(commands.map(command=>Object.freeze({...command,argv:Object.freeze([...command.argv])})))}function prepareMessagingRuntimeTarget(mode){if(mode==="clear"){removeSafeRootFile(MESSAGING_RUNTIME_PLAN_FILE);return}requireSafeExistingRootTarget(MESSAGING_RUNTIME_PLAN_FILE);try{import_node_fs3.default.unlinkSync(MESSAGING_RUNTIME_PLAN_FILE)}catch(error){if(error.code!=="ENOENT"){fail5("could not prepare the messaging runtime-plan target")}}}function verifyMessagingRuntimeTarget(mode){if(mode==="clear"){if(import_node_fs3.default.existsSync(MESSAGING_RUNTIME_PLAN_FILE)){fail5("clear messaging profile left a runtime-plan artifact")}return}const stat=import_node_fs3.default.lstatSync(MESSAGING_RUNTIME_PLAN_FILE);if(stat.isSymbolicLink()||!stat.isFile()||stat.nlink!==1||stat.uid!==0||stat.gid!==0||modeOf3(stat)!==420){fail5("messaging runtime-plan artifact failed root ownership validation")}}function runInternalSandboxAction(action,configurationEnvironment,applicationRuntime,extraEnvironment={}){execute(["/usr/local/bin/node",MANAGED_STARTUP_RUNTIME_EXECUTABLE,`--internal-${action}`],"sandbox",{...configurationEnvironment,...extraEnvironment},applicationRuntime)}function sealOpenClawConfiguration(configurationEnvironment,applicationRuntime){const validation=execute(["/usr/local/bin/openclaw","config","validate","--json"],"sandbox",{...configurationEnvironment,OPENCLAW_CONFIG_PATH:"/sandbox/.openclaw/openclaw.json"},applicationRuntime,true);let parsed;try{parsed=JSON.parse(validation.stdout)}catch{fail5("OpenClaw config validation did not emit JSON")}if(typeof parsed!=="object"||parsed===null||parsed.valid!==true){fail5("OpenClaw rejected the generated managed startup config")}runInternalSandboxAction("write-openclaw-hash",configurationEnvironment,applicationRuntime)}function sameStableFileMetadata(left,right){return left.dev===right.dev&&left.ino===right.ino&&left.mode===right.mode&&left.nlink===right.nlink&&left.uid===right.uid&&left.gid===right.gid&&left.size===right.size&&left.mtimeNs===right.mtimeNs&&left.ctimeNs===right.ctimeNs}function readStableRegularFileSnapshot(target,maxBytes){if(typeof import_node_fs3.default.constants.O_NOFOLLOW!=="number"){fail5("O_NOFOLLOW is unavailable for managed startup file reads")}const nonblock=typeof import_node_fs3.default.constants.O_NONBLOCK==="number"?import_node_fs3.default.constants.O_NONBLOCK:0;let descriptor;try{descriptor=import_node_fs3.default.openSync(target,import_node_fs3.default.constants.O_RDONLY|import_node_fs3.default.constants.O_NOFOLLOW|nonblock)}catch(error){if(error.code==="ENOENT")throw error;fail5(`refusing unsafe or unreadable file ${target}`)}try{const before=import_node_fs3.default.fstatSync(descriptor,{bigint:true});if(!before.isFile()||before.nlink!==1n||before.size<1n||before.size>BigInt(maxBytes)){fail5(`refusing unsafe or oversized file ${target}`)}const bytes=Buffer.alloc(Number(before.size));let offset=0;while(offset`${block.trim()}