From b2ab3cc0170777af24d187e7fcd1068d28283fae Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Sat, 18 Jul 2026 10:45:55 -0700 Subject: [PATCH 1/6] fix(installer): preserve stopped Station containers Signed-off-by: Senthil Ravichandran --- docs/get-started/dgx-station-preparation.mdx | 4 + scripts/prepare-dgx-station-host.sh | 173 +++++++++++++---- ...tall-station-container-coexistence.test.ts | 175 ++++++++++++++++++ test/install-station-dgx-os.test.ts | 2 + test/install-station-host-preparation.test.ts | 88 +++------ 5 files changed, 349 insertions(+), 93 deletions(-) create mode 100644 test/install-station-container-coexistence.test.ts diff --git a/docs/get-started/dgx-station-preparation.mdx b/docs/get-started/dgx-station-preparation.mdx index 2d6b7c4cb56..7fb74b071a5 100644 --- a/docs/get-started/dgx-station-preparation.mdx +++ b/docs/get-started/dgx-station-preparation.mdx @@ -25,6 +25,10 @@ Each profile requires `DGX_PLATFORM=DGX Server for GALAXY-GB300`, requires both By default, an unknown version, malformed or unsafe marker, unmatched no-OTA factory image, or other Station generation stops before host preparation. On an unqualified system, set `NEMOCLAW_PROVIDER` or `NEMOCLAW_NO_EXPRESS=1` explicitly to continue without Station host automation. +Station preparation preserves existing stopped Docker container records, captures their IDs before its probes, and stops if that inventory changes during preparation. +Running containers still block initial preparation and every Docker configuration change. +Before a Docker restart or host reboot, stopped containers with a restart policy other than `no` also block preparation because they could start automatically. + Use `--force-station-install` only on a genuine DGX Station GB300 when automatic Station detection rejects its DGX release metadata. The flag handles metadata variants not yet recognized by NemoClaw; it is not a general compatibility bypass. diff --git a/scripts/prepare-dgx-station-host.sh b/scripts/prepare-dgx-station-host.sh index 53d4fc6eb77..2dd7676fade 100755 --- a/scripts/prepare-dgx-station-host.sh +++ b/scripts/prepare-dgx-station-host.sh @@ -5,7 +5,7 @@ set -Eeuo pipefail umask 077 -readonly SCRIPT_VERSION="2026-07-17.4" +readonly SCRIPT_VERSION="2026-07-18.1" readonly REBOOT_REQUIRED_EXIT=10 readonly LOGIN_REQUIRED_EXIT=11 readonly MIN_FREE_KIB=$((20 * 1024 * 1024)) @@ -42,6 +42,11 @@ readonly TARGET_DKMS_VERSION="1:3.4.0-1ubuntu1" readonly ACCEPTANCE_IMAGE="docker.io/library/ubuntu@sha256:7f622ca8766bccb22f04242ecb6f19f770b2f08827dc4b8c707de5e78a6da7ab" readonly STATE_DIR="${HOME}/.local/state/station-bootstrap" readonly INSTALL_BOOT_MARKER="${STATE_DIR}/install-boot-id" +DOCKER_BASELINE_CAPTURED=0 +DOCKER_CONTAINER_BASELINE="" +DOCKER_CONTAINER_BASELINE_TOTAL=0 +DOCKER_QUERY_OUTPUT="" +DOCKER_QUERY_USES_SUDO=0 readonly -a PACKAGE_SPECS=( "dkms=${TARGET_DKMS_VERSION}" @@ -731,6 +736,110 @@ host_docker_sudo() { fi } +query_host_docker() { + local output + DOCKER_QUERY_OUTPUT="" + command -v docker >/dev/null 2>&1 || return 2 + if output="$(host_docker "$@" 2>/dev/null)"; then + DOCKER_QUERY_OUTPUT="$output" + return 0 + fi + if [[ "$MODE" == "--apply" ]] && output="$(host_docker_sudo "$@" 2>/dev/null)"; then + DOCKER_QUERY_OUTPUT="$output" + if ((DOCKER_QUERY_USES_SUDO == 0)); then + info "docker_access=sudo_until_group_membership_is_active" + DOCKER_QUERY_USES_SUDO=1 + fi + return 0 + fi + if systemctl is-active --quiet docker.service; then + fatal "Docker is active but inaccessible to this login; start a new login session with docker-group membership" + fi + fatal "Docker is installed but inactive, so existing container state cannot be verified safely; start Docker and rerun preparation" +} + +normalize_container_ids() { + LC_ALL=C sort -u | awk 'NF' +} + +container_count() { + awk 'NF { count++ } END { print count + 0 }' +} + +capture_docker_container_baseline() { + local status running running_count + ((DOCKER_BASELINE_CAPTURED == 0)) || return 0 + if query_host_docker ps -aq --no-trunc; then + DOCKER_CONTAINER_BASELINE="$(normalize_container_ids <<<"$DOCKER_QUERY_OUTPUT")" + else + status=$? + ((status == 2)) || return "$status" + DOCKER_CONTAINER_BASELINE="" + fi + DOCKER_CONTAINER_BASELINE_TOTAL="$(container_count <<<"$DOCKER_CONTAINER_BASELINE")" + DOCKER_BASELINE_CAPTURED=1 + + running="" + if command -v docker >/dev/null 2>&1; then + query_host_docker ps -q --no-trunc + running="$(normalize_container_ids <<<"$DOCKER_QUERY_OUTPUT")" + fi + running_count="$(container_count <<<"$running")" + info "docker_container_baseline_total=${DOCKER_CONTAINER_BASELINE_TOTAL} running=${running_count}" + if ((DOCKER_CONTAINER_BASELINE_TOTAL > 0)); then + warn "Existing Docker container records will be preserved during Station preparation" + fi +} + +verify_docker_container_baseline() { + local current current_total status + ((DOCKER_BASELINE_CAPTURED == 1)) || capture_docker_container_baseline + if query_host_docker ps -aq --no-trunc; then + current="$(normalize_container_ids <<<"$DOCKER_QUERY_OUTPUT")" + else + status=$? + ((status == 2)) || return "$status" + current="" + fi + current_total="$(container_count <<<"$current")" + if [[ "$current" != "$DOCKER_CONTAINER_BASELINE" ]]; then + fatal "Docker container inventory changed during Station preparation (before=${DOCKER_CONTAINER_BASELINE_TOTAL}, after=${current_total}); rerun after container activity stops" + fi + info "docker_container_baseline=preserved total=${current_total}" +} + +require_no_running_docker_containers() { + local action=${1:-this host mutation} + command -v docker >/dev/null 2>&1 || return 0 + query_host_docker ps --format '{{.ID}} {{.Names}}' + [[ -z "$DOCKER_QUERY_OUTPUT" ]] \ + || fatal "Running Docker containers block ${action}: ${DOCKER_QUERY_OUTPUT}" + info "running_docker_containers=none action=${action}" +} + +require_no_autorestarting_stopped_containers() { + local action=${1:-this Docker restart or reboot} line blockers="" + local -a container_ids=() + command -v docker >/dev/null 2>&1 || return 0 + query_host_docker ps -aq --no-trunc + while IFS= read -r line; do + [[ -n "$line" ]] && container_ids+=("$line") + done <<<"$DOCKER_QUERY_OUTPUT" + ((${#container_ids[@]} == 0)) && return 0 + query_host_docker inspect \ + --format '{{.Id}} {{.Name}} {{.State.Running}} {{.HostConfig.RestartPolicy.Name}}' \ + "${container_ids[@]}" + blockers="$(awk ' + $3 == "false" && $4 != "" && $4 != "no" { + sub(/^\//, "", $2) + print substr($1, 1, 12) " " $2 " restart=" $4 + } + ' <<<"$DOCKER_QUERY_OUTPUT")" + [[ -z "$blockers" ]] \ + || fatal "Stopped containers with restart policies block ${action}: ${blockers}" + info "autorestarting_stopped_containers=none action=${action}" +} + warn_openibd_remediation() { warn "openibd.service configures optional Mellanox RDMA networking; NemoClaw does not require RDMA" warn "Check the default route: ip route get 1.1.1.1" @@ -776,8 +885,8 @@ check_failed_units() { ((blocking == 0)) || fatal "Unqualified failed system units block Station preparation" } -check_no_workloads() { - local processes matches listeners containers="" +check_agent_and_inference_conflicts() { + local processes matches listeners processes="$(ps -eo pid=,ppid=,comm=,args=)" matches="$(awk -v self="$$" -v parent="$PPID" ' { @@ -796,19 +905,19 @@ check_no_workloads() { listeners="$(ss -H -ltn 2>/dev/null | awk '$4 ~ /:8000$/ {print}')" [[ -z "$listeners" ]] || fatal "Port 8000 is already listening: ${listeners}" - if command -v docker >/dev/null 2>&1; then - if containers="$(host_docker ps -aq 2>/dev/null)"; then - : - elif [[ "$MODE" == "--apply" ]] && containers="$(host_docker_sudo ps -aq 2>/dev/null)"; then - info "docker_access=sudo_until_group_membership_is_active" - elif systemctl is-active --quiet docker.service; then - fatal "Docker is active but inaccessible to this login; start a new login session with docker-group membership" - else - fatal "Docker is installed but inactive, so existing container state cannot be verified safely; start Docker and rerun preparation" - fi - fi - [[ -z "$containers" ]] || fatal "Existing Docker containers block host preparation: ${containers}" - info "workloads=none port_8000=free" + info "agent_inference_workloads=none port_8000=free" +} + +require_docker_mutation_quiescence() { + local action=$1 + check_agent_and_inference_conflicts + require_no_running_docker_containers "$action" +} + +require_docker_restart_quiescence() { + local action=$1 + require_docker_mutation_quiescence "$action" + require_no_autorestarting_stopped_containers "$action" } loaded_driver_version() { @@ -906,6 +1015,7 @@ common_preflight() { require_command ps require_command sed require_command sha256sum + require_command sort require_command ss require_command stat require_command systemctl @@ -924,7 +1034,9 @@ common_preflight() { check_network check_package_managers_idle check_failed_units - check_no_workloads + check_agent_and_inference_conflicts + capture_docker_container_baseline + require_no_running_docker_containers "initial Station host preparation" } verify_file_sha256() { @@ -1080,7 +1192,7 @@ install_packages() { sudo apt-get update validate_package_availability simulate_install - check_no_workloads + require_docker_restart_quiescence "Station prerequisite package installation" info "Installing pinned Station prerequisites" sudo env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ "${PACKAGE_SPECS[@]}" @@ -1106,7 +1218,7 @@ ensure_docker_group() { ensure_cdi_refresh_lifecycle() { ((CDI_LIFECYCLE_READY == 0)) || return 0 - check_no_workloads + require_docker_mutation_quiescence "enabling NVIDIA CDI refresh" sudo systemctl enable nvidia-cdi-refresh.path nvidia-cdi-refresh.service \ || fatal "Could not enable the packaged NVIDIA CDI refresh lifecycle" sudo systemctl start nvidia-cdi-refresh.path \ @@ -1126,7 +1238,7 @@ verify_cdi_refresh_lifecycle() { } refresh_cdi() { - check_no_workloads + require_docker_mutation_quiescence "refreshing NVIDIA CDI configuration" ensure_cdi_refresh_lifecycle if ! sudo systemctl restart nvidia-cdi-refresh.service; then warn "Packaged CDI refresh failed; collecting diagnostics" @@ -1269,7 +1381,7 @@ configure_docker_runtime_if_needed() { # missing-runtime state. It remains required until this acceptance probe # succeeds through a replacement Docker/NVIDIA runtime integration. warn "Docker --gpus all failed and Docker reports no NVIDIA runtime; applying the reviewed NVIDIA runtime registration" - check_no_workloads + require_docker_mutation_quiescence "configuring the NVIDIA Docker runtime" ensure_root_directory_safe /etc/docker /etc 0755 "Docker configuration directory" ensure_root_directory_safe /var/backups/station-bootstrap /var/backups 0700 "Station bootstrap backup directory" backup_dir="$(sudo mktemp -d /var/backups/station-bootstrap/docker-runtime.XXXXXXXXXX)" \ @@ -1284,14 +1396,14 @@ configure_docker_runtime_if_needed() { sudo touch "${backup_dir}/daemon.json.absent" sudo chmod 0600 "${backup_dir}/daemon.json.absent" fi - check_no_workloads + require_docker_mutation_quiescence "configuring the NVIDIA Docker runtime" if ! sudo nvidia-ctk runtime configure --runtime=docker; then fail_after_docker_runtime_rollback "$backup_dir" "$previous_daemon" "NVIDIA runtime registration failed" fi if ! root_regular_file_is_safe /etc/docker/daemon.json ""; then fail_after_docker_runtime_rollback "$backup_dir" "$previous_daemon" "NVIDIA runtime registration produced an unsafe Docker daemon configuration" fi - if ! (check_no_workloads); then + if ! (require_docker_restart_quiescence "restarting Docker after NVIDIA runtime registration"); then fail_after_docker_runtime_rollback "$backup_dir" "$previous_daemon" "A workload appeared before Docker restart" 0 fi if ! sudo systemctl restart docker.service; then @@ -1331,13 +1443,13 @@ fail_after_docker_runtime_rollback() { } finish_runtime() { - check_no_workloads + require_docker_restart_quiescence "starting or configuring the Station container runtime" sudo systemctl enable --now containerd.service docker.service ensure_docker_group ensure_acceptance_image ensure_cdi_runtime configure_docker_runtime_if_needed - [[ -z "$(sudo docker ps -aq)" ]] || fatal "Acceptance tests left a Docker container behind" + verify_docker_container_baseline info "runtime_setup=complete" } @@ -1365,8 +1477,7 @@ verify_dgx_os_runtime_sudo() { || fatal "The Station factory image failed the CDI Docker GPU visibility test" run_dgx_os_gpus_test_sudo \ || fatal "The Station factory image failed the Docker --gpus all GPU visibility test" - [[ -z "$(station_sudo_local_default_docker ps -aq)" ]] \ - || fatal "Station factory-image acceptance tests left a Docker container behind" + verify_docker_container_baseline if [[ "$STATION_HOST_PROFILE" == "stock-dgx-os" ]]; then info "DGX_OS_HOST_READY host_runtime_mutation=container_image_cache_only" else @@ -1390,8 +1501,7 @@ verify_dgx_os_runtime_user() { || fatal "The Station factory image failed the CDI Docker GPU visibility test" run_dgx_os_gpus_test_user \ || fatal "The Station factory image failed the Docker --gpus all GPU visibility test" - [[ -z "$(station_local_default_docker ps -aq)" ]] \ - || fatal "Station factory-image verification left a Docker container behind" + verify_docker_container_baseline if [[ "$STATION_HOST_PROFILE" == "stock-dgx-os" ]]; then info "DGX_OS_HOST_READY" else @@ -1411,7 +1521,7 @@ verify_apply_state() { verify_cdi_refresh_lifecycle nvidia-ctk cdi list | grep -Fxq 'nvidia.com/gpu=all' || fatal "CDI verification failed" sudo docker image inspect "$ACCEPTANCE_IMAGE" >/dev/null 2>&1 || fatal "Digest-pinned acceptance image is missing" - [[ -z "$(sudo docker ps -aq)" ]] || fatal "Verification found a leftover Docker container" + verify_docker_container_baseline info "STATION_HOST_READY" } @@ -1555,8 +1665,9 @@ run_apply() { assert_no_package_mismatches install_packages ensure_docker_group - check_no_workloads + require_docker_restart_quiescence "enabling the Station container runtime and rebooting" sudo systemctl enable containerd.service docker.service nvidia-cdi-refresh.path nvidia-cdi-refresh.service + verify_docker_container_baseline write_install_boot_marker info "APPLY_RESULT=REBOOT_REQUIRED" info "Run: sudo reboot" diff --git a/test/install-station-container-coexistence.test.ts b/test/install-station-container-coexistence.test.ts new file mode 100644 index 00000000000..871643579bd --- /dev/null +++ b/test/install-station-container-coexistence.test.ts @@ -0,0 +1,175 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { TEST_SYSTEM_PATH } from "./helpers/installer-sourced-env"; + +const REPO_ROOT = path.resolve(import.meta.dirname, ".."); +const STATION_PREPARE = path.join(REPO_ROOT, "scripts", "prepare-dgx-station-host.sh"); + +function runStationPreparation(body: string, extraEnv: Record = {}) { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-station-containers-")); + const result = spawnSync( + "bash", + ["--noprofile", "--norc", "-c", `source "$STATION_PREPARE" >/dev/null\n${body}`], + { + cwd: REPO_ROOT, + encoding: "utf-8", + env: { + HOME: home, + PATH: TEST_SYSTEM_PATH, + STATION_PREPARE, + ...extraEnv, + }, + timeout: 15_000, + killSignal: "SIGKILL", + }, + ); + return { result, output: `${result.stdout}${result.stderr}` }; +} + +describe("DGX Station Docker container coexistence", () => { + it("uses sudo to inspect containers during apply until Docker group access is active", () => { + const { result, output } = runStationPreparation( + ` +MODE='--apply' +ps() { printf '%s %s bash bash prepare-dgx-station-host.sh --apply\n' "$$" "$PPID"; } +ss() { :; } +docker() { return 1; } +sudo() { + if [[ "$1" == "-n" ]]; then shift; fi + case "$*" in + 'docker ps -aq --no-trunc'|'docker ps -q --no-trunc') return 0 ;; + *) return 1 ;; + esac +} +systemctl() { return 0; } +capture_docker_container_baseline +`, + { PATH: `${path.dirname(process.execPath)}:${TEST_SYSTEM_PATH}` }, + ); + + expect(result.status, output).toBe(0); + expect(output).toContain("docker_access=sudo_until_group_membership_is_active"); + expect(output).toContain("docker_container_baseline_total=0 running=0"); + }); + + it("fails closed when Docker is installed but its container state cannot be queried", () => { + const { result, output } = runStationPreparation( + ` +MODE='--apply' +ps() { printf '%s %s bash bash prepare-dgx-station-host.sh --apply\n' "$$" "$PPID"; } +ss() { :; } +docker() { return 1; } +sudo() { return 1; } +systemctl() { return 1; } +capture_docker_container_baseline +`, + { PATH: `${path.dirname(process.execPath)}:${TEST_SYSTEM_PATH}` }, + ); + + expect(result.status, output).not.toBe(0); + expect(output).toMatch(/container state cannot be verified safely/); + }); + + it("captures and preserves the pre-existing container baseline (#7153)", () => { + const { result, output } = runStationPreparation(` +docker() { + case "$*" in + 'ps -aq --no-trunc') printf 'bbbbbbbbbbbb\naaaaaaaaaaaa\n' ;; + 'ps -q --no-trunc') return 0 ;; + 'ps --format {{.ID}} {{.Names}}') return 0 ;; + *) return 1 ;; + esac +} +capture_docker_container_baseline +require_no_running_docker_containers "initial Station host preparation" +verify_docker_container_baseline +`); + + expect(result.status, output).toBe(0); + expect(output).toContain("docker_container_baseline_total=2 running=0"); + expect(output).toContain("Existing Docker container records will be preserved"); + expect(output).toContain("docker_container_baseline=preserved total=2"); + }); + + it("fails closed when container inventory changes after baseline capture (#7153)", () => { + const { result, output } = runStationPreparation(` +docker() { + case "$*" in + 'ps -aq --no-trunc') + if [[ -e "$HOME/inventory-changed" ]]; then + printf 'aaaaaaaaaaaa\ncccccccccccc\n' + else + printf 'aaaaaaaaaaaa\n' + fi + ;; + 'ps -q --no-trunc') return 0 ;; + *) return 1 ;; + esac +} +capture_docker_container_baseline +touch "$HOME/inventory-changed" +verify_docker_container_baseline +`); + + expect(result.status, output).not.toBe(0); + expect(output).toMatch(/container inventory changed during Station preparation/); + expect(output).toContain("before=1, after=2"); + }); + + it("blocks a running container at a Docker mutation boundary (#7153)", () => { + const { result, output } = runStationPreparation(` +docker() { + [[ "$*" == "ps --format {{.ID}} {{.Names}}" ]] || return 1 + printf 'abc123def456 active-nim\n' +} +require_no_running_docker_containers "configuring the NVIDIA Docker runtime" +`); + + expect(result.status, output).not.toBe(0); + expect(output).toMatch(/Running Docker containers block configuring the NVIDIA Docker runtime/); + expect(output).toContain("abc123def456 active-nim"); + }); + + it("blocks daemon restart when a stopped container may automatically restart (#7153)", () => { + const { result, output } = runStationPreparation(` +docker() { + case "$*" in + 'ps -aq --no-trunc') printf 'aaaaaaaaaaaaaaaaaaaaaaaa\n' ;; + 'inspect --format {{.Id}} {{.Name}} {{.State.Running}} {{.HostConfig.RestartPolicy.Name}} aaaaaaaaaaaaaaaaaaaaaaaa') + printf 'aaaaaaaaaaaaaaaaaaaaaaaa /background-job false unless-stopped\n' + ;; + *) return 1 ;; + esac +} +require_no_autorestarting_stopped_containers "restarting Docker" +`); + + expect(result.status, output).not.toBe(0); + expect(output).toMatch(/Stopped containers with restart policies block restarting Docker/); + expect(output).toContain("background-job restart=unless-stopped"); + }); + + it("permits a stopped container with restart policy no (#7153)", () => { + const { result, output } = runStationPreparation(` +docker() { + case "$*" in + 'ps -aq --no-trunc') printf 'aaaaaaaaaaaaaaaaaaaaaaaa\n' ;; + 'inspect --format {{.Id}} {{.Name}} {{.State.Running}} {{.HostConfig.RestartPolicy.Name}} aaaaaaaaaaaaaaaaaaaaaaaa') + printf 'aaaaaaaaaaaaaaaaaaaaaaaa /archived-job false no\n' + ;; + *) return 1 ;; + esac +} +require_no_autorestarting_stopped_containers "restarting Docker" +`); + + expect(result.status, output).toBe(0); + expect(output).toContain("autorestarting_stopped_containers=none action=restarting Docker"); + }); +}); diff --git a/test/install-station-dgx-os.test.ts b/test/install-station-dgx-os.test.ts index bffe5d85572..46e5870e586 100644 --- a/test/install-station-dgx-os.test.ts +++ b/test/install-station-dgx-os.test.ts @@ -922,6 +922,7 @@ sudo() { ensure_dgx_os_acceptance_image() { printf 'IMAGE_CACHE_READY\n'; } run_dgx_os_cdi_test_sudo() { printf 'CDI_TEST_OK\n'; } run_dgx_os_gpus_test_sudo() { printf 'GPUS_TEST_OK\n'; } +verify_docker_container_baseline() { printf 'CONTAINER_BASELINE_PRESERVED\n'; } verify_dgx_os_runtime_sudo `, ); @@ -931,6 +932,7 @@ verify_dgx_os_runtime_sudo expect(output).toContain("IMAGE_CACHE_READY"); expect(output).toContain("CDI_TEST_OK"); expect(output).toContain("GPUS_TEST_OK"); + expect(output).toContain("CONTAINER_BASELINE_PRESERVED"); expect(output).toContain("DGX_OS_HOST_READY host_runtime_mutation=container_image_cache_only"); expect(output).not.toContain("UNEXPECTED_SYSTEMCTL"); expect(output).not.toContain("UNEXPECTED_DOCKER"); diff --git a/test/install-station-host-preparation.test.ts b/test/install-station-host-preparation.test.ts index dcfa596ba4f..ecf04f7d458 100644 --- a/test/install-station-host-preparation.test.ts +++ b/test/install-station-host-preparation.test.ts @@ -389,7 +389,7 @@ installed_version() { } install_packages() { printf 'INSTALL_PACKAGES\n'; } ensure_docker_group() { printf 'ENSURE_DOCKER_GROUP\n'; } -check_no_workloads() { printf 'RECHECK_ALL_WORKLOADS\n'; } +require_docker_restart_quiescence() { printf 'RECHECK_RESTART_QUIESCENCE\n'; } write_install_boot_marker() { printf 'WRITE_BOOT_MARKER\n'; } sudo() { printf 'SUDO %s\n' "$*"; } run_apply @@ -400,7 +400,7 @@ run_apply expect(output).toContain("package=dkms status=approved_transition"); expect(output).toContain("INSTALL_PACKAGES"); expect(output).toContain("ENSURE_DOCKER_GROUP"); - expect(output).toContain("RECHECK_ALL_WORKLOADS"); + expect(output).toContain("RECHECK_RESTART_QUIESCENCE"); expect(output).toContain("WRITE_BOOT_MARKER"); expect(output).toContain( "systemctl enable containerd.service docker.service nvidia-cdi-refresh.path nvidia-cdi-refresh.service", @@ -415,7 +415,7 @@ run_apply configure_repositories() { printf 'CONFIGURE_REPOSITORIES\n'; } validate_package_availability() { printf 'VALIDATE_PACKAGES\n'; } simulate_install() { printf 'SIMULATE_INSTALL\n'; } -check_no_workloads() { printf 'RECHECK_ALL_WORKLOADS\n'; } +require_docker_restart_quiescence() { printf 'RECHECK_RESTART_QUIESCENCE\n'; } package_is_exact() { return 0; } sudo() { printf 'SUDO %s\n' "$*"; } install_packages @@ -425,7 +425,7 @@ install_packages expect(result.status, output).toBe(0); expect(output).toContain("apt-get update"); expect(output).toContain("apt-get install -y --no-install-recommends"); - expect(output).toContain("RECHECK_ALL_WORKLOADS"); + expect(output).toContain("RECHECK_RESTART_QUIESCENCE"); for (const spec of [ "libnvidia-container-tools=1.19.1-1", "libnvidia-container1=1.19.1-1", @@ -441,7 +441,7 @@ install_packages const { result, output } = runSourced( STATION_PREPARE, ` -check_no_workloads() { printf 'RECHECK_ALL_WORKLOADS\n'; } +require_docker_mutation_quiescence() { printf 'RECHECK_MUTATION_QUIESCENCE\n'; } sudo() { printf 'SUDO %s\n' "$*"; } run_cdi_test_sudo() { printf 'CDI_TEST\n'; return 0; } refresh_cdi() { printf 'REFRESH_CDI\n'; } @@ -450,7 +450,7 @@ ensure_cdi_runtime ); expect(result.status, output).toBe(0); - expect(output).toContain("RECHECK_ALL_WORKLOADS"); + expect(output).toContain("RECHECK_MUTATION_QUIESCENCE"); expect(output).toContain("systemctl enable nvidia-cdi-refresh.path nvidia-cdi-refresh.service"); expect(output).toContain("systemctl start nvidia-cdi-refresh.path"); expect(output).toContain("cdi_contract=pass_without_configuration_change"); @@ -490,7 +490,7 @@ ps() { printf '%s 1 bash bash /tmp/NemoClaw/scripts/install.sh\n' "$PPID" } ss() { :; } -check_no_workloads +check_agent_and_inference_conflicts `, ); expect(selfOnly.result.status, selfOnly.output).toBe(0); @@ -500,55 +500,13 @@ check_no_workloads ` ps() { printf '999 1 python python -m vllm serve model\n'; } ss() { :; } -check_no_workloads +check_agent_and_inference_conflicts `, ); expect(active.result.status, active.output).not.toBe(0); expect(active.output).toMatch(/Agent or inference workload is active/); }); - it("uses sudo to inspect containers during apply until Docker group access is active", () => { - const { result, output } = runSourced( - STATION_PREPARE, - ` -MODE='--apply' -ps() { printf '%s %s bash bash prepare-dgx-station-host.sh --apply\n' "$$" "$PPID"; } -ss() { :; } -docker() { return 1; } -sudo() { - if [[ "$1" == "-n" ]]; then shift; fi - [[ "$*" == "docker ps -aq" ]] || return 1 -} -systemctl() { return 0; } -check_no_workloads -`, - { PATH: `${path.dirname(process.execPath)}:${TEST_SYSTEM_PATH}` }, - ); - - expect(result.status, output).toBe(0); - expect(output).toContain("docker_access=sudo_until_group_membership_is_active"); - expect(output).toContain("workloads=none"); - }); - - it("fails closed when Docker is installed but its container state cannot be queried", () => { - const { result, output } = runSourced( - STATION_PREPARE, - ` -MODE='--apply' -ps() { printf '%s %s bash bash prepare-dgx-station-host.sh --apply\n' "$$" "$PPID"; } -ss() { :; } -docker() { return 1; } -sudo() { return 1; } -systemctl() { return 1; } -check_no_workloads -`, - { PATH: `${path.dirname(process.execPath)}:${TEST_SYSTEM_PATH}` }, - ); - - expect(result.status, output).not.toBe(0); - expect(output).toMatch(/container state cannot be verified safely/); - }); - it("refuses an installed CUDA keyring version that differs from the pin", () => { const { result, output } = runSourced( STATION_PREPARE, @@ -677,7 +635,7 @@ run_apply const { result, output } = runSourced( STATION_PREPARE, ` -check_no_workloads() { printf 'RECHECK_ALL_WORKLOADS\n'; } +require_docker_mutation_quiescence() { printf 'RECHECK_MUTATION_QUIESCENCE\n'; } sudo() { printf 'SUDO %s\n' "$*" if [[ "$*" == "systemctl restart nvidia-cdi-refresh.service" ]]; then return 1; fi @@ -690,7 +648,7 @@ refresh_cdi expect(result.status, output).not.toBe(0); expect(output).toContain("systemctl status nvidia-cdi-refresh.service --no-pager"); expect(output).toContain("journalctl -u nvidia-cdi-refresh.service --no-pager -n 50"); - expect(output).toContain("RECHECK_ALL_WORKLOADS"); + expect(output).toContain("RECHECK_MUTATION_QUIESCENCE"); expect(output).toMatch(/repair nvidia-cdi-refresh\.service/); expect(output).not.toContain("nvidia-ctk cdi generate"); }); @@ -699,7 +657,7 @@ refresh_cdi const { result, output } = runSourced( STATION_PREPARE, ` -check_no_workloads() { printf 'RECHECK_ALL_WORKLOADS\n'; } +require_docker_mutation_quiescence() { printf 'RECHECK_MUTATION_QUIESCENCE\n'; } sudo() { printf 'SUDO %s\n' "$*" return 0 @@ -710,7 +668,7 @@ refresh_cdi ); expect(result.status, output).not.toBe(0); - expect(output).toContain("RECHECK_ALL_WORKLOADS"); + expect(output).toContain("RECHECK_MUTATION_QUIESCENCE"); expect(output).toMatch(/completed without advertising nvidia\.com\/gpu=all/); expect(output).toContain("systemctl status nvidia-cdi-refresh.service --no-pager"); expect(output).toContain("journalctl -u nvidia-cdi-refresh.service --no-pager -n 50"); @@ -729,13 +687,13 @@ sudo() { [[ "$*" == "test -e /etc/docker/daemon.json" ]] && return 1 printf 'SUDO %s\n' "$*" } -check_no_workloads() { printf 'RECHECK_ALL_WORKLOADS\n'; return 1; } +require_docker_mutation_quiescence() { printf 'RECHECK_MUTATION_QUIESCENCE\n'; return 1; } configure_docker_runtime_if_needed `, ); expect(result.status, output).not.toBe(0); - expect(output).toContain("RECHECK_ALL_WORKLOADS"); + expect(output).toContain("RECHECK_MUTATION_QUIESCENCE"); expect(output).not.toContain("nvidia-ctk runtime configure"); expect(output).not.toContain("systemctl restart docker.service"); }); @@ -769,7 +727,8 @@ run_gpus_test_sudo() { } run_cdi_test_sudo() { return 0; } docker_has_nvidia_runtime_sudo() { return 1; } -check_no_workloads() { printf 'RECHECK_ALL_WORKLOADS\n'; } +require_docker_mutation_quiescence() { printf 'RECHECK_MUTATION_QUIESCENCE\n'; } +require_docker_restart_quiescence() { printf 'RECHECK_RESTART_QUIESCENCE\n'; } ensure_root_directory_safe() { :; } assert_root_directory_safe() { :; } assert_root_regular_file_safe() { :; } @@ -801,8 +760,12 @@ runtime_configured=0 run_gpus_test_sudo() { return 1; } run_cdi_test_sudo() { return 0; } docker_has_nvidia_runtime_sudo() { return 1; } -check_no_workloads() { - printf 'RECHECK_ALL_WORKLOADS configured=%s\n' "$runtime_configured" +require_docker_mutation_quiescence() { + printf 'RECHECK_MUTATION_QUIESCENCE configured=%s\n' "$runtime_configured" + return 0 +} +require_docker_restart_quiescence() { + printf 'RECHECK_RESTART_QUIESCENCE configured=%s\n' "$runtime_configured" [[ "$runtime_configured" == "0" ]] } ensure_root_directory_safe() { :; } @@ -825,7 +788,7 @@ configure_docker_runtime_if_needed ); expect(result.status, output).not.toBe(0); - expect(output).toContain("RECHECK_ALL_WORKLOADS configured=1"); + expect(output).toContain("RECHECK_RESTART_QUIESCENCE configured=1"); expect(output).toContain("rm -f -- /etc/docker/daemon.json"); expect(output).toMatch(/A workload appeared before Docker restart/); expect(output).toMatch(/prior Docker daemon configuration was restored/); @@ -839,7 +802,8 @@ configure_docker_runtime_if_needed run_gpus_test_sudo() { printf 'GPU_PROBE\n'; return 1; } run_cdi_test_sudo() { return 0; } docker_has_nvidia_runtime_sudo() { return 1; } -check_no_workloads() { printf 'RECHECK_ALL_WORKLOADS\n'; } +require_docker_mutation_quiescence() { printf 'RECHECK_MUTATION_QUIESCENCE\n'; } +require_docker_restart_quiescence() { printf 'RECHECK_RESTART_QUIESCENCE\n'; } ensure_root_directory_safe() { :; } assert_root_directory_safe() { :; } assert_root_regular_file_safe() { :; } @@ -866,7 +830,7 @@ configure_docker_runtime_if_needed const { result, output } = runSourced( STATION_PREPARE, ` -check_no_workloads() { printf 'RECHECK_ALL_WORKLOADS\n'; } +require_docker_mutation_quiescence() { printf 'RECHECK_MUTATION_QUIESCENCE\n'; } sudo() { printf 'SUDO %s\n' "$*"; } nvidia-ctk() { [[ "$*" == "cdi list" ]] && printf 'nvidia.com/gpu=all\n' From b4d507bc932360f49f507c9ddd409a1b1130e35c Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Sat, 18 Jul 2026 10:54:22 -0700 Subject: [PATCH 2/6] fix(installer): preserve containers during verification Signed-off-by: Senthil Ravichandran --- scripts/prepare-dgx-station-host.sh | 2 +- ...tall-station-container-coexistence.test.ts | 39 +++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/scripts/prepare-dgx-station-host.sh b/scripts/prepare-dgx-station-host.sh index 2dd7676fade..4028bd14955 100755 --- a/scripts/prepare-dgx-station-host.sh +++ b/scripts/prepare-dgx-station-host.sh @@ -1590,7 +1590,7 @@ verify_host() { docker image inspect "$ACCEPTANCE_IMAGE" >/dev/null 2>&1 || fatal "Digest-pinned acceptance image is missing; run --apply" run_cdi_test_user || fatal "CDI verification did not expose the qualified GB300: ${GPU_ROWS_ERROR}" run_gpus_test_user || fatal "Docker --gpus verification did not expose the qualified GB300: ${GPU_ROWS_ERROR}" - [[ -z "$(docker ps -aq)" ]] || fatal "Verification left a Docker container behind" + verify_docker_container_baseline info "docker=$(docker version --format '{{.Server.Version}}') expected_docker=${DOCKER_VERSION} toolkit=$(nvidia-ctk --version | head -n1) expected_toolkit=${TOOLKIT_VERSION}" info "STATION_HOST_READY" } diff --git a/test/install-station-container-coexistence.test.ts b/test/install-station-container-coexistence.test.ts index 871643579bd..c057e2cbc13 100644 --- a/test/install-station-container-coexistence.test.ts +++ b/test/install-station-container-coexistence.test.ts @@ -97,6 +97,45 @@ verify_docker_container_baseline expect(output).toContain("docker_container_baseline=preserved total=2"); }); + it("allows generic verification to preserve a stopped container baseline (#7153)", () => { + const { result, output } = runStationPreparation( + ` +DOCKER_BASELINE_CAPTURED=1 +DOCKER_CONTAINER_BASELINE='aaaaaaaaaaaaaaaaaaaaaaaa' +DOCKER_CONTAINER_BASELINE_TOTAL=1 +package_is_exact() { return 0; } +verify_gpu() { :; } +systemctl() { return 0; } +verify_cdi_refresh_lifecycle() { :; } +id() { printf 'operator docker\n'; } +nvidia-ctk() { + case "$*" in + 'cdi list') printf 'nvidia.com/gpu=all\n' ;; + '--version') printf 'NVIDIA Container Toolkit CLI version 1.19.1\n' ;; + *) return 1 ;; + esac +} +run_cdi_test_user() { return 0; } +run_gpus_test_user() { return 0; } +docker() { + case "$*" in + 'info') return 0 ;; + 'image inspect '*) return 0 ;; + 'ps -aq --no-trunc') printf 'aaaaaaaaaaaaaaaaaaaaaaaa\n' ;; + 'version --format {{.Server.Version}}') printf '29.6.1\n' ;; + *) return 1 ;; + esac +} +verify_host +`, + { USER: "operator" }, + ); + + expect(result.status, output).toBe(0); + expect(output).toContain("docker_container_baseline=preserved total=1"); + expect(output).toContain("STATION_HOST_READY"); + }); + it("fails closed when container inventory changes after baseline capture (#7153)", () => { const { result, output } = runStationPreparation(` docker() { From c5dec6d4d680eeb422c1b1fdd7cd97c93c134de6 Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Sat, 18 Jul 2026 10:58:34 -0700 Subject: [PATCH 3/6] fix(installer): reject container drift before mutation Signed-off-by: Senthil Ravichandran --- scripts/prepare-dgx-station-host.sh | 1 + test/install-station-container-coexistence.test.ts | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/prepare-dgx-station-host.sh b/scripts/prepare-dgx-station-host.sh index 4028bd14955..db153d5d101 100755 --- a/scripts/prepare-dgx-station-host.sh +++ b/scripts/prepare-dgx-station-host.sh @@ -911,6 +911,7 @@ check_agent_and_inference_conflicts() { require_docker_mutation_quiescence() { local action=$1 check_agent_and_inference_conflicts + verify_docker_container_baseline require_no_running_docker_containers "$action" } diff --git a/test/install-station-container-coexistence.test.ts b/test/install-station-container-coexistence.test.ts index c057e2cbc13..3a93221a815 100644 --- a/test/install-station-container-coexistence.test.ts +++ b/test/install-station-container-coexistence.test.ts @@ -136,8 +136,10 @@ verify_host expect(output).toContain("STATION_HOST_READY"); }); - it("fails closed when container inventory changes after baseline capture (#7153)", () => { + it("fails closed before mutation when container inventory changes after baseline capture (#7153)", () => { const { result, output } = runStationPreparation(` +ps() { printf '%s %s bash bash prepare-dgx-station-host.sh --apply\n' "$$" "$PPID"; } +ss() { :; } docker() { case "$*" in 'ps -aq --no-trunc') @@ -153,7 +155,7 @@ docker() { } capture_docker_container_baseline touch "$HOME/inventory-changed" -verify_docker_container_baseline +require_docker_mutation_quiescence "refreshing NVIDIA CDI configuration" `); expect(result.status, output).not.toBe(0); From 0a00899343ec992dd89846f45e589555469dad47 Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Sat, 18 Jul 2026 11:03:09 -0700 Subject: [PATCH 4/6] fix(installer): recheck Docker before service restarts Signed-off-by: Senthil Ravichandran --- scripts/prepare-dgx-station-host.sh | 10 ++- ...tall-station-container-coexistence.test.ts | 73 +++++++++++++++++++ 2 files changed, 82 insertions(+), 1 deletion(-) diff --git a/scripts/prepare-dgx-station-host.sh b/scripts/prepare-dgx-station-host.sh index db153d5d101..ec891a98f5d 100755 --- a/scripts/prepare-dgx-station-host.sh +++ b/scripts/prepare-dgx-station-host.sh @@ -919,6 +919,7 @@ require_docker_restart_quiescence() { local action=$1 require_docker_mutation_quiescence "$action" require_no_autorestarting_stopped_containers "$action" + require_docker_mutation_quiescence "$action" } loaded_driver_version() { @@ -1431,6 +1432,9 @@ rollback_docker_runtime_config() { sudo rm -f -- /etc/docker/daemon.json || return 1 fi if [[ "$restart_after_restore" == "1" ]]; then + if ! (require_docker_restart_quiescence "restarting Docker during runtime rollback"); then + return 1 + fi sudo systemctl restart docker.service fi } @@ -1444,7 +1448,11 @@ fail_after_docker_runtime_rollback() { } finish_runtime() { - require_docker_restart_quiescence "starting or configuring the Station container runtime" + if systemctl is-active --quiet containerd.service && systemctl is-active --quiet docker.service; then + require_docker_mutation_quiescence "enabling or configuring the Station container runtime" + else + require_docker_restart_quiescence "starting or configuring the Station container runtime" + fi sudo systemctl enable --now containerd.service docker.service ensure_docker_group ensure_acceptance_image diff --git a/test/install-station-container-coexistence.test.ts b/test/install-station-container-coexistence.test.ts index 3a93221a815..114b6a73ad7 100644 --- a/test/install-station-container-coexistence.test.ts +++ b/test/install-station-container-coexistence.test.ts @@ -196,6 +196,79 @@ require_no_autorestarting_stopped_containers "restarting Docker" expect(output).toContain("background-job restart=unless-stopped"); }); + it("rechecks running containers after restart-policy inspection (#7153)", () => { + const { result, output } = runStationPreparation(` +DOCKER_BASELINE_CAPTURED=1 +DOCKER_CONTAINER_BASELINE='aaaaaaaaaaaaaaaaaaaaaaaa' +DOCKER_CONTAINER_BASELINE_TOTAL=1 +ps() { printf '%s %s bash bash prepare-dgx-station-host.sh --apply\n' "$$" "$PPID"; } +ss() { :; } +docker() { + local running_checks + case "$*" in + 'ps -aq --no-trunc') printf 'aaaaaaaaaaaaaaaaaaaaaaaa\n' ;; + 'ps --format {{.ID}} {{.Names}}') + running_checks="$(cat "$HOME/running-checks" 2>/dev/null || printf '0')" + running_checks=$((running_checks + 1)) + printf '%s' "$running_checks" >"$HOME/running-checks" + if ((running_checks > 1)); then + printf 'aaaaaaaaaaaa background-job\n' + fi + ;; + 'inspect --format {{.Id}} {{.Name}} {{.State.Running}} {{.HostConfig.RestartPolicy.Name}} aaaaaaaaaaaaaaaaaaaaaaaa') + printf 'aaaaaaaaaaaaaaaaaaaaaaaa /background-job true no\n' + ;; + *) return 1 ;; + esac +} +require_docker_restart_quiescence "restarting Docker" +`); + + expect(result.status, output).not.toBe(0); + expect(output).toMatch(/Running Docker containers block restarting Docker/); + expect(output).toContain("background-job"); + }); + + it("does not restart Docker during rollback after quiescence is lost (#7153)", () => { + const { result, output } = runStationPreparation(` +root_regular_file_is_safe() { return 0; } +require_docker_restart_quiescence() { + printf 'ROLLBACK_RESTART_BLOCKED\n' + return 1 +} +sudo() { printf 'SUDO %s\n' "$*"; } +rollback_docker_runtime_config /var/backups/station-bootstrap/docker-runtime.TEST 0 1 +`); + + expect(result.status, output).not.toBe(0); + expect(output).toContain("ROLLBACK_RESTART_BLOCKED"); + expect(output).toContain("SUDO rm -f -- /etc/docker/daemon.json"); + expect(output).not.toContain("systemctl restart docker.service"); + }); + + it("does not apply restart-policy blocking when runtime services are already active (#7153)", () => { + const { result, output } = runStationPreparation(` +systemctl() { return 0; } +require_docker_mutation_quiescence() { printf 'MUTATION_QUIESCENCE\n'; } +require_docker_restart_quiescence() { + printf 'UNEXPECTED_RESTART_QUIESCENCE\n' + return 1 +} +sudo() { printf 'SUDO %s\n' "$*"; } +ensure_docker_group() { :; } +ensure_acceptance_image() { :; } +ensure_cdi_runtime() { :; } +configure_docker_runtime_if_needed() { :; } +verify_docker_container_baseline() { :; } +finish_runtime +`); + + expect(result.status, output).toBe(0); + expect(output).toContain("MUTATION_QUIESCENCE"); + expect(output).toContain("systemctl enable --now containerd.service docker.service"); + expect(output).not.toContain("UNEXPECTED_RESTART_QUIESCENCE"); + }); + it("permits a stopped container with restart policy no (#7153)", () => { const { result, output } = runStationPreparation(` docker() { From 7cfb1ea795d4e9117f5abfc4e94d7ed7452cb850 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sat, 18 Jul 2026 11:40:27 -0700 Subject: [PATCH 5/6] fix(installer): guard Station reboot handoffs Signed-off-by: Aaron Erickson --- scripts/prepare-dgx-station-host.sh | 28 ++-- ...tall-station-container-coexistence.test.ts | 157 +++++++++++++++++- test/install-station-host-preparation.test.ts | 2 + 3 files changed, 165 insertions(+), 22 deletions(-) diff --git a/scripts/prepare-dgx-station-host.sh b/scripts/prepare-dgx-station-host.sh index ec891a98f5d..88d01d3eace 100755 --- a/scripts/prepare-dgx-station-host.sh +++ b/scripts/prepare-dgx-station-host.sh @@ -922,6 +922,13 @@ require_docker_restart_quiescence() { require_docker_mutation_quiescence "$action" } +exit_reboot_required() { + require_docker_restart_quiescence "rebooting the Station host" + info "APPLY_RESULT=REBOOT_REQUIRED" + info "Run: sudo reboot" + exit "$REBOOT_REQUIRED_EXIT" +} + loaded_driver_version() { local loaded command -v nvidia-smi >/dev/null 2>&1 || return 0 @@ -1450,10 +1457,11 @@ fail_after_docker_runtime_rollback() { finish_runtime() { if systemctl is-active --quiet containerd.service && systemctl is-active --quiet docker.service; then require_docker_mutation_quiescence "enabling or configuring the Station container runtime" + sudo systemctl enable containerd.service docker.service else require_docker_restart_quiescence "starting or configuring the Station container runtime" + sudo systemctl enable --now containerd.service docker.service fi - sudo systemctl enable --now containerd.service docker.service ensure_docker_group ensure_acceptance_image ensure_cdi_runtime @@ -1665,7 +1673,7 @@ run_apply() { if reboot_required; then if all_packages_exact && ! driver_loaded_exact; then warn "A reboot is required before runtime setup can continue" - exit "$REBOOT_REQUIRED_EXIT" + exit_reboot_required fi fatal "An unrelated reboot is already pending" fi @@ -1678,32 +1686,24 @@ run_apply() { sudo systemctl enable containerd.service docker.service nvidia-cdi-refresh.path nvidia-cdi-refresh.service verify_docker_container_baseline write_install_boot_marker - info "APPLY_RESULT=REBOOT_REQUIRED" - info "Run: sudo reboot" - exit "$REBOOT_REQUIRED_EXIT" + exit_reboot_required fi if install_boot_marker_matches_current_boot; then warn "Package installation completed in the current boot" - info "APPLY_RESULT=REBOOT_REQUIRED" - info "Run: sudo reboot" - exit "$REBOOT_REQUIRED_EXIT" + exit_reboot_required fi driver_loaded_exact || { warn "Pinned packages are installed but driver ${DRIVER_VERSION} is not loaded" - info "APPLY_RESULT=REBOOT_REQUIRED" - info "Run: sudo reboot" - exit "$REBOOT_REQUIRED_EXIT" + exit_reboot_required } finish_runtime verify_apply_state if ((DOCKER_GROUP_ADDED == 1)); then warn "Docker group membership was added and requires a new login before onboarding" - info "APPLY_RESULT=REBOOT_REQUIRED" - info "Run: sudo reboot" - exit "$REBOOT_REQUIRED_EXIT" + exit_reboot_required fi rm -f "$INSTALL_BOOT_MARKER" info "APPLY_RESULT=COMPLETE" diff --git a/test/install-station-container-coexistence.test.ts b/test/install-station-container-coexistence.test.ts index 114b6a73ad7..7984288e0c8 100644 --- a/test/install-station-container-coexistence.test.ts +++ b/test/install-station-container-coexistence.test.ts @@ -97,15 +97,27 @@ verify_docker_container_baseline expect(output).toContain("docker_container_baseline=preserved total=2"); }); - it("allows generic verification to preserve a stopped container baseline (#7153)", () => { + it("allows public generic verification to preserve a stopped container baseline (#7153)", () => { const { result, output } = runStationPreparation( ` -DOCKER_BASELINE_CAPTURED=1 -DOCKER_CONTAINER_BASELINE='aaaaaaaaaaaaaaaaaaaaaaaa' -DOCKER_CONTAINER_BASELINE_TOTAL=1 +require_command() { :; } +check_platform() { :; } +check_secure_boot() { :; } +check_kernel_headers() { :; } +check_capacity() { :; } +check_network() { :; } +check_package_managers_idle() { :; } +check_failed_units() { :; } +check_agent_and_inference_conflicts() { :; } +driver_loaded_exact() { return 0; } package_is_exact() { return 0; } verify_gpu() { :; } -systemctl() { return 0; } +systemctl() { + case "$*" in + 'is-active --quiet nvidia-persistenced.service'|'is-active --quiet containerd.service'|'is-active --quiet docker.service') return 0 ;; + *) return 1 ;; + esac +} verify_cdi_refresh_lifecycle() { :; } id() { printf 'operator docker\n'; } nvidia-ctk() { @@ -122,16 +134,21 @@ docker() { 'info') return 0 ;; 'image inspect '*) return 0 ;; 'ps -aq --no-trunc') printf 'aaaaaaaaaaaaaaaaaaaaaaaa\n' ;; + 'ps -q --no-trunc'|'ps --format {{.ID}} {{.Names}}') return 0 ;; 'version --format {{.Server.Version}}') printf '29.6.1\n' ;; *) return 1 ;; esac } -verify_host +main --verify `, { USER: "operator" }, ); expect(result.status, output).toBe(0); + expect(output).toContain("docker_container_baseline_total=1 running=0"); + expect(output).toContain( + "running_docker_containers=none action=initial Station host preparation", + ); expect(output).toContain("docker_container_baseline=preserved total=1"); expect(output).toContain("STATION_HOST_READY"); }); @@ -163,6 +180,95 @@ require_docker_mutation_quiescence "refreshing NVIDIA CDI configuration" expect(output).toContain("before=1, after=2"); }); + it.each([ + { + name: "pending prerequisite", + setup: ` +reboot_required() { return 0; } +all_packages_exact() { return 0; } +driver_loaded_exact() { return 1; } +`, + expectedGate: "REBOOT_HANDOFF_BLOCKED check=1", + }, + { + name: "post-install", + setup: ` +reboot_required() { return 1; } +all_packages_exact() { return 1; } +require_docker_restart_quiescence() { + local checks + checks="$(cat "$HOME/reboot-gate-checks" 2>/dev/null || printf '0')" + checks=$((checks + 1)) + printf '%s' "$checks" >"$HOME/reboot-gate-checks" + if ((checks == 1)); then return 0; fi + printf 'REBOOT_HANDOFF_BLOCKED check=%s\n' "$checks" + return 1 +} +`, + expectedGate: "REBOOT_HANDOFF_BLOCKED check=2", + }, + { + name: "same-boot marker", + setup: ` +reboot_required() { return 1; } +all_packages_exact() { return 0; } +install_boot_marker_matches_current_boot() { return 0; } +`, + expectedGate: "REBOOT_HANDOFF_BLOCKED check=1", + }, + { + name: "unloaded driver", + setup: ` +reboot_required() { return 1; } +all_packages_exact() { return 0; } +install_boot_marker_matches_current_boot() { return 1; } +driver_loaded_exact() { return 1; } +`, + expectedGate: "REBOOT_HANDOFF_BLOCKED check=1", + }, + { + name: "Docker group", + setup: ` +reboot_required() { return 1; } +all_packages_exact() { return 0; } +install_boot_marker_matches_current_boot() { return 1; } +driver_loaded_exact() { return 0; } +finish_runtime() { DOCKER_GROUP_ADDED=1; } +`, + expectedGate: "REBOOT_HANDOFF_BLOCKED check=1", + }, + ])("blocks the $name reboot handoff when stopped containers may restart (#7153)", ({ + setup, + expectedGate, + }) => { + const { result, output } = runStationPreparation(` +require_command() { :; } +acquire_sudo() { :; } +common_preflight() { :; } +station_uses_factory_runtime() { return 1; } +assert_no_package_mismatches() { :; } +install_packages() { :; } +ensure_docker_group() { :; } +verify_docker_container_baseline() { :; } +write_install_boot_marker() { :; } +finish_runtime() { :; } +verify_apply_state() { :; } +sudo() { :; } +require_docker_restart_quiescence() { + printf 'REBOOT_HANDOFF_BLOCKED check=1\n' + return 1 +} +${setup} +run_apply +`); + + expect(result.status, output).not.toBe(0); + expect(result.status, output).not.toBe(10); + expect(output).toContain(expectedGate); + expect(output).not.toContain("APPLY_RESULT=REBOOT_REQUIRED"); + expect(output).not.toContain("Run: sudo reboot"); + }); + it("blocks a running container at a Docker mutation boundary (#7153)", () => { const { result, output } = runStationPreparation(` docker() { @@ -248,7 +354,12 @@ rollback_docker_runtime_config /var/backups/station-bootstrap/docker-runtime.TES it("does not apply restart-policy blocking when runtime services are already active (#7153)", () => { const { result, output } = runStationPreparation(` -systemctl() { return 0; } +systemctl() { + case "$*" in + 'is-active --quiet containerd.service'|'is-active --quiet docker.service') return 0 ;; + *) return 1 ;; + esac +} require_docker_mutation_quiescence() { printf 'MUTATION_QUIESCENCE\n'; } require_docker_restart_quiescence() { printf 'UNEXPECTED_RESTART_QUIESCENCE\n' @@ -265,10 +376,40 @@ finish_runtime expect(result.status, output).toBe(0); expect(output).toContain("MUTATION_QUIESCENCE"); - expect(output).toContain("systemctl enable --now containerd.service docker.service"); + expect(output).toContain("systemctl enable containerd.service docker.service"); + expect(output).not.toContain("systemctl enable --now containerd.service docker.service"); expect(output).not.toContain("UNEXPECTED_RESTART_QUIESCENCE"); }); + it("applies restart-policy blocking before starting an inactive runtime service (#7153)", () => { + const { result, output } = runStationPreparation(` +systemctl() { + case "$*" in + 'is-active --quiet containerd.service') return 0 ;; + 'is-active --quiet docker.service') return 1 ;; + *) return 1 ;; + esac +} +require_docker_mutation_quiescence() { + printf 'UNEXPECTED_MUTATION_QUIESCENCE\n' + return 1 +} +require_docker_restart_quiescence() { printf 'RESTART_QUIESCENCE\n'; } +sudo() { printf 'SUDO %s\n' "$*"; } +ensure_docker_group() { :; } +ensure_acceptance_image() { :; } +ensure_cdi_runtime() { :; } +configure_docker_runtime_if_needed() { :; } +verify_docker_container_baseline() { :; } +finish_runtime +`); + + expect(result.status, output).toBe(0); + expect(output).toContain("RESTART_QUIESCENCE"); + expect(output).toContain("systemctl enable --now containerd.service docker.service"); + expect(output).not.toContain("UNEXPECTED_MUTATION_QUIESCENCE"); + }); + it("permits a stopped container with restart policy no (#7153)", () => { const { result, output } = runStationPreparation(` docker() { diff --git a/test/install-station-host-preparation.test.ts b/test/install-station-host-preparation.test.ts index ecf04f7d458..4e62cee188b 100644 --- a/test/install-station-host-preparation.test.ts +++ b/test/install-station-host-preparation.test.ts @@ -621,12 +621,14 @@ install_boot_marker_matches_current_boot() { return 1; } driver_loaded_exact() { return 0; } finish_runtime() { DOCKER_GROUP_ADDED=1; printf 'FINISH_RUNTIME\n'; } verify_apply_state() { printf 'VERIFY_APPLY_STATE\n'; } +require_docker_restart_quiescence() { printf 'RECHECK_RESTART_QUIESCENCE\n'; } run_apply `, ); expect(result.status, output).toBe(10); expect(output).toContain("VERIFY_APPLY_STATE"); + expect(output).toContain("RECHECK_RESTART_QUIESCENCE"); expect(output).toContain("APPLY_RESULT=REBOOT_REQUIRED"); expect(output).toMatch(/new login before onboarding/); }); From abd9897bacd0b3125a4681c5a53941ee2ba1fa06 Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Sat, 18 Jul 2026 11:53:33 -0700 Subject: [PATCH 6/6] test(installer): assert restart guard ordering Signed-off-by: Senthil Ravichandran --- test/install-station-container-coexistence.test.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/install-station-container-coexistence.test.ts b/test/install-station-container-coexistence.test.ts index 7984288e0c8..812d8f5558f 100644 --- a/test/install-station-container-coexistence.test.ts +++ b/test/install-station-container-coexistence.test.ts @@ -407,6 +407,9 @@ finish_runtime expect(result.status, output).toBe(0); expect(output).toContain("RESTART_QUIESCENCE"); expect(output).toContain("systemctl enable --now containerd.service docker.service"); + expect(output.indexOf("RESTART_QUIESCENCE")).toBeLessThan( + output.indexOf("systemctl enable --now containerd.service docker.service"), + ); expect(output).not.toContain("UNEXPECTED_MUTATION_QUIESCENCE"); });