From 8181f00544e3cdfd68adf0d2d77ea18322aeaec6 Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Fri, 17 Jul 2026 16:27:13 -0700 Subject: [PATCH 1/2] feat(installer): add exact Station factory image profiles Signed-off-by: Senthil Ravichandran --- scripts/install.sh | 124 ++++- scripts/prepare-dgx-station-host.sh | 450 ++++++++++++++---- src/lib/inference/vllm.test.ts | 9 + src/lib/inference/vllm.ts | 18 +- test/install-express-prompt.test.ts | 120 +++++ test/install-station-dgx-os.test.ts | 298 +++++++++++- test/install-station-host-preparation.test.ts | 16 +- 7 files changed, 919 insertions(+), 116 deletions(-) diff --git a/scripts/install.sh b/scripts/install.sh index c053871f1d7..74d54083b62 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -3021,7 +3021,9 @@ detect_express_platform() { if is_station_gb300_product "$model"; then release_state="$(classify_dgx_station_release)" case "$release_state" in - generic-ubuntu | supported-dgx-os) printf "DGX Station" ;; + generic-ubuntu | supported-dgx-os | supported-colossus-baseos | supported-ai-developer-tools) + printf "DGX Station" + ;; *) printf "Unsupported DGX Station OS" ;; esac return @@ -3036,7 +3038,7 @@ validate_express_platform_boundary() { case "${1:-}" in "Unsupported DGX Station OS") if [ "${NEMOCLAW_NO_EXPRESS:-}" = "1" ] || [ -n "${NEMOCLAW_PROVIDER:-}" ]; then return 0; fi - error "This DGX Station OS image is outside the validated Station express boundary. Use generic Ubuntu 24.04 ARM64 or stock DGX OS 7.2.0, 7.4.0, or 7.5.0 on Station GB300." + error "This DGX Station OS image is outside the validated Station express boundary. Use generic Ubuntu 24.04 ARM64, stock DGX OS 7.2.0, 7.4.0, or 7.5.0, or an explicitly qualified Station factory image." ;; "Unsupported DGX Station generation") if [ "${NEMOCLAW_NO_EXPRESS:-}" = "1" ] || [ -n "${NEMOCLAW_PROVIDER:-}" ]; then return 0; fi @@ -3173,6 +3175,26 @@ validate_station_express_resume_model() { [[ ${#model} -le 255 && "$model" =~ ^[A-Za-z0-9][A-Za-z0-9._/-]*$ ]] } +validate_station_express_resume_agent() { + case "${1:-}" in + openclaw | hermes | langchain-deepagents-code) return 0 ;; + *) return 1 ;; + esac +} + +validate_station_express_resume_sandbox() { + local sandbox="${1:-}" + [[ ${#sandbox} -le 63 ]] \ + && { [[ "$sandbox" =~ ^[a-z]$ ]] || [[ "$sandbox" =~ ^[a-z][a-z0-9-]*[a-z0-9]$ ]]; } +} + +validate_station_express_resume_policy_tier() { + case "${1:-}" in + restricted | balanced | open) return 0 ;; + *) return 1 ;; + esac +} + validate_station_express_resume_revision() { [[ "${1:-}" =~ ^[0-9a-f]{40}$ ]] } @@ -3242,7 +3264,8 @@ assert_station_express_resume_directory_safe() { } load_station_express_resume() { - local state_file revision_line model_line generation_line line_count saved_revision current_revision + local state_file revision_line model_line generation_line agent_line sandbox_line policy_tier_line + local line_count saved_revision current_revision saved_agent saved_sandbox saved_policy_tier current_agent state_file="$(station_express_resume_file)" || return 1 assert_nemoclaw_state_path_safe "$state_file" [[ -e "$state_file" || -L "$state_file" ]] || return 1 @@ -3251,27 +3274,58 @@ load_station_express_resume() { revision_line="$(sed -n '1p' "$state_file")" model_line="$(sed -n '2p' "$state_file")" generation_line="$(sed -n '3p' "$state_file")" + agent_line="$(sed -n '4p' "$state_file")" + sandbox_line="$(sed -n '5p' "$state_file")" + policy_tier_line="$(sed -n '6p' "$state_file")" saved_revision="${revision_line#revision=}" NEMOCLAW_VLLM_MODEL="${model_line#model=}" _STATION_EXPRESS_RESUME_GENERATION="${generation_line#generation=}" - if [[ "$line_count" != "3" || "$revision_line" != "revision=${saved_revision}" || "$model_line" != "model=${NEMOCLAW_VLLM_MODEL}" || "$generation_line" != "generation=${_STATION_EXPRESS_RESUME_GENERATION}" ]] \ + if [[ "$line_count" == "3" ]]; then + saved_agent=openclaw + saved_sandbox=my-assistant + saved_policy_tier=balanced + elif [[ "$line_count" == "6" ]]; then + saved_agent="${agent_line#agent=}" + saved_sandbox="${sandbox_line#sandbox=}" + saved_policy_tier="${policy_tier_line#policy_tier=}" + else + error "DGX Station express resume state is invalid. Remove ${state_file} and rerun the installer." + fi + if [[ "$revision_line" != "revision=${saved_revision}" || "$model_line" != "model=${NEMOCLAW_VLLM_MODEL}" || "$generation_line" != "generation=${_STATION_EXPRESS_RESUME_GENERATION}" ]] \ + || { [[ "$line_count" == "6" ]] && [[ "$agent_line" != "agent=${saved_agent}" || "$sandbox_line" != "sandbox=${saved_sandbox}" || "$policy_tier_line" != "policy_tier=${saved_policy_tier}" ]]; } \ || ! validate_station_express_resume_revision "$saved_revision" \ || ! validate_station_express_resume_model "$NEMOCLAW_VLLM_MODEL" \ - || ! validate_station_express_resume_generation "$_STATION_EXPRESS_RESUME_GENERATION"; then + || ! validate_station_express_resume_generation "$_STATION_EXPRESS_RESUME_GENERATION" \ + || ! validate_station_express_resume_agent "$saved_agent" \ + || ! validate_station_express_resume_sandbox "$saved_sandbox" \ + || ! validate_station_express_resume_policy_tier "$saved_policy_tier"; then error "DGX Station express resume state is invalid. Remove ${state_file} and rerun the installer." fi current_revision="$(station_installer_revision)" if [[ "$current_revision" != "$saved_revision" ]]; then error "DGX Station express resume requires NemoClaw revision ${saved_revision}, but this installer is ${current_revision}. Rerun with: curl -fsSL https://www.nvidia.com/nemoclaw.sh | NEMOCLAW_INSTALL_TAG=${saved_revision} bash" fi + current_agent="${NEMOCLAW_AGENT:-openclaw}" + if [[ "$current_agent" != "$saved_agent" ]]; then + error "DGX Station express resume requires NEMOCLAW_AGENT=${saved_agent}. Rerun the exact command printed after host preparation." + fi + NEMOCLAW_SANDBOX_NAME="$saved_sandbox" + NEMOCLAW_POLICY_TIER="$saved_policy_tier" _STATION_EXPRESS_RESUME_LOADED=1 export NEMOCLAW_VLLM_MODEL + export NEMOCLAW_SANDBOX_NAME + export NEMOCLAW_POLICY_TIER export NEMOCLAW_STATION_EXPRESS_RECEIPT_GENERATION="$_STATION_EXPRESS_RESUME_GENERATION" } save_station_express_resume() { - local state_file state_dir temp_file revision generation model="${NEMOCLAW_VLLM_MODEL:-}" + local state_file state_dir temp_file revision generation + local model="${NEMOCLAW_VLLM_MODEL:-}" agent="${NEMOCLAW_AGENT:-openclaw}" + local sandbox="${NEMOCLAW_SANDBOX_NAME:-my-assistant}" policy_tier="${NEMOCLAW_POLICY_TIER:-balanced}" validate_station_express_resume_model "$model" || error "Cannot save an invalid DGX Station express model selector." + validate_station_express_resume_agent "$agent" || error "Cannot save an invalid DGX Station express agent." + validate_station_express_resume_sandbox "$sandbox" || error "Cannot save an invalid DGX Station express sandbox name." + validate_station_express_resume_policy_tier "$policy_tier" || error "Cannot save an invalid DGX Station express policy tier." revision="$(station_installer_revision)" state_file="$(station_express_resume_file)" || error "Could not resolve NemoClaw state for DGX Station express resume." state_dir="$(ensure_nemoclaw_state_dir)" || error "Could not prepare NemoClaw state for DGX Station express resume." @@ -3285,7 +3339,8 @@ save_station_express_resume() { rm -f "$temp_file" error "Could not secure DGX Station express resume state under ${state_dir}." } - if ! printf 'revision=%s\nmodel=%s\ngeneration=%s\n' "$revision" "$model" "$generation" >"$temp_file"; then + if ! printf 'revision=%s\nmodel=%s\ngeneration=%s\nagent=%s\nsandbox=%s\npolicy_tier=%s\n' \ + "$revision" "$model" "$generation" "$agent" "$sandbox" "$policy_tier" >"$temp_file"; then rm -f "$temp_file" error "Could not write DGX Station express resume state under ${state_dir}." fi @@ -3296,6 +3351,15 @@ save_station_express_resume() { assert_station_express_resume_file_safe "$state_file" _STATION_EXPRESS_RESUME_REVISION="$revision" _STATION_EXPRESS_RESUME_GENERATION="$generation" + _STATION_EXPRESS_RESUME_AGENT="$agent" + _STATION_EXPRESS_RESUME_SANDBOX="$sandbox" + _STATION_EXPRESS_RESUME_POLICY_TIER="$policy_tier" +} + +station_express_resume_command() { + printf 'curl -fsSL https://www.nvidia.com/nemoclaw.sh | NEMOCLAW_INSTALL_TAG=%s NEMOCLAW_AGENT=%s NEMOCLAW_SANDBOX_NAME=%s NEMOCLAW_POLICY_TIER=%s bash' \ + "$_STATION_EXPRESS_RESUME_REVISION" "$_STATION_EXPRESS_RESUME_AGENT" \ + "$_STATION_EXPRESS_RESUME_SANDBOX" "$_STATION_EXPRESS_RESUME_POLICY_TIER" } clear_station_express_resume() { @@ -3390,11 +3454,17 @@ run_station_host_preparation() { ensure_station_express_host() { [[ "${_SELECTED_EXPRESS_PLATFORM:-}" == "DGX Station" ]] || return 0 - if [[ "$(classify_dgx_station_release)" == "supported-dgx-os" ]]; then - info "Validating stock DGX OS GPU device visibility and the local container runtime contract. Host packages and runtime configuration will not be changed." - else - info "Checking pinned DGX Station host prerequisites. Exact matches are reused." - fi + local release_state + release_state="$(classify_dgx_station_release)" + case "$release_state" in + supported-dgx-os | supported-ai-developer-tools) + info "Validating the Station factory GPU and local container runtime. Host packages and runtime configuration will not be changed." + ;; + supported-colossus-baseos) + info "Validating the pinned BaseOS package inventory and preparing Docker access and CDI. Host packages and the NVIDIA driver will not be changed." + ;; + *) info "Checking pinned DGX Station host prerequisites. Exact matches are reused." ;; + esac local status=0 run_station_host_preparation || status=$? case "$status" in @@ -3403,14 +3473,19 @@ ensure_station_express_host() { ;; 10) save_station_express_resume - local revision - revision="${_STATION_EXPRESS_RESUME_REVISION}" warn "DGX Station host prerequisites were installed and require a reboot." info "Run: sudo reboot" info "After signing in again, rerun the accepted revision:" - info "curl -fsSL https://www.nvidia.com/nemoclaw.sh | NEMOCLAW_INSTALL_TAG=${revision} bash" + info "$(station_express_resume_command)" exit 10 ;; + 11) + save_station_express_resume + warn "Docker access was granted and requires a new login session. A reboot is not required." + info "After signing in again, rerun the accepted revision:" + info "$(station_express_resume_command)" + exit 11 + ;; *) error "DGX Station host preparation failed. Review the station-bootstrap log above, correct the reported host state, and rerun the installer." ;; @@ -3466,11 +3541,20 @@ describe_express_install() { inference_summary="managed local vLLM with NVIDIA Nemotron 3 Ultra 550B" inference_disclosure="Managed vLLM pulls the pinned Station image and approximately 352 GB model, then runs a local inference container." fi - if [[ "$(classify_dgx_station_release)" == "supported-dgx-os" ]]; then - printf " Stock DGX OS setup reuses the factory driver and container stack after local GPU device-visibility, CDI, Docker, and Buildx validation. It does not install or replace host packages or rewrite the Docker runtime.\n" - else - printf " Station host setup reuses exact prerequisite versions, applies the reviewed factory DKMS transition when present, installs missing pinned driver, Docker, and NVIDIA Container Toolkit packages, and may require one reboot.\n" - fi + case "$(classify_dgx_station_release)" in + supported-dgx-os) + printf " Stock DGX OS setup reuses the factory driver and container stack after local GPU device-visibility, CDI, Docker, and Buildx validation. It does not install or replace host packages or rewrite the Docker runtime.\n" + ;; + supported-colossus-baseos) + printf " Qualified BaseOS setup preserves the factory kernel, driver, DKMS, Docker, and NVIDIA Container Toolkit packages. It verifies their exact inventory, prepares Docker access and packaged CDI, and registers the NVIDIA Docker runtime only when the launch probe proves it is missing, with rollback on failure. It does not install host packages or require a reboot.\n" + ;; + supported-ai-developer-tools) + printf " Factory Ubuntu with NVIDIA AI Developer Tools reuses its driver and container stack after local GPU, CDI, Docker, and Buildx validation. It may add this account to the Docker group, but does not install or replace host packages, rewrite the Docker runtime, or require a reboot.\n" + ;; + *) + printf " Station host setup reuses exact prerequisite versions, applies the reviewed factory DKMS transition when present, installs missing pinned driver, Docker, and NVIDIA Container Toolkit packages, and may require one reboot.\n" + ;; + esac printf " Host setup may add this trusted local account to the docker group, which grants root-equivalent control. This flow is only for trusted single-user development hosts; shared or managed hosts require an organization-approved Docker access path.\n" printf " DGX Station remains Deferred; one DGX OS 7.5 GB300 physical validation passed, with repeat clean-host qualification and CI coverage still pending.\n" sandbox_summary="${NEMOCLAW_SANDBOX_NAME:-my-assistant}" diff --git a/scripts/prepare-dgx-station-host.sh b/scripts/prepare-dgx-station-host.sh index eb46a7e225c..d7093894d11 100755 --- a/scripts/prepare-dgx-station-host.sh +++ b/scripts/prepare-dgx-station-host.sh @@ -5,8 +5,9 @@ set -Eeuo pipefail umask 077 -readonly SCRIPT_VERSION="2026-07-17.3" +readonly SCRIPT_VERSION="2026-07-17.4" readonly REBOOT_REQUIRED_EXIT=10 +readonly LOGIN_REQUIRED_EXIT=11 readonly MIN_FREE_KIB=$((20 * 1024 * 1024)) readonly GB300_PCI_VENDOR="0x10de" readonly GB300_PCI_DEVICE="0x31c2" @@ -28,6 +29,7 @@ readonly DOCKER_KEY_SHA256="1500c1f56fa9e26b9b8f42452a553675796ade0807cdce11975e readonly DOCKER_KEY_FINGERPRINT="9DC858229FC7DD38854AE2D88D81803C0EBFCD88" readonly DRIVER_VERSION="610.43.02" +readonly BASEOS_DRIVER_VERSION="595.58.03" readonly DOCKER_VERSION="29.6.1" readonly TOOLKIT_VERSION="1.19.1" readonly FACTORY_DKMS_VERSION="3.0.11-1ubuntu13" @@ -54,6 +56,43 @@ readonly -a PACKAGE_SPECS=( "nvidia-container-toolkit-base=1.19.1-1" ) +readonly -a BASEOS_PACKAGE_SPECS=( + "dgx-release=7.5.0" + "dgx-repo=25.10-2" + "dgxstation-desktop=25.11-1" + "dgxstation-grub=25.02-1" + "dkms=3.2.2-1" + "nvidia-driver-595-open=595.58.03-0ubuntu0.24.04.1" + "containerd.io=2.2.1-1~ubuntu.24.04~noble" + "docker-buildx-plugin=0.31.1-1~ubuntu.24.04~noble" + "docker-ce=5:29.2.1-1~ubuntu.24.04~noble" + "docker-ce-cli=5:29.2.1-1~ubuntu.24.04~noble" + "libnvidia-container-tools=1.19.0-1" + "libnvidia-container1=1.19.0-1" + "nvidia-container-toolkit=1.19.0-1" + "nvidia-container-toolkit-base=1.19.0-1" + "cloud-init=25.3-0ubuntu1~24.04.1" + "fluent-bit=4.2.1" + "fwupd=1.9.33-0ubuntu1~24.04.1ubuntu1" + "sssd-common=2.9.4-1.1ubuntu6.4" +) + +readonly BASEOS_CLOUD_CFG_SHA256="038ba435093de59f4a21021caf6c921d63344e9aae3b88795ee5b2659f43f437" +readonly BASEOS_CLOUD_INIT_UNIT_SHA256="e13dd95a7bfac6407ea1ce45ed6683c0f4e84c791840d305c937d38ae77d9456" +readonly BASEOS_FLUENT_BIT_UNIT_SHA256="1854339f563e518894c156d081912595d2d6e175a1ed6692e74e88224b6bad5f" +readonly BASEOS_FLUENT_BIT_CFG_SHA256="bb380bf6103957cdd7440dfba60107b7a3f50db3c0e75c483a6bfdb5e046201c" +readonly BASEOS_FLUENT_BIT_PARSERS_SHA256="760e6a347874a6cbdc10c6cd21d82d1ee5388c8573ddfaab05ef37904749dbe1" +readonly BASEOS_FLUENT_BIT_PLUGINS_SHA256="9d5aad2c1be151b4d35de53a460f9783f98ac3cc815ebc638b0e8489f4ecd577" +readonly BASEOS_FWUPD_UNIT_SHA256="835e7c291761c247d3cd5c64652b768c6a7fdc7cc72fea1bf70fc92e4cb3cfd5" +readonly BASEOS_FWUPD_CFG_SHA256="a25bd457c86be85a286cd175d94e30fa152eb119c95b2a7db8a495886cdd7654" +readonly BASEOS_FWUPD_LVFS_TESTING_SHA256="f50a44def594f256a8192c1d048e08aa94f0287de804262f680b73fa62d97787" +readonly BASEOS_FWUPD_LVFS_SHA256="c4e62d855e41dbf777972b4249da5b2b968fc723e8c7d0d55f932ba47764e98c" +readonly BASEOS_FWUPD_VENDOR_SHA256="0f5a62990f2ddb1681349c01373b3208131e5254a0e734e6090c249c5af9a73f" +readonly BASEOS_SSSD_AUTOFS_UNIT_SHA256="d1be2c2c33e1591ac2fa0bf656bf8dc3d52083a7e9569902e777aea827baeb1f" +readonly BASEOS_SSSD_NSS_UNIT_SHA256="bd432f92436f5c1c142c5824fce66aded6b8be80db4fdfbbed60f222c3a97d9e" +readonly BASEOS_SSSD_PAM_UNIT_SHA256="6760940940471d5bb1b09652b1632db1251b90c3a028efcf11a1b731bb0ab43c" +readonly BASEOS_SSSD_PAM_PRIV_UNIT_SHA256="851fc28d7ab5ac38cd56fcad1f4125cfeb46e8ee62bbf6cbc6376c592faeb51a" + dgx_station_release_path() { printf '%s' /etc/dgx-release } @@ -142,19 +181,44 @@ dgx_station_release_value() { printf '%s' "$value" } -dgx_station_release_contents_are_supported() { - local path=$1 pretty version platform +dgx_station_release_profile() { + local path=$1 ota_pretty="" pretty version build_date platform dgx_station_release_schema_is_valid "$path" || return 1 - pretty="$(dgx_station_release_value "$path" DGX_OTA_PRETTY_NAME)" || return 1 - [[ "$pretty" == "DGX OS" ]] || return 1 - version="$(dgx_station_release_value "$path" DGX_OTA_VERSION)" || return 1 platform="$(dgx_station_release_value "$path" DGX_PLATFORM)" || return 1 + [[ "$platform" == "DGX Server for GALAXY-GB300" ]] || return 1 + + if ota_pretty="$(dgx_station_release_value "$path" DGX_OTA_PRETTY_NAME 2>/dev/null)"; then + [[ "$ota_pretty" == "DGX OS" ]] || return 1 + version="$(dgx_station_release_value "$path" DGX_OTA_VERSION)" || return 1 + case "$version" in + 7.2.0 | 7.4.0 | 7.5.0) printf '%s' supported-dgx-os ;; + *) return 1 ;; + esac + return 0 + fi - case "$version" in - 7.2.0 | 7.4.0 | 7.5.0) ;; + # No-OTA factory images are separate, exact profiles. Do not infer support + # merely from a missing OTA identity: internal BaseOS and customer images + # use different software stacks and qualification evidence. + dgx_station_release_value "$path" DGX_OTA_VERSION >/dev/null 2>&1 && return 1 + dgx_station_release_value "$path" DGX_OTA_DATE >/dev/null 2>&1 && return 1 + pretty="$(dgx_station_release_value "$path" DGX_PRETTY_NAME)" || return 1 + version="$(dgx_station_release_value "$path" DGX_SWBUILD_VERSION)" || return 1 + build_date="$(dgx_station_release_value "$path" DGX_SWBUILD_DATE)" || return 1 + + case "${pretty}|${version}|${build_date}" in + "NVIDIA DGX Server|7.5.0-GB300ws-GB200ws|2026-04-02-08-20-16") + printf '%s' supported-colossus-baseos + ;; + "NVIDIA DGX GB300WS|7.5.0|2026-06-16-11-48-10") + printf '%s' supported-ai-developer-tools + ;; *) return 1 ;; esac - [[ "$platform" == "DGX Server for GALAXY-GB300" ]] +} + +dgx_station_release_contents_are_supported() { + dgx_station_release_profile "$1" >/dev/null } dgx_station_release_is_supported() { @@ -164,11 +228,12 @@ dgx_station_release_is_supported() { } dgx_station_release_state() { - local path=${1:-"$(dgx_station_release_path)"} + local path=${1:-"$(dgx_station_release_path)"} profile if [[ ! -e "$path" && ! -L "$path" ]]; then printf '%s' generic-ubuntu - elif dgx_station_release_is_supported "$path"; then - printf '%s' supported-dgx-os + elif dgx_station_release_file_is_safe "$path" \ + && profile="$(dgx_station_release_profile "$path")"; then + printf '%s' "$profile" else printf '%s' unsupported-dgx-os fi @@ -179,6 +244,7 @@ LOG_FILE="" DOCKER_GROUP_ADDED=0 CDI_LIFECYCLE_READY=0 NETWORK_VALIDATED=0 +GPU_ROWS_ERROR="" info() { printf '[station-prepare] %s %s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$*" @@ -211,6 +277,8 @@ Usage: prepare-dgx-station-host.sh --check|--apply|--verify Exit 10 from --apply means an operator-controlled reboot is required. After the reboot, run --apply once more, followed by --verify. +Exit 11 means Docker-group membership was added. Start a new login session and +run --apply again; a reboot is not required. EOF } @@ -294,16 +362,100 @@ sssd_socket_failure_is_qualified() { [[ ! -e /etc/sssd/sssd.conf && ! -L /etc/sssd/sssd.conf ]] } +file_sha256_matches() { + local path=$1 expected=$2 actual + root_owned_file_is_not_writable_by_group_or_other "$path" || return 1 + actual="$(sha256sum "$path" 2>/dev/null | awk '{print $1}')" + [[ "$actual" == "$expected" ]] +} + +systemd_property_matches() { + local unit=$1 property=$2 expected=$3 actual + actual="$(systemctl show "$unit" -p "$property" --value 2>/dev/null)" || return 1 + [[ "$actual" == "$expected" ]] +} + +baseos_failed_unit_matches() { + local unit=$1 fragment=$2 unit_hash=$3 unit_state=$4 exec_status=${5:-} + systemd_property_matches "$unit" LoadState loaded \ + && systemd_property_matches "$unit" ActiveState failed \ + && systemd_property_matches "$unit" SubState failed \ + && systemd_property_matches "$unit" Result exit-code \ + && systemd_property_matches "$unit" FragmentPath "$fragment" \ + && systemd_property_matches "$unit" UnitFileState "$unit_state" \ + && file_sha256_matches "$fragment" "$unit_hash" \ + || return 1 + [[ -z "$exec_status" ]] || systemd_property_matches "$unit" ExecMainStatus "$exec_status" +} + +baseos_cloud_init_failure_is_qualified() { + local result=/run/cloud-init/result.json + ((NETWORK_VALIDATED == 1)) \ + && baseos_failed_unit_matches cloud-init.service \ + /usr/lib/systemd/system/cloud-init.service "$BASEOS_CLOUD_INIT_UNIT_SHA256" enabled 1 \ + && file_sha256_matches /etc/cloud/cloud.cfg "$BASEOS_CLOUD_CFG_SHA256" \ + && root_owned_file_is_not_writable_by_group_or_other "$result" \ + && grep -Fq '"datasource": "DataSourceConfigDrive ' "$result" \ + && grep -Fq "\"('bootcmd', ProcessExecutionError(" "$result" +} + +baseos_fluent_bit_failure_is_qualified() { + baseos_failed_unit_matches fluent-bit.service \ + /usr/lib/systemd/system/fluent-bit.service "$BASEOS_FLUENT_BIT_UNIT_SHA256" enabled 1 \ + && file_sha256_matches /etc/fluent-bit/fluent-bit.conf "$BASEOS_FLUENT_BIT_CFG_SHA256" \ + && file_sha256_matches /etc/fluent-bit/parsers.conf "$BASEOS_FLUENT_BIT_PARSERS_SHA256" \ + && file_sha256_matches /etc/fluent-bit/plugins.conf "$BASEOS_FLUENT_BIT_PLUGINS_SHA256" +} + +baseos_fwupd_failure_is_qualified() { + ((NETWORK_VALIDATED == 1)) \ + && baseos_failed_unit_matches fwupd-refresh.service \ + /usr/lib/systemd/system/fwupd-refresh.service "$BASEOS_FWUPD_UNIT_SHA256" static 1 \ + && file_sha256_matches /etc/fwupd/fwupd.conf "$BASEOS_FWUPD_CFG_SHA256" \ + && file_sha256_matches /etc/fwupd/remotes.d/lvfs-testing.conf "$BASEOS_FWUPD_LVFS_TESTING_SHA256" \ + && file_sha256_matches /etc/fwupd/remotes.d/lvfs.conf "$BASEOS_FWUPD_LVFS_SHA256" \ + && file_sha256_matches /etc/fwupd/remotes.d/vendor-directory.conf "$BASEOS_FWUPD_VENDOR_SHA256" +} + +baseos_sssd_socket_failure_is_qualified() { + local unit=$1 hash + [[ ! -e /etc/sssd/sssd.conf && ! -L /etc/sssd/sssd.conf ]] || return 1 + case "$unit" in + sssd-autofs.socket) hash="$BASEOS_SSSD_AUTOFS_UNIT_SHA256" ;; + sssd-nss.socket) hash="$BASEOS_SSSD_NSS_UNIT_SHA256" ;; + sssd-pam.socket) hash="$BASEOS_SSSD_PAM_UNIT_SHA256" ;; + sssd-pam-priv.socket) hash="$BASEOS_SSSD_PAM_PRIV_UNIT_SHA256" ;; + *) return 1 ;; + esac + baseos_failed_unit_matches "$unit" "/usr/lib/systemd/system/${unit}" "$hash" enabled +} + is_qualified_factory_failed_unit() { - [[ "$STATION_HOST_PROFILE" == "generic-ubuntu" ]] || return 1 - case "${1:-}" in - cloud-init.service) cloud_init_failure_is_qualified ;; - NetworkManager-wait-online.service | systemd-networkd-wait-online.service) - network_wait_failure_is_qualified + case "$STATION_HOST_PROFILE" in + generic-ubuntu) + case "${1:-}" in + cloud-init.service) cloud_init_failure_is_qualified ;; + NetworkManager-wait-online.service | systemd-networkd-wait-online.service) + network_wait_failure_is_qualified + ;; + fwupd-refresh.service) fwupd_refresh_failure_is_qualified ;; + sssd-autofs.socket | sssd-nss.socket | sssd-pam.socket | sssd-pam-priv.socket) + sssd_socket_failure_is_qualified + ;; + *) return 1 ;; + esac ;; - fwupd-refresh.service) fwupd_refresh_failure_is_qualified ;; - sssd-autofs.socket | sssd-nss.socket | sssd-pam.socket | sssd-pam-priv.socket) - sssd_socket_failure_is_qualified + colossus-baseos) + all_baseos_packages_exact || return 1 + case "${1:-}" in + cloud-init.service) baseos_cloud_init_failure_is_qualified ;; + fluent-bit.service) baseos_fluent_bit_failure_is_qualified ;; + fwupd-refresh.service) baseos_fwupd_failure_is_qualified ;; + sssd-autofs.socket | sssd-nss.socket | sssd-pam.socket | sssd-pam-priv.socket) + baseos_sssd_socket_failure_is_qualified "$1" + ;; + *) return 1 ;; + esac ;; *) return 1 ;; esac @@ -386,6 +538,29 @@ all_packages_exact() { return 0 } +all_baseos_packages_exact() { + local spec + for spec in "${BASEOS_PACKAGE_SPECS[@]}"; do + package_is_exact "$spec" || return 1 + done + return 0 +} + +verify_baseos_packages() { + local spec + for spec in "${BASEOS_PACKAGE_SPECS[@]}"; do + package_is_exact "$spec" || fatal "BaseOS package does not match the qualified image: ${spec}" + done + info "baseos_packages=exact" +} + +station_uses_factory_runtime() { + case "$STATION_HOST_PROFILE" in + stock-dgx-os | colossus-baseos | ai-developer-tools) return 0 ;; + *) return 1 ;; + esac +} + setup_log() { local log_dir="${HOME}/station-bootstrap-logs" mkdir -p "$log_dir" @@ -427,11 +602,13 @@ check_platform() { STATION_HOST_PROFILE="generic-ubuntu" ;; supported-dgx-os) STATION_HOST_PROFILE="stock-dgx-os" ;; + supported-colossus-baseos) STATION_HOST_PROFILE="colossus-baseos" ;; + supported-ai-developer-tools) STATION_HOST_PROFILE="ai-developer-tools" ;; *) - fatal "This DGX Station OS image is outside the validated boundary; use generic Ubuntu 24.04 ARM64 or stock DGX OS 7.2.0, 7.4.0, or 7.5.0 on Station GB300" + fatal "This DGX Station OS image is outside the validated boundary" ;; esac - info "platform=${product} profile=${STATION_HOST_PROFILE} os=${PRETTY_NAME} arch=${arch} kernel=$(uname -r)" + info "platform=${product} profile=${STATION_HOST_PROFILE} release=${release_state} os=${PRETTY_NAME} arch=${arch} kernel=$(uname -r)" } check_secure_boot() { @@ -474,9 +651,9 @@ check_package_managers_idle() { check_dgx_os_docker_selection() { [[ -z "${DOCKER_HOST:-}" ]] \ - || fatal "Stock DGX OS validation requires the local Docker daemon; unset DOCKER_HOST and rerun" + || fatal "Station factory-runtime validation requires the local Docker daemon; unset DOCKER_HOST and rerun" [[ -z "${DOCKER_CONTEXT:-}" || "${DOCKER_CONTEXT}" == "default" ]] \ - || fatal "Stock DGX OS validation requires the default local Docker context; unset DOCKER_CONTEXT and rerun" + || fatal "Station factory-runtime validation requires the default local Docker context; unset DOCKER_CONTEXT and rerun" } station_local_default_docker() ( @@ -489,7 +666,7 @@ station_sudo_local_default_docker() { } host_docker() { - if [[ "$STATION_HOST_PROFILE" == "stock-dgx-os" ]]; then + if station_uses_factory_runtime; then station_local_default_docker "$@" else docker "$@" @@ -497,7 +674,7 @@ host_docker() { } host_docker_sudo() { - if [[ "$STATION_HOST_PROFILE" == "stock-dgx-os" ]]; then + if station_uses_factory_runtime; then station_sudo_local_default_docker "$@" else sudo -n docker "$@" @@ -505,7 +682,7 @@ host_docker_sudo() { } check_failed_units() { - local unit failed_output blocking=0 + local unit failed_output blocking=0 qualified_label local -a units=() failed_output="$(systemctl --failed --no-legend --plain 2>/dev/null)" \ || fatal "Unable to inspect failed system services" @@ -523,7 +700,12 @@ check_failed_units() { warn "failed preparation-critical unit: ${unit}" blocking=1 elif is_qualified_factory_failed_unit "$unit"; then - warn "condition-qualified generic-image failed unit: ${unit}" + if [[ "$STATION_HOST_PROFILE" == "generic-ubuntu" ]]; then + qualified_label="generic-image" + else + qualified_label="$STATION_HOST_PROFILE" + fi + warn "condition-qualified ${qualified_label} failed unit: ${unit}" else warn "unqualified failed unit: ${unit}" blocking=1 @@ -665,12 +847,15 @@ common_preflight() { require_command stat require_command systemctl check_platform - if [[ "$STATION_HOST_PROFILE" == "generic-ubuntu" ]]; then + if station_uses_factory_runtime; then + info "factory_packages=preserved package_and_driver_mutation=disabled" + check_dgx_os_docker_selection + if [[ "$STATION_HOST_PROFILE" == "colossus-baseos" ]]; then + verify_baseos_packages + fi + else check_secure_boot check_kernel_headers - else - info "factory_runtime=validation_only package_and_driver_mutation=disabled" - check_dgx_os_docker_selection fi check_capacity check_network @@ -846,7 +1031,7 @@ install_packages() { ensure_docker_group() { local user_name=${SUDO_USER:-$USER} - getent group docker >/dev/null 2>&1 || fatal "Docker group is missing after package installation" + getent group docker >/dev/null 2>&1 || fatal "Docker group is missing on the Station host" if ! id -nG "$user_name" | tr ' ' '\n' | grep -Fxq docker; then sudo usermod -aG docker "$user_name" DOCKER_GROUP_ADDED=1 @@ -903,11 +1088,41 @@ ensure_acceptance_image() { } run_cdi_test_sudo() { - sudo docker run --rm --device nvidia.com/gpu=all "$ACCEPTANCE_IMAGE" nvidia-smi + local rows + rows="$(sudo docker run --rm --device nvidia.com/gpu=all "$ACCEPTANCE_IMAGE" nvidia-smi \ + --query-gpu=name,driver_version,ecc.errors.corrected.volatile.total,ecc.errors.uncorrected.volatile.total \ + --format=csv,noheader,nounits)" || return 1 + gpu_rows_are_valid "$rows" || { + warn "CDI container probe did not expose the qualified GB300: ${GPU_ROWS_ERROR}" + return 1 + } } run_gpus_test_sudo() { - sudo docker run --rm --gpus all "$ACCEPTANCE_IMAGE" nvidia-smi + local rows + rows="$(sudo docker run --rm --gpus all "$ACCEPTANCE_IMAGE" nvidia-smi \ + --query-gpu=name,driver_version,ecc.errors.corrected.volatile.total,ecc.errors.uncorrected.volatile.total \ + --format=csv,noheader,nounits)" || return 1 + gpu_rows_are_valid "$rows" || { + warn "Docker --gpus container probe did not expose the qualified GB300: ${GPU_ROWS_ERROR}" + return 1 + } +} + +run_cdi_test_user() { + local rows + rows="$(docker run --rm --device nvidia.com/gpu=all "$ACCEPTANCE_IMAGE" nvidia-smi \ + --query-gpu=name,driver_version,ecc.errors.corrected.volatile.total,ecc.errors.uncorrected.volatile.total \ + --format=csv,noheader,nounits)" || return 1 + gpu_rows_are_valid "$rows" +} + +run_gpus_test_user() { + local rows + rows="$(docker run --rm --gpus all "$ACCEPTANCE_IMAGE" nvidia-smi \ + --query-gpu=name,driver_version,ecc.errors.corrected.volatile.total,ecc.errors.uncorrected.volatile.total \ + --format=csv,noheader,nounits)" || return 1 + gpu_rows_are_valid "$rows" } ensure_dgx_os_acceptance_image() { @@ -918,11 +1133,41 @@ ensure_dgx_os_acceptance_image() { } run_dgx_os_cdi_test_sudo() { - station_sudo_local_default_docker run --rm --device nvidia.com/gpu=all "$ACCEPTANCE_IMAGE" nvidia-smi + local rows + rows="$(station_sudo_local_default_docker run --rm --device nvidia.com/gpu=all "$ACCEPTANCE_IMAGE" nvidia-smi \ + --query-gpu=name,driver_version,ecc.errors.corrected.volatile.total,ecc.errors.uncorrected.volatile.total \ + --format=csv,noheader,nounits)" || return 1 + gpu_rows_are_valid "$rows" || { + warn "Factory-runtime CDI probe did not expose the qualified GB300: ${GPU_ROWS_ERROR}" + return 1 + } } run_dgx_os_gpus_test_sudo() { - station_sudo_local_default_docker run --rm --gpus all "$ACCEPTANCE_IMAGE" nvidia-smi + local rows + rows="$(station_sudo_local_default_docker run --rm --gpus all "$ACCEPTANCE_IMAGE" nvidia-smi \ + --query-gpu=name,driver_version,ecc.errors.corrected.volatile.total,ecc.errors.uncorrected.volatile.total \ + --format=csv,noheader,nounits)" || return 1 + gpu_rows_are_valid "$rows" || { + warn "Factory-runtime Docker --gpus probe did not expose the qualified GB300: ${GPU_ROWS_ERROR}" + return 1 + } +} + +run_dgx_os_cdi_test_user() { + local rows + rows="$(station_local_default_docker run --rm --device nvidia.com/gpu=all "$ACCEPTANCE_IMAGE" nvidia-smi \ + --query-gpu=name,driver_version,ecc.errors.corrected.volatile.total,ecc.errors.uncorrected.volatile.total \ + --format=csv,noheader,nounits)" || return 1 + gpu_rows_are_valid "$rows" +} + +run_dgx_os_gpus_test_user() { + local rows + rows="$(station_local_default_docker run --rm --gpus all "$ACCEPTANCE_IMAGE" nvidia-smi \ + --query-gpu=name,driver_version,ecc.errors.corrected.volatile.total,ecc.errors.uncorrected.volatile.total \ + --format=csv,noheader,nounits)" || return 1 + gpu_rows_are_valid "$rows" } docker_has_nvidia_runtime_sudo() { @@ -1037,50 +1282,58 @@ check_dgx_os_runtime_commands() { require_command docker require_command nvidia-ctk require_command nvidia-smi - driver_is_loaded || fatal "Stock DGX OS did not expose a loaded NVIDIA driver" + driver_is_loaded || fatal "The Station factory image did not expose a loaded NVIDIA driver" verify_gpu - info "dgx_os_runtime_commands=present" + info "factory_runtime_commands=present" } verify_dgx_os_runtime_sudo() { check_dgx_os_runtime_commands - systemctl is-active --quiet containerd.service || fatal "containerd.service is not active on stock DGX OS" - systemctl is-active --quiet docker.service || fatal "docker.service is not active on stock DGX OS" + systemctl is-active --quiet containerd.service || fatal "containerd.service is not active on the Station factory image" + systemctl is-active --quiet docker.service || fatal "docker.service is not active on the Station factory image" station_sudo_local_default_docker info >/dev/null 2>&1 \ - || fatal "The local Docker daemon is not reachable with sudo on stock DGX OS" + || fatal "The local Docker daemon is not reachable with sudo on the Station factory image" station_sudo_local_default_docker buildx version >/dev/null 2>&1 \ - || fatal "Docker Buildx is unavailable on stock DGX OS" + || fatal "Docker Buildx is unavailable on the Station factory image" sudo nvidia-ctk cdi list | grep -Fxq 'nvidia.com/gpu=all' \ - || fatal "Stock DGX OS does not advertise the nvidia.com/gpu=all CDI device" + || fatal "The Station factory image does not advertise the nvidia.com/gpu=all CDI device" ensure_dgx_os_acceptance_image run_dgx_os_cdi_test_sudo \ - || fatal "Stock DGX OS failed the CDI Docker GPU visibility test; no host runtime configuration was changed" + || fatal "The Station factory image failed the CDI Docker GPU visibility test" run_dgx_os_gpus_test_sudo \ - || fatal "Stock DGX OS failed the Docker --gpus all GPU visibility test; no host runtime configuration was changed" + || fatal "The Station factory image failed the Docker --gpus all GPU visibility test" [[ -z "$(station_sudo_local_default_docker ps -aq)" ]] \ - || fatal "DGX OS acceptance tests left a Docker container behind" - info "DGX_OS_HOST_READY host_runtime_mutation=container_image_cache_only" + || fatal "Station factory-image acceptance tests left a Docker container behind" + if [[ "$STATION_HOST_PROFILE" == "stock-dgx-os" ]]; then + info "DGX_OS_HOST_READY host_runtime_mutation=container_image_cache_only" + else + info "STATION_FACTORY_HOST_READY" + fi } verify_dgx_os_runtime_user() { check_dgx_os_runtime_commands - systemctl is-active --quiet containerd.service || fatal "containerd.service is not active on stock DGX OS" - systemctl is-active --quiet docker.service || fatal "docker.service is not active on stock DGX OS" + systemctl is-active --quiet containerd.service || fatal "containerd.service is not active on the Station factory image" + systemctl is-active --quiet docker.service || fatal "docker.service is not active on the Station factory image" station_local_default_docker info >/dev/null 2>&1 \ || fatal "The current user cannot access the local Docker daemon; run --apply first" station_local_default_docker buildx version >/dev/null 2>&1 \ - || fatal "Docker Buildx is unavailable on stock DGX OS" + || fatal "Docker Buildx is unavailable on the Station factory image" nvidia-ctk cdi list | grep -Fxq 'nvidia.com/gpu=all' \ - || fatal "Stock DGX OS does not advertise the nvidia.com/gpu=all CDI device" + || fatal "The Station factory image does not advertise the nvidia.com/gpu=all CDI device" station_local_default_docker image inspect "$ACCEPTANCE_IMAGE" >/dev/null 2>&1 \ || fatal "Digest-pinned acceptance image is missing; run --apply" - station_local_default_docker run --rm --device nvidia.com/gpu=all "$ACCEPTANCE_IMAGE" nvidia-smi >/dev/null \ - || fatal "Stock DGX OS failed the CDI Docker GPU visibility test" - station_local_default_docker run --rm --gpus all "$ACCEPTANCE_IMAGE" nvidia-smi >/dev/null \ - || fatal "Stock DGX OS failed the Docker --gpus all GPU visibility test" + run_dgx_os_cdi_test_user \ + || fatal "The Station factory image failed the CDI Docker GPU visibility test" + run_dgx_os_gpus_test_user \ + || fatal "The Station factory image failed the Docker --gpus all GPU visibility test" [[ -z "$(station_local_default_docker ps -aq)" ]] \ - || fatal "DGX OS verification left a Docker container behind" - info "DGX_OS_HOST_READY" + || fatal "Station factory-image verification left a Docker container behind" + if [[ "$STATION_HOST_PROFILE" == "stock-dgx-os" ]]; then + info "DGX_OS_HOST_READY" + else + info "STATION_FACTORY_HOST_READY" + fi } verify_apply_state() { @@ -1099,11 +1352,13 @@ verify_apply_state() { info "STATION_HOST_READY" } -verify_gpu() { - local rows row name driver corrected uncorrected gpu_count=0 - rows="$(nvidia-smi \ - --query-gpu=name,driver_version,ecc.errors.corrected.volatile.total,ecc.errors.uncorrected.volatile.total \ - --format=csv,noheader,nounits)" || fatal "nvidia-smi failed" +gpu_rows_are_valid() { + local rows=$1 row name driver corrected uncorrected row_index=0 gb300_count=0 expected_driver="" + GPU_ROWS_ERROR="" + case "$STATION_HOST_PROFILE" in + generic-ubuntu) expected_driver="$DRIVER_VERSION" ;; + colossus-baseos) expected_driver="$BASEOS_DRIVER_VERSION" ;; + esac while IFS= read -r row; do [[ -n "${row//[[:space:]]/}" ]] || continue IFS=',' read -r name driver corrected uncorrected <<<"$row" @@ -1111,18 +1366,39 @@ verify_gpu() { driver="${driver//[[:space:]]/}" corrected="${corrected//[[:space:]]/}" uncorrected="${uncorrected//[[:space:]]/}" - [[ "$name" == *"GB300"* ]] || fatal "Expected NVIDIA GB300, found ${name}" - [[ -n "$driver" ]] || fatal "NVIDIA driver is not loaded" - if [[ "$STATION_HOST_PROFILE" == "generic-ubuntu" ]]; then - [[ "$driver" == "$DRIVER_VERSION" ]] \ - || fatal "Expected driver ${DRIVER_VERSION}, found ${driver}" + if [[ "$name" != *"GB300"* ]]; then + info "gpu_index=${row_index} gpu=${name} role=auxiliary validation=skipped" + ((row_index += 1)) + continue + fi + if [[ -z "$driver" ]]; then + GPU_ROWS_ERROR="NVIDIA driver is not loaded" + return 1 fi - [[ "$corrected" == "0" && "$uncorrected" == "0" ]] \ - || fatal "ECC must be 0/0, found corrected=${corrected} uncorrected=${uncorrected}" - ((gpu_count += 1)) - info "gpu_index=${gpu_count} gpu=${name} driver=${driver} ecc_corrected=${corrected} ecc_uncorrected=${uncorrected}" + if [[ -n "$expected_driver" && "$driver" != "$expected_driver" ]]; then + GPU_ROWS_ERROR="Expected driver ${expected_driver}, found ${driver}" + return 1 + fi + if [[ "$corrected" != "0" || "$uncorrected" != "0" ]]; then + GPU_ROWS_ERROR="ECC must be 0/0, found corrected=${corrected} uncorrected=${uncorrected}" + return 1 + fi + ((gb300_count += 1)) + info "gpu_index=${row_index} gpu=${name} role=inference driver=${driver} ecc_corrected=${corrected} ecc_uncorrected=${uncorrected}" + ((row_index += 1)) done <<<"$rows" - ((gpu_count > 0)) || fatal "nvidia-smi returned no GPU rows" + if ((gb300_count != 1)); then + GPU_ROWS_ERROR="Expected exactly one NVIDIA GB300, found ${gb300_count}" + return 1 + fi +} + +verify_gpu() { + local rows + rows="$(nvidia-smi \ + --query-gpu=name,driver_version,ecc.errors.corrected.volatile.total,ecc.errors.uncorrected.volatile.total \ + --format=csv,noheader,nounits)" || fatal "nvidia-smi failed" + gpu_rows_are_valid "$rows" || fatal "$GPU_ROWS_ERROR" } verify_host() { @@ -1139,8 +1415,8 @@ verify_host() { docker info >/dev/null 2>&1 || fatal "${user_name} cannot access Docker; start a new login session" nvidia-ctk cdi list | grep -Fxq 'nvidia.com/gpu=all' || fatal "CDI verification failed" docker image inspect "$ACCEPTANCE_IMAGE" >/dev/null 2>&1 || fatal "Digest-pinned acceptance image is missing; run --apply" - docker run --rm --device nvidia.com/gpu=all "$ACCEPTANCE_IMAGE" nvidia-smi >/dev/null - docker run --rm --gpus all "$ACCEPTANCE_IMAGE" nvidia-smi >/dev/null + run_cdi_test_user || fatal "CDI verification did not expose the qualified GB300: ${GPU_ROWS_ERROR}" + run_gpus_test_user || fatal "Docker --gpus verification did not expose the qualified GB300: ${GPU_ROWS_ERROR}" [[ -z "$(docker ps -aq)" ]] || fatal "Verification left a Docker container behind" info "docker=$(docker version --format '{{.Server.Version}}') expected_docker=${DOCKER_VERSION} toolkit=$(nvidia-ctk --version | head -n1) expected_toolkit=${TOOLKIT_VERSION}" info "STATION_HOST_READY" @@ -1148,9 +1424,9 @@ verify_host() { run_check() { common_preflight - if [[ "$STATION_HOST_PROFILE" == "stock-dgx-os" ]]; then + if station_uses_factory_runtime; then check_dgx_os_runtime_commands - info "CHECK_RESULT=READY_FOR_RUNTIME_VALIDATION" + info "CHECK_RESULT=READY_FOR_FACTORY_RUNTIME_PREPARATION" return 0 fi print_package_status @@ -1174,10 +1450,21 @@ run_apply() { acquire_sudo common_preflight - if [[ "$STATION_HOST_PROFILE" == "stock-dgx-os" ]]; then + if station_uses_factory_runtime; then [[ ! -e /var/run/reboot-required ]] \ - || fatal "A reboot is pending on stock DGX OS; reboot before running Station express install" + || fatal "A reboot is pending on the Station factory image; reboot before running Station express install" + if [[ "$STATION_HOST_PROFILE" == "colossus-baseos" ]]; then + finish_runtime + fi verify_dgx_os_runtime_sudo + if [[ "$STATION_HOST_PROFILE" != "colossus-baseos" ]]; then + ensure_docker_group + fi + if ((DOCKER_GROUP_ADDED == 1)); then + warn "Docker group membership was added and requires a new login before onboarding" + info "APPLY_RESULT=LOGIN_REQUIRED" + exit "$LOGIN_REQUIRED_EXIT" + fi info "APPLY_RESULT=COMPLETE" return 0 fi @@ -1243,7 +1530,10 @@ run_verify() { require_command docker require_command nvidia-ctk require_command nvidia-smi - if [[ "$STATION_HOST_PROFILE" == "stock-dgx-os" ]]; then + if station_uses_factory_runtime; then + if [[ "$STATION_HOST_PROFILE" == "colossus-baseos" ]]; then + verify_cdi_refresh_lifecycle + fi verify_dgx_os_runtime_user return 0 fi diff --git a/src/lib/inference/vllm.test.ts b/src/lib/inference/vllm.test.ts index 5063953a007..ea4b2dc1234 100644 --- a/src/lib/inference/vllm.test.ts +++ b/src/lib/inference/vllm.test.ts @@ -519,6 +519,14 @@ describe("vLLM run command", () => { expect(flags).not.toContain("device=0,1"); expect(flags).not.toContain(`'"device=0,1"'`); }); + + it("fails closed instead of exposing all GPUs when Station GB300 detection is empty", () => { + mocks.getGpuIndicesByName.mockReturnValue([]); + const profile = detectVllmProfile({ platform: "station", type: "nvidia" }); + + expect(profile).not.toBeNull(); + expect(() => profile!.buildDockerRunFlags!()).toThrow(/requires an NVIDIA GB300 GPU/); + }); }); describe("managed vLLM ownership", () => { @@ -680,6 +688,7 @@ describe("installVllm model resolution", () => { it("installs the complete Nemotron Ultra Station recipe without another selection", async () => { process.env.NEMOCLAW_VLLM_MODEL = "nemotron-3-ultra-550b-a55b"; + mocks.getGpuIndicesByName.mockReturnValue([0]); const profile = detectVllmProfile({ platform: "station", type: "nvidia" })!; const beforeInstall = vi.fn(); const promptFn = vi.fn<(q: string) => Promise>(); diff --git a/src/lib/inference/vllm.ts b/src/lib/inference/vllm.ts index 47ba2ef3fe4..1ba7ac1e478 100644 --- a/src/lib/inference/vllm.ts +++ b/src/lib/inference/vllm.ts @@ -274,14 +274,14 @@ const STATION_PROFILE: VllmProfile = { dockerRunFlags: SPARK_PROFILE.dockerRunFlags, buildDockerRunFlags: () => { const indices = getGpuIndicesByName(/GB300/i); + if (indices.length === 0) { + throw new Error( + "DGX Station managed vLLM requires an NVIDIA GB300 GPU, but none was detected", + ); + } // Docker parses --gpus as CSV, so multi-device values must retain // double quotes inside the argv token to keep the comma in one field. - const gpuFlag = - indices.length === 0 - ? "all" - : indices.length === 1 - ? `device=${indices[0]}` - : `"device=${indices.join(",")}"`; + const gpuFlag = indices.length === 1 ? `device=${indices[0]}` : `"device=${indices.join(",")}"`; return vllmDockerRunFlags(gpuFlag); }, pullTimeoutSec: SPARK_PROFILE.pullTimeoutSec, @@ -658,13 +658,13 @@ function startContainer( model: VllmModelDef, ): { ok: boolean; reason?: string } { emit(`Starting vLLM container (${profile.containerName})`); - const resolvedFlags = profile.buildDockerRunFlags - ? profile.buildDockerRunFlags() - : profile.dockerRunFlags; // The explicit download completed before this long-lived container starts, // so do not retain the host Hugging Face token in the serving process. let runArgs: string[]; try { + const resolvedFlags = profile.buildDockerRunFlags + ? profile.buildDockerRunFlags() + : profile.dockerRunFlags; runArgs = buildVllmRunArgs(profile, model, resolvedFlags); } catch (err) { return { ok: false, reason: (err as Error).message }; diff --git a/test/install-express-prompt.test.ts b/test/install-express-prompt.test.ts index 46e42e6e07c..fe0250adb3e 100644 --- a/test/install-express-prompt.test.ts +++ b/test/install-express-prompt.test.ts @@ -232,6 +232,30 @@ detect_express_platform ].join("\n"); } + function noOtaFactoryRelease(profile: "colossus-baseos" | "ai-developer-tools") { + const identity = + profile === "colossus-baseos" + ? { + pretty: "NVIDIA DGX Server", + version: "7.5.0-GB300ws-GB200ws", + buildDate: "2026-04-02-08-20-16", + } + : { + pretty: "NVIDIA DGX GB300WS", + version: "7.5.0", + buildDate: "2026-06-16-11-48-10", + }; + return [ + 'DGX_NAME="DGX Server"', + `DGX_PRETTY_NAME="${identity.pretty}"`, + `DGX_SWBUILD_DATE="${identity.buildDate}"`, + `DGX_SWBUILD_VERSION="${identity.version}"`, + 'DGX_PLATFORM="DGX Server for GALAXY-GB300"', + 'DGX_SERIAL_NUMBER="host-specific-value"', + "", + ].join("\n"); + } + it("parses and documents the DGX Station DeepSeek override", () => { const result = spawnSync("bash", [INSTALLER_PAYLOAD, "--station-deepseek", "--help"], { cwd: path.join(import.meta.dirname, ".."), @@ -317,6 +341,44 @@ detect_express_platform expect(output).toMatch(/STATION_EXPRESS=1/); }); + it.each([ + [ + "supported-colossus-baseos", + "Qualified BaseOS setup preserves the factory kernel, driver, DKMS, Docker, and NVIDIA Container Toolkit packages", + ], + [ + "supported-ai-developer-tools", + "Factory Ubuntu with NVIDIA AI Developer Tools reuses its driver and container stack", + ], + ])("describes the %s Station mutation boundary before consent", (release, expected) => { + const result = spawnSync( + "bash", + [ + "--noprofile", + "--norc", + "-c", + `source "$INSTALLER_UNDER_TEST" >/dev/null +classify_dgx_station_release() { printf '%s' "$FACTORY_RELEASE"; } +describe_express_install 'DGX Station'`, + ], + { + encoding: "utf-8", + env: { + ...process.env, + HOME: fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-station-consent-")), + PATH: TEST_SYSTEM_PATH, + INSTALLER_UNDER_TEST: INSTALLER_PAYLOAD, + FACTORY_RELEASE: release, + }, + }, + ); + const output = `${result.stdout}${result.stderr}`; + + expect(result.status, output).toBe(0); + expect(output).toContain(expected); + expect(output).not.toContain("installs missing pinned driver"); + }); + it("normalizes the canonical Ultra served alias to the registered model slug", () => { const result = runExpressPromptWithTty("\n", "pipe", "DGX Station", { NEMOCLAW_VLLM_MODEL: "nvidia/nemotron-3-ultra-550b-a55b", @@ -344,6 +406,51 @@ detect_express_platform ); }); + it("preserves complete Station Express intent across a Docker-group relogin", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-station-relogin-")); + const revision = "a".repeat(40); + const generation = "0123456789abcdef0123456789abcdef"; + const result = spawnSync( + "bash", + [ + "--noprofile", + "--norc", + "-c", + `source "$INSTALLER_UNDER_TEST" >/dev/null +_SELECTED_EXPRESS_PLATFORM='DGX Station' +NEMOCLAW_VLLM_MODEL='deepseek-v4-flash' +classify_dgx_station_release() { printf 'supported-ai-developer-tools'; } +station_installer_revision() { printf '${revision}'; } +station_express_resume_generation() { printf '${generation}'; } +run_station_host_preparation() { return 11; } +ensure_station_express_host`, + ], + { + encoding: "utf-8", + env: { + ...process.env, + HOME: home, + PATH: TEST_SYSTEM_PATH, + INSTALLER_UNDER_TEST: INSTALLER_PAYLOAD, + NEMOCLAW_AGENT: "Hermes", + NEMOCLAW_SANDBOX_NAME: "custom-agent", + NEMOCLAW_POLICY_TIER: "restricted", + }, + }, + ); + const output = `${result.stdout}${result.stderr}`; + + expect(result.status, output).toBe(11); + expect(fs.readFileSync(path.join(home, ".nemoclaw", "station-express-resume"), "utf8")).toBe( + `revision=${revision}\nmodel=deepseek-v4-flash\ngeneration=${generation}\n` + + "agent=hermes\nsandbox=custom-agent\npolicy_tier=restricted\n", + ); + expect(output).toContain("A reboot is not required"); + expect(output).toContain( + `NEMOCLAW_INSTALL_TAG=${revision} NEMOCLAW_AGENT=hermes NEMOCLAW_SANDBOX_NAME=custom-agent NEMOCLAW_POLICY_TIER=restricted bash`, + ); + }); + it("allows a matching explicit DeepSeek model with the Station demo override", () => { const result = runExpressPromptWithTty("\n", "pipe", "DGX Station", { STATION_DEEPSEEK: "1", @@ -704,6 +811,19 @@ detect_express_platform expect(result.stdout).toBe("DGX Station"); }); + it.each([ + "colossus-baseos", + "ai-developer-tools", + ] as const)("recognizes the exact no-OTA %s Station profile", (profile) => { + const result = detectExpressPlatformForStockDgxRelease( + "DGX Station GB300", + noOtaFactoryRelease(profile), + ); + + expect(result.status, `${result.stdout}${result.stderr}`).toBe(0); + expect(result.stdout).toBe("DGX Station"); + }); + it.each([ ["unreviewed version", stockDgxRelease("7.6.0")], ["wrong DGX platform", stockDgxRelease("7.5.0", "DGX Server for GALAXY-GB200")], diff --git a/test/install-station-dgx-os.test.ts b/test/install-station-dgx-os.test.ts index 98e632de83d..745ffc7950f 100644 --- a/test/install-station-dgx-os.test.ts +++ b/test/install-station-dgx-os.test.ts @@ -76,6 +76,44 @@ function writeDgxReleaseHistory(historyLines: string[]) { return release; } +function writeNoOtaFactoryRelease( + profile: "colossus-baseos" | "ai-developer-tools", + overrides: Partial<{ pretty: string; version: string; buildDate: string; platform: string }> = {}, +) { + const defaults = + profile === "colossus-baseos" + ? { + pretty: "NVIDIA DGX Server", + version: "7.5.0-GB300ws-GB200ws", + buildDate: "2026-04-02-08-20-16", + } + : { + pretty: "NVIDIA DGX GB300WS", + version: "7.5.0", + buildDate: "2026-06-16-11-48-10", + }; + const fields = { + ...defaults, + platform: "DGX Server for GALAXY-GB300", + ...overrides, + }; + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-station-factory-release-")); + const target = path.join(dir, "dgx-release"); + fs.writeFileSync( + target, + [ + 'DGX_NAME="DGX Server"', + `DGX_PRETTY_NAME="${fields.pretty}"`, + `DGX_SWBUILD_DATE="${fields.buildDate}"`, + `DGX_SWBUILD_VERSION="${fields.version}"`, + `DGX_PLATFORM="${fields.platform}"`, + 'DGX_SERIAL_NUMBER="host-specific-value"', + "", + ].join("\n"), + ); + return target; +} + describe("DGX Station stock DGX OS classification", () => { it.each([ "7.2.0", @@ -92,8 +130,66 @@ describe("DGX Station stock DGX OS classification", () => { expect(result.status, output).toBe(0); }); + it.each([ + ["supported-colossus-baseos", writeNoOtaFactoryRelease("colossus-baseos")], + ["supported-ai-developer-tools", writeNoOtaFactoryRelease("ai-developer-tools")], + ])("accepts the exact no-OTA factory profile as %s", (expected, release) => { + const { result, output } = runSourced( + STATION_PREPARE, + ` +stat() { printf '0|0|644|256\n'; } +dgx_station_release_state "$DGX_RELEASE" +`, + { DGX_RELEASE: release }, + ); + + expect(result.status, output).toBe(0); + expect(result.stdout).toBe(expected); + }); + + it.each([ + [ + "BaseOS build version drift", + writeNoOtaFactoryRelease("colossus-baseos", { version: "7.5.0" }), + ], + [ + "BaseOS build date drift", + writeNoOtaFactoryRelease("colossus-baseos", { buildDate: "2026-04-03-00-00-00" }), + ], + [ + "AI Developer Tools product drift", + writeNoOtaFactoryRelease("ai-developer-tools", { pretty: "NVIDIA DGX Server" }), + ], + [ + "AI Developer Tools build date drift", + writeNoOtaFactoryRelease("ai-developer-tools", { buildDate: "2026-06-17-00-00-00" }), + ], + [ + "factory platform drift", + writeNoOtaFactoryRelease("ai-developer-tools", { + platform: "DGX Server for GALAXY-GB200", + }), + ], + ])("rejects no-OTA factory identity with %s", (_scenario, release) => { + const { result, output } = runSourced( + STATION_PREPARE, + ` +stat() { printf '0|0|644|256\n'; } +dgx_station_release_state "$DGX_RELEASE" +`, + { DGX_RELEASE: release }, + ); + + expect(result.status, output).toBe(0); + expect(result.stdout).toBe("unsupported-dgx-os"); + }); + it.each([ ["unreviewed version", writeDgxReleaseFixture("7.6.0")], + [ + "unproven Station platform identity", + writeDgxReleaseFixture("7.5.0", 'DGX_PLATFORM="Not Specified"'), + ], ["missing DGX_OTA_PRETTY_NAME", writeDgxReleaseFixture("7.5.0", "", null)], ["BaseOS identity", writeDgxReleaseFixture("7.5.0", "", "NVIDIA BaseOS")], ["unknown field", writeDgxReleaseFixture("7.5.0", 'PAYLOAD="$(touch /tmp/nope)"')], @@ -288,12 +384,102 @@ dgx_station_release_state "$DGX_RELEASE" expect(result.status, `${result.stdout}${result.stderr}`).toBe(0); expect(original.status, `${original.stdout}${original.stderr}`).toBe(0); expect(result.stdout).toBe(original.stdout); - expect(result.stdout).toMatch(/^(generic-ubuntu|supported-dgx-os|unsupported-dgx-os)$/); + expect(result.stdout).toMatch( + /^(generic-ubuntu|supported-dgx-os|supported-colossus-baseos|supported-ai-developer-tools|unsupported-dgx-os)$/, + ); expect(result.stderr).toBe(""); }); }); describe("DGX Station stock DGX OS runtime validation", () => { + it("requires the exact qualified BaseOS package inventory", () => { + const exact = runSourced( + STATION_PREPARE, + ` +installed_version() { + local spec + for spec in "\${BASEOS_PACKAGE_SPECS[@]}"; do + if [[ "\${spec%%=*}" == "$1" ]]; then printf '%s' "\${spec#*=}"; return; fi + done +} +all_baseos_packages_exact +`, + ); + expect(exact.result.status, exact.output).toBe(0); + + const drifted = runSourced( + STATION_PREPARE, + ` +installed_version() { + if [[ "$1" == "docker-ce" ]]; then printf '5:30.0.0-1~ubuntu.24.04~noble'; return; fi + local spec + for spec in "\${BASEOS_PACKAGE_SPECS[@]}"; do + if [[ "\${spec%%=*}" == "$1" ]]; then printf '%s' "\${spec#*=}"; return; fi + done +} +all_baseos_packages_exact +`, + ); + expect(drifted.result.status, drifted.output).not.toBe(0); + }); + + it("allows BaseOS failures only with exact packages and the expected cause fingerprint", () => { + const qualified = runSourced( + STATION_PREPARE, + ` +STATION_HOST_PROFILE=colossus-baseos +all_baseos_packages_exact() { return 0; } +baseos_fluent_bit_failure_is_qualified() { return 0; } +is_qualified_factory_failed_unit fluent-bit.service +`, + ); + expect(qualified.result.status, qualified.output).toBe(0); + + const drifted = runSourced( + STATION_PREPARE, + ` +STATION_HOST_PROFILE=colossus-baseos +all_baseos_packages_exact() { return 1; } +baseos_fluent_bit_failure_is_qualified() { return 0; } +is_qualified_factory_failed_unit fluent-bit.service +`, + ); + expect(drifted.result.status, drifted.output).not.toBe(0); + + const changedCause = runSourced( + STATION_PREPARE, + ` +STATION_HOST_PROFILE=colossus-baseos +all_baseos_packages_exact() { return 0; } +baseos_fluent_bit_failure_is_qualified() { return 1; } +is_qualified_factory_failed_unit fluent-bit.service +`, + ); + expect(changedCause.result.status, changedCause.output).not.toBe(0); + }); + + it("rejects a BaseOS failed unit when any systemd or unit-file fingerprint drifts", () => { + const exact = runSourced( + STATION_PREPARE, + ` +systemd_property_matches() { return 0; } +file_sha256_matches() { return 0; } +baseos_failed_unit_matches cloud-init.service /usr/lib/systemd/system/cloud-init.service HASH enabled 1 +`, + ); + expect(exact.result.status, exact.output).toBe(0); + + const drifted = runSourced( + STATION_PREPARE, + ` +systemd_property_matches() { [[ "$2" != Result ]]; } +file_sha256_matches() { return 0; } +baseos_failed_unit_matches cloud-init.service /usr/lib/systemd/system/cloud-init.service HASH enabled 1 +`, + ); + expect(drifted.result.status, drifted.output).not.toBe(0); + }); + it("keeps stock DGX OS out of the generic package mutation path", () => { const { result, output } = runSourced( STATION_PREPARE, @@ -304,6 +490,7 @@ require_command() { acquire_sudo() { :; } common_preflight() { STATION_HOST_PROFILE=stock-dgx-os; } verify_dgx_os_runtime_sudo() { printf 'DGX_OS_VALIDATED\n'; } +ensure_docker_group() { printf 'DOCKER_GROUP_PRESENT\n'; } install_packages() { printf 'GENERIC_PACKAGE_MUTATION\n'; return 1; } finish_runtime() { printf 'GENERIC_RUNTIME_MUTATION\n'; return 1; } run_apply @@ -318,6 +505,49 @@ run_apply expect(output).not.toContain("UNEXPECTED_REQUIREMENT"); }); + it("reconciles only the qualified BaseOS container runtime", () => { + const { result, output } = runSourced( + STATION_PREPARE, + ` +require_command() { + [[ "$1" == "sudo" ]] || { printf 'UNEXPECTED_REQUIREMENT %s\n' "$1"; return 1; } +} +acquire_sudo() { :; } +common_preflight() { STATION_HOST_PROFILE=colossus-baseos; } +finish_runtime() { printf 'BASEOS_RUNTIME_RECONCILED\n'; } +verify_dgx_os_runtime_sudo() { printf 'BASEOS_RUNTIME_VALIDATED\n'; } +install_packages() { printf 'PACKAGE_MUTATION\n'; return 1; } +run_apply +`, + ); + + expect(result.status, output).toBe(0); + expect(output).toContain("BASEOS_RUNTIME_RECONCILED"); + expect(output).toContain("BASEOS_RUNTIME_VALIDATED"); + expect(output).toContain("APPLY_RESULT=COMPLETE"); + expect(output).not.toContain("PACKAGE_MUTATION"); + expect(output).not.toContain("UNEXPECTED_REQUIREMENT"); + }); + + it("returns a relogin result instead of requesting a reboot for factory Docker access", () => { + const { result, output } = runSourced( + STATION_PREPARE, + ` +require_command() { :; } +acquire_sudo() { :; } +common_preflight() { STATION_HOST_PROFILE=ai-developer-tools; } +verify_dgx_os_runtime_sudo() { printf 'FACTORY_RUNTIME_VALIDATED\n'; } +ensure_docker_group() { DOCKER_GROUP_ADDED=1; } +run_apply +`, + ); + + expect(result.status, output).toBe(11); + expect(output).toContain("FACTORY_RUNTIME_VALIDATED"); + expect(output).toContain("APPLY_RESULT=LOGIN_REQUIRED"); + expect(output).not.toContain("REBOOT_REQUIRED"); + }); + it("accepts a healthy non-610 factory driver only for stock DGX OS", () => { const stock = runSourced( STATION_PREPARE, @@ -342,10 +572,74 @@ verify_gpu expect(generic.output).toContain("Expected driver 610.43.02, found 595.71.05"); }); + it("validates the GB300 and permits an auxiliary RTX GPU with unavailable ECC", () => { + const { result, output } = runSourced( + STATION_PREPARE, + ` +STATION_HOST_PROFILE=ai-developer-tools +nvidia-smi() { + printf 'NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition, 610.43.03, [N/A], [N/A]\n' + printf 'NVIDIA GB300, 610.43.03, 0, 0\n' +} +verify_gpu +`, + ); + + expect(result.status, output).toBe(0); + expect(output).toContain("gpu_index=0 gpu=NVIDIA RTX PRO 6000"); + expect(output).toContain("role=auxiliary validation=skipped"); + expect(output).toContain("gpu_index=1 gpu=NVIDIA GB300 role=inference"); + }); + + it("requires both factory container probes to expose the GB300 on a mixed-GPU host", () => { + const mixed = runSourced( + STATION_PREPARE, + ` +STATION_HOST_PROFILE=ai-developer-tools +station_sudo_local_default_docker() { + printf 'NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition, 610.43.03, [N/A], [N/A]\n' + printf 'NVIDIA GB300, 610.43.03, 0, 0\n' +} +run_dgx_os_cdi_test_sudo +run_dgx_os_gpus_test_sudo +`, + ); + expect(mixed.result.status, mixed.output).toBe(0); + expect(mixed.output).toContain("gpu_index=1 gpu=NVIDIA GB300 role=inference"); + + const rtxOnly = runSourced( + STATION_PREPARE, + ` +STATION_HOST_PROFILE=ai-developer-tools +station_sudo_local_default_docker() { + printf 'NVIDIA RTX PRO 6000 Blackwell Max-Q Workstation Edition, 610.43.03, [N/A], [N/A]\n' +} +run_dgx_os_cdi_test_sudo +`, + ); + expect(rtxOnly.result.status, rtxOnly.output).not.toBe(0); + expect(rtxOnly.output).toContain("Expected exactly one NVIDIA GB300, found 0"); + }); + + it("requires the qualified BaseOS driver to be loaded", () => { + const { result, output } = runSourced( + STATION_PREPARE, + ` +STATION_HOST_PROFILE=colossus-baseos +nvidia-smi() { printf 'NVIDIA GB300, 595.71.05, 0, 0\n'; } +verify_gpu +`, + ); + + expect(result.status, output).not.toBe(0); + expect(output).toContain("Expected driver 595.58.03, found 595.71.05"); + }); + it("validates stock DGX OS device visibility without rewriting host runtime state", () => { const { result, output } = runSourced( STATION_PREPARE, ` +STATION_HOST_PROFILE=stock-dgx-os check_dgx_os_runtime_commands() { printf 'COMMANDS_OK\n'; } systemctl() { case "$*" in @@ -430,7 +724,7 @@ verify_dgx_os_runtime_sudo }); it.each([ - ["wrong GPU", "NVIDIA GB200, 595.71.05, 0, 0", /Expected NVIDIA GB300/], + ["wrong GPU", "NVIDIA GB200, 595.71.05, 0, 0", /Expected exactly one NVIDIA GB300, found 0/], ["non-zero volatile ECC", "NVIDIA GB300, 595.71.05, 1, 0", /ECC must be 0\/0/], [ "a failing second GPU row", diff --git a/test/install-station-host-preparation.test.ts b/test/install-station-host-preparation.test.ts index feb71bbc181..8b9827c79f2 100644 --- a/test/install-station-host-preparation.test.ts +++ b/test/install-station-host-preparation.test.ts @@ -73,11 +73,15 @@ printf 'RESULT PROVIDER=%s STATION_EXPRESS=%s\n' "\${NEMOCLAW_PROVIDER:-}" "\${N } describe("DGX Station host preparation", () => { - it("uses the documented plain-Ubuntu driver-injection probe for CDI and --gpus", () => { + it("uses the documented plain-Ubuntu probe and verifies the GB300 row", () => { const { result, output } = runSourced( STATION_PREPARE, ` -sudo() { printf 'SUDO %s\\n' "$*"; } +STATION_HOST_PROFILE=generic-ubuntu +sudo() { + printf 'SUDO %s\\n' "$*" >&2 + printf 'NVIDIA GB300, 610.43.02, 0, 0\\n' +} run_cdi_test_sudo run_gpus_test_sudo `, @@ -87,9 +91,10 @@ run_gpus_test_sudo "docker.io/library/ubuntu@sha256:7f622ca8766bccb22f04242ecb6f19f770b2f08827dc4b8c707de5e78a6da7ab"; expect(result.status, output).toBe(0); expect(output).toContain( - `SUDO docker run --rm --device nvidia.com/gpu=all ${image} nvidia-smi`, + `SUDO docker run --rm --device nvidia.com/gpu=all ${image} nvidia-smi --query-gpu=`, ); - expect(output).toContain(`SUDO docker run --rm --gpus all ${image} nvidia-smi`); + expect(output).toContain(`SUDO docker run --rm --gpus all ${image} nvidia-smi --query-gpu=`); + expect(output).toContain("gpu=NVIDIA GB300 role=inference"); }); it.each([ @@ -1106,7 +1111,8 @@ ensure_station_express_host expect(result.status, output).toBe(10); expect(fs.readFileSync(stateFile, "utf-8")).toBe( - `revision=${STATION_REVISION}\nmodel=nemotron-3-ultra-550b-a55b\ngeneration=${STATION_GENERATION}\n`, + `revision=${STATION_REVISION}\nmodel=nemotron-3-ultra-550b-a55b\ngeneration=${STATION_GENERATION}\n` + + "agent=openclaw\nsandbox=my-assistant\npolicy_tier=balanced\n", ); expect(fs.statSync(stateFile).mode & 0o777).toBe(0o600); expect(output).toContain(`NEMOCLAW_INSTALL_TAG=${STATION_REVISION}`); From d3201f14178b417f864806ced2a496ece2b7550b Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Fri, 17 Jul 2026 16:43:45 -0700 Subject: [PATCH 2/2] test(inference): qualify Station GPU fixture Signed-off-by: Senthil Ravichandran --- src/lib/inference/vllm-install-storage.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/inference/vllm-install-storage.test.ts b/src/lib/inference/vllm-install-storage.test.ts index 5d5ab970e8f..54e9fa0a62f 100644 --- a/src/lib/inference/vllm-install-storage.test.ts +++ b/src/lib/inference/vllm-install-storage.test.ts @@ -145,6 +145,7 @@ describe("managed vLLM install storage", () => { beforeEach(() => { vi.clearAllMocks(); + mocks.getGpuIndicesByName.mockReturnValue([0]); logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); errSpy = vi.spyOn(console, "error").mockImplementation(() => {}); mkdirSpy = vi.spyOn(fs, "mkdirSync").mockImplementation(() => undefined);