diff --git a/scripts/prepare-dgx-station-host.sh b/scripts/prepare-dgx-station-host.sh index a392a72d9d9..53d4fc6eb77 100755 --- a/scripts/prepare-dgx-station-host.sh +++ b/scripts/prepare-dgx-station-host.sh @@ -731,6 +731,17 @@ host_docker_sudo() { fi } +warn_openibd_remediation() { + warn "openibd.service configures optional Mellanox RDMA networking; NemoClaw does not require RDMA" + warn "Check the default route: ip route get 1.1.1.1" + warn "Check NFS mount options: findmnt -rn -t nfs,nfs4 -o TARGET,OPTIONS" + warn "These checks are not exhaustive; confirm no RDMA-backed networking, storage, or workloads are in use" + warn "If this host does not use RDMA, run: sudo systemctl disable openibd.service" + warn "After disabling the unused service, reboot and rerun the NemoClaw installer" + warn "If this host uses RDMA, repair OpenIB/OFED before rerunning the installer" + warn "NemoClaw did not change systemd or networking state" +} + check_failed_units() { local unit failed_output blocking=0 qualified_label local -a units=() @@ -758,6 +769,7 @@ check_failed_units() { warn "condition-qualified ${qualified_label} failed unit: ${unit}" else warn "unqualified failed unit: ${unit}" + [[ "$unit" != "openibd.service" ]] || warn_openibd_remediation blocking=1 fi done diff --git a/test/install-station-openibd-remediation.test.ts b/test/install-station-openibd-remediation.test.ts new file mode 100644 index 00000000000..c702fc2fac0 --- /dev/null +++ b/test/install-station-openibd-remediation.test.ts @@ -0,0 +1,77 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { TEST_SYSTEM_PATH } from "./helpers/installer-sourced-env"; + +const REPO_ROOT = path.resolve(import.meta.dirname, ".."); +const STATION_PREPARE = path.join(REPO_ROOT, "scripts", "prepare-dgx-station-host.sh"); + +function checkFailedUnit(unit: string) { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-station-openibd-")); + const result = spawnSync( + "bash", + [ + "--noprofile", + "--norc", + "-c", + ` +source "$SCRIPT_UNDER_TEST" >/dev/null +systemctl() { + if [[ "$*" != "--failed --no-legend --plain" ]]; then + printf 'unexpected systemctl call: %s\\n' "$*" >&2 + exit 97 + fi + printf '%s loaded failed failed fixture\\n' "$FAILED_UNIT" +} +check_failed_units +`, + ], + { + cwd: REPO_ROOT, + encoding: "utf-8", + env: { + HOME: home, + PATH: TEST_SYSTEM_PATH, + SCRIPT_UNDER_TEST: STATION_PREPARE, + FAILED_UNIT: unit, + }, + timeout: 15_000, + killSignal: "SIGKILL", + }, + ); + return { result, output: `${result.stdout}${result.stderr}` }; +} + +describe("DGX Station OpenIB remediation", () => { + it("explains how owners can resolve a failed OpenIB service without changing it (#7151)", () => { + const openibd = checkFailedUnit("openibd.service"); + + expect(openibd.result.error, openibd.output).toBeUndefined(); + expect(openibd.result.status, openibd.output).toBe(1); + expect(openibd.output).toMatch(/unqualified failed unit: openibd.service/); + expect(openibd.output).toMatch(/NemoClaw does not require RDMA/); + expect(openibd.output).toMatch(/ip route get 1\.1\.1\.1/); + expect(openibd.output).toMatch(/findmnt -rn -t nfs,nfs4 -o TARGET,OPTIONS/); + expect(openibd.output).toMatch(/checks are not exhaustive/); + expect(openibd.output).toMatch(/sudo systemctl disable openibd.service/); + expect(openibd.output).toMatch(/repair OpenIB\/OFED/); + expect(openibd.output).toMatch(/NemoClaw did not change systemd or networking state/); + expect(openibd.output).toMatch(/Unqualified failed system units block Station preparation/); + }); + + it("keeps unrelated failed units on the generic fail-closed path (#7151)", () => { + const unrelated = checkFailedUnit("ssh.service"); + + expect(unrelated.result.error, unrelated.output).toBeUndefined(); + expect(unrelated.result.status, unrelated.output).toBe(1); + expect(unrelated.output).toMatch(/unqualified failed unit: ssh.service/); + expect(unrelated.output).not.toMatch(/NemoClaw does not require RDMA/); + expect(unrelated.output).not.toMatch(/sudo systemctl disable openibd.service/); + expect(unrelated.output).toMatch(/Unqualified failed system units block Station preparation/); + }); +});