From 4b3cfdd53dfc61415c081e5d714dac455c16ace3 Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Sat, 18 Jul 2026 08:31:15 -0700 Subject: [PATCH 1/4] fix(installer): explain failed OpenIB service Signed-off-by: Senthil Ravichandran --- scripts/prepare-dgx-station-host.sh | 11 +++ ...nstall-station-openibd-remediation.test.ts | 68 +++++++++++++++++++ 2 files changed, 79 insertions(+) create mode 100644 test/install-station-openibd-remediation.test.ts diff --git a/scripts/prepare-dgx-station-host.sh b/scripts/prepare-dgx-station-host.sh index a392a72d9d9..184c4111a8d 100755 --- a/scripts/prepare-dgx-station-host.sh +++ b/scripts/prepare-dgx-station-host.sh @@ -731,6 +731,16 @@ host_docker_sudo() { fi } +warn_openibd_remediation() { + warn "openibd.service configures optional Mellanox RDMA networking; NemoClaw does not require RDMA" + warn "Check the default route: ip route get 1.1.1.1" + warn "Check NFS mount options: findmnt -rn -t nfs,nfs4 -o TARGET,OPTIONS" + warn "If this host does not use RDMA, run: sudo systemctl disable openibd.service" + warn "After disabling the unused service, reboot and rerun the NemoClaw installer" + warn "If this host uses RDMA, repair OpenIB/OFED before rerunning the installer" + warn "NemoClaw did not change systemd or networking state" +} + check_failed_units() { local unit failed_output blocking=0 qualified_label local -a units=() @@ -758,6 +768,7 @@ check_failed_units() { warn "condition-qualified ${qualified_label} failed unit: ${unit}" else warn "unqualified failed unit: ${unit}" + [[ "$unit" != "openibd.service" ]] || warn_openibd_remediation blocking=1 fi done diff --git a/test/install-station-openibd-remediation.test.ts b/test/install-station-openibd-remediation.test.ts new file mode 100644 index 00000000000..7134ea8bab5 --- /dev/null +++ b/test/install-station-openibd-remediation.test.ts @@ -0,0 +1,68 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { TEST_SYSTEM_PATH } from "./helpers/installer-sourced-env"; + +const REPO_ROOT = path.resolve(import.meta.dirname, ".."); +const STATION_PREPARE = path.join(REPO_ROOT, "scripts", "prepare-dgx-station-host.sh"); + +function checkFailedUnit(unit: string) { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-station-openibd-")); + const result = spawnSync( + "bash", + [ + "--noprofile", + "--norc", + "-c", + ` +source "$SCRIPT_UNDER_TEST" >/dev/null +systemctl() { printf '%s loaded failed failed fixture\\n' "$FAILED_UNIT"; } +check_failed_units +`, + ], + { + cwd: REPO_ROOT, + encoding: "utf-8", + env: { + HOME: home, + PATH: TEST_SYSTEM_PATH, + SCRIPT_UNDER_TEST: STATION_PREPARE, + FAILED_UNIT: unit, + }, + timeout: 15_000, + killSignal: "SIGKILL", + }, + ); + return { result, output: `${result.stdout}${result.stderr}` }; +} + +describe("DGX Station OpenIB remediation", () => { + it("explains how owners can resolve a failed OpenIB service without changing it (#7151)", () => { + const openibd = checkFailedUnit("openibd.service"); + + expect(openibd.result.status, openibd.output).not.toBe(0); + expect(openibd.output).toMatch(/unqualified failed unit: openibd.service/); + expect(openibd.output).toMatch(/NemoClaw does not require RDMA/); + expect(openibd.output).toMatch(/ip route get 1\.1\.1\.1/); + expect(openibd.output).toMatch(/findmnt -rn -t nfs,nfs4 -o TARGET,OPTIONS/); + expect(openibd.output).toMatch(/sudo systemctl disable openibd.service/); + expect(openibd.output).toMatch(/repair OpenIB\/OFED/); + expect(openibd.output).toMatch(/NemoClaw did not change systemd or networking state/); + expect(openibd.output).toMatch(/Unqualified failed system units block Station preparation/); + }); + + it("keeps unrelated failed units on the generic fail-closed path (#7151)", () => { + const unrelated = checkFailedUnit("ssh.service"); + + expect(unrelated.result.status, unrelated.output).not.toBe(0); + expect(unrelated.output).toMatch(/unqualified failed unit: ssh.service/); + expect(unrelated.output).not.toMatch(/NemoClaw does not require RDMA/); + expect(unrelated.output).not.toMatch(/sudo systemctl disable openibd.service/); + expect(unrelated.output).toMatch(/Unqualified failed system units block Station preparation/); + }); +}); From 3c3573003839102ce16d58a82ce6d91bd32b2211 Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Sat, 18 Jul 2026 08:37:30 -0700 Subject: [PATCH 2/4] test(installer): reject interrupted OpenIB fixtures Signed-off-by: Senthil Ravichandran --- test/install-station-openibd-remediation.test.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/install-station-openibd-remediation.test.ts b/test/install-station-openibd-remediation.test.ts index 7134ea8bab5..f7e81bd9c12 100644 --- a/test/install-station-openibd-remediation.test.ts +++ b/test/install-station-openibd-remediation.test.ts @@ -45,6 +45,8 @@ describe("DGX Station OpenIB remediation", () => { it("explains how owners can resolve a failed OpenIB service without changing it (#7151)", () => { const openibd = checkFailedUnit("openibd.service"); + expect(openibd.result.error, openibd.output).toBeUndefined(); + expect(openibd.result.status, openibd.output).not.toBeNull(); expect(openibd.result.status, openibd.output).not.toBe(0); expect(openibd.output).toMatch(/unqualified failed unit: openibd.service/); expect(openibd.output).toMatch(/NemoClaw does not require RDMA/); @@ -59,6 +61,8 @@ describe("DGX Station OpenIB remediation", () => { it("keeps unrelated failed units on the generic fail-closed path (#7151)", () => { const unrelated = checkFailedUnit("ssh.service"); + expect(unrelated.result.error, unrelated.output).toBeUndefined(); + expect(unrelated.result.status, unrelated.output).not.toBeNull(); expect(unrelated.result.status, unrelated.output).not.toBe(0); expect(unrelated.output).toMatch(/unqualified failed unit: ssh.service/); expect(unrelated.output).not.toMatch(/NemoClaw does not require RDMA/); From 151ea640eef50176be852061662c4a1a993ff00a Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Sat, 18 Jul 2026 08:44:57 -0700 Subject: [PATCH 3/4] fix(installer): clarify OpenIB usage checks Signed-off-by: Senthil Ravichandran --- scripts/prepare-dgx-station-host.sh | 1 + test/install-station-openibd-remediation.test.ts | 15 ++++++++++----- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/scripts/prepare-dgx-station-host.sh b/scripts/prepare-dgx-station-host.sh index 184c4111a8d..53d4fc6eb77 100755 --- a/scripts/prepare-dgx-station-host.sh +++ b/scripts/prepare-dgx-station-host.sh @@ -735,6 +735,7 @@ warn_openibd_remediation() { warn "openibd.service configures optional Mellanox RDMA networking; NemoClaw does not require RDMA" warn "Check the default route: ip route get 1.1.1.1" warn "Check NFS mount options: findmnt -rn -t nfs,nfs4 -o TARGET,OPTIONS" + warn "These checks are not exhaustive; confirm no RDMA-backed networking, storage, or workloads are in use" warn "If this host does not use RDMA, run: sudo systemctl disable openibd.service" warn "After disabling the unused service, reboot and rerun the NemoClaw installer" warn "If this host uses RDMA, repair OpenIB/OFED before rerunning the installer" diff --git a/test/install-station-openibd-remediation.test.ts b/test/install-station-openibd-remediation.test.ts index f7e81bd9c12..c702fc2fac0 100644 --- a/test/install-station-openibd-remediation.test.ts +++ b/test/install-station-openibd-remediation.test.ts @@ -21,7 +21,13 @@ function checkFailedUnit(unit: string) { "-c", ` source "$SCRIPT_UNDER_TEST" >/dev/null -systemctl() { printf '%s loaded failed failed fixture\\n' "$FAILED_UNIT"; } +systemctl() { + if [[ "$*" != "--failed --no-legend --plain" ]]; then + printf 'unexpected systemctl call: %s\\n' "$*" >&2 + exit 97 + fi + printf '%s loaded failed failed fixture\\n' "$FAILED_UNIT" +} check_failed_units `, ], @@ -46,12 +52,12 @@ describe("DGX Station OpenIB remediation", () => { const openibd = checkFailedUnit("openibd.service"); expect(openibd.result.error, openibd.output).toBeUndefined(); - expect(openibd.result.status, openibd.output).not.toBeNull(); - expect(openibd.result.status, openibd.output).not.toBe(0); + expect(openibd.result.status, openibd.output).toBe(1); expect(openibd.output).toMatch(/unqualified failed unit: openibd.service/); expect(openibd.output).toMatch(/NemoClaw does not require RDMA/); expect(openibd.output).toMatch(/ip route get 1\.1\.1\.1/); expect(openibd.output).toMatch(/findmnt -rn -t nfs,nfs4 -o TARGET,OPTIONS/); + expect(openibd.output).toMatch(/checks are not exhaustive/); expect(openibd.output).toMatch(/sudo systemctl disable openibd.service/); expect(openibd.output).toMatch(/repair OpenIB\/OFED/); expect(openibd.output).toMatch(/NemoClaw did not change systemd or networking state/); @@ -62,8 +68,7 @@ describe("DGX Station OpenIB remediation", () => { const unrelated = checkFailedUnit("ssh.service"); expect(unrelated.result.error, unrelated.output).toBeUndefined(); - expect(unrelated.result.status, unrelated.output).not.toBeNull(); - expect(unrelated.result.status, unrelated.output).not.toBe(0); + expect(unrelated.result.status, unrelated.output).toBe(1); expect(unrelated.output).toMatch(/unqualified failed unit: ssh.service/); expect(unrelated.output).not.toMatch(/NemoClaw does not require RDMA/); expect(unrelated.output).not.toMatch(/sudo systemctl disable openibd.service/); From 3dc37d6a48c58bc4826b6d402586d071945626aa Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Sat, 18 Jul 2026 09:14:04 -0700 Subject: [PATCH 4/4] chore(ci): refresh PR-bound checks Signed-off-by: Aaron Erickson