Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions scripts/prepare-dgx-station-host.sh
Original file line number Diff line number Diff line change
Expand Up @@ -731,6 +731,17 @@ host_docker_sudo() {
fi
}

warn_openibd_remediation() {
warn "openibd.service configures optional Mellanox RDMA networking; NemoClaw does not require RDMA"
warn "Check the default route: ip route get 1.1.1.1"
warn "Check NFS mount options: findmnt -rn -t nfs,nfs4 -o TARGET,OPTIONS"
warn "These checks are not exhaustive; confirm no RDMA-backed networking, storage, or workloads are in use"
warn "If this host does not use RDMA, run: sudo systemctl disable openibd.service"
warn "After disabling the unused service, reboot and rerun the NemoClaw installer"
warn "If this host uses RDMA, repair OpenIB/OFED before rerunning the installer"
warn "NemoClaw did not change systemd or networking state"
}

check_failed_units() {
local unit failed_output blocking=0 qualified_label
local -a units=()
Expand Down Expand Up @@ -758,6 +769,7 @@ check_failed_units() {
warn "condition-qualified ${qualified_label} failed unit: ${unit}"
else
warn "unqualified failed unit: ${unit}"
[[ "$unit" != "openibd.service" ]] || warn_openibd_remediation
blocking=1
fi
done
Expand Down
77 changes: 77 additions & 0 deletions test/install-station-openibd-remediation.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { spawnSync } from "node:child_process";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { describe, expect, it } from "vitest";
import { TEST_SYSTEM_PATH } from "./helpers/installer-sourced-env";

const REPO_ROOT = path.resolve(import.meta.dirname, "..");
const STATION_PREPARE = path.join(REPO_ROOT, "scripts", "prepare-dgx-station-host.sh");

function checkFailedUnit(unit: string) {
const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-station-openibd-"));
const result = spawnSync(
"bash",
[
"--noprofile",
"--norc",
"-c",
`
source "$SCRIPT_UNDER_TEST" >/dev/null
systemctl() {
if [[ "$*" != "--failed --no-legend --plain" ]]; then
printf 'unexpected systemctl call: %s\\n' "$*" >&2
exit 97
fi
printf '%s loaded failed failed fixture\\n' "$FAILED_UNIT"
}
check_failed_units
`,
],
{
cwd: REPO_ROOT,
encoding: "utf-8",
env: {
HOME: home,
PATH: TEST_SYSTEM_PATH,
SCRIPT_UNDER_TEST: STATION_PREPARE,
FAILED_UNIT: unit,
},
timeout: 15_000,
killSignal: "SIGKILL",
},
);
return { result, output: `${result.stdout}${result.stderr}` };
}

describe("DGX Station OpenIB remediation", () => {
it("explains how owners can resolve a failed OpenIB service without changing it (#7151)", () => {
const openibd = checkFailedUnit("openibd.service");

expect(openibd.result.error, openibd.output).toBeUndefined();
expect(openibd.result.status, openibd.output).toBe(1);
expect(openibd.output).toMatch(/unqualified failed unit: openibd.service/);
expect(openibd.output).toMatch(/NemoClaw does not require RDMA/);
expect(openibd.output).toMatch(/ip route get 1\.1\.1\.1/);
expect(openibd.output).toMatch(/findmnt -rn -t nfs,nfs4 -o TARGET,OPTIONS/);
expect(openibd.output).toMatch(/checks are not exhaustive/);
expect(openibd.output).toMatch(/sudo systemctl disable openibd.service/);
expect(openibd.output).toMatch(/repair OpenIB\/OFED/);
expect(openibd.output).toMatch(/NemoClaw did not change systemd or networking state/);
expect(openibd.output).toMatch(/Unqualified failed system units block Station preparation/);
});

it("keeps unrelated failed units on the generic fail-closed path (#7151)", () => {
const unrelated = checkFailedUnit("ssh.service");

expect(unrelated.result.error, unrelated.output).toBeUndefined();
expect(unrelated.result.status, unrelated.output).toBe(1);
expect(unrelated.output).toMatch(/unqualified failed unit: ssh.service/);
expect(unrelated.output).not.toMatch(/NemoClaw does not require RDMA/);
expect(unrelated.output).not.toMatch(/sudo systemctl disable openibd.service/);
expect(unrelated.output).toMatch(/Unqualified failed system units block Station preparation/);
});
});