From 79b04a723fc7eb70bdb1e4e87c1e6443a30a4371 Mon Sep 17 00:00:00 2001 From: sethforprivacy Date: Thu, 27 Aug 2026 09:38:20 -0400 Subject: [PATCH] fix: wrun() must pass ssh -n, and harden the fabric preflight Two silent-failure classes found bringing a second pair up. 1. wrun() inherits the caller's stdin. Any script that calls it from inside a heredoc-fed 'bash -s' has the rest of the heredoc consumed by ssh: the worker step succeeds and every later line vanishes with no error and a zero exit. ./stop.sh is the common case. Fixed with 'ssh -n' at the call site -- deliberately NOT in SSH_OPTS, which scp and 'rsync -e' share and where -n would break rsync's transport. 2. The fabric preflight only pinged default-size and only warned. So an MTU-1500 end passed silently and then capped the whole TP group (RoCE derives its IB MTU from the netdev MTU); an unreachable fabric peer warned and then hung in rendezvous instead of failing fast; and a wrong *_CX7_IB name was never checked, degrading NCCL silently. Now asserts an 8972-byte unfragmented ping end-to-end, reads the netdev MTU on both nodes, requires the configured RoCE device in ibv_devices on both nodes, and errors instead of warning. Validated on a known-good 2x GB10 pair: all four assertions pass, and a 20000-byte ping is correctly rejected, so the MTU check has teeth. --- start.sh | 41 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/start.sh b/start.sh index 6777e76..8bea427 100755 --- a/start.sh +++ b/start.sh @@ -330,7 +330,12 @@ derive_effective_api_key() { # ---- Remote helpers (worker = spark2 via ssh alias) ------------------------- wrun() { # remote shell snippet - ssh "${SSH_OPTS[@]}" "${WORKER_SSH}" "$1" + # -n (stdin from /dev/null) belongs HERE and not in SSH_OPTS: scp and rsync -e + # share SSH_OPTS, and -n breaks rsync's transport. Without it, ssh inherits the + # caller's stdin and silently consumes it -- running any script that calls wrun + # from inside `ssh host bash -s < on the worker, each arg shell-quoted local q=() a @@ -471,7 +476,39 @@ preflight() { # $1 = action; port checks only matter when serving if ping -c 1 -W 2 "${WORKER_CX7_IP}" >/dev/null 2>&1; then ok "CX7 link head ${HEAD_CX7_IP} ↔ worker ${WORKER_CX7_IP} reachable" else - warn "cannot ping ${WORKER_CX7_IP} — 2-node rendezvous may fail" + error "cannot reach ${WORKER_CX7_IP} over the CX7 fabric — TP rendezvous will hang, not fail fast" + exit 1 + fi + + # MTU 9000, asserted end-to-end rather than read from one side. RoCE derives its + # IB MTU from the netdev MTU, so a single end left at 1500 silently caps the whole + # TP group -- and a default-size ping cannot see it. + if ping -c 1 -W 2 -M do -s 8972 "${WORKER_CX7_IP}" >/dev/null 2>&1; then + ok "fabric MTU 9000 verified end-to-end (8972B unfragmented)" + else + error "MTU 9000 not usable head→${WORKER_CX7_IP}: an 8972-byte unfragmented ping failed." + error " check both ends: ip link show ${HEAD_CX7_IF} / ${WORKER_CX7_IF} (expect mtu 9000)" + exit 1 + fi + local hmtu; hmtu="$(cat "/sys/class/net/${HEAD_CX7_IF}/mtu" 2>/dev/null || echo 0)" + (( hmtu >= 9000 )) || { error "head ${HEAD_CX7_IF} MTU is ${hmtu}, expected 9000"; exit 1; } + local wmtu; wmtu="$(wrun "cat /sys/class/net/${WORKER_CX7_IF}/mtu 2>/dev/null || echo 0")" + (( wmtu >= 9000 )) || { error "worker ${WORKER_CX7_IF} MTU is ${wmtu}, expected 9000"; exit 1; } + ok "netdev MTU head ${hmtu} / worker ${wmtu}" + + # The configured RoCE device must actually exist on each node: NCCL_IB_HCA naming + # a device that is not present degrades silently instead of erroring. + if ibv_devices 2>/dev/null | grep -qw "${HEAD_CX7_IB}"; then + ok "head RoCE device ${HEAD_CX7_IB} present" + else + error "head RoCE device '${HEAD_CX7_IB}' not in ibv_devices — fix HEAD_CX7_IB" + exit 1 + fi + if wrun "ibv_devices 2>/dev/null | grep -qw '${WORKER_CX7_IB}'"; then + ok "worker RoCE device ${WORKER_CX7_IB} present" + else + error "worker RoCE device '${WORKER_CX7_IB}' not in ibv_devices — fix WORKER_CX7_IB" + exit 1 fi # Disk: needs depend on what is already cached (weights are ~135 GB)