Skip to content

[rocshmem] mpiexec_deadlock_analysis w/o NFS #6617

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
abouteiller merged 2 commits into from
Jun 26, 2026
Merged

[rocshmem] mpiexec_deadlock_analysis w/o NFS #6617

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
301c5af
Add a mode for mpiexec_deadlock_analysis to collect the dataset w/o
abouteiller May 30, 2026
dfff1ec
Merge branch 'develop' into users/abouteiller/debug/deadlock-detect-w…
abouteiller Jun 26, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
109 changes: 91 additions & 18 deletions projects/rocshmem/scripts/debug/attach_deadlock_analysis.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@
# --check-lanes Enable per-lane register divergence check inside each group
# --color Force ANSI color output (default when stdout is a tty)
# --no-color Disable ANSI color output
# --stdout After analysis, emit all output files to stdout wrapped in
# BEGIN/END FILE markers (used by mpiexec --tag-output collection)
#
# Options (via environment variables):
# ROCGDB Path to rocgdb binary (default: rocgdb)
Expand Down Expand Up @@ -73,6 +75,7 @@ OUTPUT_DIR=""
CULL_ENV=""
CHECK_LANES_ENV="0"
COLOR_ENV=""
STDOUT_MODE="0"

# Parse options from remaining arguments
_args=("${@:2}")
Expand All @@ -98,6 +101,8 @@ while [[ ${_i} -lt ${#_args[@]} ]]; do
COLOR_ENV="always"
elif [[ "${_arg}" == "--no-color" ]]; then
COLOR_ENV="never"
elif [[ "${_arg}" == "--stdout" ]]; then
STDOUT_MODE="1"
else
echo "ERROR: Unknown option '${_arg}'." >&2
exit 1
Expand All @@ -109,6 +114,42 @@ OUTPUT_DIR="${OUTPUT_DIR:-rocshmem_deadlock_analysis_${TIMESTAMP}}"

ROCGDB="${ROCGDB:-rocgdb}"
GDB_TIMEOUT="${ROCSHMEM_GDB_TIMEOUT:-120}"
if [[ ! "${GDB_TIMEOUT}" =~ ^[0-9]+$ ]]; then
echo "ERROR: ROCSHMEM_GDB_TIMEOUT must be a non-negative integer, got: '${GDB_TIMEOUT}'" >&2
exit 1
fi

# ---------------------------------------------------------------------------
# Color support for the script's own output
# ---------------------------------------------------------------------------

# Resolve color mode: explicit flag > env var > tty auto-detect
_use_color=0
if [[ "${COLOR_ENV}" == "always" ]]; then
_use_color=1
elif [[ "${COLOR_ENV}" == "never" ]]; then
_use_color=0
elif [[ -t 1 ]]; then
_use_color=1
fi

if [[ ${_use_color} -eq 1 ]]; then
_c_reset=$'\033[0m'
_c_ok=$'\033[1;32m' # bold green — OK
_c_warn=$'\033[1;33m' # bold yellow — WARN
_c_bad=$'\033[1;31m' # bold red — TIMEOUT / SKIP / bad counts
_c_header=$'\033[1;34m' # bold blue — section headers
_c_hint=$'\033[1;36m' # bold cyan — HINT text (matches rocgdb)
_c_fn=$'\033[1;32m' # bold green — rocSHMEM function name
else
_c_reset=''
_c_ok=''
_c_warn=''
_c_bad=''
_c_header=''
_c_hint=''
_c_fn=''
fi

# ---------------------------------------------------------------------------
# Prerequisite checks
Expand Down Expand Up @@ -172,7 +213,8 @@ for PID in "${PIDS[@]}"; do
# Read the MPI PE rank from the process environment (null-separated entries).
# Falls back to "unknown" if the process has already exited or is not an MPI rank.
PE_RANK="$(tr '\0' '\n' < "/proc/${PID}/environ" 2>/dev/null \
| sed -n 's/^OMPI_COMM_WORLD_RANK=//p')"
| sed -n 's/^OMPI_COMM_WORLD_RANK=//p' \
| tr -cd '0-9' | head -c 10)"
PE_RANK="${PE_RANK:-unknown}"

OUT_FILE="${OUTPUT_DIR}/pe${PE_RANK}_pid_${PID}.txt"
Expand All @@ -190,11 +232,15 @@ for PID in "${PIDS[@]}"; do
# Run rocgdb in batch mode.
# ROCSHMEM_DEADLOCK_AUTO_ANALYZE=1 causes the Python script to run the
# analysis immediately upon attach (the process is stopped on attach).
_gdb_env=(
ROCSHMEM_DEADLOCK_AUTO_ANALYZE=1
ROCSHMEM_DEADLOCK_CULL="${CULL_ENV}"
ROCSHMEM_DEADLOCK_CHECK_LANES="${CHECK_LANES_ENV}"
)
[[ -n "${COLOR_ENV}" ]] && _gdb_env+=(ROCSHMEM_DEADLOCK_COLOR="${COLOR_ENV}")

timeout "${GDB_TIMEOUT}" \
env ROCSHMEM_DEADLOCK_AUTO_ANALYZE=1 \
ROCSHMEM_DEADLOCK_CULL="${CULL_ENV}" \
ROCSHMEM_DEADLOCK_CHECK_LANES="${CHECK_LANES_ENV}" \
${COLOR_ENV:+ROCSHMEM_DEADLOCK_COLOR="${COLOR_ENV}"} \
env "${_gdb_env[@]}" \
"${ROCGDB}" -batch \
-p "${PID}" \
-x "${GDB_SCRIPT}" \
Expand All @@ -204,17 +250,17 @@ for PID in "${PIDS[@]}"; do
# Check if we got useful output regardless of exit code
# (rocgdb often returns non-zero even on success in batch mode)
if grep -q 'Unique backtrace groups:' "${OUT_FILE}" 2>/dev/null; then
echo "OK"
printf "${_c_ok}OK${_c_reset}\n"
(( ANALYZED++ )) || true
elif [[ ${GDB_EXIT} -eq 124 ]]; then
echo "TIMEOUT (${GDB_TIMEOUT}s)"
printf "${_c_bad}TIMEOUT${_c_reset} (${GDB_TIMEOUT}s)\n"
echo "TIMEOUT: rocgdb did not finish within ${GDB_TIMEOUT}s" >> "${OUT_FILE}"
(( FAILED++ )) || true
elif grep -q 'No GPU wavefront threads found' "${OUT_FILE}" 2>/dev/null; then
echo "OK (no GPU threads)"
printf "${_c_ok}OK${_c_reset} (no GPU threads)\n"
(( ANALYZED++ )) || true
else
echo "WARN (rocgdb exit=${GDB_EXIT}; see ${OUT_FILE})"
printf "${_c_warn}WARN${_c_reset} (rocgdb exit=${GDB_EXIT}; see ${OUT_FILE})\n"
(( FAILED++ )) || true
fi
done
Expand All @@ -224,13 +270,14 @@ done
# ---------------------------------------------------------------------------

echo ""
echo "=== Attach and Analyze Summary ==="
printf "${_c_header}=== Attach and Analyze Summary ===${_c_reset}\n"
printf "Processes analyzed: %d / %d\n" "${ANALYZED}" "${#PIDS[@]}"
echo ""

for PID in "${PIDS[@]}"; do
PE_RANK="$(tr '\0' '\n' < "/proc/${PID}/environ" 2>/dev/null \
| sed -n 's/^OMPI_COMM_WORLD_RANK=//p')"
| sed -n 's/^OMPI_COMM_WORLD_RANK=//p' \
| tr -cd '0-9' | head -c 10)"
PE_RANK="${PE_RANK:-unknown}"
OUT_FILE="${OUTPUT_DIR}/pe${PE_RANK}_pid_${PID}.txt"
# If the process has exited since the analysis loop, fall back to any file
Expand All @@ -240,7 +287,7 @@ for PID in "${PIDS[@]}"; do
fi

if [[ -z "${OUT_FILE}" || ! -f "${OUT_FILE}" ]]; then
printf " PE %-4s PID %-8s SKIPPED (no output file)\n" "${PE_RANK}" "${PID}"
printf " PE %-4s PID %-8s ${_c_bad}SKIPPED${_c_reset} (no output file)\n" "${PE_RANK}" "${PID}"
continue
fi

Expand All @@ -250,14 +297,22 @@ for PID in "${PIDS[@]}"; do
| awk '{print $NF}' | head -1)"
NUM_GROUPS="${NUM_GROUPS:-?}"

# Strip ANSI escape sequences before processing so color codes in the
# output file don't appear in the plain-text summary.
# Strip ANSI when extracting text for sorting/matching; re-apply the
# script's own colors for display so the summary is consistently colored
# regardless of whether rocgdb wrote colors into the output file.
_strip_ansi='s/\x1b\[[0-9;]*m//g'

# Always extract wavefront count inside rocSHMEM from the Summary section.
ROCSHMEM_WFS="$(grep 'wavefront(s) inside rocSHMEM' "${OUT_FILE}" 2>/dev/null \
| sed "${_strip_ansi}" | awk '{print $1}' | head -1)"

# Extract the dominant rocSHMEM API function (most frequently occurring).
STUCK_FN="$(grep '\[rocSHMEM\] Stuck in:' "${OUT_FILE}" 2>/dev/null \
| sed "${_strip_ansi}" \
| sort | uniq -c | sort -rn | head -1 \
| sed 's/^[[:space:]]*[0-9]*[[:space:]]*//' \
| sed 's/.*\[rocSHMEM\] Stuck in:[[:space:]]*//')"

# Determine the dominant stuck pattern from [HINT] lines
# (most frequently occurring hint).
HINT="$(grep '\[HINT\]' "${OUT_FILE}" 2>/dev/null \
Expand All @@ -266,8 +321,12 @@ for PID in "${PIDS[@]}"; do
| sed 's/^[[:space:]]*[0-9]*[[:space:]]*//' \
| sed 's/^\[HINT\][[:space:]]*//')"

# Apply script-level colors to the extracted plain-text fields.
[[ -n "${STUCK_FN}" ]] && STUCK_FN="${_c_fn}${STUCK_FN}${_c_reset}"
[[ -n "${HINT}" ]] && HINT="${_c_hint}${HINT}${_c_reset}"

if [[ -n "${ROCSHMEM_WFS}" ]]; then
DOMINANT="${ROCSHMEM_WFS} wf(s) in rocSHMEM"
DOMINANT="${_c_bad}${ROCSHMEM_WFS}${_c_reset} wf(s) in ${STUCK_FN:-rocSHMEM}"
if [[ -n "${HINT}" ]]; then
DOMINANT="${DOMINANT}; ${HINT}"
else
Expand All @@ -276,16 +335,30 @@ for PID in "${PIDS[@]}"; do
elif grep -q 'No GPU wavefront threads found' "${OUT_FILE}" 2>/dev/null; then
DOMINANT="(no GPU threads found)"
elif grep -q 'SKIP\|TIMEOUT' "${OUT_FILE}" 2>/dev/null; then
DOMINANT="$(grep -m1 'SKIP\|TIMEOUT' "${OUT_FILE}" | sed 's/^[[:space:]]*//')"
DOMINANT="${_c_bad}$(grep -m1 'SKIP\|TIMEOUT' "${OUT_FILE}" \
| sed "${_strip_ansi}" | sed 's/^[[:space:]]*//')${_c_reset}"
else
DOMINANT="${HINT:-0 wf(s) in rocSHMEM}"
fi

printf " PE %-4s PID %-8s groups=%-3s %s\n" "${PE_RANK}" "${PID}" "${NUM_GROUPS}" "${DOMINANT}"
done

echo ""
echo "Full output in: ${OUTPUT_DIR}/"
if [[ "${STDOUT_MODE}" -eq 0 ]]; then
echo ""
echo "Full output in: ${OUTPUT_DIR}/"
fi

# In --stdout mode, emit every output file to stdout wrapped in markers so that
# the caller (mpiexec --tag-output) can reconstruct the files on the head node.
if [[ "${STDOUT_MODE}" -eq 1 ]]; then
for _f in "${OUTPUT_DIR}"/pe*_pid_*.txt; do
[[ -f "${_f}" ]] || continue
echo "=== BEGIN FILE: $(basename "${_f}") ==="
cat "${_f}"
echo "=== END FILE: $(basename "${_f}") ==="
done
fi

# Exit with non-zero if any process failed analysis
[[ ${FAILED} -eq 0 ]]
79 changes: 70 additions & 9 deletions projects/rocshmem/scripts/debug/mpiexec_deadlock_analysis.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,14 +32,20 @@
# ./mpiexec_deadlock_analysis.sh <executable_name> [options] [-- <mpiexec-args>...]
#
# Options:
# --directory <dir> Shared output directory visible from all nodes
# (default: ./rocshmem_deadlock_analysis_<timestamp>)
# --directory <dir> Output directory. With a shared filesystem (default)
# this must be visible from all nodes at the same path.
# With --no-shared-fs it is a local path on the head node
# only (default: ./rocshmem_deadlock_analysis_<timestamp>)
# --cull Cull groups stuck in GPU barriers / gridsync
# --cull=p1,p2,... Cull groups matching any of the given substrings
# --check-lanes Enable per-lane register divergence check inside each group
# --color Force ANSI color output
# --no-color Disable ANSI color output
# --no-coalesce Skip the cross-rank coalescing pass
# --no-shared-fs Do not assume a shared filesystem. Each node writes to a
# local /tmp directory; mpiexec --tag-output is used to stream
# all output to the head node, where it is reconstructed into
# per-PE files before the coalescing pass.
# -- <mpiexec-args>... All arguments after -- are forwarded verbatim to
# mpiexec (e.g. -np 4 -H node1,node2 --hostfile hosts.txt)
#
Expand All @@ -52,12 +58,12 @@
# ROCGDB Path to rocgdb binary (default: rocgdb)
# ROCSHMEM_GDB_TIMEOUT Per-process timeout in seconds (default: 120)
#
# The output directory must reside on a shared filesystem (NFS, Lustre, etc.)
# accessible with the same path from all nodes.
#
# Examples:
# ./mpiexec_deadlock_analysis.sh rocshmem_functional_tests \
# --directory /shared/analysis --cull
# # No shared filesystem — output streamed via mpiexec --tag-output:
# ./mpiexec_deadlock_analysis.sh rocshmem_functional_tests \
# --no-shared-fs --directory /tmp/local_analysis
# # Explicit node list via mpiexec arguments:
# ./mpiexec_deadlock_analysis.sh rocshmem_functional_tests \
# --directory /shared/analysis -- -np 4 -H node1,node2,node3,node4
Expand Down Expand Up @@ -87,6 +93,7 @@ CULL_ARG=""
CHECK_LANES_ARG=""
COLOR_ARG=""
RUN_COALESCE=1
NO_SHARED_FS=0
MPIEXEC_EXTRA_ARGS=()

_args=("${@:2}")
Expand Down Expand Up @@ -116,6 +123,8 @@ while [[ ${_i} -lt ${#_args[@]} ]]; do
COLOR_ARG="--no-color"
elif [[ "${_arg}" == "--no-coalesce" ]]; then
RUN_COALESCE=0
elif [[ "${_arg}" == "--no-shared-fs" ]]; then
NO_SHARED_FS=1
else
echo "ERROR: Unknown option '${_arg}'." >&2
exit 1
Expand Down Expand Up @@ -157,14 +166,24 @@ mkdir -p "${OUTPUT_DIR}" || { echo "ERROR: Cannot create output directory: ${OUT
echo "Deploying deadlock analysis..."
echo "Executable: ${EXECUTABLE}"
echo "Output directory: ${OUTPUT_DIR}"
[[ ${NO_SHARED_FS} -eq 1 ]] && echo "Filesystem mode: no shared fs (--output-filename)"
[[ -n "${CULL_ARG}" ]] && echo "Cull option: ${CULL_ARG}"
[[ -n "${CHECK_LANES_ARG}" ]] && echo "Check lanes: yes"
[[ -n "${COLOR_ARG}" ]] && echo "Color: ${COLOR_ARG}"
[[ ${#MPIEXEC_EXTRA_ARGS[@]} -gt 0 ]] && echo "mpiexec args: ${MPIEXEC_EXTRA_ARGS[*]}"
echo ""

# Build the attach script argument list
_attach_args=("${EXECUTABLE}" "--directory" "${OUTPUT_DIR}")
# In no-shared-fs mode each node writes to a local /tmp directory (same path
# on every node, based on the timestamp) and emits file contents to stdout via
# the --stdout flag. mpiexec --output-filename captures each rank's stdout
# into a separate file on the head node.
if [[ ${NO_SHARED_FS} -eq 1 ]]; then
_node_local_dir="/tmp/rocshmem_deadlock_${TIMESTAMP}"
_mpi_output_prefix="${OUTPUT_DIR}/mpi_rank_output"
_attach_args=("${EXECUTABLE}" "--directory" "${_node_local_dir}" "--stdout")
else
_attach_args=("${EXECUTABLE}" "--directory" "${OUTPUT_DIR}")
fi
[[ -n "${CULL_ARG}" ]] && _attach_args+=("${CULL_ARG}")
[[ -n "${CHECK_LANES_ARG}" ]] && _attach_args+=("${CHECK_LANES_ARG}")
[[ -n "${COLOR_ARG}" ]] && _attach_args+=("${COLOR_ARG}")
Expand All @@ -174,8 +193,16 @@ _attach_args=("${EXECUTABLE}" "--directory" "${OUTPUT_DIR}")
# forwarded automatically via the inherited environment.
# MPIEXEC_EXTRA_ARGS (from arguments after --) supply the node list / count,
# e.g. -np 4 -H node1,node2 or --hostfile hosts.txt.
"${MPIEXEC}" "${MPIEXEC_EXTRA_ARGS[@]}" -pernode \
bash "${ATTACH_SCRIPT}" "${_attach_args[@]}"
if [[ ${NO_SHARED_FS} -eq 1 ]]; then
# --output-filename <prefix> writes each rank's stdout to
# <prefix>.<jobid>.<rank> on the head node (OpenMPI convention).
"${MPIEXEC}" "${MPIEXEC_EXTRA_ARGS[@]}" -pernode \
--output-filename "${_mpi_output_prefix}" \
bash "${ATTACH_SCRIPT}" "${_attach_args[@]}"
else
"${MPIEXEC}" "${MPIEXEC_EXTRA_ARGS[@]}" -pernode \
bash "${ATTACH_SCRIPT}" "${_attach_args[@]}"
fi
MPIEXEC_EXIT=$?

echo ""
Expand All @@ -184,6 +211,40 @@ if [[ ${MPIEXEC_EXIT} -ne 0 ]]; then
"Some nodes may have failed." >&2
fi

# ---------------------------------------------------------------------------
# Reconstruct pe*_pid_*.txt files from mpiexec --output-filename captures
# ---------------------------------------------------------------------------

if [[ ${NO_SHARED_FS} -eq 1 ]]; then
echo "Reconstructing per-PE output files from mpiexec output..."
_reconstructed=0
# OpenMPI names files <prefix>.<jobid>.<rank>; glob broadly to handle
# varying formats across MPI implementations.
for _rank_file in "${_mpi_output_prefix}"*; do
[[ -f "${_rank_file}" ]] || continue
# Extract each embedded pe*_pid_*.txt block using awk.
# fname is validated against the expected pattern before use to
# prevent a compromised node from writing outside OUTPUT_DIR via
# a path-traversal payload in the BEGIN FILE marker.
awk -v outdir="${OUTPUT_DIR}" '
/^=== BEGIN FILE: / {
fname = $4
if (fname !~ /^pe[A-Za-z0-9]*_pid_[0-9]+\.txt$/) { fname = ""; next }
next
}
/^=== END FILE: / { fname = ""; next }
fname != "" { print > (outdir "/" fname) }
' "${_rank_file}"
(( _reconstructed++ )) || true
done
if [[ ${_reconstructed} -eq 0 ]]; then
echo "WARNING: No mpiexec output files found under ${_mpi_output_prefix}*" >&2
else
echo "Reconstructed output from ${_reconstructed} rank file(s)."
fi
echo ""
fi

# ---------------------------------------------------------------------------
# Cross-rank coalescing pass
# ---------------------------------------------------------------------------
Expand Down
7 changes: 6 additions & 1 deletion projects/rocshmem/scripts/debug/rocgdb_deadlock_analysis.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -994,11 +994,13 @@ def report(self):
# The innermost deadlock frame is frame #0 when a known hint pattern
# is present — highlight it so it stands out in deep call chains.
hint_frame_idx = None
hint_frame_text = None
if hint:
for idx, frame in enumerate(frames):
for substring, _ in _HINT_RULES:
if substring in frame:
hint_frame_idx = idx
hint_frame_text = frame
break
if hint_frame_idx is not None:
break
Expand All @@ -1022,7 +1024,10 @@ def report(self):
other_wf_count += len(entries)

if hint:
print(f' {c.HINT}[HINT] {hint}{c.RESET}', file=out)
detected = f' (at: {hint_frame_text.strip()})' if hint_frame_text else ''
print(f' {c.HINT}[HINT] {hint}{detected}{c.RESET}', file=out)
elif frames:
print(f' {c.LOC}[STUCK] Innermost frame: {frames[0].strip()}{c.RESET}', file=out)

print('', file=out)

Expand Down
Loading