From 13e642cfd545fa804f8b207e7dbaa7daa61018cd Mon Sep 17 00:00:00 2001
From: Donglai Dai <donglai.dai@amd.com>
Date: Fri, 18 Sep 2026 20:04:13 -0400
Subject: [PATCH 1/5] ci(rccl): build device-api with rocSHMEM GIN
 (AICOMRCCL-2221)

The device-api job compiled RCCL with --rocshmem-gin but never installed
it, so the cmake config glob died before any benchmark, and rccl-tests
did not RDC-link the binaries that load that library.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 projects/rccl-tests/CMakeLists.txt            |  9 +-
 projects/rccl-tests/src/CMakeLists.txt        | 66 +++++++------
 .../rccl/tools/ci/lib/device-api-tests.json   | 20 ++--
 projects/rccl/tools/ci/lib/device-api.sbatch  | 95 +++++++++++++++----
 projects/rccl/tools/ci/run-device-api-ci.sh   | 51 ++++++----
 5 files changed, 156 insertions(+), 85 deletions(-)

diff --git a/projects/rccl-tests/CMakeLists.txt b/projects/rccl-tests/CMakeLists.txt
index 7b54fced4abd..fcb97dc7307d 100644
--- a/projects/rccl-tests/CMakeLists.txt
+++ b/projects/rccl-tests/CMakeLists.txt
@@ -238,12 +238,11 @@ if (ENABLE_ROCSHMEM_GIN)
     if (NOT ROCSHMEM_SOURCE_DIR AND EXISTS "${_mono_root}/projects/rocshmem/CMakeLists.txt")
         set(ROCSHMEM_SOURCE_DIR "${_mono_root}/projects/rocshmem")
     endif()
-    # Installed rocshmem headers (rocshmem_config.h) are needed globally:
-    # RCCL's public nccl_device.h → gin_anvil_sdma.h → anvil_device.hpp
-    # → log.hpp → rocshmem/rocshmem_config.h.  Must come before source
-    # includes so the generated config is found first.
+    # Installed rocshmem headers (rocshmem_config.h) for GIN-SDMA perf binaries
+    # that set NCCL_GIN_ANVIL_SDMA_ENABLE=1. Must come before source includes
+    # so the generated config is found first.
     include_directories(BEFORE ${ROCSHMEM_INSTALL_DIR}/include)
-    # Internal rocshmem headers (sdma/anvil_device.hpp) needed globally.
+    # Internal rocshmem headers (sdma/anvil_device.hpp) for those same TUs.
     if (ROCSHMEM_SOURCE_DIR)
         include_directories(${ROCSHMEM_SOURCE_DIR}/src)
     endif()
diff --git a/projects/rccl-tests/src/CMakeLists.txt b/projects/rccl-tests/src/CMakeLists.txt
index 3f671325cce2..1b238dcce638 100644
--- a/projects/rccl-tests/src/CMakeLists.txt
+++ b/projects/rccl-tests/src/CMakeLists.txt
@@ -145,27 +145,6 @@ endif()
 add_rccl_test(all_gather)
 add_rccl_test(all_reduce)
 add_rccl_test(alltoall)
-if (ENABLE_ROCSHMEM_GIN AND TARGET roc::rocshmem)
-    target_compile_definitions(alltoall_perf PRIVATE ENABLE_ROCSHMEM_GIN)
-    target_compile_options(alltoall_perf PRIVATE -fgpu-rdc $<$<CONFIG:Debug>:-ggdb -O0>)
-    # --whole-archive: librccl.so (ENABLE_ROCSHMEM_GIN) leaves rocshmem host
-    # symbols unresolved (--allow-shlib-undefined).  The executable must pull
-    # in ALL rocshmem symbols (not just those it references directly) so that
-    # -rdynamic exports them for librccl.so to resolve at runtime.
-    target_link_libraries(alltoall_perf PRIVATE
-        -Wl,--whole-archive roc::rocshmem -Wl,--no-whole-archive)
-    target_link_options(alltoall_perf PRIVATE -fgpu-rdc --hip-link -rdynamic)
-
-    # GIN-SDMA hybrid AllGather (all_gather_perf -D 3): same device-RDC link as
-    # alltoall_perf so the GinHybridAllGatherKernel resolves the GIN device backend.
-    if (ENABLE_DEVICE_API)
-      target_compile_definitions(all_gather_perf PRIVATE ENABLE_DEVICE_API)
-    endif()
-    target_compile_options(all_gather_perf PRIVATE -fgpu-rdc $<$<CONFIG:Debug>:-ggdb -O0>)
-    target_link_libraries(all_gather_perf PRIVATE
-        -Wl,--whole-archive roc::rocshmem -Wl,--no-whole-archive)
-    target_link_options(all_gather_perf PRIVATE -fgpu-rdc --hip-link -rdynamic)
-endif()
 if (ENABLE_ROCSHMEM_GIN AND TARGET roc::rocshmem AND RCCL_SOURCE_DIR)
     add_executable(gin_rocshmem_gda_factory_test ${CMAKE_CURRENT_SOURCE_DIR}/gin_rocshmem_gda_factory_test.cu)
     set_source_files_properties(${CMAKE_CURRENT_SOURCE_DIR}/gin_rocshmem_gda_factory_test.cu PROPERTIES LANGUAGE CXX)
@@ -185,16 +164,6 @@ if (ENABLE_ROCSHMEM_GIN AND TARGET roc::rocshmem AND RCCL_SOURCE_DIR)
 endif()
 add_rccl_test(alltoallv)
 add_rccl_test(broadcast)
-if (ENABLE_ROCSHMEM_GIN AND TARGET roc::rocshmem)
-    # GIN-SDMA hybrid Broadcast (broadcast_perf -D 3): same device-RDC link as
-    # alltoall_perf/all_gather_perf so the Gin*BroadcastKernel variants resolve
-    # the GIN device backend (SDMA puts + LSA/LL peer stores).
-    target_compile_definitions(broadcast_perf PRIVATE ENABLE_ROCSHMEM_GIN)
-    target_compile_options(broadcast_perf PRIVATE -fgpu-rdc $<$<CONFIG:Debug>:-ggdb -O0>)
-    target_link_libraries(broadcast_perf PRIVATE
-        -Wl,--whole-archive roc::rocshmem -Wl,--no-whole-archive)
-    target_link_options(broadcast_perf PRIVATE -fgpu-rdc --hip-link -rdynamic)
-endif()
 add_rccl_test(gather)
 add_rccl_test(hypercube)
 add_rccl_test(reduce_scatter)
@@ -202,3 +171,38 @@ add_rccl_test(reduce)
 add_rccl_test(scatter)
 add_rccl_test(sendrecv)
 add_rccl_test(all_reduce_bias)
+if (ENABLE_ROCSHMEM_GIN AND TARGET roc::rocshmem)
+    function(_rccl_tests_link_rocshmem_gin_rdc target)
+        target_compile_definitions(${target} PRIVATE ENABLE_ROCSHMEM_GIN NCCL_GIN_ANVIL_SDMA_ENABLE=1)
+        target_compile_options(${target} PRIVATE -fgpu-rdc $<$<CONFIG:Debug>:-ggdb -O0>)
+        # --whole-archive: librccl.so (ENABLE_ROCSHMEM_GIN) leaves rocshmem host
+        # symbols unresolved (--allow-shlib-undefined).  The executable must pull
+        # in ALL rocshmem symbols (not just those it references directly) so that
+        # -rdynamic exports them for librccl.so to resolve at runtime.
+        target_link_libraries(${target} PRIVATE
+            -Wl,--whole-archive roc::rocshmem -Wl,--no-whole-archive)
+        target_link_options(${target} PRIVATE -fgpu-rdc --hip-link -rdynamic)
+    endfunction()
+
+    # GIN-SDMA hybrid AllGather (all_gather_perf -D 3) needs ENABLE_DEVICE_API
+    # so the GinHybridAllGatherKernel is compiled into this binary.
+    if (ENABLE_DEVICE_API)
+      target_compile_definitions(all_gather_perf PRIVATE ENABLE_DEVICE_API)
+    endif()
+
+    # Device-api CI builds one --rocshmem-gin librccl.so and runs the matrix
+    # against it. Every launched binary must RDC-link rocSHMEM so GIN host
+    # symbols resolve from the executable. all_reduce_perf is omitted: it
+    # still does not build under ENABLE_ROCSHMEM_GIN (gin.sbatch uses
+    # `make -k` for the same reason).
+    foreach(_gin_rdc_bin IN ITEMS
+        all_gather_perf
+        alltoall_perf
+        broadcast_perf
+        gather_perf
+        reduce_perf
+        scatter_perf
+        sendrecv_perf)
+      _rccl_tests_link_rocshmem_gin_rdc(${_gin_rdc_bin})
+    endforeach()
+endif()
diff --git a/projects/rccl/tools/ci/lib/device-api-tests.json b/projects/rccl/tools/ci/lib/device-api-tests.json
index 721fcc14b9d3..0f13baa57788 100644
--- a/projects/rccl/tools/ci/lib/device-api-tests.json
+++ b/projects/rccl/tools/ci/lib/device-api-tests.json
@@ -10,7 +10,9 @@
     "  {LOGDIR} in debug_env is replaced by the runner with the per-job debug dir,",
     "  so NCCL_DEBUG_FILE writes per-rank logs there instead of flooding stdout.",
     "suites:     [ { name, bins:[...], args, env:[...] (optional) } ]",
-    "Switched from YAML to JSON because PyYAML is not installed on the CI hosts."
+    "Switched from YAML to JSON because PyYAML is not installed on the CI hosts.",
+    "LSA all_reduce_perf (-D 1/-D 2) is omitted: that binary still does not build",
+    "under ENABLE_ROCSHMEM_GIN (same as gin.sbatch). Re-add those suites when it does."
   ],
 
   "bench_args": "-b 8 -e 1G -f 2 -g 1",
@@ -25,22 +27,18 @@
       "bins": ["broadcast_perf", "reduce_perf", "sendrecv_perf", "scatter_perf", "gather_perf"],
       "args": "-R 2"
     },
-    {
-      "name": "lsa-d1",
-      "bins": ["all_reduce_perf"],
-      "args": "-R 2 -D 1"
-    },
-    {
-      "name": "lsa-d2",
-      "bins": ["all_reduce_perf"],
-      "args": "-R 2 -D 2"
-    },
     {
       "name": "gin-d3",
       "bins": ["alltoall_perf"],
       "args": "-R 2 -D 3",
       "env": ["NCCL_DMABUF_ENABLE=1", "NCCL_GIN_TYPE=2", "HSA_NO_SCRATCH_RECLAIM=1", "NCCL_ENV_PLUGIN=none", "RCCL_ENABLE_INTRANET=1"]
     },
+    {
+      "name": "gin-d3-allgather",
+      "bins": ["all_gather_perf"],
+      "args": "-R 2 -D 3",
+      "env": ["NCCL_DMABUF_ENABLE=1", "NCCL_GIN_TYPE=6", "HSA_NO_SCRATCH_RECLAIM=1", "NCCL_ENV_PLUGIN=none", "RCCL_ENABLE_INTRANET=1"]
+    },
     {
       "name": "gin-d4",
       "bins": ["alltoall_perf"],
diff --git a/projects/rccl/tools/ci/lib/device-api.sbatch b/projects/rccl/tools/ci/lib/device-api.sbatch
index 81549ecd1883..a4442734b132 100755
--- a/projects/rccl/tools/ci/lib/device-api.sbatch
+++ b/projects/rccl/tools/ci/lib/device-api.sbatch
@@ -15,9 +15,10 @@
 # On one 8-GPU node:
 #   1. consume ROCm (provisioned on the head node) via .ci-out/rocm.env
 #   2. lib/build-ompi.sh     build OpenMPI (--with-rocm) -> ompi.env
-#   3. projects/rccl/install.sh   build RCCL (in-tree, release)
-#   4. rccl-tests install.sh      build rccl-tests (ENABLE_DEVICE_API=1)
-#   5. run-device-api-ci.sh       run the device-API benchmark suite
+#   3. lib/build-rocshmem.sh   build rocSHMEM -> rocshmem.env
+#   4. projects/rccl/install.sh   build RCCL (in-tree, release, --rocshmem-gin)
+#   5. rccl-tests (cmake)         build rccl-tests (ENABLE_DEVICE_API + ENABLE_ROCSHMEM_GIN)
+#   6. run-device-api-ci.sh       run the device-API benchmark suite
 #
 # Required env (from the workflow via `sbatch --export=ALL,...`):
 #   RCCL_DEVICE_API_WORKDIR   GHA workspace (rocm-systems root)
@@ -62,7 +63,7 @@ build_jobs="${RCCL_BUILD_JOBS}"
 {
 # 1) ROCm: provisioned on the head node from TheRock CI artifacts; consume the
 # workspace tree via rocm.env.
-echo "==> [1/5] use ROCm (provisioned from TheRock artifacts on head node)"
+echo "==> [1/6] use ROCm (provisioned from TheRock artifacts on head node)"
 rocm_env="${workdir}/.ci-out/rocm.env"
 if [[ ! -f "${rocm_env}" ]]; then
   echo "ERROR: ${rocm_env} not found. The workflow must run the 'Provision ROCm" >&2
@@ -83,7 +84,7 @@ echo "    hipcc: $("${ROCM_PATH}/bin/hipcc" --version 2>/dev/null | head -n1)"
 
 # 2) OpenMPI: built here against ROCM_PATH into the per-job workspace cache
 # (no shared cache), then ompi.env (MPI_HOME) is sourced below.
-echo "==> [2/5] build OpenMPI"
+echo "==> [2/6] build OpenMPI"
 "${ci_dir}/lib/build-ompi.sh"
 set -a
 # shellcheck source=/dev/null  # runtime fragment written by build-ompi.sh
@@ -108,28 +109,82 @@ echo "==> verified hipcc = ${resolved_hipcc}"
 # shellcheck source=/dev/null
 source "${ci_dir}/lib/ensure-python-yaml.sh"
 
-# 3) RCCL via its own install.sh (-l = local gpu arch only).
-echo "==> [3/5] build RCCL (targets=${amdgpu_targets})"
-( cd "${workdir}/projects/rccl" \
+# 3) rocSHMEM (in-tree, IPC+SDMA) -> rocshmem.env (ROCSHMEM_INSTALL_DIR)
+echo "==> [3/6] build rocSHMEM"
+GPU_TARGETS="${amdgpu_targets}" "${ci_dir}/lib/build-rocshmem.sh"
+set -a
+# shellcheck source=/dev/null
+source "${workdir}/.ci-out/rocshmem.env"
+set +a
+
+# 4) RCCL with GIN / rocSHMEM (device-API + GIN-SDMA -D 3 paths need rocshmem in the executable).
+# --prefix is required: without an install tree, lib/cmake/rccl/*config.cmake is
+# never written and the glob below aborts the job under pipefail before any test.
+echo "==> [4/6] build RCCL (GIN, targets=${amdgpu_targets})"
+rccl_prefix="${workdir}/.ci-out/rccl-install"
+rm -rf "${rccl_prefix}"
+(
+  cd "${workdir}/projects/rccl" \
   && MPI_PATH="${MPI_HOME}" ./install.sh -lj "${build_jobs}" \
-       --amdgpu_targets="${amdgpu_targets}" )
-rccl_prefix="${workdir}/projects/rccl/build/release"
+       --amdgpu_targets="${amdgpu_targets}" \
+       --prefix="${rccl_prefix}" \
+       --rocshmem-gin --no_clean \
+       --cmake-options "-DROCSHMEM_INSTALL_DIR=${ROCSHMEM_INSTALL_DIR}"
+)
+if [[ ! -f "${rccl_prefix}/lib/librccl.so" && ! -f "${rccl_prefix}/lib/librccl.so.1" ]]; then
+  echo "ERROR: no librccl.so under ${rccl_prefix}/lib" >&2
+  ls -la "${rccl_prefix}/lib" 2>/dev/null >&2 || true
+  exit 1
+fi
+rccl_cfg="$(ls "${rccl_prefix}"/lib*/cmake/rccl/*[Cc]onfig.cmake 2>/dev/null | head -n1)"
+if [[ -z "${rccl_cfg}" ]]; then
+  echo "ERROR: no RCCL cmake config under ${rccl_prefix}/lib*/cmake/rccl" >&2
+  ls -laR "${rccl_prefix}"/lib*/cmake 2>/dev/null >&2 || true
+  exit 1
+fi
+rccl_cmake_dir="$(dirname "${rccl_cfg}")"
+echo "    in-tree RCCL cmake config: ${rccl_cmake_dir}"
 
-# 4) rccl-tests: built in-tree from projects/rccl-tests (must match
-# RCCL_TESTS_DIR in run-device-api-ci.sh). install.sh wipes its own build/.
-echo "==> [4/5] build rccl-tests (ENABLE_DEVICE_API=1)"
-rccl_tests_dir="${workdir}/projects/rccl-tests"
-( cd "${rccl_tests_dir}" \
-  && ENABLE_DEVICE_API=1 ./install.sh \
-       --mpi --mpi_home="${MPI_HOME}" --rccl_home="${rccl_prefix}" \
-       --gpu_targets="${amdgpu_targets}" )
+# 5) rccl-tests via cmake so every device-api matrix binary RDC-links rocshmem
+# (install.sh/Makefile path does not enable ENABLE_ROCSHMEM_GIN). Needed because
+# this job's librccl.so is --rocshmem-gin. RCCL_TESTS_BIN_DIR is consumed by
+# run-device-api-ci.sh.
+echo "==> [5/6] build rccl-tests (ENABLE_DEVICE_API + ENABLE_ROCSHMEM_GIN)"
+rccl_tests_src="${workdir}/projects/rccl-tests"
+rccl_tests_build="${workdir}/.ci-out/rccl-tests-build"
+rm -rf "${rccl_tests_build}"; mkdir -p "${rccl_tests_build}"
+(
+  cd "${rccl_tests_build}"
+  cmake -DCMAKE_BUILD_TYPE=Release -DUSE_MPI=ON \
+    -DENABLE_DEVICE_API=ON -DENABLE_ROCSHMEM_GIN=ON \
+    -DROCSHMEM_INSTALL_DIR="${ROCSHMEM_INSTALL_DIR}" \
+    -DRCCL_DIR="${rccl_cmake_dir}" \
+    -DCMAKE_PREFIX_PATH="${rccl_prefix};${MPI_HOME}" \
+    -DGPU_TARGETS="${amdgpu_targets}" \
+    "${rccl_tests_src}"
+  # all_reduce_perf still fails under ENABLE_ROCSHMEM_GIN; -k keeps the
+  # matrix binaries. Do not require all_reduce_perf (see gin.sbatch).
+  make -j"${build_jobs}" -k || true
+)
+required_perf_bins=(
+  all_gather_perf alltoall_perf broadcast_perf
+  gather_perf reduce_perf scatter_perf sendrecv_perf
+)
+for perf_bin in "${required_perf_bins[@]}"; do
+  if [[ ! -x "${rccl_tests_build}/${perf_bin}" ]]; then
+    echo "ERROR: ${perf_bin} not built under ${rccl_tests_build}" >&2
+    ls -la "${rccl_tests_build}" 2>/dev/null >&2 || true
+    exit 1
+  fi
+done
+export RCCL_TESTS_BIN_DIR="${rccl_tests_build}"
 } > "${build_out}" 2> "${build_err}"
 echo "==> build phase logs: ${build_out} / ${build_err}"
 
 # --- test phase -> test-%j.{out,err} ---
 {
-# 5) Benchmarks.
-echo "==> [5/5] run benchmarks"
+# 6) Benchmarks.
+echo "==> [6/6] run benchmarks"
 # GIN/RoCE QP creation pins memory; a low memlock limit makes ibv_create_qp fail with ENOMEM. Raise it for the mpirun ranks and log it.
 ulimit -l unlimited 2>/dev/null || echo "WARN: could not raise RLIMIT_MEMLOCK (ulimit -l)"
 echo "==> RLIMIT_MEMLOCK (ulimit -l) = $(ulimit -l)"
diff --git a/projects/rccl/tools/ci/run-device-api-ci.sh b/projects/rccl/tools/ci/run-device-api-ci.sh
index aae310f913d8..1675860a20d0 100755
--- a/projects/rccl/tools/ci/run-device-api-ci.sh
+++ b/projects/rccl/tools/ci/run-device-api-ci.sh
@@ -1,12 +1,13 @@
 #!/usr/bin/env bash
-# Run the RCCL device-API benchmark suite (symmetric memory / LSA device kernels
-# / GIN proxy) against the freshly built rccl-tests tree. Mirrors the legacy
-# rocJenkins "device-api" testCommand.
+# Run the RCCL device-API benchmark suite (symmetric memory / GIN proxy)
+# against the freshly built rccl-tests tree. Mirrors the legacy rocJenkins
+# "device-api" testCommand. LSA all_reduce_perf (-D 1/-D 2) is not launched:
+# that binary still does not build under ENABLE_ROCSHMEM_GIN.
 #
 # Consumes ROCM_PATH (rocm.env), MPI_HOME (ompi.env), the in-tree RCCL build at
-# projects/rccl/build/release, and rccl-tests/build. ROCM_PATH / MPI_HOME come
-# from the environment (device-api.sbatch exports them) or, when run standalone,
-# from $WORKDIR/.ci-out/{rocm,ompi}.env.
+# projects/rccl/build/release, and rccl-tests/build. Build paths come from the
+# environment (device-api.sbatch exports them) or, when run standalone, from
+# $WORKDIR/.ci-out/{rocm,ompi,rocshmem}.env.
 #
 # Each bench is wrapped in `timeout` so a hung mpirun/driver can't wedge the job;
 # failures are collected and surfaced at the end (exit non-zero iff any failed).
@@ -17,7 +18,9 @@
 # debug_env to every run.
 #
 # Environment overrides:
-#   ROCM_PATH / MPI_HOME   Else read from .ci-out/{rocm,ompi}.env
+#   ROCM_PATH / MPI_HOME / ROCSHMEM_INSTALL_DIR
+#                          Else read from matching .ci-out/*.env fragments
+#   RCCL_TESTS_BIN_DIR     rccl-tests build directory (default: rccl-tests/build)
 #   NP                     MPI ranks per run         (default: 8)
 #   BENCH_ARGS             Common bench args         (default: from JSON bench_args)
 #   BENCH_TIMEOUT          Per-bench wall-clock cap  (default: 600s)
@@ -41,11 +44,12 @@ RCCL_LIB_DIR="${WORKDIR}/projects/rccl/build/release"
 RCCL_TESTS_DIR="${WORKDIR}/projects/rccl-tests"
 CONFIG="${CONFIG:-${script_dir}/lib/device-api-tests.json}"
 
-# Prefer the build stages' env fragments over any ambient ROCM_PATH / MPI_HOME.
-# shellcheck source=/dev/null  # runtime fragment written by the provision step
-[[ -f "${WORKDIR}/.ci-out/rocm.env" ]] && source "${WORKDIR}/.ci-out/rocm.env"
-# shellcheck source=/dev/null  # runtime fragment written by build-ompi.sh
-[[ -f "${WORKDIR}/.ci-out/ompi.env" ]] && source "${WORKDIR}/.ci-out/ompi.env"
+# Prefer the build stages' env fragments over ambient build paths.
+for frag in rocm ompi rocshmem; do
+  env_file="${WORKDIR}/.ci-out/${frag}.env"
+  # shellcheck source=/dev/null  # runtime fragment written by the build stages
+  [[ -f "${env_file}" ]] && source "${env_file}"
+done
 
 : "${ROCM_PATH:?run-device-api-ci.sh: ROCM_PATH unset (provisioned via rocm.env / sbatch)}"
 : "${MPI_HOME:?run-device-api-ci.sh: MPI_HOME unset (run build-ompi.sh / via sbatch)}"
@@ -72,14 +76,25 @@ fi
 cd "${RCCL_TESTS_DIR}"
 
 export PATH="${MPI_HOME}/bin:${ROCM_PATH}/bin:${PATH}"
-export LD_LIBRARY_PATH="${RCCL_LIB_DIR}:${MPI_HOME}/lib:${ROCM_PATH}/lib:${LD_LIBRARY_PATH:-}"
+if [[ -n "${ROCSHMEM_INSTALL_DIR:-}" ]]; then
+  export LD_LIBRARY_PATH="${RCCL_LIB_DIR}:${ROCSHMEM_INSTALL_DIR}/lib:${MPI_HOME}/lib:${ROCM_PATH}/lib:${LD_LIBRARY_PATH:-}"
+else
+  export LD_LIBRARY_PATH="${RCCL_LIB_DIR}:${MPI_HOME}/lib:${ROCM_PATH}/lib:${LD_LIBRARY_PATH:-}"
+fi
 
-PERF_DIR=build
-if [[ ! -d "${PERF_DIR}" || ! -f "${PERF_DIR}/all_reduce_perf" ]]; then
+if [[ -n "${RCCL_TESTS_BIN_DIR:-}" ]]; then
+  PERF_DIR="${RCCL_TESTS_BIN_DIR}"
+else
+  PERF_DIR="${RCCL_TESTS_DIR}/build"
+fi
+# all_reduce_perf is not a valid sentinel here: it still fails to build under
+# ENABLE_ROCSHMEM_GIN. alltoall_perf is the binary this GIN job always needs.
+if [[ ! -d "${PERF_DIR}" || ! -x "${PERF_DIR}/alltoall_perf" ]]; then
   echo "rccl-tests perf binaries not found under ${PERF_DIR}"
-  ls -la
+  ls -la "${PERF_DIR}" 2>/dev/null || ls -la "${RCCL_TESTS_DIR}"
   exit 1
 fi
+echo "==> rccl-tests perf dir = ${PERF_DIR}"
 
 PARSER="${script_dir}/lib/parse_device_api_config.py"
 [[ -f "${CONFIG}" ]] || { echo "ERROR: test-matrix config not found: ${CONFIG}" >&2; exit 1; }
@@ -136,7 +151,7 @@ run_bench() {
   set +e
   timeout --kill-after="${BENCH_KILL_AFTER}" "${BENCH_TIMEOUT}" \
     mpirun -np "${NP}" ${env_flags} -x LD_LIBRARY_PATH \
-      "./${PERF_DIR}/${bin}" ${BENCH_ARGS} ${extra_args}
+      "${PERF_DIR}/${bin}" ${BENCH_ARGS} ${extra_args}
   local rc=$?
   set -e
   if [[ ${rc} -ne 0 ]]; then
@@ -175,7 +190,7 @@ echo "All device-api benchmark runs succeeded."
 # Requires ENABLE_DEVICE_API=ON rccl-tests build (alltoall_perf with -B flags).
 run_devtime_smoke() {
   local pytest_dir="${RCCL_TESTS_DIR}/test"
-  local perf_bin="${RCCL_TESTS_DIR}/${PERF_DIR}/alltoall_perf"
+  local perf_bin="${PERF_DIR}/alltoall_perf"
   if [[ ! -x "${perf_bin}" ]]; then
     echo "WARN: skip devtime smoke: ${perf_bin} not found"
     return 0

From ccaec740222a77fbf985fff7e9646d951906e210 Mon Sep 17 00:00:00 2001
From: Donglai Dai <donglai.dai@amd.com>
Date: Sat, 26 Sep 2026 18:29:05 -0400
Subject: [PATCH 2/5] ci(rccl): run device-api allgather on Anvil SDMA
 (AICOMRCCL-2221)

GIN type 6 selects rocSHMEM GDA, so gin-d3-allgather never reached the SDMA kernel this job builds. Point it at type 7 and keep the longer rocSHMEM build from dying on a 2h SLURM limit or a missing rccl.env.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 projects/rccl-tests/CMakeLists.txt            |  9 ++++---
 projects/rccl-tests/src/CMakeLists.txt        |  6 -----
 .../rccl/tools/ci/lib/device-api-tests.json   |  2 +-
 projects/rccl/tools/ci/lib/device-api.sbatch  | 22 +++++++++++++---
 projects/rccl/tools/ci/run-device-api-ci.sh   | 26 +++++++++++++------
 5 files changed, 42 insertions(+), 23 deletions(-)

diff --git a/projects/rccl-tests/CMakeLists.txt b/projects/rccl-tests/CMakeLists.txt
index fcb97dc7307d..7b54fced4abd 100644
--- a/projects/rccl-tests/CMakeLists.txt
+++ b/projects/rccl-tests/CMakeLists.txt
@@ -238,11 +238,12 @@ if (ENABLE_ROCSHMEM_GIN)
     if (NOT ROCSHMEM_SOURCE_DIR AND EXISTS "${_mono_root}/projects/rocshmem/CMakeLists.txt")
         set(ROCSHMEM_SOURCE_DIR "${_mono_root}/projects/rocshmem")
     endif()
-    # Installed rocshmem headers (rocshmem_config.h) for GIN-SDMA perf binaries
-    # that set NCCL_GIN_ANVIL_SDMA_ENABLE=1. Must come before source includes
-    # so the generated config is found first.
+    # Installed rocshmem headers (rocshmem_config.h) are needed globally:
+    # RCCL's public nccl_device.h → gin_anvil_sdma.h → anvil_device.hpp
+    # → log.hpp → rocshmem/rocshmem_config.h.  Must come before source
+    # includes so the generated config is found first.
     include_directories(BEFORE ${ROCSHMEM_INSTALL_DIR}/include)
-    # Internal rocshmem headers (sdma/anvil_device.hpp) for those same TUs.
+    # Internal rocshmem headers (sdma/anvil_device.hpp) needed globally.
     if (ROCSHMEM_SOURCE_DIR)
         include_directories(${ROCSHMEM_SOURCE_DIR}/src)
     endif()
diff --git a/projects/rccl-tests/src/CMakeLists.txt b/projects/rccl-tests/src/CMakeLists.txt
index 1b238dcce638..12056a84b4fb 100644
--- a/projects/rccl-tests/src/CMakeLists.txt
+++ b/projects/rccl-tests/src/CMakeLists.txt
@@ -184,12 +184,6 @@ if (ENABLE_ROCSHMEM_GIN AND TARGET roc::rocshmem)
         target_link_options(${target} PRIVATE -fgpu-rdc --hip-link -rdynamic)
     endfunction()
 
-    # GIN-SDMA hybrid AllGather (all_gather_perf -D 3) needs ENABLE_DEVICE_API
-    # so the GinHybridAllGatherKernel is compiled into this binary.
-    if (ENABLE_DEVICE_API)
-      target_compile_definitions(all_gather_perf PRIVATE ENABLE_DEVICE_API)
-    endif()
-
     # Device-api CI builds one --rocshmem-gin librccl.so and runs the matrix
     # against it. Every launched binary must RDC-link rocSHMEM so GIN host
     # symbols resolve from the executable. all_reduce_perf is omitted: it
diff --git a/projects/rccl/tools/ci/lib/device-api-tests.json b/projects/rccl/tools/ci/lib/device-api-tests.json
index 0f13baa57788..1c0a64fcb87b 100644
--- a/projects/rccl/tools/ci/lib/device-api-tests.json
+++ b/projects/rccl/tools/ci/lib/device-api-tests.json
@@ -37,7 +37,7 @@
       "name": "gin-d3-allgather",
       "bins": ["all_gather_perf"],
       "args": "-R 2 -D 3",
-      "env": ["NCCL_DMABUF_ENABLE=1", "NCCL_GIN_TYPE=6", "HSA_NO_SCRATCH_RECLAIM=1", "NCCL_ENV_PLUGIN=none", "RCCL_ENABLE_INTRANET=1"]
+      "env": ["NCCL_DMABUF_ENABLE=1", "NCCL_GIN_ENABLE=1", "NCCL_GIN_TYPE=7", "HSA_NO_SCRATCH_RECLAIM=1", "NCCL_ENV_PLUGIN=none", "RCCL_ENABLE_INTRANET=1"]
     },
     {
       "name": "gin-d4",
diff --git a/projects/rccl/tools/ci/lib/device-api.sbatch b/projects/rccl/tools/ci/lib/device-api.sbatch
index a4442734b132..5668b4bd3fe1 100755
--- a/projects/rccl/tools/ci/lib/device-api.sbatch
+++ b/projects/rccl/tools/ci/lib/device-api.sbatch
@@ -4,7 +4,7 @@
 #SBATCH --ntasks=8
 #SBATCH --partition=interactive
 #SBATCH --exclusive
-#SBATCH --time=02:00:00
+#SBATCH --time=04:00:00
 #SBATCH --output=%x-%j.out
 #SBATCH --error=%x-%j.err
 #SBATCH --propagate=NONE
@@ -136,7 +136,7 @@ if [[ ! -f "${rccl_prefix}/lib/librccl.so" && ! -f "${rccl_prefix}/lib/librccl.s
   ls -la "${rccl_prefix}/lib" 2>/dev/null >&2 || true
   exit 1
 fi
-rccl_cfg="$(ls "${rccl_prefix}"/lib*/cmake/rccl/*[Cc]onfig.cmake 2>/dev/null | head -n1)"
+rccl_cfg="$(ls "${rccl_prefix}"/lib*/cmake/rccl/*[Cc]onfig.cmake 2>/dev/null | head -n1 || true)"
 if [[ -z "${rccl_cfg}" ]]; then
   echo "ERROR: no RCCL cmake config under ${rccl_prefix}/lib*/cmake/rccl" >&2
   ls -laR "${rccl_prefix}"/lib*/cmake 2>/dev/null >&2 || true
@@ -162,10 +162,17 @@ rm -rf "${rccl_tests_build}"; mkdir -p "${rccl_tests_build}"
     -DCMAKE_PREFIX_PATH="${rccl_prefix};${MPI_HOME}" \
     -DGPU_TARGETS="${amdgpu_targets}" \
     "${rccl_tests_src}"
-  # all_reduce_perf still fails under ENABLE_ROCSHMEM_GIN; -k keeps the
-  # matrix binaries. Do not require all_reduce_perf (see gin.sbatch).
+  # -k: all_reduce_perf still fails under ENABLE_ROCSHMEM_GIN (see gin.sbatch).
+  # This configure also builds gin_rocshmem_gda_factory_test and the test/
+  # host unit target. required_perf_bins below does not check those, so a
+  # green job only proves the seven matrix binaries exist.
   make -j"${build_jobs}" -k || true
 )
+# Same seven binaries as foreach(_gin_rdc_bin) in
+# projects/rccl-tests/src/CMakeLists.txt and the union of bins in
+# lib/device-api-tests.json. A suite bin missing from that foreach still
+# builds and passes this -x check, then fails at load on an unresolved
+# rocSHMEM host symbol.
 required_perf_bins=(
   all_gather_perf alltoall_perf broadcast_perf
   gather_perf reduce_perf scatter_perf sendrecv_perf
@@ -178,6 +185,13 @@ for perf_bin in "${required_perf_bins[@]}"; do
   fi
 done
 export RCCL_TESTS_BIN_DIR="${rccl_tests_build}"
+# Persist for a standalone run-device-api-ci.sh (the in-process export above
+# does not survive a new shell). Mirrors gin.sbatch's .ci-out/rccl.env.
+{
+  printf 'export RCCL_INSTALL_PREFIX=%q\n' "${rccl_prefix}"
+  printf 'export RCCL_TESTS_BIN_DIR=%q\n' "${rccl_tests_build}"
+  printf 'export RCCL_TESTS_DIR=%q\n' "${rccl_tests_src}"
+} > "${workdir}/.ci-out/rccl.env"
 } > "${build_out}" 2> "${build_err}"
 echo "==> build phase logs: ${build_out} / ${build_err}"
 
diff --git a/projects/rccl/tools/ci/run-device-api-ci.sh b/projects/rccl/tools/ci/run-device-api-ci.sh
index 1675860a20d0..a35155677b38 100755
--- a/projects/rccl/tools/ci/run-device-api-ci.sh
+++ b/projects/rccl/tools/ci/run-device-api-ci.sh
@@ -4,10 +4,13 @@
 # "device-api" testCommand. LSA all_reduce_perf (-D 1/-D 2) is not launched:
 # that binary still does not build under ENABLE_ROCSHMEM_GIN.
 #
-# Consumes ROCM_PATH (rocm.env), MPI_HOME (ompi.env), the in-tree RCCL build at
-# projects/rccl/build/release, and rccl-tests/build. Build paths come from the
-# environment (device-api.sbatch exports them) or, when run standalone, from
-# $WORKDIR/.ci-out/{rocm,ompi,rocshmem}.env.
+# Consumes ROCM_PATH (rocm.env), MPI_HOME (ompi.env), ROCSHMEM_INSTALL_DIR
+# (rocshmem.env), and rccl.env (RCCL_INSTALL_PREFIX, RCCL_TESTS_BIN_DIR,
+# RCCL_TESTS_DIR). Build paths come from the environment (device-api.sbatch
+# exports them) or, when run standalone, from
+# $WORKDIR/.ci-out/{rocm,ompi,rocshmem,rccl}.env.
+# librccl.so comes from RCCL_INSTALL_PREFIX/lib when that is set, else
+# projects/rccl/build/release. RCCL_TESTS_BIN_DIR defaults to rccl-tests/build.
 #
 # Each bench is wrapped in `timeout` so a hung mpirun/driver can't wedge the job;
 # failures are collected and surfaced at the end (exit non-zero iff any failed).
@@ -18,9 +21,10 @@
 # debug_env to every run.
 #
 # Environment overrides:
-#   ROCM_PATH / MPI_HOME / ROCSHMEM_INSTALL_DIR
+#   ROCM_PATH / MPI_HOME / ROCSHMEM_INSTALL_DIR / RCCL_INSTALL_PREFIX
 #                          Else read from matching .ci-out/*.env fragments
-#   RCCL_TESTS_BIN_DIR     rccl-tests build directory (default: rccl-tests/build)
+#   RCCL_TESTS_BIN_DIR     rccl-tests build directory (default: rccl-tests/build;
+#                          device-api.sbatch writes it to .ci-out/rccl.env)
 #   NP                     MPI ranks per run         (default: 8)
 #   BENCH_ARGS             Common bench args         (default: from JSON bench_args)
 #   BENCH_TIMEOUT          Per-bench wall-clock cap  (default: 600s)
@@ -40,17 +44,23 @@ script_dir="$(cd "$(dirname "$0")" && pwd)"
 RCCL_DIR="$(cd "${script_dir}/../.." && pwd)"
 WORKDIR="$(cd "${RCCL_DIR}/../.." && pwd)"
 
-RCCL_LIB_DIR="${WORKDIR}/projects/rccl/build/release"
 RCCL_TESTS_DIR="${WORKDIR}/projects/rccl-tests"
 CONFIG="${CONFIG:-${script_dir}/lib/device-api-tests.json}"
 
 # Prefer the build stages' env fragments over ambient build paths.
-for frag in rocm ompi rocshmem; do
+# rccl.env is written by device-api.sbatch (RCCL_INSTALL_PREFIX, RCCL_TESTS_BIN_DIR).
+for frag in rocm ompi rocshmem rccl; do
   env_file="${WORKDIR}/.ci-out/${frag}.env"
   # shellcheck source=/dev/null  # runtime fragment written by the build stages
   [[ -f "${env_file}" ]] && source "${env_file}"
 done
 
+if [[ -n "${RCCL_INSTALL_PREFIX:-}" ]]; then
+  RCCL_LIB_DIR="${RCCL_INSTALL_PREFIX}/lib"
+else
+  RCCL_LIB_DIR="${WORKDIR}/projects/rccl/build/release"
+fi
+
 : "${ROCM_PATH:?run-device-api-ci.sh: ROCM_PATH unset (provisioned via rocm.env / sbatch)}"
 : "${MPI_HOME:?run-device-api-ci.sh: MPI_HOME unset (run build-ompi.sh / via sbatch)}"
 

From 3104488f205efae18ad7d44522a20c38fc703364 Mon Sep 17 00:00:00 2001
From: Donglai Dai <donglai.dai@amd.com>
Date: Sat, 26 Sep 2026 21:34:38 -0400
Subject: [PATCH 3/5] ci(rccl): keep non-GIN device-api coverage alongside GIN
 (AICOMRCCL-2221)

Restore the ENABLE_DEVICE_API install.sh build and LSA matrix so this job
still covers device-API-without-rocSHMEM, and match gin.sbatch's build-log
handling for the remaining Argus nits.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 .../tools/ci/lib/device-api-lsa-tests.json    |  39 +++++++
 .../rccl/tools/ci/lib/device-api-tests.json   |  16 +--
 projects/rccl/tools/ci/lib/device-api.sbatch  | 109 ++++++++++++++----
 projects/rccl/tools/ci/run-device-api-ci.sh   |  56 +++++----
 4 files changed, 170 insertions(+), 50 deletions(-)
 create mode 100644 projects/rccl/tools/ci/lib/device-api-lsa-tests.json

diff --git a/projects/rccl/tools/ci/lib/device-api-lsa-tests.json b/projects/rccl/tools/ci/lib/device-api-lsa-tests.json
new file mode 100644
index 000000000000..f7aea7037657
--- /dev/null
+++ b/projects/rccl/tools/ci/lib/device-api-lsa-tests.json
@@ -0,0 +1,39 @@
+{
+  "_comment": [
+    "Non-GIN device-API matrix for run-device-api-ci.sh.",
+    "Built with ENABLE_DEVICE_API=1 ./install.sh (no ENABLE_ROCSHMEM_GIN), so",
+    "all_reduce_perf and the LSA kernels (-D 1/-D 2) are available.",
+    "device-api.sbatch runs this matrix first, then device-api-tests.json against",
+    "the cmake ENABLE_ROCSHMEM_GIN build. Launch path is ${RCCL_TESTS_BIN_DIR}/<bin>",
+    "(absolute; default rccl-tests/build for this matrix).",
+    "bench_args: common bench flags | base_env: -x on every run",
+    "debug_env:  -x added only when RCCL_CI_DEBUG=1",
+    "  {LOGDIR} in debug_env is replaced by the runner with the per-job debug dir,",
+    "  so NCCL_DEBUG_FILE writes per-rank logs there instead of flooding stdout.",
+    "suites:     [ { name, bins:[...], args, env:[...] (optional) } ]"
+  ],
+
+  "bench_args": "-b 8 -e 1G -f 2 -g 1",
+
+  "base_env": ["NCCL_CUMEM_ENABLE=1", "HSA_FORCE_FINE_GRAIN_PCIE=1"],
+
+  "debug_env": ["NCCL_DEBUG=INFO", "NCCL_DEBUG_SUBSYS=INIT,NET", "NCCL_DEBUG_FILE={LOGDIR}/nccl-debug.%h.%p.log"],
+
+  "suites": [
+    {
+      "name": "symmetric",
+      "bins": ["broadcast_perf", "reduce_perf", "sendrecv_perf", "scatter_perf", "gather_perf"],
+      "args": "-R 2"
+    },
+    {
+      "name": "lsa-d1",
+      "bins": ["all_reduce_perf"],
+      "args": "-R 2 -D 1"
+    },
+    {
+      "name": "lsa-d2",
+      "bins": ["all_reduce_perf"],
+      "args": "-R 2 -D 2"
+    }
+  ]
+}
diff --git a/projects/rccl/tools/ci/lib/device-api-tests.json b/projects/rccl/tools/ci/lib/device-api-tests.json
index 1c0a64fcb87b..3bd9062d95fe 100644
--- a/projects/rccl/tools/ci/lib/device-api-tests.json
+++ b/projects/rccl/tools/ci/lib/device-api-tests.json
@@ -1,8 +1,11 @@
 {
   "_comment": [
-    "Device-API benchmark matrix for run-device-api-ci.sh.",
+    "GIN device-API benchmark matrix for run-device-api-ci.sh.",
     "Add a mode by appending a suites entry. Each suite runs every bin as:",
-    "  mpirun -np $NP <base_env + suite env> -x LD_LIBRARY_PATH ./build/<bin> $bench_args <args>",
+    "  mpirun -np $NP <base_env + suite env> -x LD_LIBRARY_PATH ${RCCL_TESTS_BIN_DIR}/<bin> $bench_args <args>",
+    "RCCL_TESTS_BIN_DIR is the absolute cmake build dir (.ci-out/rccl-tests-build",
+    "for this job). Non-GIN / LSA suites live in device-api-lsa-tests.json and run",
+    "against the ENABLE_DEVICE_API=1 ./install.sh build.",
     "After the matrix, run-device-api-ci.sh runs pytest test/test_AllToAllDevtime.py",
     "(GIN -D 3 device-timing smoke: -B 1/2 and -B 2 -H 1; ENABLE_DEVICE_API build).",
     "bench_args: common bench flags | base_env: -x on every run",
@@ -10,9 +13,7 @@
     "  {LOGDIR} in debug_env is replaced by the runner with the per-job debug dir,",
     "  so NCCL_DEBUG_FILE writes per-rank logs there instead of flooding stdout.",
     "suites:     [ { name, bins:[...], args, env:[...] (optional) } ]",
-    "Switched from YAML to JSON because PyYAML is not installed on the CI hosts.",
-    "LSA all_reduce_perf (-D 1/-D 2) is omitted: that binary still does not build",
-    "under ENABLE_ROCSHMEM_GIN (same as gin.sbatch). Re-add those suites when it does."
+    "Switched from YAML to JSON because PyYAML is not installed on the CI hosts."
   ],
 
   "bench_args": "-b 8 -e 1G -f 2 -g 1",
@@ -22,11 +23,6 @@
   "debug_env": ["NCCL_DEBUG=INFO", "NCCL_DEBUG_SUBSYS=INIT,NET", "NCCL_DEBUG_FILE={LOGDIR}/nccl-debug.%h.%p.log"],
 
   "suites": [
-    {
-      "name": "symmetric",
-      "bins": ["broadcast_perf", "reduce_perf", "sendrecv_perf", "scatter_perf", "gather_perf"],
-      "args": "-R 2"
-    },
     {
       "name": "gin-d3",
       "bins": ["alltoall_perf"],
diff --git a/projects/rccl/tools/ci/lib/device-api.sbatch b/projects/rccl/tools/ci/lib/device-api.sbatch
index 5668b4bd3fe1..e34615652b52 100755
--- a/projects/rccl/tools/ci/lib/device-api.sbatch
+++ b/projects/rccl/tools/ci/lib/device-api.sbatch
@@ -17,8 +17,11 @@
 #   2. lib/build-ompi.sh     build OpenMPI (--with-rocm) -> ompi.env
 #   3. lib/build-rocshmem.sh   build rocSHMEM -> rocshmem.env
 #   4. projects/rccl/install.sh   build RCCL (in-tree, release, --rocshmem-gin)
-#   5. rccl-tests (cmake)         build rccl-tests (ENABLE_DEVICE_API + ENABLE_ROCSHMEM_GIN)
-#   6. run-device-api-ci.sh       run the device-API benchmark suite
+#   5. rccl-tests (install.sh)    ENABLE_DEVICE_API=1, no ENABLE_ROCSHMEM_GIN
+#                                  (symmetric + LSA all_reduce_perf coverage)
+#   6. rccl-tests (cmake)         ENABLE_DEVICE_API + ENABLE_ROCSHMEM_GIN
+#                                  (GIN -D 3/-D 4 + allgather Anvil SDMA)
+#   7. run-device-api-ci.sh       LSA matrix, then GIN matrix
 #
 # Required env (from the workflow via `sbatch --export=ALL,...`):
 #   RCCL_DEVICE_API_WORKDIR   GHA workspace (rocm-systems root)
@@ -59,11 +62,13 @@ fi
 export RCCL_BUILD_JOBS
 build_jobs="${RCCL_BUILD_JOBS}"
 
-# --- build phase -> build-%j.{out,err} ---
-{
+# --- build phase -> build-%j.{out,err}. Runs in a subshell so any failure (a
+# hard `exit` or a set -e abort) is caught below and skips the test phase. ---
+build_rc=0
+(
 # 1) ROCm: provisioned on the head node from TheRock CI artifacts; consume the
 # workspace tree via rocm.env.
-echo "==> [1/6] use ROCm (provisioned from TheRock artifacts on head node)"
+echo "==> [1/7] use ROCm (provisioned from TheRock artifacts on head node)"
 rocm_env="${workdir}/.ci-out/rocm.env"
 if [[ ! -f "${rocm_env}" ]]; then
   echo "ERROR: ${rocm_env} not found. The workflow must run the 'Provision ROCm" >&2
@@ -84,7 +89,7 @@ echo "    hipcc: $("${ROCM_PATH}/bin/hipcc" --version 2>/dev/null | head -n1)"
 
 # 2) OpenMPI: built here against ROCM_PATH into the per-job workspace cache
 # (no shared cache), then ompi.env (MPI_HOME) is sourced below.
-echo "==> [2/6] build OpenMPI"
+echo "==> [2/7] build OpenMPI"
 "${ci_dir}/lib/build-ompi.sh"
 set -a
 # shellcheck source=/dev/null  # runtime fragment written by build-ompi.sh
@@ -110,7 +115,7 @@ echo "==> verified hipcc = ${resolved_hipcc}"
 source "${ci_dir}/lib/ensure-python-yaml.sh"
 
 # 3) rocSHMEM (in-tree, IPC+SDMA) -> rocshmem.env (ROCSHMEM_INSTALL_DIR)
-echo "==> [3/6] build rocSHMEM"
+echo "==> [3/7] build rocSHMEM"
 GPU_TARGETS="${amdgpu_targets}" "${ci_dir}/lib/build-rocshmem.sh"
 set -a
 # shellcheck source=/dev/null
@@ -119,8 +124,8 @@ set +a
 
 # 4) RCCL with GIN / rocSHMEM (device-API + GIN-SDMA -D 3 paths need rocshmem in the executable).
 # --prefix is required: without an install tree, lib/cmake/rccl/*config.cmake is
-# never written and the glob below aborts the job under pipefail before any test.
-echo "==> [4/6] build RCCL (GIN, targets=${amdgpu_targets})"
+# never written and the rccl_cfg check below fails the job before any test.
+echo "==> [4/7] build RCCL (GIN, targets=${amdgpu_targets})"
 rccl_prefix="${workdir}/.ci-out/rccl-install"
 rm -rf "${rccl_prefix}"
 (
@@ -145,12 +150,38 @@ fi
 rccl_cmake_dir="$(dirname "${rccl_cfg}")"
 echo "    in-tree RCCL cmake config: ${rccl_cmake_dir}"
 
-# 5) rccl-tests via cmake so every device-api matrix binary RDC-links rocshmem
+# 5) rccl-tests via install.sh with ENABLE_DEVICE_API only. This is the
+# device-API-without-rocSHMEM build: all_reduce_perf and the LSA kernels need
+# it, and it was previously the only CI path that exercised that configure.
+# install.sh always writes into projects/rccl-tests/build.
+echo "==> [5/7] build rccl-tests (ENABLE_DEVICE_API via install.sh)"
+rccl_tests_src="${workdir}/projects/rccl-tests"
+rccl_tests_lsa_build="${rccl_tests_src}/build"
+(
+  cd "${rccl_tests_src}" \
+  && ./install.sh --mpi --enable-device-api \
+       --mpi_home="${MPI_HOME}" --rccl_home="${rccl_prefix}" \
+       --rocm_home="${ROCM_PATH}" \
+       --gpu_targets="${amdgpu_targets}"
+)
+# Symmetric + LSA suites in device-api-lsa-tests.json.
+required_lsa_bins=(
+  all_reduce_perf broadcast_perf reduce_perf
+  sendrecv_perf scatter_perf gather_perf
+)
+for perf_bin in "${required_lsa_bins[@]}"; do
+  if [[ ! -x "${rccl_tests_lsa_build}/${perf_bin}" ]]; then
+    echo "ERROR: ${perf_bin} not built under ${rccl_tests_lsa_build}" >&2
+    ls -la "${rccl_tests_lsa_build}" 2>/dev/null >&2 || true
+    exit 1
+  fi
+done
+
+# 6) rccl-tests via cmake so every GIN matrix binary RDC-links rocshmem
 # (install.sh/Makefile path does not enable ENABLE_ROCSHMEM_GIN). Needed because
 # this job's librccl.so is --rocshmem-gin. RCCL_TESTS_BIN_DIR is consumed by
-# run-device-api-ci.sh.
-echo "==> [5/6] build rccl-tests (ENABLE_DEVICE_API + ENABLE_ROCSHMEM_GIN)"
-rccl_tests_src="${workdir}/projects/rccl-tests"
+# run-device-api-ci.sh for the GIN matrix.
+echo "==> [6/7] build rccl-tests (ENABLE_DEVICE_API + ENABLE_ROCSHMEM_GIN)"
 rccl_tests_build="${workdir}/.ci-out/rccl-tests-build"
 rm -rf "${rccl_tests_build}"; mkdir -p "${rccl_tests_build}"
 (
@@ -184,24 +215,62 @@ for perf_bin in "${required_perf_bins[@]}"; do
     exit 1
   fi
 done
-export RCCL_TESTS_BIN_DIR="${rccl_tests_build}"
 # Persist for a standalone run-device-api-ci.sh (the in-process export above
 # does not survive a new shell). Mirrors gin.sbatch's .ci-out/rccl.env.
 {
   printf 'export RCCL_INSTALL_PREFIX=%q\n' "${rccl_prefix}"
   printf 'export RCCL_TESTS_BIN_DIR=%q\n' "${rccl_tests_build}"
+  printf 'export RCCL_TESTS_LSA_BIN_DIR=%q\n' "${rccl_tests_lsa_build}"
   printf 'export RCCL_TESTS_DIR=%q\n' "${rccl_tests_src}"
 } > "${workdir}/.ci-out/rccl.env"
-} > "${build_out}" 2> "${build_err}"
+) > "${build_out}" 2> "${build_err}" || build_rc=$?
 echo "==> build phase logs: ${build_out} / ${build_err}"
 
+if [[ "${build_rc}" -ne 0 ]]; then
+  echo "ERROR: build phase failed (rc=${build_rc}); skipping device-api tests." >&2
+  echo "----- last 60 lines of ${build_err} -----" >&2
+  tail -n 60 "${build_err}" >&2 || true
+  exit "${build_rc}"
+fi
+
 # --- test phase -> test-%j.{out,err} ---
+# Source build fragments so ROCM_PATH / MPI_HOME / ROCSHMEM / RCCL paths are set
+# even though the build ran in a subshell.
+set -a
+# shellcheck source=/dev/null
+source "${workdir}/.ci-out/rocm.env"
+# shellcheck source=/dev/null
+source "${workdir}/.ci-out/ompi.env"
+# shellcheck source=/dev/null
+source "${workdir}/.ci-out/rocshmem.env"
+# shellcheck source=/dev/null
+source "${workdir}/.ci-out/rccl.env"
+set +a
+
+test_rc=0
 {
-# 6) Benchmarks.
-echo "==> [6/6] run benchmarks"
-# GIN/RoCE QP creation pins memory; a low memlock limit makes ibv_create_qp fail with ENOMEM. Raise it for the mpirun ranks and log it.
+# 7) Benchmarks: LSA/symmetric against the install.sh build, then GIN against
+# the cmake ENABLE_ROCSHMEM_GIN build. Skip the GIN AllToAllDevtime smoke on the
+# LSA pass; the GIN pass runs it.
+echo "==> [7/7] run benchmarks"
 ulimit -l unlimited 2>/dev/null || echo "WARN: could not raise RLIMIT_MEMLOCK (ulimit -l)"
 echo "==> RLIMIT_MEMLOCK (ulimit -l) = $(ulimit -l)"
-"${ci_dir}/run-device-api-ci.sh"
-} > "${test_out}" 2> "${test_err}"
+
+echo "==> LSA / symmetric matrix (install.sh ENABLE_DEVICE_API build)"
+CONFIG="${ci_dir}/lib/device-api-lsa-tests.json" \
+RCCL_TESTS_BIN_DIR="${RCCL_TESTS_LSA_BIN_DIR}" \
+RCCL_CI_SKIP_DEVTIME=1 \
+  "${ci_dir}/run-device-api-ci.sh"
+
+echo "==> GIN matrix (cmake ENABLE_ROCSHMEM_GIN build)"
+CONFIG="${ci_dir}/lib/device-api-tests.json" \
+RCCL_TESTS_BIN_DIR="${RCCL_TESTS_BIN_DIR}" \
+  "${ci_dir}/run-device-api-ci.sh"
+} > "${test_out}" 2> "${test_err}" || test_rc=$?
 echo "==> test phase logs: ${test_out} / ${test_err}"
+if [[ "${test_rc}" -ne 0 ]]; then
+  echo "ERROR: device-api tests failed (rc=${test_rc})" >&2
+  echo "----- last 60 lines of ${test_err} -----" >&2
+  tail -n 60 "${test_err}" >&2 || true
+  exit "${test_rc}"
+fi
diff --git a/projects/rccl/tools/ci/run-device-api-ci.sh b/projects/rccl/tools/ci/run-device-api-ci.sh
index a35155677b38..fdc873efa02f 100755
--- a/projects/rccl/tools/ci/run-device-api-ci.sh
+++ b/projects/rccl/tools/ci/run-device-api-ci.sh
@@ -1,8 +1,7 @@
 #!/usr/bin/env bash
-# Run the RCCL device-API benchmark suite (symmetric memory / GIN proxy)
-# against the freshly built rccl-tests tree. Mirrors the legacy rocJenkins
-# "device-api" testCommand. LSA all_reduce_perf (-D 1/-D 2) is not launched:
-# that binary still does not build under ENABLE_ROCSHMEM_GIN.
+# Run the RCCL device-API benchmark suite (symmetric memory / LSA device
+# kernels / GIN proxy) against a freshly built rccl-tests tree. Mirrors the
+# legacy rocJenkins "device-api" testCommand.
 #
 # Consumes ROCM_PATH (rocm.env), MPI_HOME (ompi.env), ROCSHMEM_INSTALL_DIR
 # (rocshmem.env), and rccl.env (RCCL_INSTALL_PREFIX, RCCL_TESTS_BIN_DIR,
@@ -12,13 +11,20 @@
 # librccl.so comes from RCCL_INSTALL_PREFIX/lib when that is set, else
 # projects/rccl/build/release. RCCL_TESTS_BIN_DIR defaults to rccl-tests/build.
 #
+# device-api.sbatch invokes this twice:
+#   1. CONFIG=device-api-lsa-tests.json against the install.sh ENABLE_DEVICE_API
+#      build (symmetric + LSA), with RCCL_CI_SKIP_DEVTIME=1
+#   2. CONFIG=device-api-tests.json against the cmake ENABLE_ROCSHMEM_GIN build
+#      (GIN suites + AllToAllDevtime smoke)
+#
 # Each bench is wrapped in `timeout` so a hung mpirun/driver can't wedge the job;
 # failures are collected and surfaced at the end (exit non-zero iff any failed).
 # After the JSON matrix, runs pytest test/test_AllToAllDevtime.py (GIN AllToAll
-# in-kernel device-timing smoke: modes -B 1, -B 2, -B 2 -H 1).
+# in-kernel device-timing smoke: modes -B 1, -B 2, -B 2 -H 1) unless
+# RCCL_CI_SKIP_DEVTIME=1.
 #
-# Test matrix lives in lib/device-api-tests.json; RCCL_CI_DEBUG=1 adds its
-# debug_env to every run.
+# Test matrix lives in lib/device-api-tests.json by default; RCCL_CI_DEBUG=1
+# adds its debug_env to every run.
 #
 # Environment overrides:
 #   ROCM_PATH / MPI_HOME / ROCSHMEM_INSTALL_DIR / RCCL_INSTALL_PREFIX
@@ -33,6 +39,7 @@
 #   RCCL_CI_DEBUG          Set to 1 to add debug_env to every run
 #   RCCL_CI_DEBUG_DIR      Dir for NCCL_DEBUG_FILE output when RCCL_CI_DEBUG=1
 #                          (default: ${SLURM_SUBMIT_DIR:-$PWD}/nccl-debug)
+#   RCCL_CI_SKIP_DEVTIME   Set to 1 to skip test_AllToAllDevtime.py
 
 set -euo pipefail
 
@@ -49,11 +56,18 @@ CONFIG="${CONFIG:-${script_dir}/lib/device-api-tests.json}"
 
 # Prefer the build stages' env fragments over ambient build paths.
 # rccl.env is written by device-api.sbatch (RCCL_INSTALL_PREFIX, RCCL_TESTS_BIN_DIR).
+# Keep an already-exported RCCL_TESTS_BIN_DIR: device-api.sbatch sets it per
+# matrix (install.sh LSA build vs cmake GIN build) and rccl.env would clobber it.
+_saved_tests_bin_dir="${RCCL_TESTS_BIN_DIR-}"
 for frag in rocm ompi rocshmem rccl; do
   env_file="${WORKDIR}/.ci-out/${frag}.env"
   # shellcheck source=/dev/null  # runtime fragment written by the build stages
   [[ -f "${env_file}" ]] && source "${env_file}"
 done
+if [[ -n "${_saved_tests_bin_dir}" ]]; then
+  RCCL_TESTS_BIN_DIR="${_saved_tests_bin_dir}"
+fi
+unset _saved_tests_bin_dir
 
 if [[ -n "${RCCL_INSTALL_PREFIX:-}" ]]; then
   RCCL_LIB_DIR="${RCCL_INSTALL_PREFIX}/lib"
@@ -86,22 +100,19 @@ fi
 cd "${RCCL_TESTS_DIR}"
 
 export PATH="${MPI_HOME}/bin:${ROCM_PATH}/bin:${PATH}"
-if [[ -n "${ROCSHMEM_INSTALL_DIR:-}" ]]; then
-  export LD_LIBRARY_PATH="${RCCL_LIB_DIR}:${ROCSHMEM_INSTALL_DIR}/lib:${MPI_HOME}/lib:${ROCM_PATH}/lib:${LD_LIBRARY_PATH:-}"
-else
-  export LD_LIBRARY_PATH="${RCCL_LIB_DIR}:${MPI_HOME}/lib:${ROCM_PATH}/lib:${LD_LIBRARY_PATH:-}"
-fi
+export LD_LIBRARY_PATH="${RCCL_LIB_DIR}${ROCSHMEM_INSTALL_DIR:+:${ROCSHMEM_INSTALL_DIR}/lib}:${MPI_HOME}/lib:${ROCM_PATH}/lib:${LD_LIBRARY_PATH:-}"
 
-if [[ -n "${RCCL_TESTS_BIN_DIR:-}" ]]; then
-  PERF_DIR="${RCCL_TESTS_BIN_DIR}"
-else
-  PERF_DIR="${RCCL_TESTS_DIR}/build"
+PERF_DIR="${RCCL_TESTS_BIN_DIR:-${RCCL_TESTS_DIR}/build}"
+# Prefer alltoall_perf (always built for both matrices). Fall back to
+# all_reduce_perf for a pure LSA/install.sh tree that somehow omitted it.
+if [[ ! -d "${PERF_DIR}" ]]; then
+  echo "rccl-tests perf binaries not found under ${PERF_DIR}"
+  ls -la "${RCCL_TESTS_DIR}" 2>/dev/null || true
+  exit 1
 fi
-# all_reduce_perf is not a valid sentinel here: it still fails to build under
-# ENABLE_ROCSHMEM_GIN. alltoall_perf is the binary this GIN job always needs.
-if [[ ! -d "${PERF_DIR}" || ! -x "${PERF_DIR}/alltoall_perf" ]]; then
+if [[ ! -x "${PERF_DIR}/alltoall_perf" && ! -x "${PERF_DIR}/all_reduce_perf" ]]; then
   echo "rccl-tests perf binaries not found under ${PERF_DIR}"
-  ls -la "${PERF_DIR}" 2>/dev/null || ls -la "${RCCL_TESTS_DIR}"
+  ls -la "${PERF_DIR}" 2>/dev/null || true
   exit 1
 fi
 echo "==> rccl-tests perf dir = ${PERF_DIR}"
@@ -196,6 +207,11 @@ fi
 
 echo "All device-api benchmark runs succeeded."
 
+if [[ -n "${RCCL_CI_SKIP_DEVTIME:-}" ]]; then
+  echo "==> skip AllToAllDevtime smoke (RCCL_CI_SKIP_DEVTIME=${RCCL_CI_SKIP_DEVTIME})"
+  exit 0
+fi
+
 # GIN AllToAll in-kernel device-timing smoke (wall_clock64 / *TimedKernel).
 # Requires ENABLE_DEVICE_API=ON rccl-tests build (alltoall_perf with -B flags).
 run_devtime_smoke() {

From 7c6cc228763843b6de152aa076def805cb83d2c3 Mon Sep 17 00:00:00 2001
From: Donglai Dai <donglai.dai@amd.com>
Date: Sat, 26 Sep 2026 21:46:50 -0400
Subject: [PATCH 4/5] ci(rccl): drop non-GIN LSA suites from device-api
 coverage (AICOMRCCL-2221)

Keep the ENABLE_DEVICE_API install.sh build for the symmetric matrix, and
do not run or require all_reduce_perf -D 1/-D 2 in this job.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 ...ests.json => device-api-nongin-tests.json} | 14 +------
 .../rccl/tools/ci/lib/device-api-tests.json   |  4 +-
 projects/rccl/tools/ci/lib/device-api.sbatch  | 38 +++++++++----------
 projects/rccl/tools/ci/run-device-api-ci.sh   | 18 ++++-----
 4 files changed, 31 insertions(+), 43 deletions(-)
 rename projects/rccl/tools/ci/lib/{device-api-lsa-tests.json => device-api-nongin-tests.json} (80%)

diff --git a/projects/rccl/tools/ci/lib/device-api-lsa-tests.json b/projects/rccl/tools/ci/lib/device-api-nongin-tests.json
similarity index 80%
rename from projects/rccl/tools/ci/lib/device-api-lsa-tests.json
rename to projects/rccl/tools/ci/lib/device-api-nongin-tests.json
index f7aea7037657..bae550ca682f 100644
--- a/projects/rccl/tools/ci/lib/device-api-lsa-tests.json
+++ b/projects/rccl/tools/ci/lib/device-api-nongin-tests.json
@@ -1,8 +1,8 @@
 {
   "_comment": [
     "Non-GIN device-API matrix for run-device-api-ci.sh.",
-    "Built with ENABLE_DEVICE_API=1 ./install.sh (no ENABLE_ROCSHMEM_GIN), so",
-    "all_reduce_perf and the LSA kernels (-D 1/-D 2) are available.",
+    "Built with ENABLE_DEVICE_API=1 ./install.sh (no ENABLE_ROCSHMEM_GIN).",
+    "LSA all_reduce_perf (-D 1/-D 2) is not part of this job.",
     "device-api.sbatch runs this matrix first, then device-api-tests.json against",
     "the cmake ENABLE_ROCSHMEM_GIN build. Launch path is ${RCCL_TESTS_BIN_DIR}/<bin>",
     "(absolute; default rccl-tests/build for this matrix).",
@@ -24,16 +24,6 @@
       "name": "symmetric",
       "bins": ["broadcast_perf", "reduce_perf", "sendrecv_perf", "scatter_perf", "gather_perf"],
       "args": "-R 2"
-    },
-    {
-      "name": "lsa-d1",
-      "bins": ["all_reduce_perf"],
-      "args": "-R 2 -D 1"
-    },
-    {
-      "name": "lsa-d2",
-      "bins": ["all_reduce_perf"],
-      "args": "-R 2 -D 2"
     }
   ]
 }
diff --git a/projects/rccl/tools/ci/lib/device-api-tests.json b/projects/rccl/tools/ci/lib/device-api-tests.json
index 3bd9062d95fe..da3916a747a6 100644
--- a/projects/rccl/tools/ci/lib/device-api-tests.json
+++ b/projects/rccl/tools/ci/lib/device-api-tests.json
@@ -4,8 +4,8 @@
     "Add a mode by appending a suites entry. Each suite runs every bin as:",
     "  mpirun -np $NP <base_env + suite env> -x LD_LIBRARY_PATH ${RCCL_TESTS_BIN_DIR}/<bin> $bench_args <args>",
     "RCCL_TESTS_BIN_DIR is the absolute cmake build dir (.ci-out/rccl-tests-build",
-    "for this job). Non-GIN / LSA suites live in device-api-lsa-tests.json and run",
-    "against the ENABLE_DEVICE_API=1 ./install.sh build.",
+    "for this job). The non-GIN symmetric matrix is device-api-nongin-tests.json,",
+    "run against the ENABLE_DEVICE_API=1 ./install.sh build.",
     "After the matrix, run-device-api-ci.sh runs pytest test/test_AllToAllDevtime.py",
     "(GIN -D 3 device-timing smoke: -B 1/2 and -B 2 -H 1; ENABLE_DEVICE_API build).",
     "bench_args: common bench flags | base_env: -x on every run",
diff --git a/projects/rccl/tools/ci/lib/device-api.sbatch b/projects/rccl/tools/ci/lib/device-api.sbatch
index e34615652b52..e17b330021b9 100755
--- a/projects/rccl/tools/ci/lib/device-api.sbatch
+++ b/projects/rccl/tools/ci/lib/device-api.sbatch
@@ -18,10 +18,10 @@
 #   3. lib/build-rocshmem.sh   build rocSHMEM -> rocshmem.env
 #   4. projects/rccl/install.sh   build RCCL (in-tree, release, --rocshmem-gin)
 #   5. rccl-tests (install.sh)    ENABLE_DEVICE_API=1, no ENABLE_ROCSHMEM_GIN
-#                                  (symmetric + LSA all_reduce_perf coverage)
+#                                  (symmetric matrix; LSA all_reduce is not run)
 #   6. rccl-tests (cmake)         ENABLE_DEVICE_API + ENABLE_ROCSHMEM_GIN
 #                                  (GIN -D 3/-D 4 + allgather Anvil SDMA)
-#   7. run-device-api-ci.sh       LSA matrix, then GIN matrix
+#   7. run-device-api-ci.sh       non-GIN matrix, then GIN matrix
 #
 # Required env (from the workflow via `sbatch --export=ALL,...`):
 #   RCCL_DEVICE_API_WORKDIR   GHA workspace (rocm-systems root)
@@ -151,12 +151,11 @@ rccl_cmake_dir="$(dirname "${rccl_cfg}")"
 echo "    in-tree RCCL cmake config: ${rccl_cmake_dir}"
 
 # 5) rccl-tests via install.sh with ENABLE_DEVICE_API only. This is the
-# device-API-without-rocSHMEM build: all_reduce_perf and the LSA kernels need
-# it, and it was previously the only CI path that exercised that configure.
-# install.sh always writes into projects/rccl-tests/build.
+# device-API-without-rocSHMEM build. LSA all_reduce_perf (-D 1/-D 2) is not
+# part of this job. install.sh always writes into projects/rccl-tests/build.
 echo "==> [5/7] build rccl-tests (ENABLE_DEVICE_API via install.sh)"
 rccl_tests_src="${workdir}/projects/rccl-tests"
-rccl_tests_lsa_build="${rccl_tests_src}/build"
+rccl_tests_nongin_build="${rccl_tests_src}/build"
 (
   cd "${rccl_tests_src}" \
   && ./install.sh --mpi --enable-device-api \
@@ -164,15 +163,14 @@ rccl_tests_lsa_build="${rccl_tests_src}/build"
        --rocm_home="${ROCM_PATH}" \
        --gpu_targets="${amdgpu_targets}"
 )
-# Symmetric + LSA suites in device-api-lsa-tests.json.
-required_lsa_bins=(
-  all_reduce_perf broadcast_perf reduce_perf
-  sendrecv_perf scatter_perf gather_perf
+# Symmetric suites in device-api-nongin-tests.json. all_reduce_perf is not required.
+required_nongin_bins=(
+  broadcast_perf reduce_perf sendrecv_perf scatter_perf gather_perf
 )
-for perf_bin in "${required_lsa_bins[@]}"; do
-  if [[ ! -x "${rccl_tests_lsa_build}/${perf_bin}" ]]; then
-    echo "ERROR: ${perf_bin} not built under ${rccl_tests_lsa_build}" >&2
-    ls -la "${rccl_tests_lsa_build}" 2>/dev/null >&2 || true
+for perf_bin in "${required_nongin_bins[@]}"; do
+  if [[ ! -x "${rccl_tests_nongin_build}/${perf_bin}" ]]; then
+    echo "ERROR: ${perf_bin} not built under ${rccl_tests_nongin_build}" >&2
+    ls -la "${rccl_tests_nongin_build}" 2>/dev/null >&2 || true
     exit 1
   fi
 done
@@ -220,7 +218,7 @@ done
 {
   printf 'export RCCL_INSTALL_PREFIX=%q\n' "${rccl_prefix}"
   printf 'export RCCL_TESTS_BIN_DIR=%q\n' "${rccl_tests_build}"
-  printf 'export RCCL_TESTS_LSA_BIN_DIR=%q\n' "${rccl_tests_lsa_build}"
+  printf 'export RCCL_TESTS_NONGIN_BIN_DIR=%q\n' "${rccl_tests_nongin_build}"
   printf 'export RCCL_TESTS_DIR=%q\n' "${rccl_tests_src}"
 } > "${workdir}/.ci-out/rccl.env"
 ) > "${build_out}" 2> "${build_err}" || build_rc=$?
@@ -249,16 +247,16 @@ set +a
 
 test_rc=0
 {
-# 7) Benchmarks: LSA/symmetric against the install.sh build, then GIN against
+# 7) Benchmarks: symmetric against the install.sh build, then GIN against
 # the cmake ENABLE_ROCSHMEM_GIN build. Skip the GIN AllToAllDevtime smoke on the
-# LSA pass; the GIN pass runs it.
+# non-GIN pass; the GIN pass runs it. LSA all_reduce (-D 1/-D 2) is not run.
 echo "==> [7/7] run benchmarks"
 ulimit -l unlimited 2>/dev/null || echo "WARN: could not raise RLIMIT_MEMLOCK (ulimit -l)"
 echo "==> RLIMIT_MEMLOCK (ulimit -l) = $(ulimit -l)"
 
-echo "==> LSA / symmetric matrix (install.sh ENABLE_DEVICE_API build)"
-CONFIG="${ci_dir}/lib/device-api-lsa-tests.json" \
-RCCL_TESTS_BIN_DIR="${RCCL_TESTS_LSA_BIN_DIR}" \
+echo "==> non-GIN symmetric matrix (install.sh ENABLE_DEVICE_API build)"
+CONFIG="${ci_dir}/lib/device-api-nongin-tests.json" \
+RCCL_TESTS_BIN_DIR="${RCCL_TESTS_NONGIN_BIN_DIR}" \
 RCCL_CI_SKIP_DEVTIME=1 \
   "${ci_dir}/run-device-api-ci.sh"
 
diff --git a/projects/rccl/tools/ci/run-device-api-ci.sh b/projects/rccl/tools/ci/run-device-api-ci.sh
index fdc873efa02f..361af8bddf75 100755
--- a/projects/rccl/tools/ci/run-device-api-ci.sh
+++ b/projects/rccl/tools/ci/run-device-api-ci.sh
@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
-# Run the RCCL device-API benchmark suite (symmetric memory / LSA device
-# kernels / GIN proxy) against a freshly built rccl-tests tree. Mirrors the
-# legacy rocJenkins "device-api" testCommand.
+# Run the RCCL device-API benchmark suite (symmetric memory / GIN proxy)
+# against a freshly built rccl-tests tree. Mirrors the legacy rocJenkins
+# "device-api" testCommand. LSA all_reduce_perf (-D 1/-D 2) is not launched.
 #
 # Consumes ROCM_PATH (rocm.env), MPI_HOME (ompi.env), ROCSHMEM_INSTALL_DIR
 # (rocshmem.env), and rccl.env (RCCL_INSTALL_PREFIX, RCCL_TESTS_BIN_DIR,
@@ -12,8 +12,8 @@
 # projects/rccl/build/release. RCCL_TESTS_BIN_DIR defaults to rccl-tests/build.
 #
 # device-api.sbatch invokes this twice:
-#   1. CONFIG=device-api-lsa-tests.json against the install.sh ENABLE_DEVICE_API
-#      build (symmetric + LSA), with RCCL_CI_SKIP_DEVTIME=1
+#   1. CONFIG=device-api-nongin-tests.json against the install.sh ENABLE_DEVICE_API
+#      build (symmetric only; no LSA), with RCCL_CI_SKIP_DEVTIME=1
 #   2. CONFIG=device-api-tests.json against the cmake ENABLE_ROCSHMEM_GIN build
 #      (GIN suites + AllToAllDevtime smoke)
 #
@@ -57,7 +57,7 @@ CONFIG="${CONFIG:-${script_dir}/lib/device-api-tests.json}"
 # Prefer the build stages' env fragments over ambient build paths.
 # rccl.env is written by device-api.sbatch (RCCL_INSTALL_PREFIX, RCCL_TESTS_BIN_DIR).
 # Keep an already-exported RCCL_TESTS_BIN_DIR: device-api.sbatch sets it per
-# matrix (install.sh LSA build vs cmake GIN build) and rccl.env would clobber it.
+# matrix (install.sh non-GIN build vs cmake GIN build) and rccl.env would clobber it.
 _saved_tests_bin_dir="${RCCL_TESTS_BIN_DIR-}"
 for frag in rocm ompi rocshmem rccl; do
   env_file="${WORKDIR}/.ci-out/${frag}.env"
@@ -103,14 +103,14 @@ export PATH="${MPI_HOME}/bin:${ROCM_PATH}/bin:${PATH}"
 export LD_LIBRARY_PATH="${RCCL_LIB_DIR}${ROCSHMEM_INSTALL_DIR:+:${ROCSHMEM_INSTALL_DIR}/lib}:${MPI_HOME}/lib:${ROCM_PATH}/lib:${LD_LIBRARY_PATH:-}"
 
 PERF_DIR="${RCCL_TESTS_BIN_DIR:-${RCCL_TESTS_DIR}/build}"
-# Prefer alltoall_perf (always built for both matrices). Fall back to
-# all_reduce_perf for a pure LSA/install.sh tree that somehow omitted it.
+# alltoall_perf is built for both matrices. Fall back to broadcast_perf, which
+# the non-GIN symmetric matrix always launches.
 if [[ ! -d "${PERF_DIR}" ]]; then
   echo "rccl-tests perf binaries not found under ${PERF_DIR}"
   ls -la "${RCCL_TESTS_DIR}" 2>/dev/null || true
   exit 1
 fi
-if [[ ! -x "${PERF_DIR}/alltoall_perf" && ! -x "${PERF_DIR}/all_reduce_perf" ]]; then
+if [[ ! -x "${PERF_DIR}/alltoall_perf" && ! -x "${PERF_DIR}/broadcast_perf" ]]; then
   echo "rccl-tests perf binaries not found under ${PERF_DIR}"
   ls -la "${PERF_DIR}" 2>/dev/null || true
   exit 1

From 2b6f1bf543ba2c1829337a9691aed1c1a0de53c9 Mon Sep 17 00:00:00 2001
From: Donglai Dai <donglai.dai@amd.com>
Date: Sun, 27 Sep 2026 00:00:08 -0400
Subject: [PATCH 5/5] ci(rccl): record each device-api matrix exit status
 (AICOMRCCL-2221)

A single || on the test brace group ignored set -e for the non-GIN run, so
only the GIN matrix status was retained. Capture each invocation separately.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 projects/rccl/tools/ci/lib/device-api.sbatch | 21 +++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

diff --git a/projects/rccl/tools/ci/lib/device-api.sbatch b/projects/rccl/tools/ci/lib/device-api.sbatch
index e17b330021b9..516955c42dad 100755
--- a/projects/rccl/tools/ci/lib/device-api.sbatch
+++ b/projects/rccl/tools/ci/lib/device-api.sbatch
@@ -245,11 +245,15 @@ source "${workdir}/.ci-out/rocshmem.env"
 source "${workdir}/.ci-out/rccl.env"
 set +a
 
-test_rc=0
+nongin_rc=0
+gin_rc=0
 {
 # 7) Benchmarks: symmetric against the install.sh build, then GIN against
 # the cmake ENABLE_ROCSHMEM_GIN build. Skip the GIN AllToAllDevtime smoke on the
 # non-GIN pass; the GIN pass runs it. LSA all_reduce (-D 1/-D 2) is not run.
+# Capture each matrix with its own || rc=$?: set -e is ignored on the left of ||
+# inside a compound command, so a single trailing || on the brace group would
+# keep only the second matrix's status.
 echo "==> [7/7] run benchmarks"
 ulimit -l unlimited 2>/dev/null || echo "WARN: could not raise RLIMIT_MEMLOCK (ulimit -l)"
 echo "==> RLIMIT_MEMLOCK (ulimit -l) = $(ulimit -l)"
@@ -258,17 +262,20 @@ echo "==> non-GIN symmetric matrix (install.sh ENABLE_DEVICE_API build)"
 CONFIG="${ci_dir}/lib/device-api-nongin-tests.json" \
 RCCL_TESTS_BIN_DIR="${RCCL_TESTS_NONGIN_BIN_DIR}" \
 RCCL_CI_SKIP_DEVTIME=1 \
-  "${ci_dir}/run-device-api-ci.sh"
+  "${ci_dir}/run-device-api-ci.sh" || nongin_rc=$?
 
 echo "==> GIN matrix (cmake ENABLE_ROCSHMEM_GIN build)"
 CONFIG="${ci_dir}/lib/device-api-tests.json" \
 RCCL_TESTS_BIN_DIR="${RCCL_TESTS_BIN_DIR}" \
-  "${ci_dir}/run-device-api-ci.sh"
-} > "${test_out}" 2> "${test_err}" || test_rc=$?
+  "${ci_dir}/run-device-api-ci.sh" || gin_rc=$?
+} > "${test_out}" 2> "${test_err}"
 echo "==> test phase logs: ${test_out} / ${test_err}"
-if [[ "${test_rc}" -ne 0 ]]; then
-  echo "ERROR: device-api tests failed (rc=${test_rc})" >&2
+if [[ "${nongin_rc}" -ne 0 || "${gin_rc}" -ne 0 ]]; then
+  echo "ERROR: device-api tests failed (nongin_rc=${nongin_rc} gin_rc=${gin_rc})" >&2
   echo "----- last 60 lines of ${test_err} -----" >&2
   tail -n 60 "${test_err}" >&2 || true
-  exit "${test_rc}"
+  if [[ "${nongin_rc}" -ne 0 ]]; then
+    exit "${nongin_rc}"
+  fi
+  exit "${gin_rc}"
 fi