ci(rccl): build device-api with rocSHMEM GIN

projects/rccl-tests/CMakeLists.txt

@@ -238,12 +238,11 @@ if (ENABLE_ROCSHMEM_GIN)
     if (NOT ROCSHMEM_SOURCE_DIR AND EXISTS "${_mono_root}/projects/rocshmem/CMakeLists.txt")
         set(ROCSHMEM_SOURCE_DIR "${_mono_root}/projects/rocshmem")
     endif()
-    # Installed rocshmem headers (rocshmem_config.h) are needed globally:
-    # RCCL's public nccl_device.h → gin_anvil_sdma.h → anvil_device.hpp
-    # → log.hpp → rocshmem/rocshmem_config.h.  Must come before source
-    # includes so the generated config is found first.
+    # Installed rocshmem headers (rocshmem_config.h) for GIN-SDMA perf binaries
+    # that set NCCL_GIN_ANVIL_SDMA_ENABLE=1. Must come before source includes
+    # so the generated config is found first.
     include_directories(BEFORE ${ROCSHMEM_INSTALL_DIR}/include)
-    # Internal rocshmem headers (sdma/anvil_device.hpp) needed globally.
+    # Internal rocshmem headers (sdma/anvil_device.hpp) for those same TUs.
     if (ROCSHMEM_SOURCE_DIR)
         include_directories(${ROCSHMEM_SOURCE_DIR}/src)
     endif()

projects/rccl-tests/src/CMakeLists.txt

@@ -145,27 +145,6 @@ endif()
 add_rccl_test(all_gather)
 add_rccl_test(all_reduce)
 add_rccl_test(alltoall)
-if (ENABLE_ROCSHMEM_GIN AND TARGET roc::rocshmem)
-    target_compile_definitions(alltoall_perf PRIVATE ENABLE_ROCSHMEM_GIN)
-    target_compile_options(alltoall_perf PRIVATE -fgpu-rdc $<$<CONFIG:Debug>:-ggdb -O0>)
-    # --whole-archive: librccl.so (ENABLE_ROCSHMEM_GIN) leaves rocshmem host
-    # symbols unresolved (--allow-shlib-undefined).  The executable must pull
-    # in ALL rocshmem symbols (not just those it references directly) so that
-    # -rdynamic exports them for librccl.so to resolve at runtime.
-    target_link_libraries(alltoall_perf PRIVATE
-        -Wl,--whole-archive roc::rocshmem -Wl,--no-whole-archive)
-    target_link_options(alltoall_perf PRIVATE -fgpu-rdc --hip-link -rdynamic)
-
-    # GIN-SDMA hybrid AllGather (all_gather_perf -D 3): same device-RDC link as
-    # alltoall_perf so the GinHybridAllGatherKernel resolves the GIN device backend.
-    if (ENABLE_DEVICE_API)
-      target_compile_definitions(all_gather_perf PRIVATE ENABLE_DEVICE_API)
-    endif()
-    target_compile_options(all_gather_perf PRIVATE -fgpu-rdc $<$<CONFIG:Debug>:-ggdb -O0>)
-    target_link_libraries(all_gather_perf PRIVATE
-        -Wl,--whole-archive roc::rocshmem -Wl,--no-whole-archive)
-    target_link_options(all_gather_perf PRIVATE -fgpu-rdc --hip-link -rdynamic)
-endif()
 if (ENABLE_ROCSHMEM_GIN AND TARGET roc::rocshmem AND RCCL_SOURCE_DIR)
     add_executable(gin_rocshmem_gda_factory_test ${CMAKE_CURRENT_SOURCE_DIR}/gin_rocshmem_gda_factory_test.cu)
     set_source_files_properties(${CMAKE_CURRENT_SOURCE_DIR}/gin_rocshmem_gda_factory_test.cu PROPERTIES LANGUAGE CXX)
@@ -185,20 +164,45 @@ if (ENABLE_ROCSHMEM_GIN AND TARGET roc::rocshmem AND RCCL_SOURCE_DIR)
 endif()
 add_rccl_test(alltoallv)
 add_rccl_test(broadcast)
-if (ENABLE_ROCSHMEM_GIN AND TARGET roc::rocshmem)
-    # GIN-SDMA hybrid Broadcast (broadcast_perf -D 3): same device-RDC link as
-    # alltoall_perf/all_gather_perf so the Gin*BroadcastKernel variants resolve
-    # the GIN device backend (SDMA puts + LSA/LL peer stores).
-    target_compile_definitions(broadcast_perf PRIVATE ENABLE_ROCSHMEM_GIN)
-    target_compile_options(broadcast_perf PRIVATE -fgpu-rdc $<$<CONFIG:Debug>:-ggdb -O0>)
-    target_link_libraries(broadcast_perf PRIVATE
-        -Wl,--whole-archive roc::rocshmem -Wl,--no-whole-archive)
-    target_link_options(broadcast_perf PRIVATE -fgpu-rdc --hip-link -rdynamic)
-endif()
 add_rccl_test(gather)
 add_rccl_test(hypercube)
 add_rccl_test(reduce_scatter)
 add_rccl_test(reduce)
 add_rccl_test(scatter)
 add_rccl_test(sendrecv)
 add_rccl_test(all_reduce_bias)
+if (ENABLE_ROCSHMEM_GIN AND TARGET roc::rocshmem)
+    function(_rccl_tests_link_rocshmem_gin_rdc target)
+        target_compile_definitions(${target} PRIVATE ENABLE_ROCSHMEM_GIN NCCL_GIN_ANVIL_SDMA_ENABLE=1)
+        target_compile_options(${target} PRIVATE -fgpu-rdc $<$<CONFIG:Debug>:-ggdb -O0>)
+        # --whole-archive: librccl.so (ENABLE_ROCSHMEM_GIN) leaves rocshmem host
+        # symbols unresolved (--allow-shlib-undefined).  The executable must pull
+        # in ALL rocshmem symbols (not just those it references directly) so that
+        # -rdynamic exports them for librccl.so to resolve at runtime.
+        target_link_libraries(${target} PRIVATE
+            -Wl,--whole-archive roc::rocshmem -Wl,--no-whole-archive)
+        target_link_options(${target} PRIVATE -fgpu-rdc --hip-link -rdynamic)
+    endfunction()
+
+    # GIN-SDMA hybrid AllGather (all_gather_perf -D 3) needs ENABLE_DEVICE_API
+    # so the GinHybridAllGatherKernel is compiled into this binary.
+    if (ENABLE_DEVICE_API)
+      target_compile_definitions(all_gather_perf PRIVATE ENABLE_DEVICE_API)
+    endif()
+
+    # Device-api CI builds one --rocshmem-gin librccl.so and runs the matrix
+    # against it. Every launched binary must RDC-link rocSHMEM so GIN host
+    # symbols resolve from the executable. all_reduce_perf is omitted: it
+    # still does not build under ENABLE_ROCSHMEM_GIN (gin.sbatch uses
+    # `make -k` for the same reason).
+    foreach(_gin_rdc_bin IN ITEMS
+        all_gather_perf
+        alltoall_perf
+        broadcast_perf
+        gather_perf
+        reduce_perf
+        scatter_perf
+        sendrecv_perf)
+      _rccl_tests_link_rocshmem_gin_rdc(${_gin_rdc_bin})
+    endforeach()
+endif()

projects/rccl/tools/ci/lib/device-api-tests.json

@@ -10,7 +10,9 @@
     "  {LOGDIR} in debug_env is replaced by the runner with the per-job debug dir,",
     "  so NCCL_DEBUG_FILE writes per-rank logs there instead of flooding stdout.",
     "suites:     [ { name, bins:[...], args, env:[...] (optional) } ]",
-    "Switched from YAML to JSON because PyYAML is not installed on the CI hosts."
+    "Switched from YAML to JSON because PyYAML is not installed on the CI hosts.",
+    "LSA all_reduce_perf (-D 1/-D 2) is omitted: that binary still does not build",
+    "under ENABLE_ROCSHMEM_GIN (same as gin.sbatch). Re-add those suites when it does."
   ],
 
   "bench_args": "-b 8 -e 1G -f 2 -g 1",
@@ -25,22 +27,18 @@
       "bins": ["broadcast_perf", "reduce_perf", "sendrecv_perf", "scatter_perf", "gather_perf"],
       "args": "-R 2"
     },
-    {
-      "name": "lsa-d1",
-      "bins": ["all_reduce_perf"],
-      "args": "-R 2 -D 1"
-    },
-    {
-      "name": "lsa-d2",
-      "bins": ["all_reduce_perf"],
-      "args": "-R 2 -D 2"
-    },
     {
       "name": "gin-d3",
       "bins": ["alltoall_perf"],
       "args": "-R 2 -D 3",
       "env": ["NCCL_DMABUF_ENABLE=1", "NCCL_GIN_TYPE=2", "HSA_NO_SCRATCH_RECLAIM=1", "NCCL_ENV_PLUGIN=none", "RCCL_ENABLE_INTRANET=1"]
     },
+    {
+      "name": "gin-d3-allgather",
+      "bins": ["all_gather_perf"],
+      "args": "-R 2 -D 3",
+      "env": ["NCCL_DMABUF_ENABLE=1", "NCCL_GIN_TYPE=6", "HSA_NO_SCRATCH_RECLAIM=1", "NCCL_ENV_PLUGIN=none", "RCCL_ENABLE_INTRANET=1"]
+    },
     {
       "name": "gin-d4",
       "bins": ["alltoall_perf"],

projects/rccl/tools/ci/lib/device-api.sbatch

@@ -15,9 +15,10 @@
 # On one 8-GPU node:
 #   1. consume ROCm (provisioned on the head node) via .ci-out/rocm.env
 #   2. lib/build-ompi.sh     build OpenMPI (--with-rocm) -> ompi.env
-#   3. projects/rccl/install.sh   build RCCL (in-tree, release)
-#   4. rccl-tests install.sh      build rccl-tests (ENABLE_DEVICE_API=1)
-#   5. run-device-api-ci.sh       run the device-API benchmark suite
+#   3. lib/build-rocshmem.sh   build rocSHMEM -> rocshmem.env
+#   4. projects/rccl/install.sh   build RCCL (in-tree, release, --rocshmem-gin)
+#   5. rccl-tests (cmake)         build rccl-tests (ENABLE_DEVICE_API + ENABLE_ROCSHMEM_GIN)
+#   6. run-device-api-ci.sh       run the device-API benchmark suite
 #
 # Required env (from the workflow via `sbatch --export=ALL,...`):
 #   RCCL_DEVICE_API_WORKDIR   GHA workspace (rocm-systems root)
@@ -62,7 +63,7 @@ build_jobs="${RCCL_BUILD_JOBS}"
 {
 # 1) ROCm: provisioned on the head node from TheRock CI artifacts; consume the
 # workspace tree via rocm.env.
-echo "==> [1/5] use ROCm (provisioned from TheRock artifacts on head node)"
+echo "==> [1/6] use ROCm (provisioned from TheRock artifacts on head node)"
 rocm_env="${workdir}/.ci-out/rocm.env"
 if [[ ! -f "${rocm_env}" ]]; then
   echo "ERROR: ${rocm_env} not found. The workflow must run the 'Provision ROCm" >&2
@@ -83,7 +84,7 @@ echo "    hipcc: $("${ROCM_PATH}/bin/hipcc" --version 2>/dev/null | head -n1)"
 
 # 2) OpenMPI: built here against ROCM_PATH into the per-job workspace cache
 # (no shared cache), then ompi.env (MPI_HOME) is sourced below.
-echo "==> [2/5] build OpenMPI"
+echo "==> [2/6] build OpenMPI"
 "${ci_dir}/lib/build-ompi.sh"
 set -a
 # shellcheck source=/dev/null  # runtime fragment written by build-ompi.sh
@@ -108,28 +109,82 @@ echo "==> verified hipcc = ${resolved_hipcc}"
 # shellcheck source=/dev/null
 source "${ci_dir}/lib/ensure-python-yaml.sh"
 
-# 3) RCCL via its own install.sh (-l = local gpu arch only).
-echo "==> [3/5] build RCCL (targets=${amdgpu_targets})"
-( cd "${workdir}/projects/rccl" \
+# 3) rocSHMEM (in-tree, IPC+SDMA) -> rocshmem.env (ROCSHMEM_INSTALL_DIR)
+echo "==> [3/6] build rocSHMEM"
+GPU_TARGETS="${amdgpu_targets}" "${ci_dir}/lib/build-rocshmem.sh"
+set -a
+# shellcheck source=/dev/null
+source "${workdir}/.ci-out/rocshmem.env"
+set +a
+
+# 4) RCCL with GIN / rocSHMEM (device-API + GIN-SDMA -D 3 paths need rocshmem in the executable).
+# --prefix is required: without an install tree, lib/cmake/rccl/*config.cmake is
+# never written and the glob below aborts the job under pipefail before any test.
+echo "==> [4/6] build RCCL (GIN, targets=${amdgpu_targets})"
+rccl_prefix="${workdir}/.ci-out/rccl-install"
+rm -rf "${rccl_prefix}"
+(
+  cd "${workdir}/projects/rccl" \
   && MPI_PATH="${MPI_HOME}" ./install.sh -lj "${build_jobs}" \
-       --amdgpu_targets="${amdgpu_targets}" )
-rccl_prefix="${workdir}/projects/rccl/build/release"
+       --amdgpu_targets="${amdgpu_targets}" \
+       --prefix="${rccl_prefix}" \
+       --rocshmem-gin --no_clean \
+       --cmake-options "-DROCSHMEM_INSTALL_DIR=${ROCSHMEM_INSTALL_DIR}"
+)
+if [[ ! -f "${rccl_prefix}/lib/librccl.so" && ! -f "${rccl_prefix}/lib/librccl.so.1" ]]; then
+  echo "ERROR: no librccl.so under ${rccl_prefix}/lib" >&2
+  ls -la "${rccl_prefix}/lib" 2>/dev/null >&2 || true
+  exit 1
+fi
+rccl_cfg="$(ls "${rccl_prefix}"/lib*/cmake/rccl/*[Cc]onfig.cmake 2>/dev/null | head -n1)"
+if [[ -z "${rccl_cfg}" ]]; then
+  echo "ERROR: no RCCL cmake config under ${rccl_prefix}/lib*/cmake/rccl" >&2
+  ls -laR "${rccl_prefix}"/lib*/cmake 2>/dev/null >&2 || true
+  exit 1
+fi
+rccl_cmake_dir="$(dirname "${rccl_cfg}")"
+echo "    in-tree RCCL cmake config: ${rccl_cmake_dir}"
 
-# 4) rccl-tests: built in-tree from projects/rccl-tests (must match
-# RCCL_TESTS_DIR in run-device-api-ci.sh). install.sh wipes its own build/.
-echo "==> [4/5] build rccl-tests (ENABLE_DEVICE_API=1)"
-rccl_tests_dir="${workdir}/projects/rccl-tests"
-( cd "${rccl_tests_dir}" \
-  && ENABLE_DEVICE_API=1 ./install.sh \
-       --mpi --mpi_home="${MPI_HOME}" --rccl_home="${rccl_prefix}" \
-       --gpu_targets="${amdgpu_targets}" )
+# 5) rccl-tests via cmake so every device-api matrix binary RDC-links rocshmem
+# (install.sh/Makefile path does not enable ENABLE_ROCSHMEM_GIN). Needed because
+# this job's librccl.so is --rocshmem-gin. RCCL_TESTS_BIN_DIR is consumed by
+# run-device-api-ci.sh.
+echo "==> [5/6] build rccl-tests (ENABLE_DEVICE_API + ENABLE_ROCSHMEM_GIN)"
+rccl_tests_src="${workdir}/projects/rccl-tests"
+rccl_tests_build="${workdir}/.ci-out/rccl-tests-build"
+rm -rf "${rccl_tests_build}"; mkdir -p "${rccl_tests_build}"
+(
+  cd "${rccl_tests_build}"
+  cmake -DCMAKE_BUILD_TYPE=Release -DUSE_MPI=ON \
+    -DENABLE_DEVICE_API=ON -DENABLE_ROCSHMEM_GIN=ON \
+    -DROCSHMEM_INSTALL_DIR="${ROCSHMEM_INSTALL_DIR}" \
+    -DRCCL_DIR="${rccl_cmake_dir}" \
+    -DCMAKE_PREFIX_PATH="${rccl_prefix};${MPI_HOME}" \
+    -DGPU_TARGETS="${amdgpu_targets}" \
+    "${rccl_tests_src}"
+  # all_reduce_perf still fails under ENABLE_ROCSHMEM_GIN; -k keeps the
+  # matrix binaries. Do not require all_reduce_perf (see gin.sbatch).
+  make -j"${build_jobs}" -k || true
+)
+required_perf_bins=(
+  all_gather_perf alltoall_perf broadcast_perf
+  gather_perf reduce_perf scatter_perf sendrecv_perf
+)
+for perf_bin in "${required_perf_bins[@]}"; do
+  if [[ ! -x "${rccl_tests_build}/${perf_bin}" ]]; then
+    echo "ERROR: ${perf_bin} not built under ${rccl_tests_build}" >&2
+    ls -la "${rccl_tests_build}" 2>/dev/null >&2 || true
+    exit 1
+  fi
+done
+export RCCL_TESTS_BIN_DIR="${rccl_tests_build}"
 } > "${build_out}" 2> "${build_err}"
 echo "==> build phase logs: ${build_out} / ${build_err}"
 
 # --- test phase -> test-%j.{out,err} ---
 {
-# 5) Benchmarks.
-echo "==> [5/5] run benchmarks"
+# 6) Benchmarks.
+echo "==> [6/6] run benchmarks"
 # GIN/RoCE QP creation pins memory; a low memlock limit makes ibv_create_qp fail with ENOMEM. Raise it for the mpirun ranks and log it.
 ulimit -l unlimited 2>/dev/null || echo "WARN: could not raise RLIMIT_MEMLOCK (ulimit -l)"
 echo "==> RLIMIT_MEMLOCK (ulimit -l) = $(ulimit -l)"

projects/rccl/tools/ci/run-device-api-ci.sh

@@ -1,12 +1,13 @@
 #!/usr/bin/env bash
-# Run the RCCL device-API benchmark suite (symmetric memory / LSA device kernels
-# / GIN proxy) against the freshly built rccl-tests tree. Mirrors the legacy
-# rocJenkins "device-api" testCommand.
+# Run the RCCL device-API benchmark suite (symmetric memory / GIN proxy)
+# against the freshly built rccl-tests tree. Mirrors the legacy rocJenkins
+# "device-api" testCommand. LSA all_reduce_perf (-D 1/-D 2) is not launched:
+# that binary still does not build under ENABLE_ROCSHMEM_GIN.
 #
 # Consumes ROCM_PATH (rocm.env), MPI_HOME (ompi.env), the in-tree RCCL build at
-# projects/rccl/build/release, and rccl-tests/build. ROCM_PATH / MPI_HOME come
-# from the environment (device-api.sbatch exports them) or, when run standalone,
-# from $WORKDIR/.ci-out/{rocm,ompi}.env.
+# projects/rccl/build/release, and rccl-tests/build. Build paths come from the
+# environment (device-api.sbatch exports them) or, when run standalone, from
+# $WORKDIR/.ci-out/{rocm,ompi,rocshmem}.env.
 #
 # Each bench is wrapped in `timeout` so a hung mpirun/driver can't wedge the job;
 # failures are collected and surfaced at the end (exit non-zero iff any failed).
@@ -17,7 +18,9 @@
 # debug_env to every run.
 #
 # Environment overrides:
-#   ROCM_PATH / MPI_HOME   Else read from .ci-out/{rocm,ompi}.env
+#   ROCM_PATH / MPI_HOME / ROCSHMEM_INSTALL_DIR
+#                          Else read from matching .ci-out/*.env fragments
+#   RCCL_TESTS_BIN_DIR     rccl-tests build directory (default: rccl-tests/build)
 #   NP                     MPI ranks per run         (default: 8)
 #   BENCH_ARGS             Common bench args         (default: from JSON bench_args)
 #   BENCH_TIMEOUT          Per-bench wall-clock cap  (default: 600s)
@@ -41,11 +44,12 @@ RCCL_LIB_DIR="${WORKDIR}/projects/rccl/build/release"
 RCCL_TESTS_DIR="${WORKDIR}/projects/rccl-tests"
 CONFIG="${CONFIG:-${script_dir}/lib/device-api-tests.json}"
 
-# Prefer the build stages' env fragments over any ambient ROCM_PATH / MPI_HOME.
-# shellcheck source=/dev/null  # runtime fragment written by the provision step
-[[ -f "${WORKDIR}/.ci-out/rocm.env" ]] && source "${WORKDIR}/.ci-out/rocm.env"
-# shellcheck source=/dev/null  # runtime fragment written by build-ompi.sh
-[[ -f "${WORKDIR}/.ci-out/ompi.env" ]] && source "${WORKDIR}/.ci-out/ompi.env"
+# Prefer the build stages' env fragments over ambient build paths.
+for frag in rocm ompi rocshmem; do
+  env_file="${WORKDIR}/.ci-out/${frag}.env"
+  # shellcheck source=/dev/null  # runtime fragment written by the build stages
+  [[ -f "${env_file}" ]] && source "${env_file}"
+done
 
 : "${ROCM_PATH:?run-device-api-ci.sh: ROCM_PATH unset (provisioned via rocm.env / sbatch)}"
 : "${MPI_HOME:?run-device-api-ci.sh: MPI_HOME unset (run build-ompi.sh / via sbatch)}"
@@ -72,14 +76,25 @@ fi
 cd "${RCCL_TESTS_DIR}"
 
 export PATH="${MPI_HOME}/bin:${ROCM_PATH}/bin:${PATH}"
-export LD_LIBRARY_PATH="${RCCL_LIB_DIR}:${MPI_HOME}/lib:${ROCM_PATH}/lib:${LD_LIBRARY_PATH:-}"
+if [[ -n "${ROCSHMEM_INSTALL_DIR:-}" ]]; then
+  export LD_LIBRARY_PATH="${RCCL_LIB_DIR}:${ROCSHMEM_INSTALL_DIR}/lib:${MPI_HOME}/lib:${ROCM_PATH}/lib:${LD_LIBRARY_PATH:-}"
+else
+  export LD_LIBRARY_PATH="${RCCL_LIB_DIR}:${MPI_HOME}/lib:${ROCM_PATH}/lib:${LD_LIBRARY_PATH:-}"
+fi
 
-PERF_DIR=build
-if [[ ! -d "${PERF_DIR}" || ! -f "${PERF_DIR}/all_reduce_perf" ]]; then
+if [[ -n "${RCCL_TESTS_BIN_DIR:-}" ]]; then
+  PERF_DIR="${RCCL_TESTS_BIN_DIR}"
+else
+  PERF_DIR="${RCCL_TESTS_DIR}/build"
+fi
+# all_reduce_perf is not a valid sentinel here: it still fails to build under
+# ENABLE_ROCSHMEM_GIN. alltoall_perf is the binary this GIN job always needs.
+if [[ ! -d "${PERF_DIR}" || ! -x "${PERF_DIR}/alltoall_perf" ]]; then
   echo "rccl-tests perf binaries not found under ${PERF_DIR}"
-  ls -la
+  ls -la "${PERF_DIR}" 2>/dev/null || ls -la "${RCCL_TESTS_DIR}"
   exit 1
 fi
+echo "==> rccl-tests perf dir = ${PERF_DIR}"
 
 PARSER="${script_dir}/lib/parse_device_api_config.py"
 [[ -f "${CONFIG}" ]] || { echo "ERROR: test-matrix config not found: ${CONFIG}" >&2; exit 1; }
@@ -136,7 +151,7 @@ run_bench() {
   set +e
   timeout --kill-after="${BENCH_KILL_AFTER}" "${BENCH_TIMEOUT}" \
     mpirun -np "${NP}" ${env_flags} -x LD_LIBRARY_PATH \
-      "./${PERF_DIR}/${bin}" ${BENCH_ARGS} ${extra_args}
+      "${PERF_DIR}/${bin}" ${BENCH_ARGS} ${extra_args}
   local rc=$?
   set -e
   if [[ ${rc} -ne 0 ]]; then
@@ -175,7 +190,7 @@ echo "All device-api benchmark runs succeeded."
 # Requires ENABLE_DEVICE_API=ON rccl-tests build (alltoall_perf with -B flags).
 run_devtime_smoke() {
   local pytest_dir="${RCCL_TESTS_DIR}/test"
-  local perf_bin="${RCCL_TESTS_DIR}/${PERF_DIR}/alltoall_perf"
+  local perf_bin="${PERF_DIR}/alltoall_perf"
   if [[ ! -x "${perf_bin}" ]]; then
     echo "WARN: skip devtime smoke: ${perf_bin} not found"
     return 0