rccl-tests: GIN device-API CI and harness cleanup (AICOMRCCL-2221)

projects/rccl-tests/CMakeLists.txt

@@ -238,12 +238,11 @@ if (ENABLE_ROCSHMEM_GIN)
     if (NOT ROCSHMEM_SOURCE_DIR AND EXISTS "${_mono_root}/projects/rocshmem/CMakeLists.txt")
         set(ROCSHMEM_SOURCE_DIR "${_mono_root}/projects/rocshmem")
     endif()
-    # Installed rocshmem headers (rocshmem_config.h) are needed globally:
-    # RCCL's public nccl_device.h → gin_anvil_sdma.h → anvil_device.hpp
-    # → log.hpp → rocshmem/rocshmem_config.h.  Must come before source
-    # includes so the generated config is found first.
+    # Installed rocshmem headers (rocshmem_config.h) for GIN-SDMA perf binaries
+    # that set NCCL_GIN_ANVIL_SDMA_ENABLE=1. Must come before source includes
+    # so the generated config is found first.
     include_directories(BEFORE ${ROCSHMEM_INSTALL_DIR}/include)
-    # Internal rocshmem headers (sdma/anvil_device.hpp) needed globally.
+    # Internal rocshmem headers (sdma/anvil_device.hpp) for those same TUs.
     if (ROCSHMEM_SOURCE_DIR)
         include_directories(${ROCSHMEM_SOURCE_DIR}/src)
     endif()

projects/rccl-tests/src/CMakeLists.txt

@@ -145,27 +145,6 @@ endif()
 add_rccl_test(all_gather)
 add_rccl_test(all_reduce)
 add_rccl_test(alltoall)
-if (ENABLE_ROCSHMEM_GIN AND TARGET roc::rocshmem)
-    target_compile_definitions(alltoall_perf PRIVATE ENABLE_ROCSHMEM_GIN)
-    target_compile_options(alltoall_perf PRIVATE -fgpu-rdc $<$<CONFIG:Debug>:-ggdb -O0>)
-    # --whole-archive: librccl.so (ENABLE_ROCSHMEM_GIN) leaves rocshmem host
-    # symbols unresolved (--allow-shlib-undefined).  The executable must pull
-    # in ALL rocshmem symbols (not just those it references directly) so that
-    # -rdynamic exports them for librccl.so to resolve at runtime.
-    target_link_libraries(alltoall_perf PRIVATE
-        -Wl,--whole-archive roc::rocshmem -Wl,--no-whole-archive)
-    target_link_options(alltoall_perf PRIVATE -fgpu-rdc --hip-link -rdynamic)
-
-    # GIN-SDMA hybrid AllGather (all_gather_perf -D 3): same device-RDC link as
-    # alltoall_perf so the GinHybridAllGatherKernel resolves the GIN device backend.
-    if (ENABLE_DEVICE_API)
-      target_compile_definitions(all_gather_perf PRIVATE ENABLE_DEVICE_API)
-    endif()
-    target_compile_options(all_gather_perf PRIVATE -fgpu-rdc $<$<CONFIG:Debug>:-ggdb -O0>)
-    target_link_libraries(all_gather_perf PRIVATE
-        -Wl,--whole-archive roc::rocshmem -Wl,--no-whole-archive)
-    target_link_options(all_gather_perf PRIVATE -fgpu-rdc --hip-link -rdynamic)
-endif()
 if (ENABLE_ROCSHMEM_GIN AND TARGET roc::rocshmem AND RCCL_SOURCE_DIR)
     add_executable(gin_rocshmem_gda_factory_test ${CMAKE_CURRENT_SOURCE_DIR}/gin_rocshmem_gda_factory_test.cu)
     set_source_files_properties(${CMAKE_CURRENT_SOURCE_DIR}/gin_rocshmem_gda_factory_test.cu PROPERTIES LANGUAGE CXX)
@@ -185,20 +164,45 @@ if (ENABLE_ROCSHMEM_GIN AND TARGET roc::rocshmem AND RCCL_SOURCE_DIR)
 endif()
 add_rccl_test(alltoallv)
 add_rccl_test(broadcast)
-if (ENABLE_ROCSHMEM_GIN AND TARGET roc::rocshmem)
-    # GIN-SDMA hybrid Broadcast (broadcast_perf -D 3): same device-RDC link as
-    # alltoall_perf/all_gather_perf so the Gin*BroadcastKernel variants resolve
-    # the GIN device backend (SDMA puts + LSA/LL peer stores).
-    target_compile_definitions(broadcast_perf PRIVATE ENABLE_ROCSHMEM_GIN)
-    target_compile_options(broadcast_perf PRIVATE -fgpu-rdc $<$<CONFIG:Debug>:-ggdb -O0>)
-    target_link_libraries(broadcast_perf PRIVATE
-        -Wl,--whole-archive roc::rocshmem -Wl,--no-whole-archive)
-    target_link_options(broadcast_perf PRIVATE -fgpu-rdc --hip-link -rdynamic)
-endif()
 add_rccl_test(gather)
 add_rccl_test(hypercube)
 add_rccl_test(reduce_scatter)
 add_rccl_test(reduce)
 add_rccl_test(scatter)
 add_rccl_test(sendrecv)
 add_rccl_test(all_reduce_bias)
+if (ENABLE_ROCSHMEM_GIN AND TARGET roc::rocshmem)
+    function(_rccl_tests_link_rocshmem_gin_rdc target)
+        target_compile_definitions(${target} PRIVATE ENABLE_ROCSHMEM_GIN NCCL_GIN_ANVIL_SDMA_ENABLE=1)
+        target_compile_options(${target} PRIVATE -fgpu-rdc $<$<CONFIG:Debug>:-ggdb -O0>)
+        # --whole-archive: librccl.so (ENABLE_ROCSHMEM_GIN) leaves rocshmem host
+        # symbols unresolved (--allow-shlib-undefined).  The executable must pull
+        # in ALL rocshmem symbols (not just those it references directly) so that
+        # -rdynamic exports them for librccl.so to resolve at runtime.
+        target_link_libraries(${target} PRIVATE
+            -Wl,--whole-archive roc::rocshmem -Wl,--no-whole-archive)
+        target_link_options(${target} PRIVATE -fgpu-rdc --hip-link -rdynamic)
+    endfunction()
+
+    # GIN-SDMA hybrid AllGather (all_gather_perf -D 3) needs ENABLE_DEVICE_API
+    # so the GinHybridAllGatherKernel is compiled into this binary.
+    if (ENABLE_DEVICE_API)
+      target_compile_definitions(all_gather_perf PRIVATE ENABLE_DEVICE_API)
+    endif()
+
+    # Device-api CI builds one --rocshmem-gin librccl.so and runs the matrix
+    # against it. Every launched binary must RDC-link rocSHMEM so GIN host
+    # symbols resolve from the executable. all_reduce_perf is omitted: it
+    # still does not build under ENABLE_ROCSHMEM_GIN (gin.sbatch uses
+    # `make -k` for the same reason).
+    foreach(_gin_rdc_bin IN ITEMS
+        all_gather_perf
+        alltoall_perf
+        broadcast_perf
+        gather_perf
+        reduce_perf
+        scatter_perf
+        sendrecv_perf)
+      _rccl_tests_link_rocshmem_gin_rdc(${_gin_rdc_bin})
+    endforeach()
+endif()