UCT/GDA: Use gpunetio submodule

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "src/uct/ib/mlx5/gdaki/gpunetio"]
-	path = src/uct/ib/mlx5/gdaki/gpunetio
+	path = external/gpunetio
 	url = https://github.com/NVIDIA-DOCA/gpunetio

Makefile.am

@@ -41,8 +41,7 @@ SUBDIRS = \
 	src/ucm \
 	src/ucs \
 	src/uct \
-	src/ucp \
-	stubs
+	src/ucp
 
 SUBDIRS += \
 	src/tools/vfs \

autogen.sh

@@ -25,7 +25,7 @@ done
 rm -rf autom4te.cache
 mkdir -p config/m4 config/aux
 
-git submodule update --init src/uct/ib/mlx5/gdaki/gpunetio
+git submodule update --init external/gpunetio
 
 autoreconf -v --install || exit 1
 rm -rf autom4te.cache

bindings/java/src/test/java/org/openucx/jucx/UcxTest.java

@@ -37,11 +37,15 @@ public UcpMemory getMemory() {
 
         public void setData(String data) throws Exception {
             if (memory.getMemType() == UcsConstants.MEMORY_TYPE.UCS_MEMORY_TYPE_CUDA) {
-                ByteBuffer srcBuffer = ByteBuffer.allocateDirect(data.length() * 2);
+                ByteBuffer srcBuffer = ByteBuffer.allocateDirect((int)memory.getLength());
+                while (srcBuffer.hasRemaining()) {
+                    srcBuffer.put((byte) 0);
+                }
+                srcBuffer.clear();
                 srcBuffer.asCharBuffer().put(data);
                 UcpRequest send = selfEp.sendTaggedNonBlocking(srcBuffer, 0, null);
                 worker.progressRequest(worker.recvTaggedNonBlocking(memory.getAddress(),
-                        data.length() * 2L, 0, 0, null));
+                        memory.getLength(), 0, 0, null));
                 worker.progressRequest(send);
             } else {
                 buffer.asCharBuffer().put(data);

buildlib/pr/cuda/test_malloc_hook.sh

@@ -67,7 +67,8 @@ test_malloc_hook_mode() {
 	UCX_MEM_LOG_LEVEL=diag ./test/gtest/gtest --gtest_filter='cuda_hooks.*'
 
 	# Test hooks for device-side variables
-	./test/apps/test_cuda_get_symbol_address
+	[ -x ./test/apps/test_cuda_get_symbol_address ] && \
+	    ./test/apps/test_cuda_get_symbol_address
 
 	unset UCX_MEM_CUDA_HOOK_MODE
 }

config/m4/cuda.m4

@@ -3,8 +3,7 @@
 # See file LICENSE for terms.
 #
 
-NVCC_CUDA_MIN_REQUIRED_MAJOR=10
-NVCC_CUDA_MIN_REQUIRED_MINOR=2
+NVCC_CUDA_MIN_REQUIRED=12.2
 
 ARCH9_CODE="-gencode=arch=compute_70,code=sm_70"
 ARCH10_CODE="-gencode=arch=compute_75,code=sm_75"
@@ -48,13 +47,24 @@ AC_DEFUN([AC_LANG_COMPILER(CUDA)], [
 # Check for nvcc compiler support
 AC_DEFUN([UCX_CUDA_CHECK_NVCC], [
     AS_IF([test "x$NVCC" != "x"], [
-        CUDA_MAJOR_VERSION=`$NVCC --version | grep release | sed 's/.*release //' | sed 's/\,.*//' |  cut -d "." -f 1`
-        CUDA_MINOR_VERSION=`$NVCC --version | grep release | sed 's/.*release //' | sed 's/\,.*//' |  cut -d "." -f 2`
-        AC_MSG_RESULT([Detected CUDA version: $CUDA_MAJOR_VERSION.$CUDA_MINOR_VERSION])
-        AS_IF([test $CUDA_MAJOR_VERSION -lt $NVCC_CUDA_MIN_REQUIRED_MAJOR -o \( $CUDA_MAJOR_VERSION -eq $NVCC_CUDA_MIN_REQUIRED_MAJOR -a $CUDA_MINOR_VERSION -lt $NVCC_CUDA_MIN_REQUIRED_MINOR \)],
-            [AC_MSG_WARN([Minimum required CUDA version for device code: $NVCC_CUDA_MIN_REQUIRED_MAJOR.$NVCC_CUDA_MIN_REQUIRED_MINOR])
-             NVCC=""
-            ])
+        CUDA_VERSION=$($NVCC --version | grep release | sed 's/.*release //' | sed 's/\,.*//')
+        CUDA_MAJOR_VERSION=$(echo $CUDA_VERSION | cut -d "." -f 1)
+        CUDA_MINOR_VERSION=$(echo $CUDA_VERSION | cut -d "." -f 2)
+        AC_MSG_RESULT([Detected CUDA version: $CUDA_VERSION])
+        AS_VERSION_COMPARE([$CUDA_VERSION], [$NVCC_CUDA_MIN_REQUIRED],
+              [AC_MSG_WARN([Minimum required CUDA version for device code: $NVCC_CUDA_MIN_REQUIRED])
+               NVCC=""])
+
+        NVCC_CXX_DIALECT=c++17
+        cxx_dialect_ver=201703L
+        AS_VERSION_COMPARE([$CUDA_VERSION], [13.0],
+              [NVCC_CXX_DIALECT=c++11
+               cxx_dialect_ver=201103L
+               NVCCFLAGS="$NVCCFLAGS -DCCCL_IGNORE_DEPRECATED_CPP_DIALECT"])
+
+        AS_VERSION_COMPARE([$CUDA_VERSION], [12.9], [],
+              [NVCCFLAGS="$NVCCFLAGS -D_LIBCUDACXX_ATOMIC_UNSAFE_AUTOMATIC_STORAGE"],
+              [NVCCFLAGS="$NVCCFLAGS -D_LIBCUDACXX_ATOMIC_UNSAFE_AUTOMATIC_STORAGE"])
 
         AS_IF([test "x$NVCC" != "x"], [
                 AC_ARG_WITH([nvcc-gencode],
@@ -85,17 +95,18 @@ AC_DEFUN([UCX_CUDA_CHECK_NVCC], [
                                  [*],
                                      [NVCC_ARCH="${ARCH9_CODE} ${ARCH9_PTX}"])],
                         [NVCC_ARCH="$with_nvcc_gencode"])
+
                 BASE_NVCCFLAGS="$BASE_NVCCFLAGS $NVCC_ARCH"
-                AC_MSG_CHECKING([$NVCC needs explicit c++11 option])
+                AC_MSG_CHECKING([$NVCC needs explicit $NVCC_CXX_DIALECT option])
                 AC_LANG_PUSH([CUDA])
                 AC_COMPILE_IFELSE([AC_LANG_SOURCE([[
-                    #if __cplusplus < 201103L
-                    #error missing C++11
+                    #if __cplusplus < $cxx_dialect_ver
+                    #error missing $NVCC_CXX_DIALECT
                     #endif
                 ]])],
                 [AC_MSG_RESULT([no])],
                 [AC_MSG_RESULT([yes])
-                BASE_NVCCFLAGS="$BASE_NVCCFLAGS -std=c++11"])
+                BASE_NVCCFLAGS="$BASE_NVCCFLAGS -std=$NVCC_CXX_DIALECT"])
                 AC_LANG_POP
 
                 AC_MSG_CHECKING([$NVCC can compile])
@@ -109,6 +120,18 @@ AC_DEFUN([UCX_CUDA_CHECK_NVCC], [
                 [AC_MSG_RESULT([no])
                 NVCC=""])
                 AC_LANG_POP
+
+                AC_MSG_CHECKING([checking cuda/atomic support])
+                AC_LANG_PUSH([CUDA])
+                AC_COMPILE_IFELSE([AC_LANG_SOURCE([[
+                      #include <cuda/atomic>
+                      int v;
+                      cuda::atomic_ref<int> ref{v};
+                   ]])],
+                   [AC_MSG_RESULT([yes])],
+                   [AC_MSG_RESULT([no])
+                    NVCC=""])
+                AC_LANG_POP
             ])
         ])
 
@@ -232,10 +255,4 @@ AS_IF([test "x$cuda_checked" != "xyes"],
         UCX_CUDA_CHECK_NVCC
    ]) # "x$cuda_checked" != "xyes"
 
-   # Set stubs include path for pkg-config
-   # e.g. UCX Device API headers include <uct/ib/mlx5/gdaki/gdaki.cuh>
-   # from a separate package. Stubs provide fallback when not installed.
-   STUB_CFLAGS="-I\${includedir}/stubs"
-   AC_SUBST([STUB_CFLAGS])
-
 ]) # UCX_CHECK_CUDA

configure.ac

@@ -198,7 +198,6 @@ AS_IF([test "x$with_docs_only" = xyes],
      AM_CONDITIONAL([HAVE_CUDA_STATIC], [false])
      AM_CONDITIONAL([HAVE_NVCC], [false])
      AM_CONDITIONAL([HAVE_GDR_COPY], [false])
-     AM_CONDITIONAL([HAVE_GPUNETIO], [false])
      AM_CONDITIONAL([HAVE_ROCM], [false])
      AM_CONDITIONAL([HAVE_HIP], [false])
      AM_CONDITIONAL([HAVE_XPMEM], [false])
@@ -367,7 +366,6 @@ AC_SUBST([build_bindings], [${build_bindings}])
 # Final output
 #
 AC_CONFIG_FILES([Makefile
-                 stubs/Makefile
                  docs/doxygen/header.tex
                  src/uct/api/version.h
                  ])

debian/control.in

@@ -63,15 +63,3 @@ Description: Unified Communication X - gdrcopy support
  .
  This package provides UCX support for using gdrcopy - A low-latency GPU
  memory copy library based on NVIDIA GPUDirect RDMA technology.
-
-Package: ucx-ib-mlx5-gda
-Section: libs
-Depends: ${misc:Depends}, ${shlibs:Depends}
-Architecture: any
-Build-Profiles: <gda>
-Description: Unified Communication X - GPU Direct Async support
- UCX is a communication library implementing high-performance messaging.
- .
- Provide GPU Direct Async support for UCX. GPU Direct Async allows GPU kernels
- to initiate network communications directly without explicit synchronization
- with the host CPU.

debian/ucx-cuda.install

@@ -1,2 +1,4 @@
 usr/lib/ucx/libucx_perftest_cuda.*
 usr/lib/ucx/libuc?_cuda.*
+usr/lib/ucx/libuct_ib_mlx5_gda.*
+usr/include/uct/ib/mlx5/gdaki/*

src/tools/perf/cuda/Makefile.am

@@ -14,9 +14,8 @@ libucx_perftest_cuda_la_LIBADD   = $(CUDART_LIBS)
 libucx_perftest_cuda_la_SOURCES  = cuda_alloc.c
 
 if HAVE_NVCC
-if HAVE_GPUNETIO # temporarily all device code depends on doca
 
-NVCC_EXTRA_FLAGS = $(CUDA_CPPFLAGS) $(GPUNETIO_CFLAGS)
+NVCC_EXTRA_FLAGS = $(CUDA_CPPFLAGS)
 
 libucx_perftest_cuda_la_LIBADD  += \
     $(abs_top_builddir)/src/tools/perf/lib/libucxperf.la \
@@ -30,7 +29,6 @@ noinst_HEADERS = \
     cuda_kernel.cuh
 
 include $(top_srcdir)/config/cuda.am
-endif # HAVE_GPUNETIO
 endif # HAVE_NVCC
 
 include $(top_srcdir)/config/module.am

src/tools/perf/cuda/ucp_cuda_kernel.cu

@@ -10,6 +10,7 @@
 
 #include "cuda_kernel.cuh"
 #include "curand_kernel.h"
+
 #include <ucp/api/device/ucp_host.h>
 #include <ucp/api/device/ucp_device_impl.h>
 #include <tools/perf/lib/ucp_tests.h>

src/ucs/sys/device_code.h

@@ -33,6 +33,14 @@
 #define UCS_DEVICE_NUM_THREADS_IN_WARP 32
 
 
+/* nvcc does not provide __builtin_ia32_prefetch used by GCC's x86 intrinsic headers.
+   Redirect to the generic __builtin_prefetch so those headers compile. */
+#if defined(__NVCC__) && !defined(__builtin_ia32_prefetch)
+  #define __builtin_ia32_prefetch(_p, _rw, _loc, _cache) \
+    __builtin_prefetch((_p), (_rw), (_loc))
+#endif
+
+
 /**
  * @brief Cooperation level when calling device functions.
  */

src/uct/api/device/uct_device_impl.h

@@ -13,10 +13,17 @@
 #include <uct/cuda/cuda_ipc/cuda_ipc.cuh>
 #include <ucs/sys/device_code.h>
 
-#include <uct/ib/mlx5/gdaki/gdaki.cuh>
+#if __has_include(<uct/ib/mlx5/gdaki/gdaki.cuh>) && __has_include(<infiniband/mlx5dv.h>)
+#  include <uct/ib/mlx5/gdaki/gdaki.cuh>
+#  define UCT_RC_MLX5_GDA_SUPPORTED 1
+#else
+#  define UCT_RC_MLX5_GDA_SUPPORTED 0
+#endif
 
 union uct_device_completion {
+#if UCT_RC_MLX5_GDA_SUPPORTED
     uct_rc_gda_completion_t   rc_gda;
+#endif
     uct_cuda_ipc_completion_t cuda_ipc;
 };
 
@@ -56,12 +63,15 @@ UCS_F_DEVICE ucs_status_t uct_device_ep_put_single(
         const void *address, uint64_t remote_address, size_t length,
         unsigned channel_id, uint64_t flags, uct_device_completion_t *comp)
 {
+#if UCT_RC_MLX5_GDA_SUPPORTED
     if (device_ep->uct_tl_id == UCT_DEVICE_TL_RC_MLX5_GDA) {
         return uct_rc_mlx5_gda_ep_put_single<level>(device_ep, mem_elem,
                                                     address, remote_address,
                                                     length, channel_id, flags,
                                                     comp);
-    } else if (device_ep->uct_tl_id == UCT_DEVICE_TL_CUDA_IPC) {
+    } else
+#endif
+    if (device_ep->uct_tl_id == UCT_DEVICE_TL_CUDA_IPC) {
         return uct_cuda_ipc_ep_put_single<level>(device_ep, mem_elem, address,
                                                  remote_address, length, flags,
                                                  comp);
@@ -109,12 +119,15 @@ uct_device_ep_put(uct_device_ep_h device_ep,
                   uint64_t remote_address, size_t length, unsigned channel_id,
                   uint64_t flags, uct_device_completion_t *comp)
 {
+#if UCT_RC_MLX5_GDA_SUPPORTED
     if (device_ep->uct_tl_id == UCT_DEVICE_TL_RC_MLX5_GDA) {
         return uct_rc_mlx5_gda_ep_put_single<level>(device_ep, src_uct_elem,
                                                     mem_elem, address,
                                                     remote_address, length,
                                                     channel_id, flags, comp);
-    } else if (device_ep->uct_tl_id == UCT_DEVICE_TL_CUDA_IPC) {
+    } else
+#endif
+    if (device_ep->uct_tl_id == UCT_DEVICE_TL_CUDA_IPC) {
         return uct_cuda_ipc_ep_put_single<level>(device_ep, mem_elem, address,
                                                  remote_address, length, flags,
                                                  comp);
@@ -157,11 +170,14 @@ UCS_F_DEVICE ucs_status_t uct_device_ep_atomic_add(
         uint64_t inc_value, uint64_t remote_address, unsigned channel_id,
         uint64_t flags, uct_device_completion_t *comp)
 {
+#if UCT_RC_MLX5_GDA_SUPPORTED
     if (device_ep->uct_tl_id == UCT_DEVICE_TL_RC_MLX5_GDA) {
         return uct_rc_mlx5_gda_ep_atomic_add<level>(device_ep, mem_elem,
                                                     inc_value, remote_address,
                                                     channel_id, flags, comp);
-    } else if (device_ep->uct_tl_id == UCT_DEVICE_TL_CUDA_IPC) {
+    } else
+#endif
+    if (device_ep->uct_tl_id == UCT_DEVICE_TL_CUDA_IPC) {
         return uct_cuda_ipc_ep_atomic_add<level>(device_ep, mem_elem, inc_value,
                                                  remote_address, flags, comp);
     }
@@ -247,14 +263,17 @@ UCS_F_DEVICE ucs_status_t uct_device_ep_put_multi(
         uint64_t counter_inc_value, uint64_t counter_remote_address,
         unsigned channel_id, uint64_t flags, uct_device_completion_t *comp)
 {
+#if UCT_RC_MLX5_GDA_SUPPORTED
     if (device_ep->uct_tl_id == UCT_DEVICE_TL_RC_MLX5_GDA) {
         return uct_rc_mlx5_gda_ep_put_multi<level>(device_ep, mem_list,
                                                    mem_list_count, addresses,
                                                    remote_addresses, lengths,
                                                    counter_inc_value,
                                                    counter_remote_address,
                                                    channel_id, flags, comp);
-    } else if (device_ep->uct_tl_id == UCT_DEVICE_TL_CUDA_IPC) {
+    } else
+#endif
+    if (device_ep->uct_tl_id == UCT_DEVICE_TL_CUDA_IPC) {
         return uct_cuda_ipc_ep_put_multi<level>(device_ep, mem_list,
                                                 mem_list_count, addresses,
                                                 remote_addresses, lengths,
@@ -331,13 +350,16 @@ UCS_F_DEVICE ucs_status_t uct_device_ep_put_multi_partial(
         uint64_t counter_inc_value, uint64_t counter_remote_address,
         unsigned channel_id, uint64_t flags, uct_device_completion_t *comp)
 {
+#if UCT_RC_MLX5_GDA_SUPPORTED
     if (device_ep->uct_tl_id == UCT_DEVICE_TL_RC_MLX5_GDA) {
         return uct_rc_mlx5_gda_ep_put_multi_partial<level>(
                 device_ep, mem_list, mem_list_indices, mem_list_count,
                 addresses, remote_addresses, local_offsets, remote_offsets,
                 lengths, counter_index, counter_inc_value,
                 counter_remote_address, channel_id, flags, comp);
-    } else if (device_ep->uct_tl_id == UCT_DEVICE_TL_CUDA_IPC) {
+    } else
+#endif
+    if (device_ep->uct_tl_id == UCT_DEVICE_TL_CUDA_IPC) {
         return uct_cuda_ipc_ep_put_multi_partial<level>(
                 device_ep, mem_list, mem_list_indices, mem_list_count,
                 addresses, remote_addresses, local_offsets, remote_offsets,
@@ -357,9 +379,11 @@ UCS_F_DEVICE ucs_status_t uct_device_ep_put_multi_partial(
 template<ucs_device_level_t level>
 UCS_F_DEVICE void uct_device_ep_progress(uct_device_ep_h device_ep)
 {
+#if UCT_RC_MLX5_GDA_SUPPORTED
     if (device_ep->uct_tl_id == UCT_DEVICE_TL_RC_MLX5_GDA) {
         uct_rc_mlx5_gda_ep_progress<level>(device_ep);
     }
+#endif
 }
 
 
@@ -379,9 +403,11 @@ template<ucs_device_level_t level>
 UCS_F_DEVICE ucs_status_t uct_device_ep_check_completion(
         uct_device_ep_h device_ep, uct_device_completion_t *comp)
 {
+#if UCT_RC_MLX5_GDA_SUPPORTED
     if (device_ep->uct_tl_id == UCT_DEVICE_TL_RC_MLX5_GDA) {
         return uct_rc_mlx5_gda_ep_check_completion<level>(device_ep, comp);
     }
+#endif
 
     return UCS_ERR_UNSUPPORTED;
 }

src/uct/ib/mlx5/gdaki/Makefile.am

@@ -3,13 +3,12 @@
 # See file LICENSE for terms.
 #
 
-if HAVE_GPUNETIO
+if HAVE_CUDA
 
 module_LTLIBRARIES             = libuct_ib_mlx5_gda.la
 libuct_ib_mlx5_gda_la_CPPFLAGS = $(BASE_CPPFLAGS) $(IBVERBS_CPPFLAGS) \
                                  $(CUDA_CPPFLAGS)
-libuct_ib_mlx5_gda_la_CFLAGS   = $(BASE_CFLAGS) $(LT_CFLAGS) $(CUDA_CFLAGS) \
-                                 $(GPUNETIO_CFLAGS)
+libuct_ib_mlx5_gda_la_CFLAGS   = $(BASE_CFLAGS) $(LT_CFLAGS) $(CUDA_CFLAGS)
 libuct_ib_mlx5_gda_la_LDFLAGS  = $(IBVERBS_LDFLAGS) $(CUDA_LDFLAGS) \
                                  -version-info $(SOVERSION)
 libuct_ib_mlx5_gda_la_LIBADD   = $(top_builddir)/src/ucs/libucs.la \
@@ -25,7 +24,12 @@ libuct_ib_mlx5_gda_la_SOURCES = \
 
 nobase_dist_libuct_ib_mlx5_gda_la_HEADERS = \
 	gdaki.cuh \
-	gdaki_dev.h
+	gdaki_dev.h \
+	gpunetio/common/doca_gpunetio_verbs_def.h \
+	gpunetio/common/doca_gpunetio_verbs_dev.h \
+	gpunetio/device/doca_gpunetio_dev_verbs_common.cuh \
+	gpunetio/device/doca_gpunetio_dev_verbs_qp.cuh \
+	gpunetio/device/doca_gpunetio_dev_verbs_cq.cuh
 
 noinst_HEADERS = \
 	gdaki.h