[P2P]: add CXI transport endpoint

.github/workflows/nixlbench_test_internode.sh

@@ -181,5 +181,11 @@ if [ $OVERALL_STATUS -ne 0 ]; then
     exit 1
 fi
 
+if $HOST_ETCD; then
+    ETCD_GRACE_SECONDS="${NIXL_HOST_ETCD_GRACE_SECONDS:-10}"
+    echo "=== Keeping etcd alive for ${ETCD_GRACE_SECONDS}s for peer shutdown ==="
+    sleep "${ETCD_GRACE_SECONDS}"
+fi
+
 echo ""
 echo "=== All inter-node nixlbench tests PASSED ==="

.github/workflows/uccl-build-test-gb10.yml

@@ -116,6 +116,18 @@ jobs:
           ssh-keyscan -p "${{secrets.UCCL_DEV_SPARK0_PORT}}" -H "${{secrets.UCCL_DEV_HOSTNAME}}" >> ~/.ssh/known_hosts
           ssh-keyscan -p "${{secrets.UCCL_DEV_SPARK1_PORT}}" -H "${{secrets.UCCL_DEV_HOSTNAME}}" >> ~/.ssh/known_hosts
 
+      - name: Install CXI system dependencies on gb10
+        shell: bash
+        run: |
+          set -euo pipefail
+          for host in spark0 spark1; do
+            ssh -o BatchMode=yes "$host" bash -s << 'EOF'
+              set -euo pipefail
+              sudo apt-get update
+              sudo apt-get install -y libfabric-dev libhwloc-dev
+          EOF
+          done
+
       - name: Sync pushed code to gb10
         shell: bash
         run: |

.github/workflows/uccl-build-test-gh200.yml

@@ -114,6 +114,18 @@ jobs:
           ssh-keyscan -H "${{secrets.UCCL_GH200_HOST0}}" >> ~/.ssh/known_hosts
           ssh-keyscan -H "${{secrets.UCCL_GH200_HOST1}}" >> ~/.ssh/known_hosts
 
+      - name: Install CXI system dependencies on gh200
+        shell: bash
+        run: |
+          set -euo pipefail
+          for host in gh0 gh1; do
+            ssh -o BatchMode=yes "$host" bash -s << 'EOF'
+              set -euo pipefail
+              sudo apt-get update
+              sudo apt-get install -y libfabric-dev libhwloc-dev
+          EOF
+          done
+
       - name: Sync pushed code to gh200
         shell: bash
         run: |
@@ -243,6 +255,8 @@ jobs:
               ln -sf "${NVIDIA_NCCL_LIB}/libnccl.so.2" "${NIXL_INSTALL_DIR}/lib/libnccl.so"
             fi
             export LD_LIBRARY_PATH="${NIXL_INSTALL_DIR}/lib:${NIXL_INSTALL_DIR}/lib/aarch64-linux-gnu:${NIXL_INSTALL_DIR}/lib/aarch64-linux-gnu/plugins:${CONDA_ENV_DIR}/lib:${NVIDIA_NCCL_LIB}:${LD_LIBRARY_PATH:-}"
+            export UCCL_SOCKET_IFNAME=enx3c18a0d51943
+            export NCCL_SOCKET_IFNAME=enx3c18a0d51943
 
             bash .github/workflows/nixlbench_test.sh "${NIXL_INSTALL_DIR}" 2>&1 | tee nixlbench.log
           EOF

build_inner.sh

@@ -204,7 +204,7 @@ build_ep() {
   elif [[ "$TARGET" == roc[67] || "$TARGET" == cu* || "$TARGET" == "therock" ]]; then
     cd ep
     # This may be needed if you traverse through different git commits
-    # make clean && rm -r build || true
+    make clean && rm -r build || true
     extra_env=()
     if [[ "$TARGET" == "therock" ]]; then
       # On TheRock, ROCm comes from a pip-installed rocm-sdk wheel; expose its

p2p/Makefile

@@ -1,10 +1,10 @@
 # Makefile for UCCL P2P Engine nanobind project (stable ABI)
 #
 # Single binary, runtime transport selection via UCCL_P2P_TRANSPORT env var.
-# libibverbs, libnccl, and libefa are loaded dynamically at runtime via
-# dlopen/dlsym (see *_dl.h). No transport libraries are linked.
+# libibverbs, libnccl, libefa, and libfabric/CXI are loaded dynamically at
+# runtime via dlopen/dlsym (see *_dl.h).
 #
-# Runtime: UCCL_P2P_TRANSPORT=rdma|nccl|efa (default: rdma)
+# Runtime: UCCL_P2P_TRANSPORT=rdma|nccl|efa|cxi (default: rdma)
 
 # DietGPU optional integration
 USE_DIETGPU ?= $(shell echo $${USE_DIETGPU:-0})
@@ -14,6 +14,7 @@ CUDA_HOME ?= /usr/local/cuda
 CUDA_INC  := $(CUDA_HOME)/include
 CUDA_LIB  := $(CUDA_HOME)/lib64
 EFA_HOME ?= /opt/amazon/efa
+LIBFABRIC_HOME ?= /opt/libfabric
 
 # Building with the following settings:
 # NOTE: -libverbs, -lnccl, -lefa are NOT linked at compile time.
@@ -23,7 +24,7 @@ LIBS = -L ${CUDA_HOME}/lib64 -lcudart -lcuda -lpthread -lz -lelf -ldl
 CXX := g++
 CXXFLAGS := -O3 -shared -std=c++17 -fPIC \
 	-I. -I./util -I./rdma -I./rdma/providers -I./nccl -I../include -I$(CUDA_INC) \
-	-I$(EFA_HOME)/include \
+	-I$(EFA_HOME)/include -I$(LIBFABRIC_HOME)/include \
 	-MMD -MP \
 	-Wno-pointer-arith -Wno-sign-compare -Wno-unused-variable \
 	-Wl,-rpath=/usr/lib/x86_64-linux-gnu
@@ -75,9 +76,9 @@ PREFIX ?= /usr/local
 LIBDIR ?= $(PREFIX)/lib
 INCDIR ?= $(PREFIX)/include
 
-# NO transport libraries linked (-libverbs, -lnccl, -lefa).
-# All ibv_* / nccl* / efadv_* symbols are provided by the dlsym wrappers
-# in rdma/ibverbs_dl.cc, nccl/nccl_dl.cc, rdma/efadv_dl.cc.
+# No transport libraries are linked (-libverbs, -lnccl, -lefa, -lfabric).
+# All ibv_* / nccl* / efadv_* / fi_* symbols are provided by dlsym wrappers in
+# rdma/ibverbs_dl.cc, nccl/nccl_dl.cc, rdma/efadv_dl.cc, and cxi/fabric_dl.cc.
 # The --wrap flags redirect compat-layer calls from verbs.h to our wrappers.
 IBV_WRAP_FLAGS := -Wl,--wrap=ibv_get_device_list \
                   -Wl,--wrap=ibv_query_port \
@@ -96,9 +97,11 @@ P2P_SHARED_LIB := libuccl_p2p.so
 CAPI_SOURCE := uccl_engine.cc
 CAPI_HEADER := uccl_engine.h
 CAPI_OBJECT := $(CAPI_SOURCE:.cc=.o)
-# dlsym wrapper objects provide ibv_*/nccl*/efadv_* symbols at link time
-DL_SOURCES := rdma/ibverbs_dl.cc nccl/nccl_dl.cc rdma/efadv_dl.cc
-DL_OBJECTS := rdma/ibverbs_dl.o nccl/nccl_dl.o rdma/efadv_dl.o
+# dlsym wrapper objects provide ibv_*/nccl*/efadv_*/fi_* symbols at link time
+DL_SOURCES := rdma/ibverbs_dl.cc nccl/nccl_dl.cc rdma/efadv_dl.cc \
+	cxi/fabric_dl.cc
+DL_OBJECTS := rdma/ibverbs_dl.o nccl/nccl_dl.o rdma/efadv_dl.o \
+	cxi/fabric_dl.o
 P2P_IMPL_SOURCES := \
 	util/adaptive_sleeper.cc \
 	util/common.cc \
@@ -114,7 +117,8 @@ P2P_IMPL_SOURCES := \
 	rdma/rdma_endpoint.cc \
 	util/seq_num.cc \
 	rdma/providers/efa_data_channel_impl.cc \
-	rdma/providers/ib_data_channel_impl.cc
+	rdma/providers/ib_data_channel_impl.cc \
+	cxi/cxi_endpoint.cc
 P2P_IMPL_OBJECTS := $(P2P_IMPL_SOURCES:.cc=.o)
 SOURCES := engine.cc engine_api.cc nccl/nccl_endpoint.cc $(P2P_IMPL_SOURCES)
 CORE_OBJECT := engine.o nccl/nccl_endpoint.o $(P2P_IMPL_OBJECTS) $(DL_OBJECTS)
@@ -176,6 +180,7 @@ clean:
 	rm -f $(OBJECTS) $(CAPI_OBJECT) $(DL_OBJECTS) $(NB_OBJECTS) $(P2P_SHARED_LIB)
 	rm -f $(DEPS)
 	rm -f nccl/*.o nccl/*.d rdma/*.o rdma/*.d rdma/providers/*.o rdma/providers/*.d util/*.o util/*.d
+	rm -f cxi/*.o cxi/*.d
 	rm -f $(P2P_PYTHON_EXT)
 
 # Test the module

p2p/README.md

@@ -64,6 +64,13 @@ To enable AWS EFA support, you can do the same as above, and specify `UCCL_P2P_T
 
 To enable GCP TCPX support, you can refer to [NIXL_plugin_readme.md](./NIXL_plugin_readme.md).
 
+To enable HPE Slingshot/CXI support, set `UCCL_P2P_TRANSPORT=cxi` at runtime:
+```bash
+sudo apt install libfabric-dev libhwloc-dev -y
+make -j install
+UCCL_P2P_TRANSPORT=cxi UCCL_P2P_DISABLE_IPC=1 torchrun ...
+```
+
 To build with DietGPU float compression support, you can:
 ```bash
 USE_DIETGPU=1 make -j install
@@ -92,7 +99,7 @@ torchrun --nnodes=2 --nproc_per_node=1 --node-rank=1 --master_addr=<IP addr> ben
 Notes: 
 * You may consider exporting `GLOO_SOCKET_IFNAME=xxx NCCL_SOCKET_IFNAME=xxx` if triggering Gloo connectFullMesh failure.
 * You may consider exporting `UCCL_P2P_RDMA_GID_INDEX` if your cluster requires it for NCCL to run (usually 1, or 3 in some testbed).
-* You can specify `UCCL_P2P_TRANSPORT=ib|efa|nccl|tcp|tcpx` at runtime to choose different network backends. The default is `ib` that works for NVIDIA, Broadcom, AMD, and Intel RDMA NICs. 
+* You can specify `UCCL_P2P_TRANSPORT=ib|efa|nccl|tcp|tcpx|cxi` at runtime to choose different network backends. The default is `ib` that works for NVIDIA, Broadcom, AMD, and Intel RDMA NICs.
 * **You must first import `torch` before importing `uccl.p2p` for AMD GPUs**, otherwise, `RuntimeError: No HIP GPUs are available` will occur. We guess this is because torch does some extra init for AMD GPUs, in order for Pybind-C++ code to work. 
 * One-sided network write is the default in `benchmark_uccl.py`; use `--mode read` for RDMA read.
 * To benchmark one-sided IPC write (GPU-to-GPU or CPU-to-GPU), `torchrun --nproc_per_node=2 benchmarks/benchmark_uccl.py --write-ipc`. Use `--device cpu --pinned` for CPU source buffers.
@@ -107,7 +114,15 @@ Notes:
 | UCCL_P2P_RDMA_SL | Service level in RDMA network | 8/3 (EFA/IB) |
 | UCCL_P2P_RDMA_TC | Traffic class in RDMA network | 104 (IB) |
 | UCCL_P2P_RDMA_DEV | RDMA devices forced to use (instead of auto-selecting based on PCIe affinity) | none (eg, `irdma-mkp0,irdma-mkp1`) |
-| UCCL_P2P_TRANSPORT | Network backend to use at runtime | ib (others: efa/nccl/tcp/tcpx) |
+| UCCL_P2P_TRANSPORT | Network backend to use at runtime | ib (others: efa/nccl/tcp/tcpx/cxi) |
+| UCCL_CXI_DOMAIN | CXI/libfabric domain to use when `UCCL_P2P_TRANSPORT=cxi` | auto from GPU index, eg `cxi0` |
+| UCCL_CXI_DEVICE_INDEX | CXI device index used for automatic domain selection | GPU index modulo 4 |
+| UCCL_CXI_THREADING | libfabric threading hint for the CXI domain | endpoint |
+| UCCL_CXI_TX_QUEUE_SIZE | CXI transmit queue size | 4096 |
+| UCCL_CXI_RX_QUEUE_SIZE | CXI receive queue size | 4096 |
+| UCCL_CXI_CQ_SIZE | CXI completion queue size | 8192 |
+| UCCL_P2P_MAX_INFLIGHT_OPS | Maximum one-sided in-flight operations; CXI defaults lower than RDMA | 32 for CXI, otherwise internal maximum |
+| UCCL_LIBFABRIC_SO | Override libfabric shared-library path for the CXI dlsym wrapper | auto-detect `libfabric.so` / `libfabric.so.1` |
 | UCCL_P2P_COMPRESS_STRATEGY | DietGPU compression strategy (requires `USE_DIETGPU=1` build) | none |
 | UCCL_RDMA_ADAPTIVE_SLEEP | Enable adaptive sleeping on proxy threads, by putting the proxy threads into a sleeping state if there have been no new work requests / RDMA completion events after 120s. | null |