From 6a984e7d907c8132753bc3b54a2cb0dd75692c4f Mon Sep 17 00:00:00 2001 From: Michal Shalev Date: Tue, 9 Dec 2025 08:12:39 -0800 Subject: [PATCH 1/2] NIXL_EP: Enable CUDA IPC NVLINK backend Signed-off-by: Michal Shalev --- .gitlab/build.sh | 4 ---- examples/device/ep/tests/elastic/elastic.py | 4 ---- 2 files changed, 8 deletions(-) diff --git a/.gitlab/build.sh b/.gitlab/build.sh index 816e44cda1..d6b0e85635 100755 --- a/.gitlab/build.sh +++ b/.gitlab/build.sh @@ -253,10 +253,6 @@ export PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig:${INSTALL_DIR}/lib64/pkgcon export NIXL_PLUGIN_DIR="${INSTALL_DIR}/lib/$ARCH-linux-gnu/plugins" export CMAKE_PREFIX_PATH="${INSTALL_DIR}:${CMAKE_PREFIX_PATH}" -# Disabling CUDA IPC not to use NVLINK, as it slows down local -# UCX transfers and can cause contention with local collectives. -export UCX_TLS=^cuda_ipc - # shellcheck disable=SC2086 meson setup nixl_build --prefix=${INSTALL_DIR} -Ducx_path=${UCX_INSTALL_DIR} -Dbuild_docs=true -Drust=false ${EXTRA_BUILD_ARGS} -Dlibfabric_path="${LIBFABRIC_INSTALL_DIR}" --buildtype=debug ninja -j"$NPROC" -C nixl_build && ninja -j"$NPROC" -C nixl_build install diff --git a/examples/device/ep/tests/elastic/elastic.py b/examples/device/ep/tests/elastic/elastic.py index 0e347d7c7e..5a8daf9175 100644 --- a/examples/device/ep/tests/elastic/elastic.py +++ b/examples/device/ep/tests/elastic/elastic.py @@ -636,10 +636,6 @@ def main(): args = parser.parse_args() - # TODO_Itay: remove this once NIXL supports NVLink on multiple workers - assert ( - args.nvlink_backend != "nixl" - ), "NIXL does not support NVLink on multiple workers yet" rank_server_process = None if not args.rank_server: print("Starting rank server locally", flush=True) From 86c77dcc7da610eb6a29d9d638f286b8a5cc449f Mon Sep 17 00:00:00 2001 From: Michal Shalev Date: Tue, 23 Dec 2025 21:01:56 +0200 Subject: [PATCH 2/2] Re-add disabling cuda_ipc from build.sh script Signed-off-by: Michal Shalev --- .gitlab/build.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitlab/build.sh b/.gitlab/build.sh index d6b0e85635..816e44cda1 100755 --- a/.gitlab/build.sh +++ b/.gitlab/build.sh @@ -253,6 +253,10 @@ export PKG_CONFIG_PATH="${INSTALL_DIR}/lib/pkgconfig:${INSTALL_DIR}/lib64/pkgcon export NIXL_PLUGIN_DIR="${INSTALL_DIR}/lib/$ARCH-linux-gnu/plugins" export CMAKE_PREFIX_PATH="${INSTALL_DIR}:${CMAKE_PREFIX_PATH}" +# Disabling CUDA IPC not to use NVLINK, as it slows down local +# UCX transfers and can cause contention with local collectives. +export UCX_TLS=^cuda_ipc + # shellcheck disable=SC2086 meson setup nixl_build --prefix=${INSTALL_DIR} -Ducx_path=${UCX_INSTALL_DIR} -Dbuild_docs=true -Drust=false ${EXTRA_BUILD_ARGS} -Dlibfabric_path="${LIBFABRIC_INSTALL_DIR}" --buildtype=debug ninja -j"$NPROC" -C nixl_build && ninja -j"$NPROC" -C nixl_build install