NIXLBENCH: Add test in CI

.ci/jenkins/lib/build-matrix.yaml

@@ -71,6 +71,12 @@ steps:
     run: |
       .gitlab/test_python.sh ${NIXL_INSTALL_DIR}
 
+  - name: Test Nixlbench
+    parallel: false
+    timeout: "${TEST_TIMEOUT}"
+    run: |
+      .gitlab/test_nixlbench.sh ${NIXL_INSTALL_DIR}
+
   - name: Test Rust
     parallel: false
     timeout: "${TEST_TIMEOUT}"

.ci/jenkins/lib/test-matrix.yaml

@@ -111,6 +111,14 @@ steps:
       docker rm -f "${JOB_BASE_NAME}-${BUILD_ID}-${axis_index}"
       docker image rm -f "${JOB_BASE_NAME}-${BUILD_ID}-${axis_index}"
 
+  - name: Test Nixlbench
+    parallel: false
+    run: |
+      timeout ${TEST_TIMEOUT}m docker exec -w ${WORKSPACE} "${JOB_BASE_NAME}-${BUILD_ID}-${axis_index}" /bin/bash -c ".gitlab/test_nixlbench.sh ${INSTALL_DIR}"
+    onfail: |
+      docker rm -f "${JOB_BASE_NAME}-${BUILD_ID}-${axis_index}"
+      docker image rm -f "${JOB_BASE_NAME}-${BUILD_ID}-${axis_index}"
+
   - name: Test Rust
     parallel: false
     run: |

.gitlab/build.sh

@@ -55,6 +55,7 @@ $SUDO apt-get -qq install -y curl \
                              numactl \
                              autotools-dev \
                              automake \
+                             git \
                              libtool \
                              libz-dev \
                              libiberty-dev \
@@ -126,6 +127,7 @@ curl -fSsL "https://github.com/openucx/ucx/tarball/${UCX_VERSION}" | tar xz
 )
 
 ( \
+  cd /tmp && \
   git clone --recurse-submodules --depth 1 --shallow-submodules https://github.com/aws/aws-sdk-cpp.git --branch 1.11.581 && \
   mkdir aws_sdk_build && \
   cd aws_sdk_build && \
@@ -148,8 +150,11 @@ export UCX_TLS=^cuda_ipc
 
 # shellcheck disable=SC2086
 meson setup nixl_build --prefix=${INSTALL_DIR} -Ducx_path=${UCX_INSTALL_DIR} -Dbuild_docs=true -Drust=false ${EXTRA_BUILD_ARGS}
-cd nixl_build && ninja && ninja install
+ninja -C nixl_build && ninja -C nixl_build install
 
 # TODO(kapila): Copy the nixl.pc file to the install directory if needed.
 # cp ${BUILD_DIR}/nixl.pc ${INSTALL_DIR}/lib/pkgconfig/nixl.pc
 
+cd benchmark/nixlbench
+meson setup nixlbench_build -Dnixl_path=${INSTALL_DIR} -Dprefix=${INSTALL_DIR}
+ninja -C nixlbench_build && ninja -C nixlbench_build install

.gitlab/test_nixlbench.sh

@@ -0,0 +1,89 @@
+#!/bin/sh
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# shellcheck disable=SC1091
+. "$(dirname "$0")/../.ci/scripts/common.sh"
+
+set -e
+set -x
+
+# Parse commandline arguments with first argument being the install directory.
+INSTALL_DIR=$1
+
+if [ -z "$INSTALL_DIR" ]; then
+    echo "Usage: $0 <install_dir>"
+    exit 1
+fi
+
+ARCH=$(uname -m)
+[ "$ARCH" = "arm64" ] && ARCH="aarch64"
+
+export LD_LIBRARY_PATH=${INSTALL_DIR}/lib:${INSTALL_DIR}/lib/$ARCH-linux-gnu:${INSTALL_DIR}/lib/$ARCH-linux-gnu/plugins:/usr/local/lib:$LD_LIBRARY_PATH
+export LD_LIBRARY_PATH=/usr/local/cuda/lib64:/usr/local/cuda/lib64/stubs:/usr/local/cuda-12.8/compat:$LD_LIBRARY_PATH
+export LD_LIBRARY_PATH=/usr/local/cuda/compat/lib.real:$LD_LIBRARY_PATH
+
+export CPATH=${INSTALL_DIR}/include:$CPATH
+export PATH=${INSTALL_DIR}/bin:$PATH
+export PKG_CONFIG_PATH=${INSTALL_DIR}/lib/pkgconfig:$PKG_CONFIG_PATH
+export NIXL_PLUGIN_DIR=${INSTALL_DIR}/lib/$ARCH-linux-gnu/plugins
+
+echo "==== Show system info ===="
+env
+nvidia-smi topo -m || true
+ibv_devinfo || true
+uname -a || true
+
+echo "==== Running ETCD server ===="
+etcd_port=$(get_next_tcp_port)
+etcd_peer_port=$(get_next_tcp_port)
+export NIXL_ETCD_ENDPOINTS="http://127.0.0.1:${etcd_port}"
+export NIXL_ETCD_PEER_URLS="http://127.0.0.1:${etcd_peer_port}"
+etcd --listen-client-urls ${NIXL_ETCD_ENDPOINTS} --advertise-client-urls ${NIXL_ETCD_ENDPOINTS} \
+     --listen-peer-urls ${NIXL_ETCD_PEER_URLS} --initial-advertise-peer-urls ${NIXL_ETCD_PEER_URLS} \
+     --initial-cluster default=${NIXL_ETCD_PEER_URLS} &
+sleep 5
+
+echo "==== Running Nixlbench tests ===="
+cd ${INSTALL_DIR}
+
+run_nixlbench() {
+    args="$@"
+    ./bin/nixlbench --etcd-endpoints ${NIXL_ETCD_ENDPOINTS} --initiator_seg_type DRAM --target_seg_type DRAM --filepath /tmp --total_buffer_size 80000000 --start_block_size 4096 --max_block_size 16384 --start_batch_size 1 --max_batch_size 4 $args
+}
+
+run_nixlbench_one_worker() {
+    benchmark_group=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1)
+    args="$@"
+    run_nixlbench --benchmark_group $benchmark_group $args
+}
+
+run_nixlbench_two_workers() {
+    benchmark_group=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1)
+    args="$@"
+    run_nixlbench --benchmark_group $benchmark_group $args &
+    pid=$!
+    sleep 1
+    run_nixlbench --benchmark_group $benchmark_group $args
+    wait $pid
+}
+
+run_nixlbench_two_workers --backend UCX --op_type READ
+run_nixlbench_two_workers --backend UCX --op_type WRITE
+run_nixlbench_one_worker --backend POSIX --op_type READ
+run_nixlbench_one_worker --backend POSIX --op_type WRITE
+
+pkill etcd

benchmark/nixlbench/meson.build

@@ -82,10 +82,6 @@ if cuda_available
     endif
 endif
 
-
-# UCX
-ucx_dep = dependency('ucx')
-
 # GFlags
 gflags_dep = dependency('gflags', required: true)
 
@@ -206,7 +202,7 @@ if nvshmem_available
                  ]
 
     if etcd_available
-        nvcc_args += ['-letcd-cpp-api', '-lcpprest']
+        nvcc_args += ['-letcd-cpp-api-core', '-lcpprest']
         etcd_rt = meson.current_build_dir() + '/src/runtime/etcd/libetcd_rt.a.p/etcd_rt.cpp.o'
         nvcc_cmd_files += etcd_rt
     endif

benchmark/nixlbench/meson_options.txt

@@ -13,9 +13,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-option('cudapath_inc', type: 'string', value: '/usr/local/cuda/include', description: 'Include path for CUDA')
-option('cudapath_lib', type: 'string', value: '/usr/local/cuda/lib64/', description: 'Library path for CUDA')
-option('cudapath_stub', type: 'string', value: '/usr/local/cuda/lib64/stubs', description: 'Extra Stub path for CUDA')
+option('cudapath_inc', type: 'string', value: '', description: 'Include path for CUDA')
+option('cudapath_lib', type: 'string', value: '', description: 'Library path for CUDA')
+option('cudapath_stub', type: 'string', value: '', description: 'Extra Stub path for CUDA')
 option('etcd_inc_path', type: 'string', value: '', description: 'Path to ETCD C++ Client includes')
 option('etcd_lib_path', type: 'string', value: '', description: 'Path to ETCD C++ Client library')
 option('nixl_path', type: 'string', value: '/usr/local', description: 'Path to NiXL')