Skip to content

AZP: UCXX integration - tests + builds #11473

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
Alexey-Rivkin wants to merge 7 commits into
base: master
Choose a base branch
from
Draft

AZP: UCXX integration - tests + builds #11473

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
0b12d01
BUILD: add rapidsai CI wrapper images for UCXX
Alexey-Rivkin Jul 27, 2026
5d126ee
AZP: add UCXX build + test stages to the PR pipeline
Alexey-Rivkin Jul 27, 2026
ad107af
AZP: build the PR's UCX once per pipeline run
Alexey-Rivkin Jul 27, 2026
0e3fe54
AZP: persistent pip cache for the UCXX jobs
Alexey-Rivkin Jul 27, 2026
3745c8e
AZP: start the UCXX stages after Basic_compile, not Static_check
Alexey-Rivkin Jul 27, 2026
4ea0368
AZP: make the UCXX driver scripts fail on error
Alexey-Rivkin Jul 29, 2026
3f39051
AZP: pin the UCX-PR toolchain compiler to gcc 14
Alexey-Rivkin Jul 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions buildlib/azure-pipelines-pr.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,5 +31,13 @@ pr:
- buildlib/tools/perf_results.py
- buildlib/tools/perf-common.yml

resources:
repositories:
- repository: ucxx
type: github
name: rapidsai/ucxx
endpoint: Mellanox-lab
ref: refs/tags/v0.52.00a

extends:
template: pr/main.yml
10 changes: 10 additions & 0 deletions buildlib/dockers/rapidsai-ci-conda.Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
# Azure wrapper around rapidsai/ci-conda: chmod /opt/conda so the non-root UID Azure runs
# steps as can use conda/python (rapidsai owns it as root); + adds gdb for stack capture.

ARG BASE_IMAGE=rapidsai/ci-conda:26.10-latest
FROM ${BASE_IMAGE}

RUN chmod -R o+rwX /opt/conda \
&& apt-get update \
&& apt-get install -y --no-install-recommends gdb \
&& rm -rf /var/lib/apt/lists/*
10 changes: 10 additions & 0 deletions buildlib/dockers/rapidsai-ci-wheel.Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
# Azure wrapper around rapidsai/ci-wheel: chmod /pyenv so the non-root UID Azure runs
# steps as can write there (rapidsai owns it as root); + adds gdb for stack capture.
# Default base = cuda13; cuda12 image built by overriding BASE_IMAGE to the cuda12.9.2 tag.

ARG BASE_IMAGE=rapidsai/ci-wheel:26.10-cuda13.3.0-rockylinux8-py3.11
FROM ${BASE_IMAGE}

RUN chmod -R o+rwX /pyenv \
&& dnf install -y gdb \
&& dnf clean all
79 changes: 79 additions & 0 deletions buildlib/pr/main.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@ variables:
DOCKER_OPT_IB: --ulimit memlock=-1:-1 --device=/dev/infiniband/ --net=host
DOCKER_OPT_GPU: --gpus all --device=/dev/gdrdrv --ipc=host $(DOCKER_OPT_IB)
DOCKER_OPT_ARGS: --cap-add=SYS_PTRACE
# Persistent pip cache for the UCXX jobs (the wheel test env alone downloads
# ~2 GB of cuda/rapids wheels). Nodes without the dir degrade gracefully.
UCXX_PIP_CACHE_VOL: -v /scrap/pip-cache:/pip-cache

resources:
containers:
Expand Down Expand Up @@ -261,6 +264,27 @@ resources:
- container: centos10stream
image: rdmz-harbor.rdmz.labs.mlnx/hpcx/x86_64/centos10stream/builder:inbox
options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_VOLUMES)
- container: ucxx_rapidsai_ci_conda
# Thin wrapper of rapidsai/ci-conda; see buildlib/dockers/rapidsai-ci-conda.Dockerfile.
image: rdmz-harbor.rdmz.labs.mlnx/ucx/rapidsai-ci-conda:26.10-azp-1
options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_VOLUMES) $(UCXX_PIP_CACHE_VOL)
- container: ucxx_rapidsai_ci_conda_gpu
# No IB/host-net: with IB, UCX binds rc_mlx5 and the AM/tag tests hang here.
image: rdmz-harbor.rdmz.labs.mlnx/ucx/rapidsai-ci-conda:26.10-azp-1
options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_VOLUMES) $(UCXX_PIP_CACHE_VOL) --gpus all --ipc=host
# Wheel images are CUDA-pinned: one base per CUDA version (see rapidsai-ci-wheel.Dockerfile).
- container: ucxx_rapidsai_ci_wheel_cuda13
image: rdmz-harbor.rdmz.labs.mlnx/ucx/rapidsai-ci-wheel:26.10-cuda13-azp-1
options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_VOLUMES) $(UCXX_PIP_CACHE_VOL)
- container: ucxx_rapidsai_ci_wheel_cuda13_gpu
image: rdmz-harbor.rdmz.labs.mlnx/ucx/rapidsai-ci-wheel:26.10-cuda13-azp-1
options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_VOLUMES) $(UCXX_PIP_CACHE_VOL) --gpus all --ipc=host
- container: ucxx_rapidsai_ci_wheel_cuda12
image: rdmz-harbor.rdmz.labs.mlnx/ucx/rapidsai-ci-wheel:26.10-cuda12-azp-1
options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_VOLUMES) $(UCXX_PIP_CACHE_VOL)
- container: ucxx_rapidsai_ci_wheel_cuda12_gpu
image: rdmz-harbor.rdmz.labs.mlnx/ucx/rapidsai-ci-wheel:26.10-cuda12-azp-1
options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_VOLUMES) $(UCXX_PIP_CACHE_VOL) --gpus all --ipc=host

stages:
- stage: Codestyle
Expand Down Expand Up @@ -349,6 +373,61 @@ stages:
demands: ucx_docker -equals yes
container: coverity_rh7

- template: ucxx_ucx.yml
parameters:
dependsOn: [Basic_compile]
slices:
- { name: x86_64_cuda12, demands: ucx_docker, rapids_cuda_version: '12.9.2' }
- { name: x86_64_cuda13, demands: ucx_docker, rapids_cuda_version: '13.3.0' }
- { name: aarch64_cuda12, demands: ucx_arm64, rapids_cuda_version: '12.9.2' }
- { name: aarch64_cuda13, demands: ucx_arm64, rapids_cuda_version: '13.3.0' }

- template: ucxx_build.yml
parameters:
dependsOn: [UCXX_ucx]
# Wheel GPU tests run on x86 only (no arm64 GPU runner); cuda12 + cuda13.
wheel_tests_ucxx_slices:
- { name: x86_64_cuda12_py311, container: ucxx_rapidsai_ci_wheel_cuda12_gpu, libucxx_slice: x86_64_cuda12_py311, ucxx_slice: x86_64_cuda12_py311,
demands: ucx_gpu, rapids_cuda_version: '12.9.2', rapids_py_version: '3.11' }
- { name: x86_64_cuda13_py311, container: ucxx_rapidsai_ci_wheel_cuda13_gpu, libucxx_slice: x86_64_cuda13_py311, ucxx_slice: x86_64_cuda13_py311,
demands: ucx_gpu, rapids_cuda_version: '13.3.0', rapids_py_version: '3.11' }
conda_cpp_slices:
- { name: x86_64_cuda12_py311, demands: ucx_docker, rapids_cuda_version: '12.9.2', rapids_py_version: '3.11' }
- { name: x86_64_cuda13_py311, demands: ucx_docker, rapids_cuda_version: '13.3.0', rapids_py_version: '3.11' }
- { name: aarch64_cuda12_py311, demands: ucx_arm64, rapids_cuda_version: '12.9.2', rapids_py_version: '3.11' }
- { name: aarch64_cuda13_py311, demands: ucx_arm64, rapids_cuda_version: '13.3.0', rapids_py_version: '3.11' }
conda_python_slices:
- { name: x86_64_cuda12_py311, demands: ucx_docker, rapids_cuda_version: '12.9.2', rapids_py_version: '3.11' }
- { name: x86_64_cuda13_py311, demands: ucx_docker, rapids_cuda_version: '13.3.0', rapids_py_version: '3.11' }
- { name: aarch64_cuda12_py311, demands: ucx_arm64, rapids_cuda_version: '12.9.2', rapids_py_version: '3.11' }
- { name: aarch64_cuda13_py311, demands: ucx_arm64, rapids_cuda_version: '13.3.0', rapids_py_version: '3.11' }
# libucxx + ucxx wheels for cuda12 + cuda13, x86_64 + aarch64.
wheel_libucxx_slices:
- { name: x86_64_cuda12_py311, container: ucxx_rapidsai_ci_wheel_cuda12, demands: ucx_docker, rapids_cuda_version: '12.9.2', rapids_py_version: '3.11' }
- { name: x86_64_cuda13_py311, container: ucxx_rapidsai_ci_wheel_cuda13, demands: ucx_docker, rapids_cuda_version: '13.3.0', rapids_py_version: '3.11' }
- { name: aarch64_cuda12_py311, container: ucxx_rapidsai_ci_wheel_cuda12, demands: ucx_arm64, rapids_cuda_version: '12.9.2', rapids_py_version: '3.11' }
- { name: aarch64_cuda13_py311, container: ucxx_rapidsai_ci_wheel_cuda13, demands: ucx_arm64, rapids_cuda_version: '13.3.0', rapids_py_version: '3.11' }
wheel_ucxx_slices:
- { name: x86_64_cuda12_py311, container: ucxx_rapidsai_ci_wheel_cuda12, libucxx_slice: x86_64_cuda12_py311, demands: ucx_docker, rapids_cuda_version: '12.9.2', rapids_py_version: '3.11' }
- { name: x86_64_cuda13_py311, container: ucxx_rapidsai_ci_wheel_cuda13, libucxx_slice: x86_64_cuda13_py311, demands: ucx_docker, rapids_cuda_version: '13.3.0', rapids_py_version: '3.11' }
- { name: aarch64_cuda12_py311, container: ucxx_rapidsai_ci_wheel_cuda12, libucxx_slice: aarch64_cuda12_py311, demands: ucx_arm64, rapids_cuda_version: '12.9.2', rapids_py_version: '3.11' }
- { name: aarch64_cuda13_py311, container: ucxx_rapidsai_ci_wheel_cuda13, libucxx_slice: aarch64_cuda13_py311, demands: ucx_arm64, rapids_cuda_version: '13.3.0', rapids_py_version: '3.11' }
docs_slices:
- { name: x86_64_cuda13_py311, cpp_slice: x86_64_cuda13_py311, python_slice: x86_64_cuda13_py311,
demands: ucx_docker, rapids_cuda_version: '13.3.0', rapids_py_version: '3.11' }
devcontainer_slices:
- { name: x86_64_cuda13_py311, demands: ucx_docker, rapids_cuda_version: '13.3.0', rapids_py_version: '3.11' }

- template: ucxx_tests.yml
parameters:
dependsOn: [UCXX_ucx]
slices:
- { name: x86_64_cuda13_py313, gpu: true, demands: ucx_gpu, rapids_cuda_version: '13.3.0', rapids_py_version: '3.13' }
- { name: x86_64_cuda12_py311, gpu: false, demands: ucx_docker, rapids_cuda_version: '12.9.2', rapids_py_version: '3.11' }
- { name: x86_64_cuda13_py311, gpu: false, demands: ucx_docker, rapids_cuda_version: '13.3.0', rapids_py_version: '3.11' }
- { name: aarch64_cuda12_py311, gpu: false, demands: ucx_arm64, rapids_cuda_version: '12.9.2', rapids_py_version: '3.11' }
- { name: aarch64_cuda13_py311, gpu: false, demands: ucx_arm64, rapids_cuda_version: '13.3.0', rapids_py_version: '3.11' }

- stage: Tests
dependsOn: [Basic_compile]
jobs:
Expand Down
Loading
Loading