diff --git a/buildlib/azure-pipelines-pr.yml b/buildlib/azure-pipelines-pr.yml index d446fe72ea7..b07817c85c4 100644 --- a/buildlib/azure-pipelines-pr.yml +++ b/buildlib/azure-pipelines-pr.yml @@ -31,5 +31,13 @@ pr: - buildlib/tools/perf_results.py - buildlib/tools/perf-common.yml +resources: + repositories: + - repository: ucxx + type: github + name: rapidsai/ucxx + endpoint: Mellanox-lab + ref: refs/tags/v0.52.00a + extends: template: pr/main.yml diff --git a/buildlib/dockers/rapidsai-ci-conda.Dockerfile b/buildlib/dockers/rapidsai-ci-conda.Dockerfile new file mode 100644 index 00000000000..c4f0f6aef34 --- /dev/null +++ b/buildlib/dockers/rapidsai-ci-conda.Dockerfile @@ -0,0 +1,10 @@ +# Azure wrapper around rapidsai/ci-conda: chmod /opt/conda so the non-root UID Azure runs +# steps as can use conda/python (rapidsai owns it as root); + adds gdb for stack capture. + +ARG BASE_IMAGE=rapidsai/ci-conda:26.10-latest +FROM ${BASE_IMAGE} + +RUN chmod -R o+rwX /opt/conda \ + && apt-get update \ + && apt-get install -y --no-install-recommends gdb \ + && rm -rf /var/lib/apt/lists/* diff --git a/buildlib/dockers/rapidsai-ci-wheel.Dockerfile b/buildlib/dockers/rapidsai-ci-wheel.Dockerfile new file mode 100644 index 00000000000..7d6bf1edc78 --- /dev/null +++ b/buildlib/dockers/rapidsai-ci-wheel.Dockerfile @@ -0,0 +1,10 @@ +# Azure wrapper around rapidsai/ci-wheel: chmod /pyenv so the non-root UID Azure runs +# steps as can write there (rapidsai owns it as root); + adds gdb for stack capture. +# Default base = cuda13; cuda12 image built by overriding BASE_IMAGE to the cuda12.9.2 tag. + +ARG BASE_IMAGE=rapidsai/ci-wheel:26.10-cuda13.3.0-rockylinux8-py3.11 +FROM ${BASE_IMAGE} + +RUN chmod -R o+rwX /pyenv \ + && dnf install -y gdb \ + && dnf clean all diff --git a/buildlib/pr/main.yml b/buildlib/pr/main.yml index 0e087877a5b..6734fd1cc37 100644 --- a/buildlib/pr/main.yml +++ b/buildlib/pr/main.yml @@ -3,6 +3,9 @@ variables: DOCKER_OPT_IB: --ulimit memlock=-1:-1 --device=/dev/infiniband/ --net=host DOCKER_OPT_GPU: --gpus all --device=/dev/gdrdrv --ipc=host $(DOCKER_OPT_IB) DOCKER_OPT_ARGS: --cap-add=SYS_PTRACE + # Persistent pip cache for the UCXX jobs (the wheel test env alone downloads + # ~2 GB of cuda/rapids wheels). Nodes without the dir degrade gracefully. + UCXX_PIP_CACHE_VOL: -v /scrap/pip-cache:/pip-cache resources: containers: @@ -261,6 +264,27 @@ resources: - container: centos10stream image: rdmz-harbor.rdmz.labs.mlnx/hpcx/x86_64/centos10stream/builder:inbox options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_VOLUMES) + - container: ucxx_rapidsai_ci_conda + # Thin wrapper of rapidsai/ci-conda; see buildlib/dockers/rapidsai-ci-conda.Dockerfile. + image: rdmz-harbor.rdmz.labs.mlnx/ucx/rapidsai-ci-conda:26.10-azp-1 + options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_VOLUMES) $(UCXX_PIP_CACHE_VOL) + - container: ucxx_rapidsai_ci_conda_gpu + # No IB/host-net: with IB, UCX binds rc_mlx5 and the AM/tag tests hang here. + image: rdmz-harbor.rdmz.labs.mlnx/ucx/rapidsai-ci-conda:26.10-azp-1 + options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_VOLUMES) $(UCXX_PIP_CACHE_VOL) --gpus all --ipc=host + # Wheel images are CUDA-pinned: one base per CUDA version (see rapidsai-ci-wheel.Dockerfile). + - container: ucxx_rapidsai_ci_wheel_cuda13 + image: rdmz-harbor.rdmz.labs.mlnx/ucx/rapidsai-ci-wheel:26.10-cuda13-azp-1 + options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_VOLUMES) $(UCXX_PIP_CACHE_VOL) + - container: ucxx_rapidsai_ci_wheel_cuda13_gpu + image: rdmz-harbor.rdmz.labs.mlnx/ucx/rapidsai-ci-wheel:26.10-cuda13-azp-1 + options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_VOLUMES) $(UCXX_PIP_CACHE_VOL) --gpus all --ipc=host + - container: ucxx_rapidsai_ci_wheel_cuda12 + image: rdmz-harbor.rdmz.labs.mlnx/ucx/rapidsai-ci-wheel:26.10-cuda12-azp-1 + options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_VOLUMES) $(UCXX_PIP_CACHE_VOL) + - container: ucxx_rapidsai_ci_wheel_cuda12_gpu + image: rdmz-harbor.rdmz.labs.mlnx/ucx/rapidsai-ci-wheel:26.10-cuda12-azp-1 + options: $(DOCKER_OPT_ARGS) $(DOCKER_OPT_VOLUMES) $(UCXX_PIP_CACHE_VOL) --gpus all --ipc=host stages: - stage: Codestyle @@ -349,6 +373,61 @@ stages: demands: ucx_docker -equals yes container: coverity_rh7 + - template: ucxx_ucx.yml + parameters: + dependsOn: [Basic_compile] + slices: + - { name: x86_64_cuda12, demands: ucx_docker, rapids_cuda_version: '12.9.2' } + - { name: x86_64_cuda13, demands: ucx_docker, rapids_cuda_version: '13.3.0' } + - { name: aarch64_cuda12, demands: ucx_arm64, rapids_cuda_version: '12.9.2' } + - { name: aarch64_cuda13, demands: ucx_arm64, rapids_cuda_version: '13.3.0' } + + - template: ucxx_build.yml + parameters: + dependsOn: [UCXX_ucx] + # Wheel GPU tests run on x86 only (no arm64 GPU runner); cuda12 + cuda13. + wheel_tests_ucxx_slices: + - { name: x86_64_cuda12_py311, container: ucxx_rapidsai_ci_wheel_cuda12_gpu, libucxx_slice: x86_64_cuda12_py311, ucxx_slice: x86_64_cuda12_py311, + demands: ucx_gpu, rapids_cuda_version: '12.9.2', rapids_py_version: '3.11' } + - { name: x86_64_cuda13_py311, container: ucxx_rapidsai_ci_wheel_cuda13_gpu, libucxx_slice: x86_64_cuda13_py311, ucxx_slice: x86_64_cuda13_py311, + demands: ucx_gpu, rapids_cuda_version: '13.3.0', rapids_py_version: '3.11' } + conda_cpp_slices: + - { name: x86_64_cuda12_py311, demands: ucx_docker, rapids_cuda_version: '12.9.2', rapids_py_version: '3.11' } + - { name: x86_64_cuda13_py311, demands: ucx_docker, rapids_cuda_version: '13.3.0', rapids_py_version: '3.11' } + - { name: aarch64_cuda12_py311, demands: ucx_arm64, rapids_cuda_version: '12.9.2', rapids_py_version: '3.11' } + - { name: aarch64_cuda13_py311, demands: ucx_arm64, rapids_cuda_version: '13.3.0', rapids_py_version: '3.11' } + conda_python_slices: + - { name: x86_64_cuda12_py311, demands: ucx_docker, rapids_cuda_version: '12.9.2', rapids_py_version: '3.11' } + - { name: x86_64_cuda13_py311, demands: ucx_docker, rapids_cuda_version: '13.3.0', rapids_py_version: '3.11' } + - { name: aarch64_cuda12_py311, demands: ucx_arm64, rapids_cuda_version: '12.9.2', rapids_py_version: '3.11' } + - { name: aarch64_cuda13_py311, demands: ucx_arm64, rapids_cuda_version: '13.3.0', rapids_py_version: '3.11' } + # libucxx + ucxx wheels for cuda12 + cuda13, x86_64 + aarch64. + wheel_libucxx_slices: + - { name: x86_64_cuda12_py311, container: ucxx_rapidsai_ci_wheel_cuda12, demands: ucx_docker, rapids_cuda_version: '12.9.2', rapids_py_version: '3.11' } + - { name: x86_64_cuda13_py311, container: ucxx_rapidsai_ci_wheel_cuda13, demands: ucx_docker, rapids_cuda_version: '13.3.0', rapids_py_version: '3.11' } + - { name: aarch64_cuda12_py311, container: ucxx_rapidsai_ci_wheel_cuda12, demands: ucx_arm64, rapids_cuda_version: '12.9.2', rapids_py_version: '3.11' } + - { name: aarch64_cuda13_py311, container: ucxx_rapidsai_ci_wheel_cuda13, demands: ucx_arm64, rapids_cuda_version: '13.3.0', rapids_py_version: '3.11' } + wheel_ucxx_slices: + - { name: x86_64_cuda12_py311, container: ucxx_rapidsai_ci_wheel_cuda12, libucxx_slice: x86_64_cuda12_py311, demands: ucx_docker, rapids_cuda_version: '12.9.2', rapids_py_version: '3.11' } + - { name: x86_64_cuda13_py311, container: ucxx_rapidsai_ci_wheel_cuda13, libucxx_slice: x86_64_cuda13_py311, demands: ucx_docker, rapids_cuda_version: '13.3.0', rapids_py_version: '3.11' } + - { name: aarch64_cuda12_py311, container: ucxx_rapidsai_ci_wheel_cuda12, libucxx_slice: aarch64_cuda12_py311, demands: ucx_arm64, rapids_cuda_version: '12.9.2', rapids_py_version: '3.11' } + - { name: aarch64_cuda13_py311, container: ucxx_rapidsai_ci_wheel_cuda13, libucxx_slice: aarch64_cuda13_py311, demands: ucx_arm64, rapids_cuda_version: '13.3.0', rapids_py_version: '3.11' } + docs_slices: + - { name: x86_64_cuda13_py311, cpp_slice: x86_64_cuda13_py311, python_slice: x86_64_cuda13_py311, + demands: ucx_docker, rapids_cuda_version: '13.3.0', rapids_py_version: '3.11' } + devcontainer_slices: + - { name: x86_64_cuda13_py311, demands: ucx_docker, rapids_cuda_version: '13.3.0', rapids_py_version: '3.11' } + + - template: ucxx_tests.yml + parameters: + dependsOn: [UCXX_ucx] + slices: + - { name: x86_64_cuda13_py313, gpu: true, demands: ucx_gpu, rapids_cuda_version: '13.3.0', rapids_py_version: '3.13' } + - { name: x86_64_cuda12_py311, gpu: false, demands: ucx_docker, rapids_cuda_version: '12.9.2', rapids_py_version: '3.11' } + - { name: x86_64_cuda13_py311, gpu: false, demands: ucx_docker, rapids_cuda_version: '13.3.0', rapids_py_version: '3.11' } + - { name: aarch64_cuda12_py311, gpu: false, demands: ucx_arm64, rapids_cuda_version: '12.9.2', rapids_py_version: '3.11' } + - { name: aarch64_cuda13_py311, gpu: false, demands: ucx_arm64, rapids_cuda_version: '13.3.0', rapids_py_version: '3.11' } + - stage: Tests dependsOn: [Basic_compile] jobs: diff --git a/buildlib/pr/ucxx_build.yml b/buildlib/pr/ucxx_build.yml new file mode 100644 index 00000000000..8ceaf01cb63 --- /dev/null +++ b/buildlib/pr/ucxx_build.yml @@ -0,0 +1,292 @@ +parameters: + dependsOn: [Static_check] + conda_container: ucxx_rapidsai_ci_conda + conda_cpp_slices: [] + conda_python_slices: [] + wheel_libucxx_slices: [] + wheel_ucxx_slices: [] + wheel_tests_ucxx_slices: [] + docs_slices: [] + devcontainer_slices: [] + +stages: + # UCXX results are informational: every job is continueOnError, so failures + # surface as warnings and never block the UCX PR. + - stage: UCXX_build + dependsOn: ${{ parameters.dependsOn }} + variables: + UCX_DIR: $(Agent.BuildDirectory)/ucx + UCXX_DIR: $(Agent.BuildDirectory)/ucxx + RAPIDS_BLD_OUTPUT_DIR: $(Build.ArtifactStagingDirectory) + jobs: + - ${{ each slice in parameters.conda_cpp_slices }}: + - job: ucxx_conda_cpp_build_${{ slice.name }} + continueOnError: true + workspace: + clean: all + pool: + name: MLNX + demands: ${{ slice.demands }} + displayName: 'UCXX conda-cpp-build (${{ slice.name }})' + container: ${{ parameters.conda_container }} + timeoutInMinutes: 60 + variables: + RAPIDS_CUDA_VERSION: ${{ slice.rapids_cuda_version }} + RAPIDS_PY_VERSION: ${{ slice.rapids_py_version }} + steps: + - checkout: self + path: ucx + fetchDepth: 100 + retryCountOnTaskFailure: 5 + - checkout: ucxx + path: ucxx + retryCountOnTaskFailure: 5 + - template: ucxx_ucx_fetch.yml + - bash: bash $(UCX_DIR)/buildlib/tools/build_ucxx.sh conda_cpp + displayName: Build UCXX conda C++ package + - task: PublishBuildArtifacts@1 + displayName: Publish conda-cpp artifact + inputs: + pathToPublish: $(RAPIDS_BLD_OUTPUT_DIR) + artifactName: ucxx-conda-cpp-${{ slice.name }} + + - ${{ each slice in parameters.conda_python_slices }}: + - job: ucxx_conda_python_build_${{ slice.name }} + continueOnError: true + workspace: + clean: all + pool: + name: MLNX + demands: ${{ slice.demands }} + displayName: 'UCXX conda-python-build (${{ slice.name }})' + container: ${{ parameters.conda_container }} + timeoutInMinutes: 60 + dependsOn: ucxx_conda_cpp_build_${{ slice.name }} + variables: + RAPIDS_CUDA_VERSION: ${{ slice.rapids_cuda_version }} + RAPIDS_PY_VERSION: ${{ slice.rapids_py_version }} + steps: + - checkout: self + path: ucx + fetchDepth: 100 + retryCountOnTaskFailure: 5 + - checkout: ucxx + path: ucxx + retryCountOnTaskFailure: 5 + - template: ucxx_ucx_fetch.yml + - task: DownloadBuildArtifacts@1 + displayName: Fetch conda-cpp artifact + inputs: + buildType: current + artifactName: ucxx-conda-cpp-${{ slice.name }} + downloadPath: $(System.DefaultWorkingDirectory)/_dl + - bash: | + rm -rf "$(RAPIDS_BLD_OUTPUT_DIR)" + mv "$(System.DefaultWorkingDirectory)/_dl/ucxx-conda-cpp-${{ slice.name }}" "$(RAPIDS_BLD_OUTPUT_DIR)" + displayName: Stage conda-cpp artifact + - bash: bash $(UCX_DIR)/buildlib/tools/build_ucxx.sh conda_python + displayName: Build UCXX conda Python package + - task: PublishBuildArtifacts@1 + displayName: Publish conda-python artifact + inputs: + pathToPublish: $(RAPIDS_BLD_OUTPUT_DIR) + artifactName: ucxx-conda-python-${{ slice.name }} + + - ${{ each slice in parameters.wheel_libucxx_slices }}: + - job: ucxx_wheel_libucxx_build_${{ slice.name }} + continueOnError: true + workspace: + clean: all + pool: + name: MLNX + demands: ${{ slice.demands }} + displayName: 'UCXX wheel-build-libucxx (${{ slice.name }})' + container: ${{ slice.container }} + timeoutInMinutes: 60 + variables: + RAPIDS_CUDA_VERSION: ${{ slice.rapids_cuda_version }} + RAPIDS_PY_VERSION: ${{ slice.rapids_py_version }} + steps: + - checkout: self + path: ucx + fetchDepth: 100 + retryCountOnTaskFailure: 5 + - checkout: ucxx + path: ucxx + retryCountOnTaskFailure: 5 + - template: ucxx_ucx_fetch.yml + - bash: bash $(UCX_DIR)/buildlib/tools/build_ucxx.sh wheel_libucxx + displayName: Build libucxx wheel + - task: PublishBuildArtifacts@1 + displayName: Publish libucxx wheel artifact + inputs: + pathToPublish: $(RAPIDS_BLD_OUTPUT_DIR) + artifactName: ucxx-wheel-libucxx-${{ slice.name }} + + - ${{ each slice in parameters.wheel_ucxx_slices }}: + - job: ucxx_wheel_ucxx_build_${{ slice.name }} + continueOnError: true + workspace: + clean: all + pool: + name: MLNX + demands: ${{ slice.demands }} + displayName: 'UCXX wheel-build-ucxx (${{ slice.name }})' + container: ${{ slice.container }} + timeoutInMinutes: 60 + dependsOn: ucxx_wheel_libucxx_build_${{ slice.libucxx_slice }} + variables: + RAPIDS_CUDA_VERSION: ${{ slice.rapids_cuda_version }} + RAPIDS_PY_VERSION: ${{ slice.rapids_py_version }} + WHEEL_INPUT_DIR: $(System.DefaultWorkingDirectory)/wheel-libucxx + steps: + - checkout: self + path: ucx + fetchDepth: 100 + retryCountOnTaskFailure: 5 + - checkout: ucxx + path: ucxx + retryCountOnTaskFailure: 5 + - template: ucxx_ucx_fetch.yml + - task: DownloadBuildArtifacts@1 + displayName: Fetch libucxx wheel artifact + inputs: + buildType: current + artifactName: ucxx-wheel-libucxx-${{ slice.libucxx_slice }} + downloadPath: $(System.DefaultWorkingDirectory)/_dl + - bash: | + rm -rf "$(WHEEL_INPUT_DIR)" + mv "$(System.DefaultWorkingDirectory)/_dl/ucxx-wheel-libucxx-${{ slice.libucxx_slice }}" "$(WHEEL_INPUT_DIR)" + displayName: Stage libucxx wheel artifact + - bash: bash $(UCX_DIR)/buildlib/tools/build_ucxx.sh wheel_ucxx + displayName: Build ucxx wheel + - task: PublishBuildArtifacts@1 + displayName: Publish ucxx wheel artifact + inputs: + pathToPublish: $(RAPIDS_BLD_OUTPUT_DIR) + artifactName: ucxx-wheel-ucxx-${{ slice.name }} + + - ${{ each slice in parameters.docs_slices }}: + - job: ucxx_docs_build_${{ slice.name }} + continueOnError: true + workspace: + clean: all + pool: + name: MLNX + demands: ${{ slice.demands }} + displayName: 'UCXX docs-build (${{ slice.name }})' + container: ${{ parameters.conda_container }} + timeoutInMinutes: 60 + dependsOn: + - ucxx_conda_cpp_build_${{ slice.cpp_slice }} + - ucxx_conda_python_build_${{ slice.python_slice }} + variables: + RAPIDS_CUDA_VERSION: ${{ slice.rapids_cuda_version }} + RAPIDS_PY_VERSION: ${{ slice.rapids_py_version }} + DOCS_OUT_DIR: $(Build.ArtifactStagingDirectory)/docs + steps: + - checkout: self + path: ucx + fetchDepth: 100 + retryCountOnTaskFailure: 5 + - checkout: ucxx + path: ucxx + retryCountOnTaskFailure: 5 + - template: ucxx_ucx_fetch.yml + - task: DownloadBuildArtifacts@1 + displayName: Fetch conda-cpp artifact + inputs: + buildType: current + artifactName: ucxx-conda-cpp-${{ slice.cpp_slice }} + downloadPath: $(System.DefaultWorkingDirectory)/_dl_cpp + - task: DownloadBuildArtifacts@1 + displayName: Fetch conda-python artifact + inputs: + buildType: current + artifactName: ucxx-conda-python-${{ slice.python_slice }} + downloadPath: $(System.DefaultWorkingDirectory)/_dl_py + - bash: bash $(UCX_DIR)/buildlib/tools/build_ucxx.sh docs + displayName: Build UCXX docs + env: + CPP_CHANNEL_DIR: $(System.DefaultWorkingDirectory)/_dl_cpp/ucxx-conda-cpp-${{ slice.cpp_slice }} + PYTHON_CHANNEL_DIR: $(System.DefaultWorkingDirectory)/_dl_py/ucxx-conda-python-${{ slice.python_slice }} + RAPIDS_DOCS_DIR: $(DOCS_OUT_DIR) + - task: PublishBuildArtifacts@1 + displayName: Publish docs artifact + inputs: + pathToPublish: $(DOCS_OUT_DIR) + artifactName: ucxx-docs-${{ slice.name }} + + - ${{ each slice in parameters.wheel_tests_ucxx_slices }}: + - job: ucxx_wheel_tests_ucxx_${{ slice.name }} + continueOnError: true + workspace: + clean: all + pool: + name: MLNX + demands: ${{ slice.demands }} + displayName: 'UCXX wheel-tests-ucxx (${{ slice.name }})' + container: ${{ slice.container }} + timeoutInMinutes: 90 + dependsOn: + - ucxx_wheel_libucxx_build_${{ slice.libucxx_slice }} + - ucxx_wheel_ucxx_build_${{ slice.ucxx_slice }} + variables: + RAPIDS_CUDA_VERSION: ${{ slice.rapids_cuda_version }} + RAPIDS_PY_VERSION: ${{ slice.rapids_py_version }} + LIBUCXX_WHL_DIR: $(Build.ArtifactStagingDirectory)/libucxx_whl + UCXX_WHL_DIR: $(Build.ArtifactStagingDirectory)/ucxx_whl + steps: + - checkout: self + path: ucx + fetchDepth: 100 + retryCountOnTaskFailure: 5 + - checkout: ucxx + path: ucxx + retryCountOnTaskFailure: 5 + - template: ucxx_ucx_fetch.yml + - task: DownloadBuildArtifacts@1 + displayName: Fetch libucxx wheel artifact + inputs: + buildType: current + artifactName: ucxx-wheel-libucxx-${{ slice.libucxx_slice }} + downloadPath: $(System.DefaultWorkingDirectory)/_dl_libucxx + - task: DownloadBuildArtifacts@1 + displayName: Fetch ucxx wheel artifact + inputs: + buildType: current + artifactName: ucxx-wheel-ucxx-${{ slice.ucxx_slice }} + downloadPath: $(System.DefaultWorkingDirectory)/_dl_ucxx + - bash: | + rm -rf "$(LIBUCXX_WHL_DIR)" "$(UCXX_WHL_DIR)" + mv "$(System.DefaultWorkingDirectory)/_dl_libucxx/ucxx-wheel-libucxx-${{ slice.libucxx_slice }}" "$(LIBUCXX_WHL_DIR)" + mv "$(System.DefaultWorkingDirectory)/_dl_ucxx/ucxx-wheel-ucxx-${{ slice.ucxx_slice }}" "$(UCXX_WHL_DIR)" + displayName: Stage wheels + - bash: bash $(UCX_DIR)/buildlib/tools/test_ucxx.sh test_wheel_ucxx + displayName: Run UCXX wheel tests + + - ${{ each slice in parameters.devcontainer_slices }}: + - job: ucxx_devcontainer_${{ slice.name }} + continueOnError: true + workspace: + clean: all + pool: + name: MLNX + demands: ${{ slice.demands }} + displayName: 'UCXX devcontainer (${{ slice.name }})' + container: ${{ parameters.conda_container }} + timeoutInMinutes: 15 + dependsOn: [] + variables: + RAPIDS_CUDA_VERSION: ${{ slice.rapids_cuda_version }} + RAPIDS_PY_VERSION: ${{ slice.rapids_py_version }} + steps: + - checkout: self + path: ucx + fetchDepth: 100 + retryCountOnTaskFailure: 5 + - checkout: ucxx + path: ucxx + retryCountOnTaskFailure: 5 + - bash: bash $(UCX_DIR)/buildlib/tools/build_ucxx.sh devcontainer + displayName: Validate UCXX devcontainer configs diff --git a/buildlib/pr/ucxx_tests.yml b/buildlib/pr/ucxx_tests.yml new file mode 100644 index 00000000000..f4952bc94b3 --- /dev/null +++ b/buildlib/pr/ucxx_tests.yml @@ -0,0 +1,52 @@ +parameters: + dependsOn: [Static_check] + cpu_container: ucxx_rapidsai_ci_conda + gpu_container: ucxx_rapidsai_ci_conda_gpu + slices: [] + +stages: + - stage: UCXX_tests + dependsOn: ${{ parameters.dependsOn }} + variables: + UCX_DIR: $(Agent.BuildDirectory)/ucx + UCXX_DIR: $(Agent.BuildDirectory)/ucxx + jobs: + - ${{ each slice in parameters.slices }}: + - job: ucxx_tests_${{ slice.name }} + # UCXX results are informational: failures surface as warnings and + # never block the UCX PR. + continueOnError: true + workspace: + clean: all + pool: + name: MLNX + demands: ${{ slice.demands }} + ${{ if eq(slice.gpu, true) }}: + displayName: 'UCXX GPU tests (${{ slice.name }})' + container: ${{ parameters.gpu_container }} + timeoutInMinutes: 120 + ${{ if eq(slice.gpu, false) }}: + displayName: 'UCXX tests (${{ slice.name }})' + container: ${{ parameters.cpu_container }} + timeoutInMinutes: 90 + variables: + IS_GPU: ${{ slice.gpu }} + RAPIDS_CUDA_VERSION: ${{ slice.rapids_cuda_version }} + RAPIDS_PY_VERSION: ${{ slice.rapids_py_version }} + steps: + - checkout: self + path: ucx + fetchDepth: 100 + retryCountOnTaskFailure: 5 + - checkout: ucxx + path: ucxx + retryCountOnTaskFailure: 5 + - template: ucxx_ucx_fetch.yml + - bash: bash $(UCX_DIR)/buildlib/tools/test_ucxx.sh build + displayName: Build UCXX + - bash: bash $(UCX_DIR)/buildlib/tools/test_ucxx.sh test_cpp + displayName: Run UCXX C++ tests + # GPU slices only: cupy variants need a real device. + - ${{ if eq(slice.gpu, true) }}: + - bash: bash $(UCX_DIR)/buildlib/tools/test_ucxx.sh test_python + displayName: Run UCXX Python tests diff --git a/buildlib/pr/ucxx_ucx.yml b/buildlib/pr/ucxx_ucx.yml new file mode 100644 index 00000000000..2be27b75001 --- /dev/null +++ b/buildlib/pr/ucxx_ucx.yml @@ -0,0 +1,43 @@ +parameters: + dependsOn: [Basic_compile] + conda_container: ucxx_rapidsai_ci_conda + slices: [] + +stages: + # Builds the PR's UCX once per arch/cuda pair and publishes it; the UCXX + # build and test jobs consume the artifact instead of each building UCX. + - stage: UCXX_ucx + displayName: 'UCXX: UCX build' + dependsOn: ${{ parameters.dependsOn }} + variables: + UCX_DIR: $(Agent.BuildDirectory)/ucx + jobs: + - ${{ each slice in parameters.slices }}: + - job: ucxx_ucx_pr_${{ slice.name }} + continueOnError: true + workspace: + clean: all + pool: + name: MLNX + demands: ${{ slice.demands }} + displayName: 'UCX PR build (${{ slice.name }})' + container: ${{ parameters.conda_container }} + timeoutInMinutes: 40 + variables: + RAPIDS_CUDA_VERSION: ${{ slice.rapids_cuda_version }} + steps: + - checkout: self + path: ucx + fetchDepth: 100 + retryCountOnTaskFailure: 5 + - bash: bash $(UCX_DIR)/buildlib/tools/build_ucxx.sh ucx_pr + displayName: Build UCX (PR) + env: + # host-mapped path - the artifact upload runs on the host, + # which cannot see the container's /tmp + UCX_PR_TARBALL: $(Build.ArtifactStagingDirectory)/ucx-pr.tar.gz + - task: PublishBuildArtifacts@1 + displayName: Publish UCX (PR) artifact + inputs: + pathToPublish: $(Build.ArtifactStagingDirectory)/ucx-pr.tar.gz + artifactName: ucx-pr-${{ slice.name }} diff --git a/buildlib/pr/ucxx_ucx_fetch.yml b/buildlib/pr/ucxx_ucx_fetch.yml new file mode 100644 index 00000000000..6bacf4cce47 --- /dev/null +++ b/buildlib/pr/ucxx_ucx_fetch.yml @@ -0,0 +1,16 @@ +# Steps: fetch the PR-built UCX artifact (published by the UCXX_ucx stage) +# and stage it at /tmp/ucx-pr, where the ucxx build/test scripts expect it. +steps: + - bash: echo "##vso[task.setvariable variable=UCX_PR_SLICE]$(uname -m)_cuda${RAPIDS_CUDA_VERSION%%.*}" + displayName: Resolve UCX (PR) artifact slice + - task: DownloadBuildArtifacts@1 + displayName: Fetch UCX (PR) artifact + inputs: + buildType: current + artifactName: ucx-pr-$(UCX_PR_SLICE) + downloadPath: $(System.DefaultWorkingDirectory)/_dl_ucx_pr + - bash: | + rm -rf /tmp/ucx-pr + tar -C /tmp -xzf "$(System.DefaultWorkingDirectory)/_dl_ucx_pr/ucx-pr-$(UCX_PR_SLICE)/ucx-pr.tar.gz" + /tmp/ucx-pr/bin/ucx_info -v | head -3 + displayName: Stage UCX (PR) diff --git a/buildlib/tools/build_ucxx.sh b/buildlib/tools/build_ucxx.sh new file mode 100755 index 00000000000..7a0a43f227d --- /dev/null +++ b/buildlib/tools/build_ucxx.sh @@ -0,0 +1,156 @@ +#!/bin/bash -eE +# +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# See file LICENSE for terms. +# +# Usage: build_ucxx.sh +# Env: UCXX_DIR (all phases except ucx_pr). Build phases also need +# RAPIDS_CUDA_VERSION, RAPIDS_PY_VERSION, RAPIDS_BLD_OUTPUT_DIR. +# wheel_ucxx phase also requires WHEEL_INPUT_DIR (libucxx wheel artifact dir) +# Docs phase env: CPP_CHANNEL_DIR, PYTHON_CHANNEL_DIR, RAPIDS_DOCS_DIR +# +# All packages build against the UCX built from this checkout (the PR under +# test) - see ucxx_ucx_pr.sh. The ucx_pr phase builds it once and tars it for +# the other jobs, which fetch the artifact instead of rebuilding. + +# The pipeline runs this as `bash `, which ignores the shebang options, +# so the shell flags have to be set here for the fail-loud guards to bite. +set -eE -o pipefail + +phase=${1:?phase required} + +if [ "$phase" = "ucx_pr" ]; then + : "${RAPIDS_CUDA_VERSION:?RAPIDS_CUDA_VERSION required}" + export RAPIDS_CUDA_VERSION + ucx_dir=$(cd "$(dirname "$0")/../.." && pwd) + export ucx_dir UCX_PR_PREFIX=/tmp/ucx-pr + source "$ucx_dir/buildlib/tools/ucxx_ucx_pr.sh" + build_ucx_pr_conda + echo "== UCX under test ==" + # sed, not head: head closes the pipe early, which under pipefail can turn a + # healthy ucx_info into a SIGPIPE failure. + "$UCX_PR_PREFIX/bin/ucx_info" -v | sed -n '1,3p' + # The tarball must land on an agent-mapped path (UCX_PR_TARBALL) - the + # artifact upload runs on the host, which cannot see the container's /tmp. + tar -C /tmp -czf "${UCX_PR_TARBALL:?UCX_PR_TARBALL required}" ucx-pr + exit 0 +fi + +: "${UCXX_DIR:?UCXX_DIR required}" + +case "$phase" in + devcontainer) + # Parse each .devcontainer config; verify its Dockerfile + BASE exist + # (no registry pull - the devcontainer CLI catches missing images at use). + UCXX_DIR="$UCXX_DIR" python3 - <<'PY' +import glob, json, os, sys +root = os.environ["UCXX_DIR"] +cfgs = glob.glob(os.path.join(root, ".devcontainer", "*", "devcontainer.json")) +if not cfgs: + sys.exit("ERROR: no devcontainer.json under .devcontainer/") +for cfg in cfgs: + b = json.load(open(cfg))["build"] + df = b["dockerfile"].replace("${localWorkspaceFolder}", root) + assert os.path.isfile(df), f"{cfg}: missing Dockerfile {df}" + assert b["args"]["BASE"], f"{cfg}: empty BASE" + print(f"OK {cfg}") +PY + exit 0 ;; +esac + +: "${RAPIDS_CUDA_VERSION:?RAPIDS_CUDA_VERSION required}" +: "${RAPIDS_PY_VERSION:?RAPIDS_PY_VERSION required}" +: "${RAPIDS_BLD_OUTPUT_DIR:?RAPIDS_BLD_OUTPUT_DIR required}" + +ucx_dir=$(cd "$(dirname "$0")/../.." && pwd) +export ucx_dir UCX_PR_PREFIX=/tmp/ucx-pr +source "$ucx_dir/buildlib/tools/ucxx_ucx_pr.sh" + +export RAPIDS_CUDA_VERSION RAPIDS_PY_VERSION +# upstream test-env matrices select the dependency set via this key +export RAPIDS_DEPENDENCIES=${RAPIDS_DEPENDENCIES:-latest} +# persistent pip cache (host volume); pip disables caching itself if unwritable +[ -d /pip-cache ] && export PIP_CACHE_DIR=/pip-cache +mkdir -p "$RAPIDS_BLD_OUTPUT_DIR" + +case "$phase" in + conda_*) export RAPIDS_CONDA_BLD_OUTPUT_DIR="$RAPIDS_BLD_OUTPUT_DIR" ;; + wheel_*) export RAPIDS_WHEEL_BLD_OUTPUT_DIR="$RAPIDS_BLD_OUTPUT_DIR" ;; + docs) + : "${CPP_CHANNEL_DIR:?CPP_CHANNEL_DIR required for docs phase}" + : "${PYTHON_CHANNEL_DIR:?PYTHON_CHANNEL_DIR required for docs phase}" + : "${RAPIDS_DOCS_DIR:?RAPIDS_DOCS_DIR required for docs phase}" + mkdir -p "$RAPIDS_DOCS_DIR" ;; +esac + +# The ucxx CI scripts fetch artifacts of previous jobs through +# rapids-download-from-github; shim it to the locally staged output so the +# jobs resolve this pipeline's artifacts instead of GitHub's. +mkdir -p "$HOME/.local/bin" +printf '#!/bin/bash\necho "%s"\n' "$RAPIDS_BLD_OUTPUT_DIR" \ + > "$HOME/.local/bin/rapids-download-from-github" +chmod +x "$HOME/.local/bin/rapids-download-from-github" +# Docs phase: build_docs.sh fetches both channels via rapids-download-from-github +# with per-package artifact names - dispatch on the name to the staged channel. +if [ "$phase" = "docs" ]; then + cat > "$HOME/.local/bin/rapids-download-from-github" < "$HOME/.local/bin/rapids-download-from-github" + chmod +x "$HOME/.local/bin/rapids-download-from-github" +fi + +export PATH="$HOME/.local/bin:$PATH" + +cd "$UCXX_DIR" + +# Point every ucxx build at the PR's UCX instead of released ucx. +use_pr_ucx + +# Wheel builds otherwise pick system gcc 8.5 (too old for libucxx's C++20); +# point CC/CXX at gcc-toolset-14. +if [[ "$phase" == wheel_* ]]; then + toolset=/opt/rh/gcc-toolset-14/root/usr/bin + [ -x "$toolset/gcc" ] \ + || { echo "ERROR: gcc-toolset-14 not found at $toolset (needed for libucxx C++20)" >&2; exit 1; } + export CC="$toolset/gcc" CXX="$toolset/g++" +fi + +# The PR's UCX was built once by the UCXX_ucx stage and staged at +# UCX_PR_PREFIX by the fetch step; the package builds link against it. +[ -x "$UCX_PR_PREFIX/bin/ucx_info" ] \ + || { echo "ERROR: UCX (PR) not staged at $UCX_PR_PREFIX - UCXX_ucx artifact missing?" >&2; exit 1; } + +case "$phase" in + conda_cpp) bash ci/build_cpp.sh ;; + conda_python) bash ci/build_python.sh ;; + wheel_libucxx) bash ci/build_wheel_libucxx.sh ;; + wheel_ucxx) + : "${WHEEL_INPUT_DIR:?WHEEL_INPUT_DIR required for wheel_ucxx (libucxx wheel dir)}" + bash ci/build_wheel_ucxx.sh ;; + docs) + # Upstream forces RAPIDS_DOCS_DIR=$(mktemp -d); make it default-if-unset + # so our staged output dir survives. Guard catches upstream rewording. + sed -i 's|RAPIDS_DOCS_DIR="$(mktemp -d)"|: "${RAPIDS_DOCS_DIR:=$(mktemp -d)}"|' ci/build_docs.sh + grep -q 'RAPIDS_DOCS_DIR:=' ci/build_docs.sh \ + || { echo "ERROR: docs patch did not apply to ci/build_docs.sh" >&2; exit 1; } + # The docs env carries no ucx package; provide the PR's UCX libraries. + grep -q "ucx-pr" ci/build_docs.sh \ + || sed -i 's#^conda activate docs$#conda activate docs\ncp -a /tmp/ucx-pr/lib/. "$CONDA_PREFIX/lib/"\ncp -a /tmp/ucx-pr/bin/. "$CONDA_PREFIX/bin/"\necho "UCX-PR overlaid into docs env"#' ci/build_docs.sh + grep -q "ucx-pr" ci/build_docs.sh \ + || { echo "ERROR: UCX-PR overlay patch did not apply to ci/build_docs.sh" >&2; exit 1; } + bash ci/build_docs.sh ;; + *) echo "Unknown phase: $phase" >&2; exit 1 ;; +esac diff --git a/buildlib/tools/test_ucxx.sh b/buildlib/tools/test_ucxx.sh new file mode 100755 index 00000000000..c70b0f105b7 --- /dev/null +++ b/buildlib/tools/test_ucxx.sh @@ -0,0 +1,143 @@ +#!/bin/bash -eE +# +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# See file LICENSE for terms. +# +# Usage: test_ucxx.sh +# Env: RAPIDS_CUDA_VERSION, RAPIDS_PY_VERSION, UCXX_DIR +# build|test_cpp|test_python: also IS_GPU +# test_wheel_ucxx: also LIBUCXX_WHL_DIR, UCXX_WHL_DIR +# +# Everything runs against the UCX built from this checkout (the PR under +# test): the released ucx/libucx dependencies are stripped from the ucxx tree +# (see ucxx_ucx_pr.sh), the packages build against UCX_PR_PREFIX, and the +# tests load its libraries. + +# The pipeline runs this as `bash `, which ignores the shebang options, +# so the shell flags have to be set here for the fail-loud guards to bite. +set -eE -o pipefail + +phase=${1:?phase required} +case "$phase" in + build|test_cpp|test_python) : "${IS_GPU:?IS_GPU required}" ;; +esac +: "${RAPIDS_CUDA_VERSION:?RAPIDS_CUDA_VERSION required}" +: "${RAPIDS_PY_VERSION:?RAPIDS_PY_VERSION required}" +: "${UCXX_DIR:?UCXX_DIR required}" + +ucx_dir=$(cd "$(dirname "$0")/../.." && pwd) +export ucx_dir UCX_PR_PREFIX=/tmp/ucx-pr +source "$ucx_dir/buildlib/tools/ucxx_ucx_pr.sh" + +# The PR's UCX was built once by the UCXX_ucx stage and staged at +# UCX_PR_PREFIX by the fetch step; every build and test consumes it. +[ -x "$UCX_PR_PREFIX/bin/ucx_info" ] \ + || { echo "ERROR: UCX (PR) not staged at $UCX_PR_PREFIX - UCXX_ucx artifact missing?" >&2; exit 1; } + +export RAPIDS_CUDA_VERSION RAPIDS_PY_VERSION +# upstream test-env matrices select the dependency set via this key +export RAPIDS_DEPENDENCIES=${RAPIDS_DEPENDENCIES:-latest} +# persistent pip cache (host volume); pip disables caching itself if unwritable +[ -d /pip-cache ] && export PIP_CACHE_DIR=/pip-cache +export RAPIDS_CONDA_BLD_OUTPUT_DIR=/tmp/conda-bld-output +mkdir -p "$RAPIDS_CONDA_BLD_OUTPUT_DIR" "$HOME/.local/bin" + +# The ucxx test scripts fetch the packages built by the previous step through +# rapids-download-from-github; shim it to the locally built channel. +printf '#!/bin/bash\necho "%s"\n' "$RAPIDS_CONDA_BLD_OUTPUT_DIR" \ + > "$HOME/.local/bin/rapids-download-from-github" +chmod +x "$HOME/.local/bin/rapids-download-from-github" +export PATH="$HOME/.local/bin:$PATH" + +cd "$UCXX_DIR" + +# Point every ucxx build and test at the PR's UCX instead of released ucx. +use_pr_ucx + +# Tolerate missing nvidia-smi on CPU containers. Guard catches upstream rewording. +sed -i 's#^ nvidia-smi$# command -v nvidia-smi >/dev/null \&\& nvidia-smi || echo "(no GPU)"#' ci/test_common.sh +grep -q 'command -v nvidia-smi' ci/test_common.sh \ + || { echo "ERROR: nvidia-smi patch did not apply to ci/test_common.sh" >&2; exit 1; } + +# Conda tests load the PR-built UCX: right after env activation, copy its +# libraries and tools into the env (the env carries no ucx package - the PR +# build is the only provider) and verify the lib the loader sees is the PR's. +for f in ci/test_cpp.sh ci/test_python.sh; do + grep -q "ucx-pr" "$f" \ + || sed -i 's#^conda activate test$#conda activate test\ncp -a /tmp/ucx-pr/lib/. "$CONDA_PREFIX/lib/"\ncp -a /tmp/ucx-pr/bin/. "$CONDA_PREFIX/bin/"\ncmp -s "$CONDA_PREFIX/lib/libucs.so.0" /tmp/ucx-pr/lib/libucs.so.0 || { echo "ERROR: UCX-PR overlay verification failed" >\&2; exit 1; }\necho "UCX-PR overlaid into test env"#' "$f" + grep -q "ucx-pr" "$f" \ + || { echo "ERROR: UCX-PR overlay patch did not apply to $f" >&2; exit 1; } +done + +# Forces the host driver ahead of the image's newer compat driver (MPS rejects +# a client newer than the daemon -> cuInit hangs). ubuntu: /usr/lib/-linux-gnu; +# wheel: /usr/lib64. Test-runtime only: with this in LD_LIBRARY_PATH, ld resolves +# shared-lib dependencies against the image's newer glibc and the UCX build fails, +# so the build steps run without it. +export_host_driver_override() { + local arch hostlib + arch=$(uname -m) + for hostlib in "/usr/lib/$arch-linux-gnu" /usr/lib64; do + [ -d "$hostlib" ] && export LD_LIBRARY_PATH="$hostlib:${LD_LIBRARY_PATH:-}" + done + # arm64 images carry no /usr/lib64, so the loop ends non-zero there; the + # caller runs under errexit and must not treat that as a failure. + return 0 +} + +case "$phase" in + build) + if [ "${IS_GPU,,}" = "true" ]; then + # sccache wrapper crashes CMake's compiler probe on the GPU build hosts; no-op it. + cat > "$HOME/.local/bin/rapids-configure-sccache" <<'EOF' +#!/bin/bash +export CMAKE_C_COMPILER_LAUNCHER= CMAKE_CXX_COMPILER_LAUNCHER= CMAKE_CUDA_COMPILER_LAUNCHER= RUSTC_WRAPPER= +EOF + chmod +x "$HOME/.local/bin/rapids-configure-sccache" + fi + bash ci/build_cpp.sh + bash ci/build_python.sh + ;; + + test_cpp) + export_host_driver_override + # CPU slices have no GPU device bound; CUDA-touching gtests would crash. + if [ "${IS_GPU,,}" = "true" ]; then + bash ci/test_cpp.sh + else + CUDA_VISIBLE_DEVICES= UCX_TLS=tcp,sm,self GTEST_FILTER='-RMM*.*:CCCL*.*' \ + bash ci/test_cpp.sh + fi + ;; + + test_python) + export_host_driver_override + bash ci/test_python.sh + ;; + + test_wheel_ucxx) + : "${LIBUCXX_WHL_DIR:?LIBUCXX_WHL_DIR required}" + : "${UCXX_WHL_DIR:?UCXX_WHL_DIR required}" + # The ucxx wheel carries no libucx dependency - the loader resolves the + # UCX libraries from UCX_PR_PREFIX via LD_LIBRARY_PATH, and the injected + # check asserts the runtime UCX version is the PR's. + grep -q "ucx-pr" ci/test_wheel_ucxx.sh \ + || sed -i '/^print_system_stats$/i want=$(/tmp/ucx-pr/bin/ucx_info -v | sed -n "s/^# Library version: //p")\ngot=$(python -c "import ucxx; print(*ucxx.get_ucx_version(), sep=chr(46))")\n[ "$got" = "$want" ] || { echo "ERROR: UCX version mismatch: $got != $want" >\&2; exit 1; }\necho "UCXX runs UCX-PR $got"' ci/test_wheel_ucxx.sh + grep -q "ucx-pr" ci/test_wheel_ucxx.sh \ + || { echo "ERROR: UCX-PR version check did not apply to ci/test_wheel_ucxx.sh" >&2; exit 1; } + # The wheel test installs from both the libucxx and ucxx wheelhouses via + # the download helpers; stage both wheels in one dir and point the helpers + # there so the libucxx_*.whl / ucxx_*.whl globs resolve. + wheelhouse="$RAPIDS_CONDA_BLD_OUTPUT_DIR/ucxx-wheelhouse" + mkdir -p "$wheelhouse" + cp "$LIBUCXX_WHL_DIR"/*.whl "$UCXX_WHL_DIR"/*.whl "$wheelhouse"/ + printf '#!/bin/bash\necho "%s"\n' "$wheelhouse" \ + > "$HOME/.local/bin/rapids-download-from-github" + chmod +x "$HOME/.local/bin/rapids-download-from-github" + export_host_driver_override + export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}$UCX_PR_PREFIX/lib" + bash ci/test_wheel_ucxx.sh + ;; + + *) echo "Unknown phase: $phase" >&2; exit 1 ;; +esac diff --git a/buildlib/tools/ucxx_ucx_pr.sh b/buildlib/tools/ucxx_ucx_pr.sh new file mode 100644 index 00000000000..36e0960af71 --- /dev/null +++ b/buildlib/tools/ucxx_ucx_pr.sh @@ -0,0 +1,97 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# See file LICENSE for terms. +# +# Sourced by build_ucxx.sh / test_ucxx.sh. Provides: +# build_ucx_pr_conda - build the PR's UCX into UCX_PR_PREFIX via a +# conda-forge toolchain env +# use_pr_ucx - patch the ucxx tree (cwd) so every package build and +# test consumes the PR's UCX instead of released ucx +# Requires: ucx_dir, UCX_PR_PREFIX exported by the caller; cwd = UCXX_DIR for +# use_pr_ucx. + +# Runs inside the toolchain env created by build_ucx_pr_conda (invoked there by +# name through conda run). Idempotent - the container, and thus /tmp, is shared +# by all steps of a job. +build_ucx_pr() { + [ -x "$UCX_PR_PREFIX/bin/ucx_info" ] && return 0 + # CUDA comes from the env's conda-forge cuda dev packages, pinned to + # RAPIDS_CUDA_VERSION; headers and stubs live under targets/-linux. + local cuda_opt="" conda_cuda="${CONDA_PREFIX:-}/targets/$(uname -m)-linux" + [ -e "$conda_cuda/include/cuda.h" ] && cuda_opt="--with-cuda=$conda_cuda" + echo "UCX-PR configure cuda: ${cuda_opt:-NONE (no cuda headers found)}" + (cd "$ucx_dir" \ + && ./autogen.sh \ + && ./contrib/configure-release --prefix="$UCX_PR_PREFIX" $cuda_opt \ + --enable-mt --without-java --without-go --disable-doxygen-doc \ + && make -j"$(nproc)" install) > /tmp/ucx-pr-build.log 2>&1 \ + || { tail -50 /tmp/ucx-pr-build.log >&2; echo "ERROR: UCX (PR) build failed" >&2; return 1; } +} +export -f build_ucx_pr + +# The conda image has no system toolchain; build the PR's UCX with a +# conda-forge one. Everything version-sensitive is pinned, because the env is +# solved fresh on every run and conda-forge moves under it: gcc 15+ turns the +# OpenMP 'master' deprecation into an error under UCX's -Werror, newer libtool +# releases break UCX's autogen, and sysroots past 2.17 stamp x86-64-v3 ISA +# notes that the pre-v3 CPUs of the GPU nodes refuse to load. +build_ucx_pr_conda() { + [ -x "$UCX_PR_PREFIX/bin/ucx_info" ] && return 0 + local sysroot_pkg="sysroot_linux-64" + [ "$(uname -m)" = "aarch64" ] && sysroot_pkg="sysroot_linux-aarch64" + rapids-mamba-retry create -y -n ucx-build -c conda-forge \ + gcc=14 gxx=14 binutils make autoconf automake libtool=2.4.7 "$sysroot_pkg=2.17" \ + cuda-cudart-dev cuda-driver-dev cuda-nvml-dev cuda-crt cuda-cccl \ + "cuda-version=${RAPIDS_CUDA_VERSION%.*}" \ + > /tmp/ucx-toolchain.log 2>&1 \ + || { tail -30 /tmp/ucx-toolchain.log >&2; echo "ERROR: toolchain env create failed" >&2; return 1; } + conda run -n ucx-build bash -ec build_ucx_pr \ + || { echo "ERROR: UCX (PR) build failed in toolchain env" >&2; return 1; } +} + +# Patches the ucxx tree (cwd) so packages build against and run on the PR's +# UCX: strips the released ucx/libucx dependencies from the recipes and +# dependencies.yaml, relaxes the linker checks accordingly, and points the +# builds at UCX_PR_PREFIX. Idempotent; every patch is guarded fail-loud. +use_pr_ucx() { + local f + # 1. conda recipes: drop the ucx dependency (host/run/ignore_run_exports). + for f in conda/recipes/libucxx/recipe.yaml conda/recipes/ucxx/recipe.yaml; do + sed -i -E '/^\s*- ucx(\s*|\s+[<>=].*)$/d' "$f" + grep -qE '^\s*- ucx(\s|$)' "$f" \ + && { echo "ERROR: ucx dep strip did not apply to $f" >&2; return 1; } + # linking against a lib outside the recipe deps is now intended + sed -i 's#overlinking_behavior: "error"#overlinking_behavior: "ignore"#' "$f" + # the builds find the PR's UCX via CMake, and the build-time imports of + # the fresh python module load its libraries via LD_LIBRARY_PATH + grep -q "CMAKE_PREFIX_PATH=$UCX_PR_PREFIX" "$f" \ + || sed -i -E "s#^([[:space:]]*)(\./build\.sh .*)#\1export CMAKE_PREFIX_PATH=$UCX_PR_PREFIX\n\1export LD_LIBRARY_PATH=$UCX_PR_PREFIX/lib\\\${LD_LIBRARY_PATH:+:\\\$LD_LIBRARY_PATH}\n\1\2#" "$f" + grep -q "CMAKE_PREFIX_PATH=$UCX_PR_PREFIX" "$f" \ + || { echo "ERROR: CMAKE_PREFIX_PATH inject did not apply to $f" >&2; return 1; } + done + # 2. dependencies.yaml: drop the released ucx/libucx package deps + # (feeds the wheel pyproject deps and the conda test/docs envs). + sed -i -E '/^\s*- (lib)?ucx[<>=]/d; /^\s*- libucx-cu1[23][<>=]/d' dependencies.yaml + grep -qE '^\s*- (lib)?ucx[<>=]|^\s*- libucx-cu1[23][<>=]' dependencies.yaml \ + && { echo "ERROR: ucx dep strip did not apply to dependencies.yaml" >&2; return 1; } + # 3. wheel builds: CMake finds the PR's UCX; auditwheel resolves libucp + # from it at repair time (libucp.so.0 stays excluded from the wheel). + # CMAKE_PREFIX_PATH via the environment: additive - CMake consults it on + # top of the prefix paths scikit-build-core injects (site-packages, where + # the libucxx wheel's cmake config lives). + for f in ci/build_wheel_libucxx.sh ci/build_wheel_ucxx.sh; do + grep -q "ucx-pr" "$f" && continue + sed -i "s#^export SKBUILD_CMAKE_ARGS=\"#export CMAKE_PREFIX_PATH=$UCX_PR_PREFIX\${CMAKE_PREFIX_PATH:+:\$CMAKE_PREFIX_PATH}\nexport LD_LIBRARY_PATH=$UCX_PR_PREFIX/lib\${LD_LIBRARY_PATH:+:\$LD_LIBRARY_PATH}\nexport SKBUILD_CMAKE_ARGS=\"#" "$f" + grep -q "ucx-pr" "$f" \ + || { echo "ERROR: UCX-PR wheel patch did not apply to $f" >&2; return 1; } + done + # 4. The recipe package self-tests import ucxx in a fresh ephemeral env that + # has no ucx (by design now) - skip them; the pipeline's test jobs are + # the real coverage. + for f in ci/build_cpp.sh ci/build_python.sh; do + grep -q "test skip" "$f" \ + || sed -i "s#rattler-build build#rattler-build build --test skip#" "$f" + grep -q "test skip" "$f" \ + || { echo "ERROR: rattler test-skip patch did not apply to $f" >&2; return 1; } + done + return 0 +}