From 8eda57a8e1f7a5cac3cd67598847f15e7ca87d95 Mon Sep 17 00:00:00 2001 From: niranda perera Date: Mon, 6 Jul 2026 13:34:36 -0700 Subject: [PATCH 1/4] adding logger changes Signed-off-by: niranda perera --- .../benchmarks/bench_shuffle.cpp | 10 ++++---- .../streaming/bench_streaming_shuffle.cpp | 19 +++++++-------- .../benchmarks/streaming/ndsh/utils.cpp | 23 +++++++++++-------- .../examples/example_shuffle.cpp | 8 +++---- cpp/libcudf_streaming/tests/main/mpi.cpp | 12 +++++----- cpp/libcudf_streaming/tests/main/single.cpp | 5 ++-- cpp/libcudf_streaming/tests/main/ucxx.cpp | 7 ++++-- 7 files changed, 47 insertions(+), 37 deletions(-) diff --git a/cpp/libcudf_streaming/benchmarks/bench_shuffle.cpp b/cpp/libcudf_streaming/benchmarks/bench_shuffle.cpp index f8fa5fe19b2b..0fbab9c2777b 100644 --- a/cpp/libcudf_streaming/benchmarks/bench_shuffle.cpp +++ b/cpp/libcudf_streaming/benchmarks/bench_shuffle.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -516,6 +517,8 @@ int main(int argc, char** argv) auto& stat_enabled_mr = br->device_mr_adaptor(); rmm::mr::set_current_device_resource(stat_enabled_mr); + auto log = rapidsmpf::Logger::from_options(options); + std::shared_ptr comm; auto progress_thread = std::make_shared(stats); if (args.comm_type == "mpi") { @@ -527,7 +530,7 @@ int main(int argc, char** argv) return 1; } rapidsmpf::mpi::init(&argc, &argv); - comm = std::make_shared(MPI_COMM_WORLD, options, progress_thread); + comm = std::make_shared(MPI_COMM_WORLD, progress_thread, log); #else std::cerr << "Error: MPI communicator is not available in this build." << std::endl; return 1; @@ -537,11 +540,11 @@ int main(int argc, char** argv) if (use_bootstrap) { // Launched with rrun - use bootstrap backend comm = rapidsmpf::bootstrap::create_ucxx_comm( - progress_thread, rapidsmpf::bootstrap::BackendType::AUTO, options); + progress_thread, rapidsmpf::bootstrap::BackendType::AUTO, options, log); } else { #ifdef CUDF_STREAMING_HAVE_MPI // Launched with mpirun - use MPI bootstrap - comm = rapidsmpf::ucxx::init_using_mpi(MPI_COMM_WORLD, options, progress_thread); + comm = rapidsmpf::ucxx::init_using_mpi(MPI_COMM_WORLD, options, progress_thread, log); #else std::cerr << "Error: UCXX without MPI support requires bootstrap mode." << std::endl; return 1; @@ -558,7 +561,6 @@ int main(int argc, char** argv) args.pprint(*comm); - auto& log = comm->logger(); rmm::cuda_stream_view stream = cudf::get_default_stream(); // Print benchmark/hardware info. diff --git a/cpp/libcudf_streaming/benchmarks/streaming/bench_streaming_shuffle.cpp b/cpp/libcudf_streaming/benchmarks/streaming/bench_streaming_shuffle.cpp index d15d5501db46..511b9a022135 100644 --- a/cpp/libcudf_streaming/benchmarks/streaming/bench_streaming_shuffle.cpp +++ b/cpp/libcudf_streaming/benchmarks/streaming/bench_streaming_shuffle.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -276,6 +277,7 @@ int main(int argc, char** argv) // Initialize configuration options from environment variables. rapidsmpf::config::Options options{rapidsmpf::config::get_environment_variables()}; + auto log = rapidsmpf::Logger::from_options(options); auto progress_thread = std::make_shared(); std::shared_ptr comm; @@ -288,7 +290,7 @@ int main(int argc, char** argv) return 1; } rapidsmpf::mpi::init(&argc, &argv); - comm = std::make_shared(MPI_COMM_WORLD, options, progress_thread); + comm = std::make_shared(MPI_COMM_WORLD, progress_thread, log); #else std::cerr << "Error: MPI communicator is not available in this build." << std::endl; return 1; @@ -298,11 +300,11 @@ int main(int argc, char** argv) if (use_bootstrap) { // Launched with rrun - use bootstrap backend comm = rapidsmpf::bootstrap::create_ucxx_comm( - progress_thread, rapidsmpf::bootstrap::BackendType::AUTO, options); + progress_thread, rapidsmpf::bootstrap::BackendType::AUTO, options, log); } else { #ifdef CUDF_STREAMING_HAVE_MPI // Launched with mpirun - use MPI bootstrap - comm = rapidsmpf::ucxx::init_using_mpi(MPI_COMM_WORLD, options, progress_thread); + comm = rapidsmpf::ucxx::init_using_mpi(MPI_COMM_WORLD, options, progress_thread, log); #else std::cerr << "Error: UCXX without MPI support requires bootstrap mode." << std::endl; return 1; @@ -344,7 +346,6 @@ int main(int argc, char** argv) auto& stat_enabled_mr = br->device_mr_adaptor(); rmm::mr::set_current_device_resource(stat_enabled_mr); - auto& log = *comm->logger(); rmm::cuda_stream_view stream = cudf::get_default_stream(); // Print benchmark/hardware info. @@ -361,7 +362,7 @@ int main(int argc, char** argv) ss << " PCI Bus ID: " << pci_bus_id.substr(0, pci_bus_id.find('\0')) << "\n"; ss << " Total Memory: " << rapidsmpf::format_nbytes(properties.totalGlobalMem, 0) << "\n"; ss << " Comm: " << *comm << "\n"; - log.print(ss.str()); + log->print(ss.str()); } auto ctx = std::make_shared(options, comm->logger(), br); @@ -378,7 +379,7 @@ int main(int argc, char** argv) << "/s | global throughput: " << rapidsmpf::format_nbytes(args.total_nbytes / elapsed) << "/s"; if (i < args.num_warmups) { ss << " (warmup run)"; } - log.print(ss.str()); + log->print(ss.str()); if (i >= args.num_warmups) { elapsed_vec.push_back(elapsed); } } @@ -417,18 +418,18 @@ int main(int argc, char** argv) << rapidsmpf::format_nbytes(record.total() / static_cast(total_num_runs)) << " (avg)"; } - log.print(ss.str()); + log->print(ss.str()); } auto statistics = ctx->statistics(); if (args.enable_memory_profiler) { - log.print(statistics->report({ + log->print(statistics->report({ .mr = stat_enabled_mr, .pinned_mr = pinned_mr, .header = "Statistics (of the last run):", })); } else { - log.print(statistics->report({.header = "Statistics (of the last run):"})); + log->print(statistics->report({.header = "Statistics (of the last run):"})); } #ifdef CUDF_STREAMING_HAVE_MPI diff --git a/cpp/libcudf_streaming/benchmarks/streaming/ndsh/utils.cpp b/cpp/libcudf_streaming/benchmarks/streaming/ndsh/utils.cpp index fdd2237447b8..c90f56e78dae 100644 --- a/cpp/libcudf_streaming/benchmarks/streaming/ndsh/utils.cpp +++ b/cpp/libcudf_streaming/benchmarks/streaming/ndsh/utils.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -206,6 +207,7 @@ std::pair, std::shared_ptr> cr auto environment = config::get_environment_variables(); environment["NUM_STREAMING_THREADS"] = std::to_string(arguments.num_streaming_threads); auto options = config::Options(environment); + auto log = Logger::from_options(options); auto progress_thread = std::make_shared(statistics); std::shared_ptr comm; switch (arguments.comm_type) { @@ -214,20 +216,21 @@ std::pair, std::shared_ptr> cr RAPIDSMPF_EXPECTS(!bootstrap::is_running_with_rrun(), "Can't use MPI communicator with rrun"); mpi::init(nullptr, nullptr); - comm = std::make_shared(MPI_COMM_WORLD, options, progress_thread); + comm = std::make_shared(MPI_COMM_WORLD, progress_thread, log); #else RAPIDSMPF_FAIL("MPI communicator is not available in this build", std::invalid_argument); #endif break; - case CommType::SINGLE: comm = std::make_shared(options, progress_thread); break; + case CommType::SINGLE: comm = std::make_shared(progress_thread, log); break; case CommType::UCXX: #ifdef CUDF_STREAMING_HAVE_UCXX if (bootstrap::is_running_with_rrun()) { - comm = bootstrap::create_ucxx_comm(progress_thread, bootstrap::BackendType::AUTO, options); + comm = + bootstrap::create_ucxx_comm(progress_thread, bootstrap::BackendType::AUTO, options, log); } else { #ifdef CUDF_STREAMING_HAVE_MPI mpi::init(nullptr, nullptr); - comm = ucxx::init_using_mpi(MPI_COMM_WORLD, options, progress_thread); + comm = ucxx::init_using_mpi(MPI_COMM_WORLD, options, progress_thread, log); #else RAPIDSMPF_FAIL("UCXX without MPI support requires bootstrap mode", std::invalid_argument); #endif @@ -238,13 +241,13 @@ std::pair, std::shared_ptr> cr break; default: RAPIDSMPF_FAIL("Unknown communicator type"); } - auto ctx = std::make_shared(options, comm->logger(), br); + auto ctx = std::make_shared(options, log, br); if (comm->rank() == 0) { - comm->logger()->print("Execution context on ", - comm->nranks(), - " ranks has ", - ctx->executor()->num_streaming_threads(), - " threads"); + log->print("Execution context on ", + comm->nranks(), + " ranks has ", + ctx->executor()->num_streaming_threads(), + " threads"); } return {ctx, comm}; } diff --git a/cpp/libcudf_streaming/examples/example_shuffle.cpp b/cpp/libcudf_streaming/examples/example_shuffle.cpp index eb861c34a227..06eb54343a1a 100644 --- a/cpp/libcudf_streaming/examples/example_shuffle.cpp +++ b/cpp/libcudf_streaming/examples/example_shuffle.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -35,14 +36,13 @@ int main(int argc, char** argv) // single progress thread may be used by multiple shufflers simultaneously. auto progress_thread = std::make_shared(stats); + auto log = rapidsmpf::Logger::from_options(options); + // Now we have to create a Communicator, which we will use throughout the // example. Multiple concurrent shuffles are possible on the same communicator by // providing differentiating "OpID" arguments. std::shared_ptr comm = - std::make_shared(MPI_COMM_WORLD, options, progress_thread); - - // The Communicator provides a logger. - auto& log = comm->logger(); + std::make_shared(MPI_COMM_WORLD, progress_thread, log); // We will use the same stream, memory, and buffer resource throughout the example. rmm::cuda_stream_view stream = cudf::get_default_stream(); diff --git a/cpp/libcudf_streaming/tests/main/mpi.cpp b/cpp/libcudf_streaming/tests/main/mpi.cpp index ecfc4ce1cb12..8865aebad60a 100644 --- a/cpp/libcudf_streaming/tests/main/mpi.cpp +++ b/cpp/libcudf_streaming/tests/main/mpi.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -29,8 +30,9 @@ class MPIEnvironment : public Environment { options_ = rapidsmpf::config::Options(rapidsmpf::config::get_environment_variables()); - comm_ = std::make_shared( - mpi_comm_, options_, std::make_shared()); + comm_ = std::make_shared(mpi_comm_, + std::make_shared(), + rapidsmpf::Logger::from_options(options_)); } void TearDown() override @@ -45,16 +47,14 @@ class MPIEnvironment : public Environment { std::shared_ptr split_comm() override { - // Initialize configuration options from environment variables. - rapidsmpf::config::Options options{rapidsmpf::config::get_environment_variables()}; - // Create the new split communicator int rank; RAPIDSMPF_MPI(MPI_Comm_rank(mpi_comm_, &rank)); MPI_Comm split_comm = MPI_COMM_NULL; RAPIDSMPF_MPI(MPI_Comm_split(mpi_comm_, rank, 0, &split_comm)); + auto logger = rapidsmpf::Logger::from_options(options_); return std::shared_ptr( - new rapidsmpf::MPI(split_comm, options, comm_->progress_thread()), + new rapidsmpf::MPI(split_comm, comm_->progress_thread(), std::move(logger)), // Don't leak the split handle. [comm = split_comm](rapidsmpf::MPI* x) mutable { delete x; diff --git a/cpp/libcudf_streaming/tests/main/single.cpp b/cpp/libcudf_streaming/tests/main/single.cpp index cf54b83eca5a..542db6f54861 100644 --- a/cpp/libcudf_streaming/tests/main/single.cpp +++ b/cpp/libcudf_streaming/tests/main/single.cpp @@ -7,6 +7,7 @@ #include +#include #include #include #include @@ -23,8 +24,8 @@ class SingleEnvironment : public Environment { void SetUp() override { options_ = rapidsmpf::config::Options(rapidsmpf::config::get_environment_variables()); - comm_ = - std::make_shared(options_, std::make_shared()); + comm_ = std::make_shared(std::make_shared(), + rapidsmpf::Logger::from_options(options_)); } void TearDown() override { comm_ = nullptr; } diff --git a/cpp/libcudf_streaming/tests/main/ucxx.cpp b/cpp/libcudf_streaming/tests/main/ucxx.cpp index 51ad19551ce9..b50c974947f4 100644 --- a/cpp/libcudf_streaming/tests/main/ucxx.cpp +++ b/cpp/libcudf_streaming/tests/main/ucxx.cpp @@ -10,6 +10,7 @@ #include #include +#include #include #include #include @@ -39,8 +40,10 @@ class UCXXEnvironment : public Environment { "didn't get the requested thread level support: MPI_THREAD_MULTIPLE"); options_ = rapidsmpf::config::Options(rapidsmpf::config::get_environment_variables()); - comm_ = rapidsmpf::ucxx::init_using_mpi( - MPI_COMM_WORLD, options_, std::make_shared()); + comm_ = rapidsmpf::ucxx::init_using_mpi(MPI_COMM_WORLD, + options_, + std::make_shared(), + rapidsmpf::Logger::from_options(options_)); } void TearDown() override From 6d0439a21c821c2f1cb1808edbbce6a28a0edacd Mon Sep 17 00:00:00 2001 From: niranda perera Date: Mon, 6 Jul 2026 13:51:06 -0700 Subject: [PATCH 2/4] addressing comments Signed-off-by: niranda perera --- cpp/libcudf_streaming/tests/main/mpi.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cpp/libcudf_streaming/tests/main/mpi.cpp b/cpp/libcudf_streaming/tests/main/mpi.cpp index 8865aebad60a..eedf76af8039 100644 --- a/cpp/libcudf_streaming/tests/main/mpi.cpp +++ b/cpp/libcudf_streaming/tests/main/mpi.cpp @@ -52,9 +52,10 @@ class MPIEnvironment : public Environment { RAPIDSMPF_MPI(MPI_Comm_rank(mpi_comm_, &rank)); MPI_Comm split_comm = MPI_COMM_NULL; RAPIDSMPF_MPI(MPI_Comm_split(mpi_comm_, rank, 0, &split_comm)); - auto logger = rapidsmpf::Logger::from_options(options_); + auto new_logger = rapidsmpf::Logger::from_options(options_); + new_logger->set_name(std::to_string(rank)); return std::shared_ptr( - new rapidsmpf::MPI(split_comm, comm_->progress_thread(), std::move(logger)), + new rapidsmpf::MPI(split_comm, comm_->progress_thread(), std::move(new_logger)), // Don't leak the split handle. [comm = split_comm](rapidsmpf::MPI* x) mutable { delete x; From 3da472365b5b651df1ad87ab758fb09727a96e96 Mon Sep 17 00:00:00 2001 From: niranda perera Date: Mon, 6 Jul 2026 14:05:14 -0700 Subject: [PATCH 3/4] add ci testing with rapidsmpf PR 1081 Signed-off-by: niranda perera --- ci/build_cpp.sh | 5 +- ci/build_docs.sh | 6 ++- ci/build_python.sh | 3 ++ ci/build_python_noarch.sh | 5 +- ci/build_wheel_cudf_streaming.sh | 3 ++ ci/build_wheel_libcudf_streaming.sh | 3 ++ .../third-party-integration/test.sh | 8 ++- ci/test_cpp_common.sh | 6 ++- ci/test_cudf_polars_polars_tests.sh | 5 +- ci/test_java.sh | 4 ++ ci/test_notebooks.sh | 6 ++- ci/test_python_common.sh | 16 +++++- ci/test_wheel_cudf_polars.sh | 3 ++ ci/test_wheel_cudf_streaming.sh | 5 +- ci/use_conda_packages_from_prs.sh | 49 +++++++++++++++++++ ci/use_wheels_from_prs.sh | 21 ++++++++ 16 files changed, 137 insertions(+), 11 deletions(-) create mode 100644 ci/use_conda_packages_from_prs.sh create mode 100644 ci/use_wheels_from_prs.sh diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index c3b4c5d876d1..09a99a42f0e3 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail @@ -22,6 +22,9 @@ RAPIDS_ARTIFACTS_DIR=${RAPIDS_ARTIFACTS_DIR:-"${PWD}/artifacts"} mkdir -p "${RAPIDS_ARTIFACTS_DIR}" export RAPIDS_ARTIFACTS_DIR +# TODO: Remove before merging. Use rapidsmpf conda packages from rapidsai/rapidsmpf#1081. +source ./ci/use_conda_packages_from_prs.sh + # populates `RATTLER_CHANNELS` array and `RATTLER_ARGS` array source rapids-rattler-channel-string diff --git a/ci/build_docs.sh b/ci/build_docs.sh index 59fbcf406341..236ae47dd2c4 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail @@ -15,6 +15,9 @@ rapids-logger "Create test conda environment" rapids-logger "Configuring conda strict channel priority" conda config --set channel_priority strict +# TODO: Remove before merging. Use rapidsmpf conda packages from rapidsai/rapidsmpf#1081. +source ./ci/use_conda_packages_from_prs.sh + ENV_YAML_DIR="$(mktemp -d)" rapids-logger "Downloading artifacts from previous jobs" @@ -28,6 +31,7 @@ rapids-dependency-file-generator \ --prepend-channel "${CPP_CHANNEL}" \ --prepend-channel "${PYTHON_CHANNEL}" \ --prepend-channel "${PYTHON_NOARCH_CHANNEL}" \ + "${RAPIDS_PREPENDED_CHANNEL_ARGS[@]}" \ --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee "${ENV_YAML_DIR}/env.yaml" rapids-mamba-retry env create --yes -f "${ENV_YAML_DIR}/env.yaml" -n docs diff --git a/ci/build_python.sh b/ci/build_python.sh index 3858ebc6fd7a..d8efb82a893d 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -21,6 +21,9 @@ CPP_CHANNEL=$(rapids-download-from-github "$(rapids-artifact-name conda_cpp libc RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) export RAPIDS_PACKAGE_VERSION +# TODO: Remove before merging. Use rapidsmpf conda packages from rapidsai/rapidsmpf#1081. +source ./ci/use_conda_packages_from_prs.sh + # populates `RATTLER_CHANNELS` array and `RATTLER_ARGS` array source rapids-rattler-channel-string diff --git a/ci/build_python_noarch.sh b/ci/build_python_noarch.sh index 2aab3666f3a3..9d87530d7722 100755 --- a/ci/build_python_noarch.sh +++ b/ci/build_python_noarch.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail @@ -21,6 +21,9 @@ PYTHON_CHANNEL=$(rapids-download-from-github "$(rapids-artifact-name conda_pytho RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) export RAPIDS_PACKAGE_VERSION +# TODO: Remove before merging. Use rapidsmpf conda packages from rapidsai/rapidsmpf#1081. +source ./ci/use_conda_packages_from_prs.sh + # populates `RATTLER_CHANNELS` array and `RATTLER_ARGS` array source rapids-rattler-channel-string diff --git a/ci/build_wheel_cudf_streaming.sh b/ci/build_wheel_cudf_streaming.sh index cc961508b253..f6345ba9a60b 100755 --- a/ci/build_wheel_cudf_streaming.sh +++ b/ci/build_wheel_cudf_streaming.sh @@ -21,6 +21,9 @@ echo "libcudf-streaming-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo "${LIBCUDF_STRE echo "libcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo "${LIBCUDF_WHEELHOUSE}"/libcudf_*.whl)" >> "${PIP_CONSTRAINT}" echo "pylibcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo "${PYLIBCUDF_WHEELHOUSE}"/pylibcudf_*.whl)" >> "${PIP_CONSTRAINT}" +# TODO: Remove before merging. Use rapidsmpf wheels from rapidsai/rapidsmpf#1081. +source ./ci/use_wheels_from_prs.sh + rapids-logger "Generating build requirements" rapids-dependency-file-generator \ diff --git a/ci/build_wheel_libcudf_streaming.sh b/ci/build_wheel_libcudf_streaming.sh index 2810119f587a..7d3589175b54 100755 --- a/ci/build_wheel_libcudf_streaming.sh +++ b/ci/build_wheel_libcudf_streaming.sh @@ -17,6 +17,9 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")" LIBCUDF_WHEELHOUSE=$(rapids-download-from-github "$(rapids-artifact-name wheel_cpp libcudf cudf --cuda "$RAPIDS_CUDA_VERSION")") echo "libcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo "${LIBCUDF_WHEELHOUSE}"/libcudf_*.whl)" >> "${PIP_CONSTRAINT}" +# TODO: Remove before merging. Use rapidsmpf wheels from rapidsai/rapidsmpf#1081. +source ./ci/use_wheels_from_prs.sh + rapids-logger "Generating build requirements" rapids-dependency-file-generator \ diff --git a/ci/cudf_pandas_scripts/third-party-integration/test.sh b/ci/cudf_pandas_scripts/third-party-integration/test.sh index d93438fb2895..cf8bf5f14f16 100755 --- a/ci/cudf_pandas_scripts/third-party-integration/test.sh +++ b/ci/cudf_pandas_scripts/third-party-integration/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # Common setup steps shared by Python test jobs @@ -27,6 +27,9 @@ main() { CPP_CHANNEL=$(rapids-download-from-github "$(rapids-artifact-name conda_cpp libcudf cudf --cuda "$RAPIDS_CUDA_VERSION")") PYTHON_CHANNEL=$(rapids-download-from-github "$(rapids-artifact-name conda_python cudf cudf --stable --cuda "$RAPIDS_CUDA_VERSION")") PYTHON_NOARCH_CHANNEL=$(rapids-download-from-github "$(rapids-artifact-name conda_python cudf cudf --pure --cuda "$RAPIDS_CUDA_VERSION" --arch any)") + + # TODO: Remove before merging. Use rapidsmpf conda packages from rapidsai/rapidsmpf#1081. + source ./ci/use_conda_packages_from_prs.sh fi ANY_FAILURES=0 @@ -46,7 +49,8 @@ main() { --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" \ --prepend-channel "${CPP_CHANNEL}" \ --prepend-channel "${PYTHON_CHANNEL}" \ - --prepend-channel "${PYTHON_NOARCH_CHANNEL}" | tee env.yaml + --prepend-channel "${PYTHON_NOARCH_CHANNEL}" \ + "${RAPIDS_PREPENDED_CHANNEL_ARGS[@]}" | tee env.yaml else rapids-logger "Generate Python testing dependencies" rapids-dependency-file-generator \ diff --git a/ci/test_cpp_common.sh b/ci/test_cpp_common.sh index a0607b278911..31bcee93fc1f 100755 --- a/ci/test_cpp_common.sh +++ b/ci/test_cpp_common.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail @@ -9,6 +9,9 @@ set -euo pipefail rapids-logger "Configuring conda strict channel priority" conda config --set channel_priority strict +# TODO: Remove before merging. Use rapidsmpf conda packages from rapidsai/rapidsmpf#1081. +source ./ci/use_conda_packages_from_prs.sh + rapids-logger "Downloading artifacts from previous jobs" CPP_CHANNEL=$(rapids-download-from-github "$(rapids-artifact-name conda_cpp libcudf cudf --cuda "$RAPIDS_CUDA_VERSION")") @@ -20,6 +23,7 @@ rapids-dependency-file-generator \ --output conda \ --file-key test_cpp \ --prepend-channel "${CPP_CHANNEL}" \ + "${RAPIDS_PREPENDED_CHANNEL_ARGS[@]}" \ --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee "${ENV_YAML_DIR}/env.yaml" rapids-mamba-retry env create --yes -f "${ENV_YAML_DIR}/env.yaml" -n test diff --git a/ci/test_cudf_polars_polars_tests.sh b/ci/test_cudf_polars_polars_tests.sh index e44a6ccfb2fa..70a81c4686ee 100755 --- a/ci/test_cudf_polars_polars_tests.sh +++ b/ci/test_cudf_polars_polars_tests.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail @@ -18,6 +18,9 @@ CUDF_POLARS_WHEELHOUSE=$(rapids-download-from-github "$(rapids-artifact-name whe # generate constraints (possibly pinning to oldest support versions of dependencies) rapids-generate-pip-constraints py_test_cudf_polars "${PIP_CONSTRAINT}" +# TODO: Remove before merging. Use rapidsmpf wheels from rapidsai/rapidsmpf#1081. +source ./ci/use_wheels_from_prs.sh + rapids-logger "Install libcudf, pylibcudf and cudf_polars" # notes: diff --git a/ci/test_java.sh b/ci/test_java.sh index 0e452497a5fb..d4d2dee517b5 100755 --- a/ci/test_java.sh +++ b/ci/test_java.sh @@ -9,6 +9,9 @@ set -euo pipefail rapids-logger "Configuring conda strict channel priority" conda config --set channel_priority strict +# TODO: Remove before merging. Use rapidsmpf conda packages from rapidsai/rapidsmpf#1081. +source ./ci/use_conda_packages_from_prs.sh + rapids-logger "Downloading artifacts from previous jobs" CPP_CHANNEL=$(rapids-download-from-github "$(rapids-artifact-name conda_cpp libcudf cudf --cuda "$RAPIDS_CUDA_VERSION")") @@ -20,6 +23,7 @@ rapids-dependency-file-generator \ --output conda \ --file-key test_java \ --prepend-channel "${CPP_CHANNEL}" \ + "${RAPIDS_PREPENDED_CHANNEL_ARGS[@]}" \ --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee "${ENV_YAML_DIR}/env.yaml" rapids-mamba-retry env create --yes -f "${ENV_YAML_DIR}/env.yaml" -n test diff --git a/ci/test_notebooks.sh b/ci/test_notebooks.sh index 2927ceb3ada3..8e44c335ffcd 100755 --- a/ci/test_notebooks.sh +++ b/ci/test_notebooks.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail @@ -9,6 +9,9 @@ set -euo pipefail rapids-logger "Configuring conda strict channel priority" conda config --set channel_priority strict +# TODO: Remove before merging. Use rapidsmpf conda packages from rapidsai/rapidsmpf#1081. +source ./ci/use_conda_packages_from_prs.sh + rapids-logger "Downloading artifacts from previous jobs" CPP_CHANNEL=$(rapids-download-from-github "$(rapids-artifact-name conda_cpp libcudf cudf --cuda "$RAPIDS_CUDA_VERSION")") PYTHON_CHANNEL=$(rapids-download-from-github "$(rapids-artifact-name conda_python cudf cudf --stable --cuda "$RAPIDS_CUDA_VERSION")") @@ -24,6 +27,7 @@ rapids-dependency-file-generator \ --prepend-channel "${CPP_CHANNEL}" \ --prepend-channel "${PYTHON_CHANNEL}" \ --prepend-channel "${PYTHON_NOARCH_CHANNEL}" \ + "${RAPIDS_PREPENDED_CHANNEL_ARGS[@]}" \ --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee "${ENV_YAML_DIR}/env.yaml" rapids-mamba-retry env create --yes -f "${ENV_YAML_DIR}/env.yaml" -n test diff --git a/ci/test_python_common.sh b/ci/test_python_common.sh index d6dad79905fd..7ffbe3dc1512 100755 --- a/ci/test_python_common.sh +++ b/ci/test_python_common.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # Common setup steps shared by Python test jobs @@ -11,6 +11,9 @@ set -euo pipefail rapids-logger "Configuring conda strict channel priority" conda config --set channel_priority strict +# TODO: Remove before merging. Use rapidsmpf conda packages from rapidsai/rapidsmpf#1081. +source ./ci/use_conda_packages_from_prs.sh + rapids-logger "Downloading artifacts from previous jobs" CPP_CHANNEL=$(rapids-download-from-github "$(rapids-artifact-name conda_cpp libcudf cudf --cuda "$RAPIDS_CUDA_VERSION")") PYTHON_CHANNEL=$(rapids-download-from-github "$(rapids-artifact-name conda_python cudf cudf --stable --cuda "$RAPIDS_CUDA_VERSION")") @@ -30,7 +33,16 @@ done CMD="${CMD} \ --prepend-channel \"${CPP_CHANNEL}\" \ --prepend-channel \"${PYTHON_CHANNEL}\" \ - --prepend-channel \"${PYTHON_NOARCH_CHANNEL}\" \ + --prepend-channel \"${PYTHON_NOARCH_CHANNEL}\"" + +# TODO: Remove before merging. Prepend rapidsmpf PR channels (see +# ci/use_conda_packages_from_prs.sh) ahead of the default channels in the +# generated env file so the solver prefers them over rapidsai-nightly. +for _channel in "${RAPIDS_PREPENDED_CONDA_CHANNELS[@]}"; do + CMD="${CMD} --prepend-channel \"${_channel}\"" +done + +CMD="${CMD} \ --matrix \"cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};dependencies=${RAPIDS_DEPENDENCIES}\"" eval ${CMD} | tee "${ENV_YAML_DIR}/env.yaml" diff --git a/ci/test_wheel_cudf_polars.sh b/ci/test_wheel_cudf_polars.sh index 09315bbe981c..80a7f21910f9 100755 --- a/ci/test_wheel_cudf_polars.sh +++ b/ci/test_wheel_cudf_polars.sh @@ -20,6 +20,9 @@ CUDF_STREAMING_WHEELHOUSE=$(rapids-download-from-github "$(rapids-artifact-name # generate constraints (possibly pinning to oldest support versions of dependencies) rapids-generate-pip-constraints py_test_cudf_polars "${PIP_CONSTRAINT}" +# TODO: Remove before merging. Use rapidsmpf wheels from rapidsai/rapidsmpf#1081. +source ./ci/use_wheels_from_prs.sh + read -r -a VERSIONS <<< "$(python ci/utils/get_matrix_values.py dependencies.yaml test_cudf_polars_compat polars_compat_version)" if [[ "${POLARS_VERSIONS:-all}" == "endpoints" ]] && [[ ${#VERSIONS[@]} -ge 2 ]]; then diff --git a/ci/test_wheel_cudf_streaming.sh b/ci/test_wheel_cudf_streaming.sh index 0a1210f9b04c..117d1b129f47 100755 --- a/ci/test_wheel_cudf_streaming.sh +++ b/ci/test_wheel_cudf_streaming.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail @@ -17,6 +17,9 @@ PYLIBCUDF_WHEELHOUSE=$(rapids-download-from-github "$(rapids-artifact-name wheel # generate constraints (possibly pinning to oldest support versions of dependencies) rapids-generate-pip-constraints py_test_cudf_streaming "${PIP_CONSTRAINT}" +# TODO: Remove before merging. Use rapidsmpf wheels from rapidsai/rapidsmpf#1081. +source ./ci/use_wheels_from_prs.sh + rapids-logger "Install cudf_streaming and its dependencies" rapids-pip-retry install \ diff --git a/ci/use_conda_packages_from_prs.sh b/ci/use_conda_packages_from_prs.sh new file mode 100644 index 000000000000..78633840ffbb --- /dev/null +++ b/ci/use_conda_packages_from_prs.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# download CI artifacts +LIBRAPIDSMPF_CHANNEL=$(rapids-get-pr-artifact rapidsmpf 1081 cpp conda) +RAPIDSMPF_CHANNEL=$(rapids-get-pr-artifact rapidsmpf 1081 python conda) + +# For `rattler` builds: +# +# Add these channels to the array checked by 'rapids-rattler-channel-string'. +# This ensures that when conda packages are built with strict channel priority enabled, +# the locally-downloaded packages will be preferred to remote packages (e.g. nightlies). +# +RAPIDS_PREPENDED_CONDA_CHANNELS=( + "${LIBRAPIDSMPF_CHANNEL}" + "${RAPIDSMPF_CHANNEL}" +) +export RAPIDS_PREPENDED_CONDA_CHANNELS + +# For tests and `conda-build` builds: +# +# Add these channels to the system-wide conda configuration. +# This results in PREPENDING them to conda's channel list, so +# these packages should be found first if strict channel priority is enabled. +# +for _channel in "${RAPIDS_PREPENDED_CONDA_CHANNELS[@]}" +do + conda config --system --add channels "${_channel}" +done + +# For `conda env create -f ` jobs (e.g. C++/Python/Java/notebook +# tests, docs builds): +# +# `conda env create` gives the channels listed in the environment file a HIGHER +# priority than the channels in the system/user `.condarc`. So the +# `conda config --system --add channels` calls above are NOT sufficient on their +# own: with strict channel priority the solver still prefers e.g. +# `rapidsai-nightly` (which is listed in the generated env file) over these PR +# channels, pulling in the wrong build of `librapidsmpf`/`rapidsmpf`. +# +# Expose the channels as `--prepend-channel` arguments so callers can inject +# them into the generated environment file ahead of the default channels. +RAPIDS_PREPENDED_CHANNEL_ARGS=() +for _channel in "${RAPIDS_PREPENDED_CONDA_CHANNELS[@]}" +do + RAPIDS_PREPENDED_CHANNEL_ARGS+=(--prepend-channel "${_channel}") +done +export RAPIDS_PREPENDED_CHANNEL_ARGS diff --git a/ci/use_wheels_from_prs.sh b/ci/use_wheels_from_prs.sh new file mode 100644 index 000000000000..5736c34c2a81 --- /dev/null +++ b/ci/use_wheels_from_prs.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# NOTE: This script assumes the caller has already run `source rapids-init-pip` +# (so PIP_CONSTRAINT is set) and has finished generating any of its own +# constraints. It APPENDS to PIP_CONSTRAINT so it must not be sourced before +# `rapids-generate-pip-constraints`, which overwrites that file. + +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")" + +# download wheels, store the directories holding them in variables +LIBRAPIDSMPF_WHEELHOUSE=$(rapids-get-pr-artifact rapidsmpf 1081 cpp wheel) +RAPIDSMPF_WHEELHOUSE=$(rapids-get-pr-artifact rapidsmpf 1081 python wheel) + +# append to the pip constraints file saying e.g. "whenever you encounter a +# requirement for 'librapidsmpf-cu12', use this wheel" +cat >> "${PIP_CONSTRAINT}" < Date: Mon, 6 Jul 2026 14:55:40 -0700 Subject: [PATCH 4/4] Revert "add ci testing with rapidsmpf PR 1081" This reverts commit 3da472365b5b651df1ad87ab758fb09727a96e96. --- ci/build_cpp.sh | 5 +- ci/build_docs.sh | 6 +-- ci/build_python.sh | 3 -- ci/build_python_noarch.sh | 5 +- ci/build_wheel_cudf_streaming.sh | 3 -- ci/build_wheel_libcudf_streaming.sh | 3 -- .../third-party-integration/test.sh | 8 +-- ci/test_cpp_common.sh | 6 +-- ci/test_cudf_polars_polars_tests.sh | 5 +- ci/test_java.sh | 4 -- ci/test_notebooks.sh | 6 +-- ci/test_python_common.sh | 16 +----- ci/test_wheel_cudf_polars.sh | 3 -- ci/test_wheel_cudf_streaming.sh | 5 +- ci/use_conda_packages_from_prs.sh | 49 ------------------- ci/use_wheels_from_prs.sh | 21 -------- 16 files changed, 11 insertions(+), 137 deletions(-) delete mode 100644 ci/use_conda_packages_from_prs.sh delete mode 100644 ci/use_wheels_from_prs.sh diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index 09a99a42f0e3..c3b4c5d876d1 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail @@ -22,9 +22,6 @@ RAPIDS_ARTIFACTS_DIR=${RAPIDS_ARTIFACTS_DIR:-"${PWD}/artifacts"} mkdir -p "${RAPIDS_ARTIFACTS_DIR}" export RAPIDS_ARTIFACTS_DIR -# TODO: Remove before merging. Use rapidsmpf conda packages from rapidsai/rapidsmpf#1081. -source ./ci/use_conda_packages_from_prs.sh - # populates `RATTLER_CHANNELS` array and `RATTLER_ARGS` array source rapids-rattler-channel-string diff --git a/ci/build_docs.sh b/ci/build_docs.sh index 236ae47dd2c4..59fbcf406341 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail @@ -15,9 +15,6 @@ rapids-logger "Create test conda environment" rapids-logger "Configuring conda strict channel priority" conda config --set channel_priority strict -# TODO: Remove before merging. Use rapidsmpf conda packages from rapidsai/rapidsmpf#1081. -source ./ci/use_conda_packages_from_prs.sh - ENV_YAML_DIR="$(mktemp -d)" rapids-logger "Downloading artifacts from previous jobs" @@ -31,7 +28,6 @@ rapids-dependency-file-generator \ --prepend-channel "${CPP_CHANNEL}" \ --prepend-channel "${PYTHON_CHANNEL}" \ --prepend-channel "${PYTHON_NOARCH_CHANNEL}" \ - "${RAPIDS_PREPENDED_CHANNEL_ARGS[@]}" \ --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee "${ENV_YAML_DIR}/env.yaml" rapids-mamba-retry env create --yes -f "${ENV_YAML_DIR}/env.yaml" -n docs diff --git a/ci/build_python.sh b/ci/build_python.sh index d8efb82a893d..3858ebc6fd7a 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -21,9 +21,6 @@ CPP_CHANNEL=$(rapids-download-from-github "$(rapids-artifact-name conda_cpp libc RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) export RAPIDS_PACKAGE_VERSION -# TODO: Remove before merging. Use rapidsmpf conda packages from rapidsai/rapidsmpf#1081. -source ./ci/use_conda_packages_from_prs.sh - # populates `RATTLER_CHANNELS` array and `RATTLER_ARGS` array source rapids-rattler-channel-string diff --git a/ci/build_python_noarch.sh b/ci/build_python_noarch.sh index 9d87530d7722..2aab3666f3a3 100755 --- a/ci/build_python_noarch.sh +++ b/ci/build_python_noarch.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail @@ -21,9 +21,6 @@ PYTHON_CHANNEL=$(rapids-download-from-github "$(rapids-artifact-name conda_pytho RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) export RAPIDS_PACKAGE_VERSION -# TODO: Remove before merging. Use rapidsmpf conda packages from rapidsai/rapidsmpf#1081. -source ./ci/use_conda_packages_from_prs.sh - # populates `RATTLER_CHANNELS` array and `RATTLER_ARGS` array source rapids-rattler-channel-string diff --git a/ci/build_wheel_cudf_streaming.sh b/ci/build_wheel_cudf_streaming.sh index f6345ba9a60b..cc961508b253 100755 --- a/ci/build_wheel_cudf_streaming.sh +++ b/ci/build_wheel_cudf_streaming.sh @@ -21,9 +21,6 @@ echo "libcudf-streaming-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo "${LIBCUDF_STRE echo "libcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo "${LIBCUDF_WHEELHOUSE}"/libcudf_*.whl)" >> "${PIP_CONSTRAINT}" echo "pylibcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo "${PYLIBCUDF_WHEELHOUSE}"/pylibcudf_*.whl)" >> "${PIP_CONSTRAINT}" -# TODO: Remove before merging. Use rapidsmpf wheels from rapidsai/rapidsmpf#1081. -source ./ci/use_wheels_from_prs.sh - rapids-logger "Generating build requirements" rapids-dependency-file-generator \ diff --git a/ci/build_wheel_libcudf_streaming.sh b/ci/build_wheel_libcudf_streaming.sh index 7d3589175b54..2810119f587a 100755 --- a/ci/build_wheel_libcudf_streaming.sh +++ b/ci/build_wheel_libcudf_streaming.sh @@ -17,9 +17,6 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")" LIBCUDF_WHEELHOUSE=$(rapids-download-from-github "$(rapids-artifact-name wheel_cpp libcudf cudf --cuda "$RAPIDS_CUDA_VERSION")") echo "libcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo "${LIBCUDF_WHEELHOUSE}"/libcudf_*.whl)" >> "${PIP_CONSTRAINT}" -# TODO: Remove before merging. Use rapidsmpf wheels from rapidsai/rapidsmpf#1081. -source ./ci/use_wheels_from_prs.sh - rapids-logger "Generating build requirements" rapids-dependency-file-generator \ diff --git a/ci/cudf_pandas_scripts/third-party-integration/test.sh b/ci/cudf_pandas_scripts/third-party-integration/test.sh index cf8bf5f14f16..d93438fb2895 100755 --- a/ci/cudf_pandas_scripts/third-party-integration/test.sh +++ b/ci/cudf_pandas_scripts/third-party-integration/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # Common setup steps shared by Python test jobs @@ -27,9 +27,6 @@ main() { CPP_CHANNEL=$(rapids-download-from-github "$(rapids-artifact-name conda_cpp libcudf cudf --cuda "$RAPIDS_CUDA_VERSION")") PYTHON_CHANNEL=$(rapids-download-from-github "$(rapids-artifact-name conda_python cudf cudf --stable --cuda "$RAPIDS_CUDA_VERSION")") PYTHON_NOARCH_CHANNEL=$(rapids-download-from-github "$(rapids-artifact-name conda_python cudf cudf --pure --cuda "$RAPIDS_CUDA_VERSION" --arch any)") - - # TODO: Remove before merging. Use rapidsmpf conda packages from rapidsai/rapidsmpf#1081. - source ./ci/use_conda_packages_from_prs.sh fi ANY_FAILURES=0 @@ -49,8 +46,7 @@ main() { --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" \ --prepend-channel "${CPP_CHANNEL}" \ --prepend-channel "${PYTHON_CHANNEL}" \ - --prepend-channel "${PYTHON_NOARCH_CHANNEL}" \ - "${RAPIDS_PREPENDED_CHANNEL_ARGS[@]}" | tee env.yaml + --prepend-channel "${PYTHON_NOARCH_CHANNEL}" | tee env.yaml else rapids-logger "Generate Python testing dependencies" rapids-dependency-file-generator \ diff --git a/ci/test_cpp_common.sh b/ci/test_cpp_common.sh index 31bcee93fc1f..a0607b278911 100755 --- a/ci/test_cpp_common.sh +++ b/ci/test_cpp_common.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail @@ -9,9 +9,6 @@ set -euo pipefail rapids-logger "Configuring conda strict channel priority" conda config --set channel_priority strict -# TODO: Remove before merging. Use rapidsmpf conda packages from rapidsai/rapidsmpf#1081. -source ./ci/use_conda_packages_from_prs.sh - rapids-logger "Downloading artifacts from previous jobs" CPP_CHANNEL=$(rapids-download-from-github "$(rapids-artifact-name conda_cpp libcudf cudf --cuda "$RAPIDS_CUDA_VERSION")") @@ -23,7 +20,6 @@ rapids-dependency-file-generator \ --output conda \ --file-key test_cpp \ --prepend-channel "${CPP_CHANNEL}" \ - "${RAPIDS_PREPENDED_CHANNEL_ARGS[@]}" \ --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee "${ENV_YAML_DIR}/env.yaml" rapids-mamba-retry env create --yes -f "${ENV_YAML_DIR}/env.yaml" -n test diff --git a/ci/test_cudf_polars_polars_tests.sh b/ci/test_cudf_polars_polars_tests.sh index 70a81c4686ee..e44a6ccfb2fa 100755 --- a/ci/test_cudf_polars_polars_tests.sh +++ b/ci/test_cudf_polars_polars_tests.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail @@ -18,9 +18,6 @@ CUDF_POLARS_WHEELHOUSE=$(rapids-download-from-github "$(rapids-artifact-name whe # generate constraints (possibly pinning to oldest support versions of dependencies) rapids-generate-pip-constraints py_test_cudf_polars "${PIP_CONSTRAINT}" -# TODO: Remove before merging. Use rapidsmpf wheels from rapidsai/rapidsmpf#1081. -source ./ci/use_wheels_from_prs.sh - rapids-logger "Install libcudf, pylibcudf and cudf_polars" # notes: diff --git a/ci/test_java.sh b/ci/test_java.sh index d4d2dee517b5..0e452497a5fb 100755 --- a/ci/test_java.sh +++ b/ci/test_java.sh @@ -9,9 +9,6 @@ set -euo pipefail rapids-logger "Configuring conda strict channel priority" conda config --set channel_priority strict -# TODO: Remove before merging. Use rapidsmpf conda packages from rapidsai/rapidsmpf#1081. -source ./ci/use_conda_packages_from_prs.sh - rapids-logger "Downloading artifacts from previous jobs" CPP_CHANNEL=$(rapids-download-from-github "$(rapids-artifact-name conda_cpp libcudf cudf --cuda "$RAPIDS_CUDA_VERSION")") @@ -23,7 +20,6 @@ rapids-dependency-file-generator \ --output conda \ --file-key test_java \ --prepend-channel "${CPP_CHANNEL}" \ - "${RAPIDS_PREPENDED_CHANNEL_ARGS[@]}" \ --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee "${ENV_YAML_DIR}/env.yaml" rapids-mamba-retry env create --yes -f "${ENV_YAML_DIR}/env.yaml" -n test diff --git a/ci/test_notebooks.sh b/ci/test_notebooks.sh index 8e44c335ffcd..2927ceb3ada3 100755 --- a/ci/test_notebooks.sh +++ b/ci/test_notebooks.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail @@ -9,9 +9,6 @@ set -euo pipefail rapids-logger "Configuring conda strict channel priority" conda config --set channel_priority strict -# TODO: Remove before merging. Use rapidsmpf conda packages from rapidsai/rapidsmpf#1081. -source ./ci/use_conda_packages_from_prs.sh - rapids-logger "Downloading artifacts from previous jobs" CPP_CHANNEL=$(rapids-download-from-github "$(rapids-artifact-name conda_cpp libcudf cudf --cuda "$RAPIDS_CUDA_VERSION")") PYTHON_CHANNEL=$(rapids-download-from-github "$(rapids-artifact-name conda_python cudf cudf --stable --cuda "$RAPIDS_CUDA_VERSION")") @@ -27,7 +24,6 @@ rapids-dependency-file-generator \ --prepend-channel "${CPP_CHANNEL}" \ --prepend-channel "${PYTHON_CHANNEL}" \ --prepend-channel "${PYTHON_NOARCH_CHANNEL}" \ - "${RAPIDS_PREPENDED_CHANNEL_ARGS[@]}" \ --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee "${ENV_YAML_DIR}/env.yaml" rapids-mamba-retry env create --yes -f "${ENV_YAML_DIR}/env.yaml" -n test diff --git a/ci/test_python_common.sh b/ci/test_python_common.sh index 7ffbe3dc1512..d6dad79905fd 100755 --- a/ci/test_python_common.sh +++ b/ci/test_python_common.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # Common setup steps shared by Python test jobs @@ -11,9 +11,6 @@ set -euo pipefail rapids-logger "Configuring conda strict channel priority" conda config --set channel_priority strict -# TODO: Remove before merging. Use rapidsmpf conda packages from rapidsai/rapidsmpf#1081. -source ./ci/use_conda_packages_from_prs.sh - rapids-logger "Downloading artifacts from previous jobs" CPP_CHANNEL=$(rapids-download-from-github "$(rapids-artifact-name conda_cpp libcudf cudf --cuda "$RAPIDS_CUDA_VERSION")") PYTHON_CHANNEL=$(rapids-download-from-github "$(rapids-artifact-name conda_python cudf cudf --stable --cuda "$RAPIDS_CUDA_VERSION")") @@ -33,16 +30,7 @@ done CMD="${CMD} \ --prepend-channel \"${CPP_CHANNEL}\" \ --prepend-channel \"${PYTHON_CHANNEL}\" \ - --prepend-channel \"${PYTHON_NOARCH_CHANNEL}\"" - -# TODO: Remove before merging. Prepend rapidsmpf PR channels (see -# ci/use_conda_packages_from_prs.sh) ahead of the default channels in the -# generated env file so the solver prefers them over rapidsai-nightly. -for _channel in "${RAPIDS_PREPENDED_CONDA_CHANNELS[@]}"; do - CMD="${CMD} --prepend-channel \"${_channel}\"" -done - -CMD="${CMD} \ + --prepend-channel \"${PYTHON_NOARCH_CHANNEL}\" \ --matrix \"cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};dependencies=${RAPIDS_DEPENDENCIES}\"" eval ${CMD} | tee "${ENV_YAML_DIR}/env.yaml" diff --git a/ci/test_wheel_cudf_polars.sh b/ci/test_wheel_cudf_polars.sh index 80a7f21910f9..09315bbe981c 100755 --- a/ci/test_wheel_cudf_polars.sh +++ b/ci/test_wheel_cudf_polars.sh @@ -20,9 +20,6 @@ CUDF_STREAMING_WHEELHOUSE=$(rapids-download-from-github "$(rapids-artifact-name # generate constraints (possibly pinning to oldest support versions of dependencies) rapids-generate-pip-constraints py_test_cudf_polars "${PIP_CONSTRAINT}" -# TODO: Remove before merging. Use rapidsmpf wheels from rapidsai/rapidsmpf#1081. -source ./ci/use_wheels_from_prs.sh - read -r -a VERSIONS <<< "$(python ci/utils/get_matrix_values.py dependencies.yaml test_cudf_polars_compat polars_compat_version)" if [[ "${POLARS_VERSIONS:-all}" == "endpoints" ]] && [[ ${#VERSIONS[@]} -ge 2 ]]; then diff --git a/ci/test_wheel_cudf_streaming.sh b/ci/test_wheel_cudf_streaming.sh index 117d1b129f47..0a1210f9b04c 100755 --- a/ci/test_wheel_cudf_streaming.sh +++ b/ci/test_wheel_cudf_streaming.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail @@ -17,9 +17,6 @@ PYLIBCUDF_WHEELHOUSE=$(rapids-download-from-github "$(rapids-artifact-name wheel # generate constraints (possibly pinning to oldest support versions of dependencies) rapids-generate-pip-constraints py_test_cudf_streaming "${PIP_CONSTRAINT}" -# TODO: Remove before merging. Use rapidsmpf wheels from rapidsai/rapidsmpf#1081. -source ./ci/use_wheels_from_prs.sh - rapids-logger "Install cudf_streaming and its dependencies" rapids-pip-retry install \ diff --git a/ci/use_conda_packages_from_prs.sh b/ci/use_conda_packages_from_prs.sh deleted file mode 100644 index 78633840ffbb..000000000000 --- a/ci/use_conda_packages_from_prs.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# download CI artifacts -LIBRAPIDSMPF_CHANNEL=$(rapids-get-pr-artifact rapidsmpf 1081 cpp conda) -RAPIDSMPF_CHANNEL=$(rapids-get-pr-artifact rapidsmpf 1081 python conda) - -# For `rattler` builds: -# -# Add these channels to the array checked by 'rapids-rattler-channel-string'. -# This ensures that when conda packages are built with strict channel priority enabled, -# the locally-downloaded packages will be preferred to remote packages (e.g. nightlies). -# -RAPIDS_PREPENDED_CONDA_CHANNELS=( - "${LIBRAPIDSMPF_CHANNEL}" - "${RAPIDSMPF_CHANNEL}" -) -export RAPIDS_PREPENDED_CONDA_CHANNELS - -# For tests and `conda-build` builds: -# -# Add these channels to the system-wide conda configuration. -# This results in PREPENDING them to conda's channel list, so -# these packages should be found first if strict channel priority is enabled. -# -for _channel in "${RAPIDS_PREPENDED_CONDA_CHANNELS[@]}" -do - conda config --system --add channels "${_channel}" -done - -# For `conda env create -f ` jobs (e.g. C++/Python/Java/notebook -# tests, docs builds): -# -# `conda env create` gives the channels listed in the environment file a HIGHER -# priority than the channels in the system/user `.condarc`. So the -# `conda config --system --add channels` calls above are NOT sufficient on their -# own: with strict channel priority the solver still prefers e.g. -# `rapidsai-nightly` (which is listed in the generated env file) over these PR -# channels, pulling in the wrong build of `librapidsmpf`/`rapidsmpf`. -# -# Expose the channels as `--prepend-channel` arguments so callers can inject -# them into the generated environment file ahead of the default channels. -RAPIDS_PREPENDED_CHANNEL_ARGS=() -for _channel in "${RAPIDS_PREPENDED_CONDA_CHANNELS[@]}" -do - RAPIDS_PREPENDED_CHANNEL_ARGS+=(--prepend-channel "${_channel}") -done -export RAPIDS_PREPENDED_CHANNEL_ARGS diff --git a/ci/use_wheels_from_prs.sh b/ci/use_wheels_from_prs.sh deleted file mode 100644 index 5736c34c2a81..000000000000 --- a/ci/use_wheels_from_prs.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# NOTE: This script assumes the caller has already run `source rapids-init-pip` -# (so PIP_CONSTRAINT is set) and has finished generating any of its own -# constraints. It APPENDS to PIP_CONSTRAINT so it must not be sourced before -# `rapids-generate-pip-constraints`, which overwrites that file. - -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")" - -# download wheels, store the directories holding them in variables -LIBRAPIDSMPF_WHEELHOUSE=$(rapids-get-pr-artifact rapidsmpf 1081 cpp wheel) -RAPIDSMPF_WHEELHOUSE=$(rapids-get-pr-artifact rapidsmpf 1081 python wheel) - -# append to the pip constraints file saying e.g. "whenever you encounter a -# requirement for 'librapidsmpf-cu12', use this wheel" -cat >> "${PIP_CONSTRAINT}" <