Define pandas and polars compatibility testing versions in dependencies.yaml

ci/cudf_pandas_scripts/run_tests.sh

@@ -81,34 +81,47 @@ python -m pytest -p cudf.pandas \
     -k "profiler" \
     ./python/cudf/cudf_pandas_tests/
 
-available_pandas_versions=$(python -m pip index versions pandas --json | jq '.versions')
-output=$(python ci/utils/filter_package_versions.py dependencies.yaml run_common pandas "$available_pandas_versions")
-
-# Convert the space-separated list into an array
-read -r -a versions <<< "${output}"
-
 version_lte() {
   [ "$1" = "$(echo -e "$1\n$2" | sort -V | head -n1)" ]
 }
 
+read -r -a versions <<< "$(python ci/utils/get_matrix_values.py dependencies.yaml test_cudf_pandas_compat pandas_compat_version)"
+
 if version_lte "${RAPIDS_PY_VERSION}" "3.13"; then
     for version in "${versions[@]}"; do
-        echo "Installing pandas version: ${version}"
-        # This loop tests cudf.pandas compatibility with older pandas-numpy versions,
-        # requiring numpy<2. cupy>=14 dropped support for numpy<2, so we explicitly
-        # downgrade cupy here to avoid an import failure when cupy tries
-        # to load against the older numpy.
-        set +e
-        # We're iterating over a range of versions that might not all have
-        # supported wheels, so our best bet for now is to disallow source
-        # distributions.
-        rapids-pip-retry install "numpy>=1.26,<2.0a0" "pandas==${version}" "cupy-cuda${RAPIDS_CUDA_VERSION%%.*}x<14" --only-binary=:all:
-        INSTALL_SUCCESS=$?
-        set -e
-        if [[ ${INSTALL_SUCCESS} -ne 0 ]]; then
-            echo "Failed to install pandas ${version} with numpy<2 for Python ${RAPIDS_PY_VERSION}. Skipping tests for this version."
-            continue
-        fi
+        rapids-logger "Testing cudf.pandas compatibility with pandas ${version}.*"
+
+        # Generate requirements for this pandas compat version.
+        # Each entry pins numpy<2 + the specific pandas minor line + the CUDA-appropriate cupy<14.
+        # cupy>=14 dropped support for numpy<2 (see https://github.com/cupy/cupy/issues/9709).
+        pandas_requirements_txt="pandas-compat-${version}-requirements.txt"
+        rapids-dependency-file-generator \
+            --config dependencies.yaml \
+            --file-key test_cudf_pandas_compat \
+            --output requirements \
+            --matrix "cuda=${RAPIDS_CUDA_VERSION};pandas_compat_version=${version}" \
+            > "${pandas_requirements_txt}"
+
+        env_name="venv_pandas_${version}"
+        python -m venv --clear "${env_name}"
+        # shellcheck disable=SC1090
+        source "${env_name}/bin/activate"
+
+        # notes:
+        #
+        #   * echo to expand wildcard before adding `[test,cudf-pandas-tests]` requires for pip
+        #   * need to provide --constraint="${PIP_CONSTRAINT}" because that environment variable is
+        #     ignored if any other --constraint are passed via the CLI
+        #
+        rapids-pip-retry install \
+            -v \
+            --constraint ./constraints.txt \
+            --constraint "${PIP_CONSTRAINT}" \
+            "$(echo "${CUDF_WHEELHOUSE}"/cudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)[test,cudf-pandas-tests]" \
+            "$(echo "${LIBCUDF_WHEELHOUSE}"/libcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \
+            "$(echo "${PYLIBCUDF_WHEELHOUSE}"/pylibcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \
+            -r "${pandas_requirements_txt}"
+
         python -m pytest -p cudf.pandas \
             --ignore=./python/cudf/cudf_pandas_tests/third_party_integration_tests/ \
             --numprocesses=8 \
@@ -130,6 +143,9 @@ if version_lte "${RAPIDS_PY_VERSION}" "3.13"; then
             --numprocesses=0 \
             -k "profiler" \
             ./python/cudf/cudf_pandas_tests/
+
+        deactivate
+        rm -rf "${env_name}" "${pandas_requirements_txt}"
     done
 else
     rapids-logger "Python ${RAPIDS_PY_VERSION} detected (>= 3.13). Skipping cudf.pandas compatibility tests with numpy<2"

ci/test_wheel_cudf_polars.sh

@@ -15,29 +15,11 @@ CUDF_POLARS_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="cudf_polars_${RAPIDS_PY_CUDA_SUFF
 LIBCUDF_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-github cpp)
 PYLIBCUDF_WHEELHOUSE=$(rapids-download-from-github "$(rapids-package-name "wheel_python" pylibcudf --stable --cuda "$RAPIDS_CUDA_VERSION")")
 
-rapids-logger "Installing cudf_polars and its dependencies"
-
 # generate constraints (possibly pinning to oldest support versions of dependencies)
 rapids-generate-pip-constraints py_test_cudf_polars "${PIP_CONSTRAINT}"
 
-# notes:
-#
-#   * echo to expand wildcard before adding `[test]` requires for pip
-#   * just providing --constraint="${PIP_CONSTRAINT}" to be explicit, and because
-#     that environment variable is ignored if any other --constraint are passed via the CLI
-#
-rapids-pip-retry install \
-    -v \
-    --prefer-binary \
-    --constraint "${PIP_CONSTRAINT}" \
-    "$(echo "${CUDF_POLARS_WHEELHOUSE}"/cudf_polars_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)[test,dask,ray]" \
-    "$(echo "${LIBCUDF_WHEELHOUSE}"/libcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \
-    "$(echo "${PYLIBCUDF_WHEELHOUSE}"/pylibcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)"
-
-rapids-logger "Run cudf_polars tests"
-
-available_polars_versions=$(python -m pip index versions polars --json | jq '.versions')
-POLARS_VERSIONS=$(python ci/utils/filter_package_versions.py dependencies.yaml run_cudf_polars polars "$available_polars_versions")
+read -r -a VERSIONS <<< "$(python ci/utils/get_matrix_values.py dependencies.yaml test_cudf_polars_compat polars_compat_version)"
+LATEST_VERSION="${VERSIONS[-1]}"
 
 # shellcheck disable=SC2317
 function set_exitcode()
@@ -51,14 +33,41 @@ set +e
 PASSED=()
 FAILED=()
 
-read -r -a VERSIONS <<< "${POLARS_VERSIONS}"
-LATEST_VERSION="${VERSIONS[-1]}"
-
 for version in "${VERSIONS[@]}"; do
-    rapids-logger "Installing polars==${version}"
-    rapids-pip-retry install -U "polars==${version}"
-
-    rapids-logger "Running tests for polars==${version}"
+    rapids-logger "Testing cudf_polars with polars ${version}.*"
+
+    # Generate requirements for this polars compat version.
+    polars_requirements_txt="polars-compat-${version}-requirements.txt"
+    rapids-dependency-file-generator \
+        --config dependencies.yaml \
+        --file-key test_cudf_polars_compat \
+        --output requirements \
+        --matrix "polars_compat_version=${version}" \
+        > "${polars_requirements_txt}"
+
+    env_name="venv_polars_${version}"
+    python -m venv --clear "${env_name}"
+    # shellcheck disable=SC1090
+    source "${env_name}/bin/activate"
+
+    rapids-logger "Installing cudf_polars and its dependencies for polars ${version}.*"
+
+    # notes:
+    #
+    #   * echo to expand wildcard before adding `[test]` requires for pip
+    #   * just providing --constraint="${PIP_CONSTRAINT}" to be explicit, and because
+    #     that environment variable is ignored if any other --constraint are passed via the CLI
+    #
+    rapids-pip-retry install \
+        -v \
+        --prefer-binary \
+        --constraint "${PIP_CONSTRAINT}" \
+        "$(echo "${CUDF_POLARS_WHEELHOUSE}"/cudf_polars_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)[test,dask,ray]" \
+        "$(echo "${LIBCUDF_WHEELHOUSE}"/libcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \
+        "$(echo "${PYLIBCUDF_WHEELHOUSE}"/pylibcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \
+        -r "polars-compat-${version}-requirements.txt"
+
+    rapids-logger "Running tests for polars ${version}.*"
 
     if [ "${version}" == "${LATEST_VERSION}" ]; then
         COVERAGE_ARGS=(
@@ -79,13 +88,17 @@ for version in "${VERSIONS[@]}"; do
         -ra \
         --junitxml="${RAPIDS_TESTS_DIR}/junit-cudf-polars-${version}.xml"
 
-    if [ $? -ne 0 ]; then
+    test_exitcode=$?
+    deactivate
+    rm -rf "${env_name}" "${polars_requirements_txt}"
+
+    if [ ${test_exitcode} -ne 0 ]; then
         EXITCODE=1
         FAILED+=("${version}")
-        rapids-logger "Tests failed for polars==${version}"
+        rapids-logger "Tests failed for polars ${version}.*"
     else
         PASSED+=("${version}")
-        rapids-logger "Tests passed for polars==${version}"
+        rapids-logger "Tests passed for polars ${version}.*"
     fi
 done
 

ci/utils/get_matrix_values.py

@@ -0,0 +1,43 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Extract matrix dimension values for a file key from dependencies.yaml."""
+
+import argparse
+
+import yaml
+
+
+def get_matrix_values(
+    deps_yaml_path: str, file_key: str, matrix_var: str
+) -> list[str]:
+    with open(deps_yaml_path) as f:
+        deps = yaml.safe_load(f)
+
+    file_cfg = deps.get("files", {}).get(file_key)
+    if file_cfg is None:
+        raise RuntimeError(
+            f"File key '{file_key}' not found in {deps_yaml_path}"
+        )
+
+    matrix = file_cfg.get("matrix", {})
+    values = matrix.get(matrix_var)
+    if values is None:
+        raise RuntimeError(
+            f"Matrix variable '{matrix_var}' not found in file key '{file_key}'"
+        )
+
+    return [str(v) for v in values]
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Extract matrix dimension values for a file key in dependencies.yaml"
+    )
+    parser.add_argument("deps_yaml", help="Path to dependencies.yaml")
+    parser.add_argument("file_key", help="File key in the 'files' section")
+    parser.add_argument("matrix_var", help="Matrix variable name to extract")
+    args = parser.parse_args()
+
+    values = get_matrix_values(args.deps_yaml, args.file_key, args.matrix_var)
+    print(" ".join(values))

dependencies.yaml

@@ -480,6 +480,19 @@ files:
       - depends_on_cudf
       - depends_on_cudf_polars
       - depends_on_narwhals
+  test_cudf_pandas_compat:
+    output: none
+    matrix:
+      cuda: ["12.9", "13.1"]
+      pandas_compat_version: ["3.0"]
+    includes:
+      - test_cudf_pandas_compat
+  test_cudf_polars_compat:
+    output: none
+    matrix:
+      polars_compat_version: ["1.35", "1.36", "1.37", "1.38", "1.39"]
+    includes:
+      - test_cudf_polars_compat
 channels:
   - rapidsai-nightly
   - rapidsai
@@ -1366,3 +1379,52 @@ dependencies:
       - output_types: [conda, requirements, pyproject]
         packages:
           - structlog
+  test_cudf_pandas_compat:
+    specific:
+      - output_types: requirements
+        matrices:
+          - matrix:
+              pandas_compat_version: "3.0"
+            packages:
+              - numpy>=1.26,<2.0a0
+              - pandas==3.0.*
+          - matrix:
+            packages:
+      - output_types: requirements
+        matrices:
+          - matrix:
+              cuda: "12.*"
+            packages:
+              - cupy-cuda12x<14
+          - matrix:
+              cuda: "13.*"
+            packages:
+              - cupy-cuda13x<14
+          - matrix:
+            packages:
+  test_cudf_polars_compat:
+    specific:
+      - output_types: requirements
+        matrices:
+          - matrix:
+              polars_compat_version: "1.35"
+            packages:
+              - polars==1.35.*
+          - matrix:
+              polars_compat_version: "1.36"
+            packages:
+              - polars==1.36.*
+          - matrix:
+              polars_compat_version: "1.37"
+            packages:
+              - polars==1.37.*
+          - matrix:
+              polars_compat_version: "1.38"
+            packages:
+              - polars==1.38.*
+          - matrix:
+              polars_compat_version: "1.39"
+            packages:
+              - polars==1.39.*
+          - matrix:
+            packages: