diff --git a/ci/cudf_pandas_scripts/run_tests.sh b/ci/cudf_pandas_scripts/run_tests.sh index e17c575377c0..6da9186de15e 100755 --- a/ci/cudf_pandas_scripts/run_tests.sh +++ b/ci/cudf_pandas_scripts/run_tests.sh @@ -81,34 +81,47 @@ python -m pytest -p cudf.pandas \ -k "profiler" \ ./python/cudf/cudf_pandas_tests/ -available_pandas_versions=$(python -m pip index versions pandas --json | jq '.versions') -output=$(python ci/utils/filter_package_versions.py dependencies.yaml run_common pandas "$available_pandas_versions") - -# Convert the space-separated list into an array -read -r -a versions <<< "${output}" - version_lte() { [ "$1" = "$(echo -e "$1\n$2" | sort -V | head -n1)" ] } +read -r -a versions <<< "$(python ci/utils/get_matrix_values.py dependencies.yaml test_cudf_pandas_compat pandas_compat_version)" + if version_lte "${RAPIDS_PY_VERSION}" "3.13"; then for version in "${versions[@]}"; do - echo "Installing pandas version: ${version}" - # This loop tests cudf.pandas compatibility with older pandas-numpy versions, - # requiring numpy<2. cupy>=14 dropped support for numpy<2, so we explicitly - # downgrade cupy here to avoid an import failure when cupy tries - # to load against the older numpy. - set +e - # We're iterating over a range of versions that might not all have - # supported wheels, so our best bet for now is to disallow source - # distributions. - rapids-pip-retry install "numpy>=1.26,<2.0a0" "pandas==${version}" "cupy-cuda${RAPIDS_CUDA_VERSION%%.*}x<14" --only-binary=:all: - INSTALL_SUCCESS=$? - set -e - if [[ ${INSTALL_SUCCESS} -ne 0 ]]; then - echo "Failed to install pandas ${version} with numpy<2 for Python ${RAPIDS_PY_VERSION}. Skipping tests for this version." - continue - fi + rapids-logger "Testing cudf.pandas compatibility with pandas ${version}.*" + + # Generate requirements for this pandas compat version. + # Each entry pins numpy<2 + the specific pandas minor line + the CUDA-appropriate cupy<14. + # cupy>=14 dropped support for numpy<2 (see https://github.com/cupy/cupy/pull/9406). + pandas_requirements_txt="pandas-compat-${version}-requirements.txt" + rapids-dependency-file-generator \ + --config dependencies.yaml \ + --file-key test_cudf_pandas_compat \ + --output requirements \ + --matrix "cuda=${RAPIDS_CUDA_VERSION};pandas_compat_version=${version}" \ + > "${pandas_requirements_txt}" + + env_name="venv_pandas_${version}" + python -m venv --clear "${env_name}" + # shellcheck disable=SC1090 + source "${env_name}/bin/activate" + + # notes: + # + # * echo to expand wildcard before adding `[test,cudf-pandas-tests]` requires for pip + # * need to provide --constraint="${PIP_CONSTRAINT}" because that environment variable is + # ignored if any other --constraint are passed via the CLI + # + rapids-pip-retry install \ + -v \ + --constraint ./constraints.txt \ + --constraint "${PIP_CONSTRAINT}" \ + "$(echo "${CUDF_WHEELHOUSE}"/cudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)[test,cudf-pandas-tests]" \ + "$(echo "${LIBCUDF_WHEELHOUSE}"/libcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \ + "$(echo "${PYLIBCUDF_WHEELHOUSE}"/pylibcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \ + -r "${pandas_requirements_txt}" + python -m pytest -p cudf.pandas \ --ignore=./python/cudf/cudf_pandas_tests/third_party_integration_tests/ \ --numprocesses=8 \ @@ -130,6 +143,9 @@ if version_lte "${RAPIDS_PY_VERSION}" "3.13"; then --numprocesses=0 \ -k "profiler" \ ./python/cudf/cudf_pandas_tests/ + + deactivate + rm -rf "${env_name}" "${pandas_requirements_txt}" done else rapids-logger "Python ${RAPIDS_PY_VERSION} detected (>= 3.13). Skipping cudf.pandas compatibility tests with numpy<2" diff --git a/ci/test_wheel_cudf_polars.sh b/ci/test_wheel_cudf_polars.sh index e59334ebd167..6c697af57baa 100755 --- a/ci/test_wheel_cudf_polars.sh +++ b/ci/test_wheel_cudf_polars.sh @@ -15,29 +15,11 @@ CUDF_POLARS_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="cudf_polars_${RAPIDS_PY_CUDA_SUFF LIBCUDF_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-github cpp) PYLIBCUDF_WHEELHOUSE=$(rapids-download-from-github "$(rapids-package-name "wheel_python" pylibcudf --stable --cuda "$RAPIDS_CUDA_VERSION")") -rapids-logger "Installing cudf_polars and its dependencies" - # generate constraints (possibly pinning to oldest support versions of dependencies) rapids-generate-pip-constraints py_test_cudf_polars "${PIP_CONSTRAINT}" -# notes: -# -# * echo to expand wildcard before adding `[test]` requires for pip -# * just providing --constraint="${PIP_CONSTRAINT}" to be explicit, and because -# that environment variable is ignored if any other --constraint are passed via the CLI -# -rapids-pip-retry install \ - -v \ - --prefer-binary \ - --constraint "${PIP_CONSTRAINT}" \ - "$(echo "${CUDF_POLARS_WHEELHOUSE}"/cudf_polars_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)[test,dask,ray]" \ - "$(echo "${LIBCUDF_WHEELHOUSE}"/libcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \ - "$(echo "${PYLIBCUDF_WHEELHOUSE}"/pylibcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" - -rapids-logger "Run cudf_polars tests" - -available_polars_versions=$(python -m pip index versions polars --json | jq '.versions') -POLARS_VERSIONS=$(python ci/utils/filter_package_versions.py dependencies.yaml run_cudf_polars polars "$available_polars_versions") +read -r -a VERSIONS <<< "$(python ci/utils/get_matrix_values.py dependencies.yaml test_cudf_polars_compat polars_compat_version)" +LATEST_VERSION="${VERSIONS[-1]}" # shellcheck disable=SC2317 function set_exitcode() @@ -51,14 +33,41 @@ set +e PASSED=() FAILED=() -read -r -a VERSIONS <<< "${POLARS_VERSIONS}" -LATEST_VERSION="${VERSIONS[-1]}" - for version in "${VERSIONS[@]}"; do - rapids-logger "Installing polars==${version}" - rapids-pip-retry install -U "polars==${version}" - - rapids-logger "Running tests for polars==${version}" + rapids-logger "Testing cudf_polars with polars ${version}.*" + + # Generate requirements for this polars compat version. + polars_requirements_txt="polars-compat-${version}-requirements.txt" + rapids-dependency-file-generator \ + --config dependencies.yaml \ + --file-key test_cudf_polars_compat \ + --output requirements \ + --matrix "polars_compat_version=${version}" \ + > "${polars_requirements_txt}" + + env_name="venv_polars_${version}" + python -m venv --clear "${env_name}" + # shellcheck disable=SC1090 + source "${env_name}/bin/activate" + + rapids-logger "Installing cudf_polars and its dependencies for polars ${version}.*" + + # notes: + # + # * echo to expand wildcard before adding `[test]` requires for pip + # * just providing --constraint="${PIP_CONSTRAINT}" to be explicit, and because + # that environment variable is ignored if any other --constraint are passed via the CLI + # + rapids-pip-retry install \ + -v \ + --prefer-binary \ + --constraint "${PIP_CONSTRAINT}" \ + "$(echo "${CUDF_POLARS_WHEELHOUSE}"/cudf_polars_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)[test,dask,ray]" \ + "$(echo "${LIBCUDF_WHEELHOUSE}"/libcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \ + "$(echo "${PYLIBCUDF_WHEELHOUSE}"/pylibcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \ + -r "polars-compat-${version}-requirements.txt" + + rapids-logger "Running tests for polars ${version}.*" if [ "${version}" == "${LATEST_VERSION}" ]; then COVERAGE_ARGS=( @@ -79,13 +88,17 @@ for version in "${VERSIONS[@]}"; do -ra \ --junitxml="${RAPIDS_TESTS_DIR}/junit-cudf-polars-${version}.xml" - if [ $? -ne 0 ]; then + test_exitcode=$? + deactivate + rm -rf "${env_name}" "${polars_requirements_txt}" + + if [ ${test_exitcode} -ne 0 ]; then EXITCODE=1 FAILED+=("${version}") - rapids-logger "Tests failed for polars==${version}" + rapids-logger "Tests failed for polars ${version}.*" else PASSED+=("${version}") - rapids-logger "Tests passed for polars==${version}" + rapids-logger "Tests passed for polars ${version}.*" fi done diff --git a/ci/utils/get_matrix_values.py b/ci/utils/get_matrix_values.py new file mode 100644 index 000000000000..afdc536c57e1 --- /dev/null +++ b/ci/utils/get_matrix_values.py @@ -0,0 +1,43 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 + +"""Extract matrix dimension values for a file key from dependencies.yaml.""" + +import argparse + +import yaml + + +def get_matrix_values( + deps_yaml_path: str, file_key: str, matrix_var: str +) -> list[str]: + with open(deps_yaml_path) as f: + deps = yaml.safe_load(f) + + file_cfg = deps.get("files", {}).get(file_key) + if file_cfg is None: + raise RuntimeError( + f"File key '{file_key}' not found in {deps_yaml_path}" + ) + + matrix = file_cfg.get("matrix", {}) + values = matrix.get(matrix_var) + if values is None: + raise RuntimeError( + f"Matrix variable '{matrix_var}' not found in file key '{file_key}'" + ) + + return [str(v) for v in values] + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Extract matrix dimension values for a file key in dependencies.yaml" + ) + parser.add_argument("deps_yaml", help="Path to dependencies.yaml") + parser.add_argument("file_key", help="File key in the 'files' section") + parser.add_argument("matrix_var", help="Matrix variable name to extract") + args = parser.parse_args() + + values = get_matrix_values(args.deps_yaml, args.file_key, args.matrix_var) + print(" ".join(values)) diff --git a/dependencies.yaml b/dependencies.yaml index d47fac2e3181..decbaacaef10 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -480,6 +480,19 @@ files: - depends_on_cudf - depends_on_cudf_polars - depends_on_narwhals + test_cudf_pandas_compat: + output: none + matrix: + cuda: ["12.9", "13.1"] + pandas_compat_version: ["3.0"] + includes: + - test_cudf_pandas_compat + test_cudf_polars_compat: + output: none + matrix: + polars_compat_version: ["1.35", "1.36", "1.37", "1.38", "1.39"] + includes: + - test_cudf_polars_compat channels: - rapidsai-nightly - rapidsai @@ -1375,3 +1388,52 @@ dependencies: - output_types: [conda, requirements, pyproject] packages: - structlog + test_cudf_pandas_compat: + specific: + - output_types: requirements + matrices: + - matrix: + pandas_compat_version: "3.0" + packages: + - numpy>=1.26,<2.0a0 + - pandas==3.0.* + - matrix: + packages: + - output_types: requirements + matrices: + - matrix: + cuda: "12.*" + packages: + - cupy-cuda12x<14 + - matrix: + cuda: "13.*" + packages: + - cupy-cuda13x<14 + - matrix: + packages: + test_cudf_polars_compat: + specific: + - output_types: requirements + matrices: + - matrix: + polars_compat_version: "1.35" + packages: + - polars==1.35.* + - matrix: + polars_compat_version: "1.36" + packages: + - polars==1.36.* + - matrix: + polars_compat_version: "1.37" + packages: + - polars==1.37.* + - matrix: + polars_compat_version: "1.38" + packages: + - polars==1.38.* + - matrix: + polars_compat_version: "1.39" + packages: + - polars==1.39.* + - matrix: + packages: