diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 6b0fd420b582..25b01b25a065 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -480,17 +480,6 @@ jobs: matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) build_type: pull-request script: "ci/test_wheel_cudf_polars.sh" - wheel-tests-cudf-polars-with-rapidsmpf: - needs: [wheel-build-cudf-polars, changed-files] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@main - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python_wheels && fromJSON(needs.changed-files.outputs.changed_file_groups).neither_cudf_nor_dask_cudf - with: - # This selects "ARCH=amd64 + the latest supported Python + CUDA" to minimize CI usage. - # (rapidsmpf compatibility already validated in rapidsmpf CI) - matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) - build_type: pull-request - script: "ci/test_cudf_polars_with_rapidsmpf.sh" cudf-polars-polars-tests: needs: [wheel-build-cudf-polars, changed-files] secrets: inherit diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 51ddac09d4df..b4f672c61f9c 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -153,19 +153,6 @@ jobs: date: ${{ inputs.date }} sha: ${{ inputs.sha }} script: "ci/test_wheel_cudf_polars.sh" - wheel-tests-cudf-polars-with-rapidsmpf: - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@main - with: - # This selects "ARCH=amd64 + the latest supported Python + CUDA" to minimize CI usage. - # (rapidsmpf compatibility already validated in rapidsmpf CI) - matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) - build_type: ${{ inputs.build_type }} - branch: ${{ inputs.branch }} - date: ${{ inputs.date }} - sha: ${{ inputs.sha }} - script: "ci/test_cudf_polars_with_rapidsmpf.sh" - continue-on-error: true cudf-polars-polars-tests: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@main diff --git a/ci/run_cudf_polars_pytests.sh b/ci/run_cudf_polars_pytests.sh index ce30428675e8..a1b06fd2fc56 100755 --- a/ci/run_cudf_polars_pytests.sh +++ b/ci/run_cudf_polars_pytests.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail @@ -9,17 +9,32 @@ set -euo pipefail # Support invoking run_cudf_polars_pytests.sh outside the script directory cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cudf_polars/ -# Test the "in-memory" executor +echo "Test the in-memory executor" python -m pytest --cache-clear "$@" tests --executor in-memory -# Test the default "streaming" executor +echo "Test the default streaming executor" python -m pytest --cache-clear "$@" tests --executor streaming -# Test the "streaming" executor with small blocksize +echo "Test the streaming executor with small blocksize" python -m pytest --cache-clear "$@" tests --executor streaming --blocksize-mode small -# Run experimental tests with Distributed cluster +echo "Test the (future default) streaming executor with rapidsmpf" +CUDF_POLARS__PARQUET_OPTIONS__USE_RAPIDSMPF_NATIVE=1 CUDF_POLARS__EXECUTOR__SHUFFLE_METHOD=rapidsmpf python -m pytest --cache-clear "$@" tests \ + --executor streaming \ + --blocksize-mode small \ + --cluster single \ + --runtime rapidsmpf \ + --cov-fail-under=0 # TODO? missing coverage may be due to old task-based paths + +echo "Run experimental tests with Distributed cluster" python -m pytest --cache-clear "$@" "tests/experimental" \ --executor streaming \ --cluster distributed \ --cov-fail-under=0 # No code-coverage requirement for these tests. + +echo "Run experimental tests with the distributed cluster mode and the rapidsmpf runtime" +python -m pytest --cache-clear "$@" "tests/experimental" \ + --executor streaming \ + --cluster distributed \ + --runtime rapidsmpf \ + --cov-fail-under=0 # No code-coverage requirement for these tests(?) diff --git a/ci/run_cudf_polars_with_rapidsmpf_pytests.sh b/ci/run_cudf_polars_with_rapidsmpf_pytests.sh deleted file mode 100755 index b172995a214b..000000000000 --- a/ci/run_cudf_polars_with_rapidsmpf_pytests.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -set -euo pipefail - -# Test cudf_polars with rapidsmpf integration -# This script runs experimental tests with single cluster mode and the rapidsmpf runtime - -# It is essential to cd into python/cudf_polars as `pytest-xdist` + `coverage` seem to work only at this directory level. - -# Support invoking run_cudf_polars_with_rapidsmpf_pytests.sh outside the script directory -cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cudf_polars/ - -# Run experimental tests with the "single" cluster mode and the "rapidsmpf" runtime -rapids-logger "Running experimental tests with the 'rapidsmpf' runtime and a 'single' cluster" -timeout 10m python -m pytest --cache-clear "$@" "tests/experimental" \ - --executor streaming \ - --cluster single \ - --runtime rapidsmpf - -# Run experimental tests with the "distributed" cluster mode and the "rapidsmpf" runtime -rapids-logger "Running experimental tests with the 'rapidsmpf' runtime and a 'distributed' cluster" -timeout 10m python -m pytest --cache-clear "$@" "tests/experimental" \ - --executor streaming \ - --cluster distributed \ - --runtime rapidsmpf diff --git a/ci/test_cudf_polars_with_rapidsmpf.sh b/ci/test_cudf_polars_with_rapidsmpf.sh deleted file mode 100755 index 6e9ad45afe3d..000000000000 --- a/ci/test_cudf_polars_with_rapidsmpf.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -set -euo pipefail - -source rapids-init-pip - -rapids-logger "Download wheels" - -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")" -CUDF_POLARS_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="cudf_polars_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-github python) -LIBCUDF_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libcudf_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-github cpp) -PYLIBCUDF_WHEELHOUSE=$(rapids-download-from-github "$(rapids-package-name "wheel_python" pylibcudf --stable --cuda "$RAPIDS_CUDA_VERSION")") - -rapids-logger "Installing cudf_polars and its dependencies (including rapidsmpf)" - -rapids-pip-retry install \ - -v \ - --constraint "${PIP_CONSTRAINT}" \ - "$(echo "${CUDF_POLARS_WHEELHOUSE}"/cudf_polars_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)[test,experimental,rapidsmpf]" \ - "$(echo "${LIBCUDF_WHEELHOUSE}"/libcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \ - "$(echo "${PYLIBCUDF_WHEELHOUSE}"/pylibcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" - -rapids-logger "Run cudf_polars tests with rapidsmpf" - -# Get the latest polars version for testing -POLARS_VERSION=$(python ci/utils/fetch_polars_versions.py --latest-patch-only dependencies.yaml | awk '{print $NF}') - -rapids-logger "Installing polars==${POLARS_VERSION}" -pip install -U "polars==${POLARS_VERSION}" - -# shellcheck disable=SC2317 -function set_exitcode() -{ - EXITCODE=$? -} -EXITCODE=0 -trap set_exitcode ERR -set +e - -rapids-logger "Running cudf_polars tests with rapidsmpf" - -# Run cudf_polars tests with rapidsmpf using dedicated test runner -timeout 15m ./ci/run_cudf_polars_with_rapidsmpf_pytests.sh \ - --no-cov \ - --numprocesses=8 \ - --dist=worksteal \ - -v \ - --junitxml="${RAPIDS_TESTS_DIR}/junit-cudf-polars-rapidsmpf.xml" - -trap - ERR -set -e - -if [ ${EXITCODE} != 0 ]; then - rapids-logger "cudf_polars + rapidsmpf tests FAILED: exitcode ${EXITCODE}" -else - rapids-logger "cudf_polars + rapidsmpf tests PASSED" -fi -exit ${EXITCODE} diff --git a/ci/test_wheel_cudf_polars.sh b/ci/test_wheel_cudf_polars.sh index abdaeebd1659..5865468802b6 100755 --- a/ci/test_wheel_cudf_polars.sh +++ b/ci/test_wheel_cudf_polars.sh @@ -30,7 +30,7 @@ rapids-pip-retry install \ -v \ --constraint ./constraints.txt \ --constraint "${PIP_CONSTRAINT}" \ - "$(echo "${CUDF_POLARS_WHEELHOUSE}"/cudf_polars_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)[test,experimental]" \ + "$(echo "${CUDF_POLARS_WHEELHOUSE}"/cudf_polars_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)[test,experimental,rapidsmpf]" \ "$(echo "${LIBCUDF_WHEELHOUSE}"/libcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" \ "$(echo "${PYLIBCUDF_WHEELHOUSE}"/pylibcudf_"${RAPIDS_PY_CUDA_SUFFIX}"*.whl)" diff --git a/dependencies.yaml b/dependencies.yaml index 70409ea10a14..5dc5eaccb8f1 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -115,6 +115,8 @@ files: - depends_on_cudf_kafka - depends_on_custreamz - depends_on_cudf_polars + - depends_on_rapidsmpf + - cudf_polars_trace test_java: output: none includes: diff --git a/python/cudf_polars/cudf_polars/experimental/dask_registers.py b/python/cudf_polars/cudf_polars/experimental/dask_registers.py index 15895e209db0..94334ccbc57f 100644 --- a/python/cudf_polars/cudf_polars/experimental/dask_registers.py +++ b/python/cudf_polars/cudf_polars/experimental/dask_registers.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. # SPDX-License-Identifier: Apache-2.0 """Dask function registrations such as serializers and dispatch implementations.""" @@ -215,7 +215,7 @@ def _(x: DataFrame) -> int: from rapidsmpf.integrations.dask.spilling import register_dask_serialize register_dask_serialize() # pragma: no cover; rapidsmpf dependency not included yet - except ImportError: + except ImportError: # pragma: no cover pass # Register the tokenizer for NamedExpr and DataType. This is a performance diff --git a/python/cudf_polars/tests/conftest.py b/python/cudf_polars/tests/conftest.py index 1a94eb038e13..de923c60cefd 100644 --- a/python/cudf_polars/tests/conftest.py +++ b/python/cudf_polars/tests/conftest.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations @@ -28,6 +28,11 @@ def clear_memory_resource_cache(): cudf_polars.callback.default_memory_resource.cache_clear() +@pytest.fixture +def using_rapidsmpf(): + return cudf_polars.testing.asserts.DEFAULT_RUNTIME == "rapidsmpf" + + def pytest_addoption(parser): parser.addoption( "--executor", diff --git a/python/cudf_polars/tests/experimental/rapidsmpf/test_tracing.py b/python/cudf_polars/tests/experimental/rapidsmpf/test_tracing.py index 5818609b9a39..1caef3c63f82 100644 --- a/python/cudf_polars/tests/experimental/rapidsmpf/test_tracing.py +++ b/python/cudf_polars/tests/experimental/rapidsmpf/test_tracing.py @@ -20,6 +20,7 @@ @pytest.mark.skipif(DEFAULT_CLUSTER != "single", reason="Requires 'single' cluster.") def test_structlog_streaming_node_events(): """Test that structlog emits 'Streaming Actor' events when tracing is enabled.""" + pytest.importorskip("structlog") # Run in subprocess to control CUDF_POLARS_LOG_TRACES environment variable code = textwrap.dedent("""\ import polars as pl diff --git a/python/cudf_polars/tests/expressions/test_agg.py b/python/cudf_polars/tests/expressions/test_agg.py index 5ad6f68cc2ab..e10e1969f9b6 100644 --- a/python/cudf_polars/tests/expressions/test_agg.py +++ b/python/cudf_polars/tests/expressions/test_agg.py @@ -194,7 +194,13 @@ def test_implode_agg_unsupported(): assert_ir_translation_raises(q, NotImplementedError) -def test_decimal_aggs(decimal_df: pl.LazyFrame) -> None: +def test_decimal_aggs(decimal_df: pl.LazyFrame, using_rapidsmpf, request) -> None: + request.applymarker( + pytest.mark.xfail( + condition=using_rapidsmpf, + reason="https://github.com/rapidsai/cudf/issues/21645", + ) + ) q = decimal_df.with_columns( sum=pl.col("a").sum(), min=pl.col("a").min(), diff --git a/python/cudf_polars/tests/expressions/test_len.py b/python/cudf_polars/tests/expressions/test_len.py index ccc903aa99d8..10efadd879a0 100644 --- a/python/cudf_polars/tests/expressions/test_len.py +++ b/python/cudf_polars/tests/expressions/test_len.py @@ -30,7 +30,13 @@ def test_len(dtype, empty): @pytest.mark.parametrize("data", [[1, 2, 3], [1, 2, None]]) -def test_col_len(data): +def test_col_len(data, using_rapidsmpf, request): + request.applymarker( + pytest.mark.xfail( + condition=using_rapidsmpf, + reason="https://github.com/rapidsai/cudf/issues/21645", + ) + ) data = {"a": list("xyz"), "b": data} q = pl.LazyFrame(data).select( pl.col("a").len().alias("l"), diff --git a/python/cudf_polars/tests/test_config.py b/python/cudf_polars/tests/test_config.py index 523f62622827..d369818552ee 100644 --- a/python/cudf_polars/tests/test_config.py +++ b/python/cudf_polars/tests/test_config.py @@ -3,6 +3,7 @@ from __future__ import annotations +import os import sys from typing import Any, cast @@ -381,8 +382,12 @@ def test_scheduler_deprecated() -> None: def test_validate_shuffle_method_defaults( *, + monkeypatch: pytest.MonkeyPatch, rapidsmpf_distributed_available: bool, ) -> None: + # TODO: Can remove once removed from ci/run_cudf_polars_pytests.sh + # and the default shuffle method changes to "rapidsmpf" + monkeypatch.delenv("CUDF_POLARS__EXECUTOR__SHUFFLE_METHOD", raising=False) config = ConfigOptions.from_polars_engine( pl.GPUEngine( executor="streaming", @@ -804,7 +809,12 @@ def test_cuda_stream_policy_from_env( elif runtime == "rapidsmpf": with pytest.raises(ValueError, match="The rapidsmpf streaming engine"): ConfigOptions.from_polars_engine(engine) - else: + elif ( + os.environ.get("CUDF_POLARS__EXECUTOR__SHUFFLE_METHOD", "") != "rapidsmpf" + and not rapidsmpf_single_available + ): + # CUDF_POLARS__EXECUTOR__SHUFFLE_METHOD set in ci/run_cudf_polars_pytests.sh + # Would raise since rapidsmpf shuffle requires rapidsmpf to be installed config = ConfigOptions.from_polars_engine(engine) assert config.cuda_stream_policy == env diff --git a/python/cudf_polars/tests/test_dataframescan.py b/python/cudf_polars/tests/test_dataframescan.py index e002820fe383..2986c58b5fef 100644 --- a/python/cudf_polars/tests/test_dataframescan.py +++ b/python/cudf_polars/tests/test_dataframescan.py @@ -86,7 +86,13 @@ def test_dataframescan_with_decimals(): POLARS_VERSION_LT_138, reason="height parameter added in Polars 1.38", ) -def test_dataframescan_zero_width_with_rows(): +def test_dataframescan_zero_width_with_rows(request, using_rapidsmpf): + request.applymarker( + pytest.mark.xfail( + using_rapidsmpf, + reason="https://github.com/rapidsai/cudf/issues/21644", + ) + ) df = pl.LazyFrame(height=5) q = df.select(pl.len()) assert_gpu_result_equal(q) diff --git a/python/cudf_polars/tests/test_drop_nulls.py b/python/cudf_polars/tests/test_drop_nulls.py index 0fe9b963eddb..56da63763b5b 100644 --- a/python/cudf_polars/tests/test_drop_nulls.py +++ b/python/cudf_polars/tests/test_drop_nulls.py @@ -42,7 +42,15 @@ def test_drop_null(null_data): [0, pl.col("a").mean(), pl.col("b")], ids=["scalar", "aggregation", "column_expression"], ) -def test_fill_null(null_data, value): +def test_fill_null(null_data, value, using_rapidsmpf, request): + request.applymarker( + pytest.mark.xfail( + condition=using_rapidsmpf + and "mean" in str(value) + and len(null_data.collect()) == 2, + reason="https://github.com/rapidsai/cudf/issues/21721", + ) + ) q = null_data.select(pl.col("a").fill_null(value)) assert_gpu_result_equal(q) diff --git a/python/cudf_polars/tests/test_groupby.py b/python/cudf_polars/tests/test_groupby.py index 8713185f56f5..e9400e22dbd1 100644 --- a/python/cudf_polars/tests/test_groupby.py +++ b/python/cudf_polars/tests/test_groupby.py @@ -17,6 +17,8 @@ ) from cudf_polars.utils.versions import ( POLARS_VERSION_LT_132, + POLARS_VERSION_LT_133, + POLARS_VERSION_LT_134, POLARS_VERSION_LT_136, POLARS_VERSION_LT_1321, ) @@ -126,7 +128,53 @@ def maintain_order(request): return request.param -def test_groupby(df: pl.LazyFrame, maintain_order, keys, exprs): +def test_groupby( + df: pl.LazyFrame, maintain_order, keys, exprs, using_rapidsmpf, request +): + failing_rapidsmpf_nodeids = { + 'test_groupby[maintain_order-col("key1")-col("int32").mean()]', + 'test_groupby[maintain_order-col("key1")-col("uint16_with_null").sum()-col("uint16_with_null").mean().alias("mean")]', + 'test_groupby[maintain_order-col("key2")-col("int32").mean()]', + 'test_groupby[maintain_order-col("key2")-col("uint16_with_null").sum()-col("uint16_with_null").mean().alias("mean")]', + 'test_groupby[maintain_order-col("key1")-dyn int: 1-col("int32").mean()]', + 'test_groupby[maintain_order-col("key1")-dyn int: 1-col("uint16_with_null").sum()-col("uint16_with_null").mean().alias("mean")]', + 'test_groupby[maintain_order-[(col("key1")) * (col("key2"))]-col("int32").mean()]', + 'test_groupby[maintain_order-[(col("key1")) * (col("key2"))]-col("uint16_with_null").sum()-col("uint16_with_null").mean().alias("mean")]', + 'test_groupby[maintain_order-col("key1")-col("key2")-col("int32").mean()]', + 'test_groupby[maintain_order-col("key1")-col("key2")-col("uint16_with_null").sum()-col("uint16_with_null").mean().alias("mean")]', + 'test_groupby[maintain_order-[(col("key1")) == (col("key2"))]-col("int32").mean()]', + 'test_groupby[maintain_order-[(col("key1")) == (col("key2"))]-col("uint16_with_null").sum()-col("uint16_with_null").mean().alias("mean")]', + 'test_groupby[maintain_order-col("key2")-[(col("key1")) == (1)]-col("int32").mean()]', + 'test_groupby[maintain_order-col("key2")-[(col("key1")) == (1)]-col("uint16_with_null").sum()-col("uint16_with_null").mean().alias("mean")]', + } + request.applymarker( + pytest.mark.xfail( + using_rapidsmpf and request.node.name in failing_rapidsmpf_nodeids, + reason="https://github.com/rapidsai/cudf/issues/21642 and probably mean calculation referenced in https://github.com/rapidsai/cudf/issues/21721", + ) + ) + failing_rapidsmpf_nodeids_lt_1_36 = { + 'test_groupby[maintain_order-col("key2")-[(col("key1")) == (dyn int: 1.strict_cast(Int64))]-col("uint16_with_null").sum()-col("uint16_with_null").mean().alias("mean")]' + } + request.applymarker( + pytest.mark.xfail( + using_rapidsmpf + and POLARS_VERSION_LT_136 + and request.node.name in failing_rapidsmpf_nodeids_lt_1_36, + reason="Type mismatch in columns to concatenate.", + ) + ) + segfaulting_rapidsmpf_nodeids = { + 'test_groupby[maintain_order-col("key2")-[(col("key1")) == (dyn int: 1.strict_cast(Int64))]-col("int32").mean()]' + } + if ( + using_rapidsmpf + and POLARS_VERSION_LT_136 + and request.node.name in segfaulting_rapidsmpf_nodeids + ): + pytest.skip( + "Usually raises 'Type mismatch in columns to concatenate' but can also segfault." + ) q = df.group_by(*keys, maintain_order=maintain_order).agg(*exprs) if not maintain_order: @@ -136,7 +184,14 @@ def test_groupby(df: pl.LazyFrame, maintain_order, keys, exprs): assert_gpu_result_equal(q, check_exact=False) -def test_groupby_sorted_keys(df: pl.LazyFrame, keys, exprs): +def test_groupby_sorted_keys(df: pl.LazyFrame, keys, exprs, using_rapidsmpf, request): + request.applymarker( + pytest.mark.xfail( + using_rapidsmpf, + strict=False, + reason="https://github.com/rapidsai/cudf/issues/21642 - no deterministic sort for keys", + ) + ) sorted_keys = [ key.sort(descending=descending) for key, descending in zip(keys, itertools.cycle([False, True])) @@ -382,7 +437,7 @@ def test_groupby_sum_all_null_group_returns_null(): ids=["sum", "mean", "median", "quantile-0.5"], ) def test_groupby_aggs_keep_unsupported_as_null( - request, df: pl.LazyFrame, agg_expr + request, df: pl.LazyFrame, agg_expr, using_rapidsmpf ) -> None: request.applymarker( pytest.mark.xfail( @@ -390,6 +445,15 @@ def test_groupby_aggs_keep_unsupported_as_null( reason="polars raises now", ) ) + request.applymarker( + pytest.mark.xfail( + condition=using_rapidsmpf + and "quantile" in str(agg_expr) + and POLARS_VERSION_LT_134 + and not POLARS_VERSION_LT_133, + reason="decimal precision is 38 instead of 9", + ) + ) lf = df.filter(pl.col("datetime") == date(2004, 12, 1)) q = lf.group_by("datetime").agg(agg_expr) assert_gpu_result_equal(q) diff --git a/python/cudf_polars/tests/test_join.py b/python/cudf_polars/tests/test_join.py index fc1a384957ea..a891d669c901 100644 --- a/python/cudf_polars/tests/test_join.py +++ b/python/cudf_polars/tests/test_join.py @@ -70,7 +70,16 @@ def test_join_maintain_order(left, right, maintain_order): ["c", "a"], ], ) -def test_non_coalesce_join(left, right, how, nulls_equal, join_expr): +def test_non_coalesce_join( + left, right, how, nulls_equal, join_expr, using_rapidsmpf, request +): + request.applymarker( + pytest.mark.xfail( + using_rapidsmpf, + strict=False, + reason="Non deterministic sort/join on nulls", + ) + ) query = left.join( right, on=join_expr, how=how, nulls_equal=nulls_equal, coalesce=False ) diff --git a/python/cudf_polars/tests/test_merge_sorted.py b/python/cudf_polars/tests/test_merge_sorted.py index 66c3a759f02e..ab654d8445eb 100644 --- a/python/cudf_polars/tests/test_merge_sorted.py +++ b/python/cudf_polars/tests/test_merge_sorted.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations @@ -9,17 +9,14 @@ from cudf_polars.testing.asserts import assert_gpu_result_equal -@pytest.mark.parametrize( - "descending", - [ - pytest.param( - True, - marks=pytest.mark.xfail(reason="polars/issues/21511"), - ), - False, - ], -) -def test_merge_sorted_without_nulls(descending): +@pytest.mark.parametrize("descending", [True, False]) +def test_merge_sorted_without_nulls(descending, request, using_rapidsmpf): + request.applymarker( + pytest.mark.xfail( + not using_rapidsmpf and descending, + reason="https://github.com/pola-rs/polars/issues/21511", + ) + ) df0 = pl.LazyFrame( {"name": ["steve", "elise", "bob"], "age": [42, 44, 18], "height": [5, 6, 5]} ).sort("age", descending=descending) diff --git a/python/cudf_polars/tests/test_rolling.py b/python/cudf_polars/tests/test_rolling.py index a9a0617ff719..792d83134e88 100644 --- a/python/cudf_polars/tests/test_rolling.py +++ b/python/cudf_polars/tests/test_rolling.py @@ -3,6 +3,7 @@ from __future__ import annotations +import contextlib from datetime import datetime import pytest @@ -133,7 +134,7 @@ def test_grouped_rolling(): assert_gpu_result_equal(q) -def test_grouped_rolling_unsorted_raises(): +def test_grouped_rolling_unsorted_raises(using_rapidsmpf): df = pl.LazyFrame( { "keys": [1, None, 2, 1, 2, None], @@ -145,7 +146,19 @@ def test_grouped_rolling_unsorted_raises(): with pytest.raises(pl.exceptions.ComputeError): q.collect(engine="in-memory") - with pytest.raises(RuntimeError, match="Input for grouped rolling is not sorted"): + + fallback_warning = ( + pytest.warns( + UserWarning, + match="shuffle_method=rapidsmpf-single does not support maintain_order=True", + ) + if using_rapidsmpf + else contextlib.nullcontext() + ) + with ( + pytest.raises(RuntimeError, match="Input for grouped rolling is not sorted"), + fallback_warning, + ): q.collect(engine=pl.GPUEngine(raise_on_fail=True)) diff --git a/python/cudf_polars/tests/test_scan.py b/python/cudf_polars/tests/test_scan.py index f2ab23468472..66f74b9bce12 100644 --- a/python/cudf_polars/tests/test_scan.py +++ b/python/cudf_polars/tests/test_scan.py @@ -702,7 +702,13 @@ def test_scan_tiny_file_not_compressed(tmp_path): reason="height parameter added in Polars 1.38", ) @pytest.mark.parametrize("engine", [None, NO_CHUNK_ENGINE]) -def test_scan_parquet_zero_width_with_limit(tmp_path, engine): +def test_scan_parquet_zero_width_with_limit(tmp_path, engine, request, using_rapidsmpf): + request.applymarker( + pytest.mark.xfail( + using_rapidsmpf and engine is None, + reason="https://github.com/rapidsai/cudf/issues/21644", + ) + ) path = tmp_path / "zero_width.parquet" pl.LazyFrame(height=20).sink_parquet(path) q = pl.scan_parquet(path).head(5) diff --git a/python/cudf_polars/tests/test_select.py b/python/cudf_polars/tests/test_select.py index ac9dbb90d0c0..7f85e233de34 100644 --- a/python/cudf_polars/tests/test_select.py +++ b/python/cudf_polars/tests/test_select.py @@ -87,7 +87,13 @@ def test_select_with_cse_no_agg(): assert_gpu_result_equal(query) -def test_select_with_cse_with_agg(): +def test_select_with_cse_with_agg(using_rapidsmpf, request): + request.applymarker( + pytest.mark.xfail( + condition=using_rapidsmpf, + reason="https://github.com/rapidsai/cudf/issues/21645", + ) + ) df = pl.LazyFrame({"a": [1, 2, 3]}) expr = pl.col("a") + pl.col("a") asum = pl.col("a").sum() + pl.col("a").sum() diff --git a/python/cudf_polars/tests/test_tracing.py b/python/cudf_polars/tests/test_tracing.py index f67fcda89a45..0cc6ed298d90 100644 --- a/python/cudf_polars/tests/test_tracing.py +++ b/python/cudf_polars/tests/test_tracing.py @@ -111,9 +111,9 @@ def test_log_query_plan() -> None: [sys.executable, "-c", code], env=env, stderr=subprocess.STDOUT ) - # Check for Query Plan event + # Check for Query Plan event generated from SerializablePlan assert b"Query Plan" in result assert b"scope=plan" in result or b"'scope': 'plan'" in result - assert b"ir_id" in result - assert b"ir_type" in result - assert b"children_ir_ids" in result + assert b"actor_ir_id" in result + assert b"actor_ir_type" in result + assert b"children" in result