diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fc09522bd478..35aaccc112f4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -39,7 +39,7 @@ repos: hooks: - id: mypy additional_dependencies: [ - "polars>=1.35,<1.39", + "polars>=1.35,<1.43", "numpy>=1.26", "pyarrow-stubs>=19.0", "pyarrow>=19.0.0,<24.0.0", # https://github.com/rapidsai/cudf/issues/22229 diff --git a/ci/test_narwhals.sh b/ci/test_narwhals.sh index 4c5d020593d9..13c30f178279 100755 --- a/ci/test_narwhals.sh +++ b/ci/test_narwhals.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail @@ -122,6 +122,9 @@ PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 \ # test_dtypes: narwhals' dtype mapping changed with polars 1.40 (reports Object where the test expects Int8). # test_namespace_len[polars[lazy]]: len() row count lost in zero-column streaming chunks # (https://github.com/rapidsai/cudf/issues/21428). +# test_explode_*[polars[lazy]-*]: polars 1.42 emits a DeprecationWarning when explode() is called +# without empty_as_null=True; narwhals 2.16.0 doesn't pass the kwarg yet, so filterwarnings=error +# turns this into a failure. Remove once narwhals is updated. TESTS_THAT_NEED_NARWHALS_FIX_FOR_CUDF_POLARS=" \ test_dtypes or \ test_namespace_len[polars[lazy]] or \ @@ -131,7 +134,8 @@ test_to_datetime_tz_aware[polars[lazy]-None] or \ test_truncate[polars[lazy]-1ns-expected0] or \ test_truncate_multiples[polars[lazy]-2ns-expected0] or \ ((test_nested_structures and polars and lazy) and (value0 or value1 or value3 or value4 or value6 or value7)) or \ -(test_series_from_iterable and pandas) \ +(test_series_from_iterable and pandas) or \ +(test_explode and polars and lazy) \ " rapids-logger "Run narwhals tests for cuDF Polars" diff --git a/conda/environments/all_cuda-129_arch-aarch64.yaml b/conda/environments/all_cuda-129_arch-aarch64.yaml index 02e820fce0f0..3218f3515c79 100644 --- a/conda/environments/all_cuda-129_arch-aarch64.yaml +++ b/conda/environments/all_cuda-129_arch-aarch64.yaml @@ -74,7 +74,7 @@ dependencies: - packaging - pandas>=3.0.0,<3.0.4a0 - pandoc -- polars>=1.35,<1.42 +- polars>=1.35,<1.43 - pre-commit - pyarrow>=19.0.0,<24 - pytables diff --git a/conda/environments/all_cuda-129_arch-x86_64.yaml b/conda/environments/all_cuda-129_arch-x86_64.yaml index 65a906637755..946cb18ad99b 100644 --- a/conda/environments/all_cuda-129_arch-x86_64.yaml +++ b/conda/environments/all_cuda-129_arch-x86_64.yaml @@ -74,7 +74,7 @@ dependencies: - packaging - pandas>=3.0.0,<3.0.4a0 - pandoc -- polars>=1.35,<1.42 +- polars>=1.35,<1.43 - pre-commit - pyarrow>=19.0.0,<24 - pytables diff --git a/conda/environments/all_cuda-133_arch-aarch64.yaml b/conda/environments/all_cuda-133_arch-aarch64.yaml index 49f221ee4acb..d09a6289e797 100644 --- a/conda/environments/all_cuda-133_arch-aarch64.yaml +++ b/conda/environments/all_cuda-133_arch-aarch64.yaml @@ -74,7 +74,7 @@ dependencies: - packaging - pandas>=3.0.0,<3.0.4a0 - pandoc -- polars>=1.35,<1.42 +- polars>=1.35,<1.43 - pre-commit - pyarrow>=19.0.0,<24 - pytables diff --git a/conda/environments/all_cuda-133_arch-x86_64.yaml b/conda/environments/all_cuda-133_arch-x86_64.yaml index 2c5eb31d1d86..781ff6da11ba 100644 --- a/conda/environments/all_cuda-133_arch-x86_64.yaml +++ b/conda/environments/all_cuda-133_arch-x86_64.yaml @@ -74,7 +74,7 @@ dependencies: - packaging - pandas>=3.0.0,<3.0.4a0 - pandoc -- polars>=1.35,<1.42 +- polars>=1.35,<1.43 - pre-commit - pyarrow>=19.0.0,<24 - pytables diff --git a/conda/recipes/cudf-polars/recipe.yaml b/conda/recipes/cudf-polars/recipe.yaml index 12aa1e19f162..632f5547679c 100644 --- a/conda/recipes/cudf-polars/recipe.yaml +++ b/conda/recipes/cudf-polars/recipe.yaml @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2018-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2018-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 schema_version: 1 @@ -38,7 +38,7 @@ requirements: - pylibcudf =${{ version }} - rapidsmpf =${{ minor_version }} - cudf-streaming =${{ version }} - - polars>=1.35,<1.42 + - polars>=1.35,<1.43 - packaging - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }} - if: cuda_major == "12" diff --git a/dependencies.yaml b/dependencies.yaml index 91e5c1868cbc..aa79cdd35757 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -589,7 +589,7 @@ files: test_cudf_polars_compat: output: none matrix: - polars_compat_version: ["1.35", "1.36", "1.37", "1.38", "1.39", "1.40", "1.41"] + polars_compat_version: ["1.35", "1.36", "1.37", "1.38", "1.39", "1.40", "1.41", "1.42"] includes: - test_cudf_polars_compat channels: @@ -967,7 +967,7 @@ dependencies: # 'nvidia-ml-py' provides the 'pynvml' module - &nvidia_ml_py nvidia-ml-py>=12 - packaging - - polars>=1.35,<1.42 + - polars>=1.35,<1.43 - typing_extensions>=4.0.0 run_cudf_polars_dask: common: @@ -1741,5 +1741,9 @@ dependencies: polars_compat_version: "1.41" packages: - polars==1.41.* + - matrix: + polars_compat_version: "1.42" + packages: + - polars==1.42.* - matrix: packages: diff --git a/python/cudf_polars/cudf_polars/containers/dataframe.py b/python/cudf_polars/cudf_polars/containers/dataframe.py index d9cae1d6e38b..ad75a8fde151 100644 --- a/python/cudf_polars/cudf_polars/containers/dataframe.py +++ b/python/cudf_polars/cudf_polars/containers/dataframe.py @@ -73,7 +73,7 @@ def __init__( self.stream = stream def __arrow_c_array__( - self, requested_schema: None = None + self, requested_schema: object | None = None ) -> tuple[CapsuleType, CapsuleType]: return self.obj._to_schema(self.metadata), self.obj._to_host_array( stream=self.stream diff --git a/python/cudf_polars/cudf_polars/dsl/expressions/boolean.py b/python/cudf_polars/cudf_polars/dsl/expressions/boolean.py index 93859947b425..92234805d0cd 100644 --- a/python/cudf_polars/cudf_polars/dsl/expressions/boolean.py +++ b/python/cudf_polars/cudf_polars/dsl/expressions/boolean.py @@ -65,6 +65,7 @@ class Name(IntEnum): IsNotNan = auto() IsNotNull = auto() IsNull = auto() + IsSorted = auto() IsUnique = auto() Not = auto() @@ -100,6 +101,7 @@ def __init__( BooleanFunction.Name.IsDuplicated, BooleanFunction.Name.IsFirstDistinct, BooleanFunction.Name.IsLastDistinct, + BooleanFunction.Name.IsSorted, BooleanFunction.Name.IsUnique, ) if self.name in { @@ -409,6 +411,26 @@ def do_evaluate( ), dtype=self.dtype, ) + elif self.name is BooleanFunction.Name.IsSorted: + (column,) = columns + (descending, nulls_last) = self.options + order = ( + plc.types.Order.DESCENDING if descending else plc.types.Order.ASCENDING + ) + null_order = ( + plc.types.NullOrder.AFTER if nulls_last else plc.types.NullOrder.BEFORE + ) + bool_result: bool = column.check_sorted( + order=order, null_order=null_order, stream=df.stream + ) + return Column( + plc.Column.from_scalar( + plc.Scalar.from_py(py_val=bool_result, stream=df.stream), + 1, + stream=df.stream, + ), + dtype=self.dtype, + ) elif self.name is BooleanFunction.Name.Not: (column,) = columns # Polars semantics: diff --git a/python/cudf_polars/cudf_polars/dsl/translate.py b/python/cudf_polars/cudf_polars/dsl/translate.py index 0f389e439533..341438442bb9 100644 --- a/python/cudf_polars/cudf_polars/dsl/translate.py +++ b/python/cudf_polars/cudf_polars/dsl/translate.py @@ -38,6 +38,7 @@ POLARS_VERSION_LT_139, POLARS_VERSION_LT_140, POLARS_VERSION_LT_141, + POLARS_VERSION_LT_142, ) if TYPE_CHECKING: @@ -45,7 +46,7 @@ from polars import GPUEngine - from cudf_polars.typing import NodeTraverser + from cudf_polars.typing import NodeTraverser, Slice as Zlice __all__ = ["Translator", "translate_named_expr"] @@ -164,7 +165,7 @@ def translate_ir(self, *, n: int | None = None) -> ir.IR: # IR is versioned with major.minor, minor is bumped for backwards # compatible changes (e.g. adding new nodes), major is bumped for # incompatible changes (e.g. renaming nodes). - if (version := self.visitor.version()) >= (13, 1): + if (version := self.visitor.version()) >= (14, 4): e = NotImplementedError( f"No support for polars IR {version=}" ) # pragma: no cover; no such version for now. @@ -291,10 +292,16 @@ def _drop_dyn_pred_hints( ) -> expr.Expr | None: try: node = translator.visitor.view_expression(n) - except Exception as e: + except Exception as e: # pragma: no cover if str(e) == "dynamic_pred": return None - raise # pragma: no cover + raise + if ( + not POLARS_VERSION_LT_142 + and isinstance(node, plrs._expr_nodes.Function) + and node.function_data[0] == "dynamic_pred" + ): + return None if isinstance(node, plrs._expr_nodes.BinaryExpr) and node.op in ( plrs._expr_nodes.Operator.And, plrs._expr_nodes.Operator.LogicalAnd, @@ -410,6 +417,8 @@ def _(node: plrs._ir_nodes.Scan, translator: Translator, schema: Schema) -> ir.I raise NotImplementedError( "Iceberg format is not supported in cudf-polars. Furthermore, row-level deletions are not supported." ) # pragma: no cover + if not POLARS_VERSION_LT_142 and node.hive_parts is not None: + raise NotImplementedError("Hive-partitioned scans are not supported") config_options = translator.config_options parquet_options = config_options.parquet_options @@ -657,7 +666,17 @@ def _(node: plrs._ir_nodes.Sort, translator: Translator, schema: Schema) -> ir.I order, null_order = sorting.sort_order( descending, nulls_last=nulls_last, num_keys=len(by) ) - return ir.Sort(schema, by, order, null_order, stable, node.slice, inp) + # TODO: use the DynamicPred hint from node.slice (offset, length, + # dynamic_pred_id). dynamic_pred_id is an int (u128 UUID) matching a + # DynamicPred Function node in an upstream scan. The sort actor could + # set a threshold predicate on it (e.g. col < max_seen) to prune rows + # at the source. + if not POLARS_VERSION_LT_142 and node.slice is not None: + offset, length, *_ = node.slice + zlice: Zlice | None = (offset, length) + else: + zlice = node.slice + return ir.Sort(schema, by, order, null_order, stable, zlice, inp) @_translate_ir.register diff --git a/python/cudf_polars/cudf_polars/testing/inject_gpu_engine.py b/python/cudf_polars/cudf_polars/testing/inject_gpu_engine.py index 3183b9a79e6d..8e6a212a7cba 100644 --- a/python/cudf_polars/cudf_polars/testing/inject_gpu_engine.py +++ b/python/cudf_polars/cudf_polars/testing/inject_gpu_engine.py @@ -151,7 +151,7 @@ def pytest_report_header(config: pytest.Config) -> str: "tests/unit/io/test_iceberg.py::test_scan_iceberg_nested_column_cast_deletion_rename": "Iceberg column_mapping (schema evolution) not yet implemented in cudf-polars", "tests/unit/io/test_iceberg.py::test_scan_iceberg_parquet_prefilter_with_column_mapping": "Iceberg column_mapping (schema evolution) not yet implemented in cudf-polars", "tests/unit/io/test_iceberg.py::test_fill_missing_fields_with_identity_partition_values_nested": "Iceberg partition column injection not yet implemented in cudf-polars", - "tests/unit/io/test_iceberg.py::test_scan_iceberg_fast_count": "Iceberg support not yet implemented in cudf-polars", + "tests/unit/io/test_iceberg.py::test_scan_iceberg_fast_count[native]": "Iceberg fast count from metadata not yet supported in cudf-polars", "tests/unit/io/test_iceberg.py::test_iceberg_filter_bool_26474": "Iceberg support not yet implemented in cudf-polars", "tests/unit/io/test_io_plugin.py::test_defer_validate_false": "cudf-polars always validates the IO source schema, so validate_schema=False dtype mismatches are unsupported on GPU", "tests/unit/io/test_io_plugin.py::test_datetime_io_predicate_pushdown_21790": "cudf-polars validates the IO source schema exactly and does not coerce datetime time units (us vs ns)", @@ -248,6 +248,7 @@ def pytest_report_header(config: pytest.Config) -> str: "tests/unit/sql/test_cast.py::test_cast_errors[values2-values::int1-conversion from `i64` to `i8` failed]": "Casting that raises not supported on GPU", "tests/unit/sql/test_cast.py::test_cast_errors[values5-values::int4-conversion from `str` to `i32` failed]": "Cast raises, but error user receives is wrong", "tests/unit/lazyframe/test_predicates.py::test_predicate_pushdown_split_pushable": "Casting that raises not supported on GPU", + "tests/unit/lazyframe/test_predicates.py::test_filter_contradiction_fallible_error_handling": "Casting that raises not supported on GPU", "tests/unit/sql/test_miscellaneous.py::test_read_csv": "Incorrect handling of missing_is_null in read_csv", "tests/unit/lazyframe/test_cse.py::test_cse_predicate_self_join[False]": "Debug output on stderr doesn't match, see https://github.com/rapidsai/cudf/issues/22967", "tests/unit/lazyframe/test_cse.py::test_cse_predicate_self_join[True]": "Debug output on stderr doesn't match, see https://github.com/rapidsai/cudf/issues/22967", @@ -344,6 +345,12 @@ def pytest_report_header(config: pytest.Config) -> str: "tests/unit/io/test_scan.py::test_scan_metrics[False-parquet]": "Checks to IO metric logs specific to Polars CPU", "tests/unit/io/test_scan.py::test_scan_metrics[False-csv]": "Checks to IO metric logs specific to Polars CPU", "tests/unit/io/test_scan.py::test_scan_metrics[False-ndjson]": "Checks to IO metric logs specific to Polars CPU", + # polars 1.42 updated these tests to also assert deprecated_call and strict=True ShapeError + # in the same test function. The SPMD engine fails on the strict=True collect() inside a + # pytest.raises block because the DeprecationWarning from how='horizontal' propagates differently + # across engines. Skip both runs rather than xfail (which would XPASS on in-memory). + "tests/unit/lazyframe/test_predicates.py::test_hconcat_predicate": "polars 1.42: test uses deprecated how='horizontal' with strict=True in ways that behave differently across GPU engines", + "tests/unit/functions/test_union.py::test_union_lazyframe_horizontal": "polars 1.42: test uses deprecated how='horizontal' with strict=True in ways that behave differently across GPU engines", } diff --git a/python/cudf_polars/cudf_polars/utils/versions.py b/python/cudf_polars/cudf_polars/utils/versions.py index df564318aecd..3c74ef2cb298 100644 --- a/python/cudf_polars/cudf_polars/utils/versions.py +++ b/python/cudf_polars/cudf_polars/utils/versions.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 """Version utilities so that cudf_polars supports a range of polars versions.""" @@ -18,6 +18,7 @@ POLARS_VERSION_LT_139 = POLARS_VERSION < parse("1.39.0") POLARS_VERSION_LT_140 = POLARS_VERSION < parse("1.40.0") POLARS_VERSION_LT_141 = POLARS_VERSION < parse("1.41.0") +POLARS_VERSION_LT_142 = POLARS_VERSION < parse("1.42.0") def _ensure_polars_version() -> None: diff --git a/python/cudf_polars/pyproject.toml b/python/cudf_polars/pyproject.toml index 3f9f6cbff5e9..20d46cbc649a 100644 --- a/python/cudf_polars/pyproject.toml +++ b/python/cudf_polars/pyproject.toml @@ -25,7 +25,7 @@ dependencies = [ "cudf-streaming==26.8.*,>=0.0.0a0", "nvidia-ml-py>=12", "packaging", - "polars>=1.35,<1.42", + "polars>=1.35,<1.43", "pylibcudf==26.8.*,>=0.0.0a0", "rapidsmpf==26.8.*,>=0.0.0a0", "typing_extensions>=4.0.0", diff --git a/python/cudf_polars/tests/dsl/test_serialization.py b/python/cudf_polars/tests/dsl/test_serialization.py index a845b106ea70..2cad17324682 100644 --- a/python/cudf_polars/tests/dsl/test_serialization.py +++ b/python/cudf_polars/tests/dsl/test_serialization.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations @@ -16,6 +16,7 @@ from cudf_polars.utils.versions import ( POLARS_VERSION_LT_138, POLARS_VERSION_LT_141, + POLARS_VERSION_LT_142, ) @@ -57,6 +58,9 @@ def test_from_polars_all_names(function): if POLARS_VERSION_LT_141 and function == BooleanFunction: # 'HasNulls' and 'IsEmpty' were added to polars' BooleanFunction in 1.41. cudf_polars_names_set = cudf_polars_names_set - {"HasNulls", "IsEmpty"} + if POLARS_VERSION_LT_142 and function == BooleanFunction: + # 'IsSorted' was added to polars' BooleanFunction in 1.42. + cudf_polars_names_set = cudf_polars_names_set - {"IsSorted"} assert polars_names_set == cudf_polars_names_set names = function.Name if function == StructFunction: @@ -71,6 +75,8 @@ def test_from_polars_all_names(function): BooleanFunction.Name.HasNulls, BooleanFunction.Name.IsEmpty, } + if POLARS_VERSION_LT_142 and function == BooleanFunction: + names = set(names) - {BooleanFunction.Name.IsSorted} for name in names: attr = getattr(polars_function, name.name) assert function.Name.from_polars(attr) == name diff --git a/python/cudf_polars/tests/expressions/test_booleanfunction.py b/python/cudf_polars/tests/expressions/test_booleanfunction.py index 294063eb3080..64ca99584c53 100644 --- a/python/cudf_polars/tests/expressions/test_booleanfunction.py +++ b/python/cudf_polars/tests/expressions/test_booleanfunction.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations @@ -12,6 +12,7 @@ assert_gpu_result_equal, assert_ir_translation_raises, ) +from cudf_polars.utils.versions import POLARS_VERSION_LT_142 if TYPE_CHECKING: from collections.abc import Callable @@ -294,6 +295,35 @@ def test_boolean_not_with_integers(engine: pl.GPUEngine, dtype, col): assert_gpu_result_equal(q, engine=engine) +@pytest.mark.skipif( + POLARS_VERSION_LT_142, reason="IsSorted BooleanFunction was added in polars 1.42" +) +@pytest.mark.parametrize("descending", [False, True]) +@pytest.mark.parametrize("nulls_last", [False, True]) +def test_boolean_is_sorted( + engine: pl.GPUEngine, *, descending: bool, nulls_last: bool, has_nulls: bool +) -> None: + values: list[int | None] = [1, 2, 3, 4, 5] + if has_nulls: + values[2] = None + + ldf = pl.LazyFrame( + { + "asc": pl.Series(values, dtype=pl.Int64()), + "desc": pl.Series(list(reversed(values)), dtype=pl.Int64()), + "unsorted": pl.Series([3, 1, None, 4, 2], dtype=pl.Int64()), + } + ) + + q = ldf.select( + pl.col("asc").is_sorted(descending=descending, nulls_last=nulls_last), + pl.col("desc").is_sorted(descending=descending, nulls_last=nulls_last), + pl.col("unsorted").is_sorted(descending=descending, nulls_last=nulls_last), + ) + + assert_gpu_result_equal(q, engine=engine) + + @pytest.mark.parametrize( "expr", [ diff --git a/python/cudf_polars/tests/test_scan.py b/python/cudf_polars/tests/test_scan.py index 7a9a4f2bb10f..ccbd68050588 100644 --- a/python/cudf_polars/tests/test_scan.py +++ b/python/cudf_polars/tests/test_scan.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations @@ -25,6 +25,7 @@ from cudf_polars.utils.versions import ( POLARS_VERSION_LT_138, POLARS_VERSION_LT_139, + POLARS_VERSION_LT_142, ) if TYPE_CHECKING: @@ -532,6 +533,7 @@ def test_scan_with_row_index(engine: pl.GPUEngine, tmp_path: Path) -> None: pytest.param( "foo=bar", marks=pytest.mark.xfail( + condition=POLARS_VERSION_LT_142, reason="https://github.com/pola-rs/polars/issues/27840", strict=True, ), @@ -812,3 +814,16 @@ def test_scan_parquet_is_between_literal_dtype_mismatch_22622( ) assert_gpu_result_equal(q, engine=engine) + + +@pytest.mark.skipif( + POLARS_VERSION_LT_142, + reason="hive::HivePartitionedDf not exposed in the logical plan before 1.42", +) +def test_scan_parquet_hive_partitioned_raises( + engine: pl.GPUEngine, tmp_path: Path +) -> None: + (tmp_path / "part=1").mkdir() + pl.DataFrame({"x": [1, 2, 3]}).write_parquet(tmp_path / "part=1" / "data.parquet") + q = pl.scan_parquet(tmp_path, hive_schema={"part": pl.Int32}) + assert_ir_translation_raises(q, engine, NotImplementedError)