From 59d344eaef2f7b779941a31a29ab2db5d942dea9 Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Tue, 7 Apr 2026 15:26:58 -0700 Subject: [PATCH 01/44] Pin polars >=1.38 and <1.40 --- python/cudf_polars/cudf_polars/testing/plugin.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/cudf_polars/cudf_polars/testing/plugin.py b/python/cudf_polars/cudf_polars/testing/plugin.py index ae1d2fcc1d7b..6e2968cccc68 100644 --- a/python/cudf_polars/cudf_polars/testing/plugin.py +++ b/python/cudf_polars/cudf_polars/testing/plugin.py @@ -204,7 +204,6 @@ def pytest_configure(config: pytest.Config) -> None: "tests/unit/operations/test_slice.py::test_schema_slice_on_literal_23999[lit1-0-len1-False]": "List literal loses nesting in slice: cudf#19610", "tests/unit/operations/test_slice.py::test_schema_slice_on_literal_23999[lit1-offset1-0-False]": "List literal loses nesting in slice: cudf#19610", "tests/unit/operations/test_slice.py::test_schema_slice_on_literal_23999[lit1-offset1-len1-False]": "List literal loses nesting in slice: cudf#19610", - "tests/unit/functions/test_concat.py::test_concat_horizontally_strict": "polars doesnt hand us the hconcat options. Fixed in 1.39.", } From 4b40d81c299e1f9b9c266ebdbc5971e3daf3ebff Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Tue, 7 Apr 2026 15:31:38 -0700 Subject: [PATCH 02/44] Pin polars >=1.38 and <1.40 --- .../all_cuda-129_arch-aarch64.yaml | 2 +- .../all_cuda-129_arch-x86_64.yaml | 2 +- .../all_cuda-131_arch-aarch64.yaml | 2 +- .../all_cuda-131_arch-x86_64.yaml | 2 +- conda/recipes/cudf-polars/recipe.yaml | 2 +- dependencies.yaml | 2 +- .../cudf_polars/containers/dataframe.py | 3 - .../dsl/expressions/aggregation.py | 11 - .../cudf_polars/dsl/expressions/rolling.py | 20 +- .../cudf_polars/dsl/expressions/string.py | 103 ++------ .../cudf_polars/dsl/expressions/unary.py | 25 +- python/cudf_polars/cudf_polars/dsl/ir.py | 80 ++----- .../cudf_polars/cudf_polars/dsl/translate.py | 226 ++++++++---------- .../cudf_polars/dsl/utils/aggregations.py | 49 +--- .../experimental/benchmarks/asserts.py | 7 +- .../cudf_polars/experimental/io.py | 3 +- .../cudf_polars/testing/asserts.py | 6 +- .../cudf_polars/cudf_polars/utils/versions.py | 13 +- python/cudf_polars/pyproject.toml | 2 +- .../tests/dsl/test_serialization.py | 32 +-- .../tests/experimental/test_rolling.py | 12 +- .../tests/experimental/test_select.py | 22 +- .../cudf_polars/tests/expressions/test_agg.py | 18 +- .../tests/expressions/test_booleanfunction.py | 8 +- .../tests/expressions/test_casting.py | 3 +- .../tests/expressions/test_numeric_binops.py | 7 +- .../expressions/test_numeric_unaryops.py | 10 +- .../tests/expressions/test_rolling.py | 108 ++++----- .../tests/expressions/test_sort.py | 34 --- .../tests/expressions/test_stringfunction.py | 49 +--- .../tests/expressions/test_struct.py | 51 +--- python/cudf_polars/tests/test_cache.py | 10 +- .../cudf_polars/tests/test_dataframescan.py | 5 - python/cudf_polars/tests/test_drop_nulls.py | 16 +- python/cudf_polars/tests/test_groupby.py | 19 +- python/cudf_polars/tests/test_join.py | 17 +- python/cudf_polars/tests/test_mapfunction.py | 19 +- python/cudf_polars/tests/test_rolling.py | 8 +- python/cudf_polars/tests/test_scan.py | 31 +-- python/cudf_polars/tests/test_select.py | 3 +- python/cudf_polars/tests/test_sink.py | 5 - .../tests/test_window_functions.py | 24 +- 42 files changed, 293 insertions(+), 778 deletions(-) diff --git a/conda/environments/all_cuda-129_arch-aarch64.yaml b/conda/environments/all_cuda-129_arch-aarch64.yaml index 2b49cd41f298..365804533ddf 100644 --- a/conda/environments/all_cuda-129_arch-aarch64.yaml +++ b/conda/environments/all_cuda-129_arch-aarch64.yaml @@ -67,7 +67,7 @@ dependencies: - packaging - pandas>=2.0,<2.4.0 - pandoc -- polars>=1.30,<1.39 +- polars>=1.38,<1.40 - pre-commit - pyarrow>=19.0.0 - pydata-sphinx-theme>=0.15.4 diff --git a/conda/environments/all_cuda-129_arch-x86_64.yaml b/conda/environments/all_cuda-129_arch-x86_64.yaml index b3a4b7102146..12dbf0c87431 100644 --- a/conda/environments/all_cuda-129_arch-x86_64.yaml +++ b/conda/environments/all_cuda-129_arch-x86_64.yaml @@ -67,7 +67,7 @@ dependencies: - packaging - pandas>=2.0,<2.4.0 - pandoc -- polars>=1.30,<1.39 +- polars>=1.38,<1.40 - pre-commit - pyarrow>=19.0.0 - pydata-sphinx-theme>=0.15.4 diff --git a/conda/environments/all_cuda-131_arch-aarch64.yaml b/conda/environments/all_cuda-131_arch-aarch64.yaml index 25ea42819e47..c67c1eb758ae 100644 --- a/conda/environments/all_cuda-131_arch-aarch64.yaml +++ b/conda/environments/all_cuda-131_arch-aarch64.yaml @@ -67,7 +67,7 @@ dependencies: - packaging - pandas>=2.0,<2.4.0 - pandoc -- polars>=1.30,<1.39 +- polars>=1.38,<1.40 - pre-commit - pyarrow>=19.0.0 - pydata-sphinx-theme>=0.15.4 diff --git a/conda/environments/all_cuda-131_arch-x86_64.yaml b/conda/environments/all_cuda-131_arch-x86_64.yaml index 2ccd1614d462..233797432f04 100644 --- a/conda/environments/all_cuda-131_arch-x86_64.yaml +++ b/conda/environments/all_cuda-131_arch-x86_64.yaml @@ -67,7 +67,7 @@ dependencies: - packaging - pandas>=2.0,<2.4.0 - pandoc -- polars>=1.30,<1.39 +- polars>=1.38,<1.40 - pre-commit - pyarrow>=19.0.0 - pydata-sphinx-theme>=0.15.4 diff --git a/conda/recipes/cudf-polars/recipe.yaml b/conda/recipes/cudf-polars/recipe.yaml index e3a21aa1afd8..e46ce5376231 100644 --- a/conda/recipes/cudf-polars/recipe.yaml +++ b/conda/recipes/cudf-polars/recipe.yaml @@ -36,7 +36,7 @@ requirements: - nvidia-ml-py>=12 - python - pylibcudf =${{ version }} - - polars>=1.30,<1.39 + - polars>=1.38,<1.40 - packaging - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }} - if: cuda_major == "12" diff --git a/dependencies.yaml b/dependencies.yaml index e76794400f77..1ff0001f72a3 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -817,7 +817,7 @@ dependencies: # 'nvidia-ml-py' provides the 'pynvml' module - &nvidia_ml_py nvidia-ml-py>=12 - packaging - - polars>=1.30,<1.39 + - polars>=1.38,<1.40 - typing_extensions>=4.0.0 run_cudf_polars_experimental: common: diff --git a/python/cudf_polars/cudf_polars/containers/dataframe.py b/python/cudf_polars/cudf_polars/containers/dataframe.py index d96e96cc997b..977449a971e8 100644 --- a/python/cudf_polars/cudf_polars/containers/dataframe.py +++ b/python/cudf_polars/cudf_polars/containers/dataframe.py @@ -14,7 +14,6 @@ from cudf_polars.containers import Column, DataType from cudf_polars.utils import conversion -from cudf_polars.utils.versions import POLARS_VERSION_LT_138 if TYPE_CHECKING: from collections.abc import Iterable, Mapping, Sequence, Set @@ -119,8 +118,6 @@ def copy(self) -> Self: def to_polars(self) -> pl.DataFrame: """Convert to a polars DataFrame.""" if self._num_rows_override is not None and len(self.column_map) == 0: - if POLARS_VERSION_LT_138: # pragma: no cover - return pl.DataFrame() return pl.DataFrame(height=self._num_rows_override) # If the arrow table has empty names, from_arrow produces diff --git a/python/cudf_polars/cudf_polars/dsl/expressions/aggregation.py b/python/cudf_polars/cudf_polars/dsl/expressions/aggregation.py index cca9a1562c91..238bf853fbb6 100644 --- a/python/cudf_polars/cudf_polars/dsl/expressions/aggregation.py +++ b/python/cudf_polars/cudf_polars/dsl/expressions/aggregation.py @@ -15,7 +15,6 @@ from cudf_polars.containers import Column from cudf_polars.dsl.expressions.base import ExecutionContext, Expr from cudf_polars.dsl.expressions.literal import Literal -from cudf_polars.utils.versions import POLARS_VERSION_LT_136 if TYPE_CHECKING: from rmm.pylibrmm.stream import Stream @@ -95,16 +94,6 @@ def __init__( raise NotImplementedError( f"Unreachable, {name=} is incorrectly listed in _SUPPORTED" ) # pragma: no cover - if ( - POLARS_VERSION_LT_136 - and context == ExecutionContext.FRAME - and req is not None - and not plc.aggregation.is_valid_aggregation(dtype.plc_type, req) - ): # pragma: no cover; polars may raise ahead of time - # TODO: Check which cases polars raises vs returns all-NULL column. - # For the all-NULL column cases, we could build it using Column.all_null_like - # at evaluation time. - raise NotImplementedError(f"Invalid aggregation {req} with dtype {dtype}") self.request = req op = getattr(self, f"_{name}", None) if op is None: diff --git a/python/cudf_polars/cudf_polars/dsl/expressions/rolling.py b/python/cudf_polars/cudf_polars/dsl/expressions/rolling.py index 57103bfd597c..6f16b8dfd3aa 100644 --- a/python/cudf_polars/cudf_polars/dsl/expressions/rolling.py +++ b/python/cudf_polars/cudf_polars/dsl/expressions/rolling.py @@ -22,7 +22,6 @@ offsets_to_windows, range_window_bounds, ) -from cudf_polars.utils.versions import POLARS_VERSION_LT_136 if TYPE_CHECKING: from collections.abc import Sequence @@ -79,14 +78,9 @@ def to_request( elif isinstance(value, expr.Agg): child = value.children[0] col = child.evaluate(df, context=ExecutionContext.ROLLING) - if POLARS_VERSION_LT_136 and value.name == "var": # pragma: no cover - # Polars variance produces null if nvalues <= ddof - # libcudf produces NaN. However, we can get the polars - # behaviour by setting the minimum window size to ddof + - # 1. - # - # We still need this check, polars is not hitting it because - # of https://github.com/pola-rs/polars/pull/25117 + if value.name == "var": + # Polars rolling var returns null when nvalues <= ddof; libcudf returns NaN. + # Setting min_periods = ddof+1 causes libcudf to emit null instead of NaN. min_periods = value.options + 1 else: col = value.evaluate( @@ -96,7 +90,7 @@ def to_request( return plc.rolling.RollingRequest(col.obj, min_periods, value.agg_request) -class RollingWindow(Expr): # pragma: no cover; polars >1.36 uses AExpr::Rolling now +class RollingWindow(Expr): __slots__ = ( "closed_window", "following_ordinal", @@ -146,12 +140,10 @@ def __init__( raise NotImplementedError( "Incorrect handling of empty groups for list collection" ) - if POLARS_VERSION_LT_136 and not plc.rolling.is_valid_rolling_aggregation( + if not plc.rolling.is_valid_rolling_aggregation( agg.dtype.plc_type, agg.agg_request ): - raise NotImplementedError( - f"Unsupported rolling aggregation {agg}" - ) # pragma: no cover; polars may raise ahead of time + raise NotImplementedError(f"Unsupported rolling aggregation {agg}") def do_evaluate( # noqa: D102 self, df: DataFrame, *, context: ExecutionContext = ExecutionContext.FRAME diff --git a/python/cudf_polars/cudf_polars/dsl/expressions/string.py b/python/cudf_polars/cudf_polars/dsl/expressions/string.py index ebc2e8f950ad..33de97eac9f0 100644 --- a/python/cudf_polars/cudf_polars/dsl/expressions/string.py +++ b/python/cudf_polars/cudf_polars/dsl/expressions/string.py @@ -12,7 +12,10 @@ from enum import IntEnum, auto from typing import TYPE_CHECKING, Any, ClassVar, cast -from polars import Struct as pl_Struct, polars # type: ignore[attr-defined] +from polars import ( # type: ignore[attr-defined] # noqa: TC002 + Struct as pl_Struct, + polars, +) from polars.exceptions import InvalidOperationError import pylibcudf as plc @@ -21,11 +24,6 @@ from cudf_polars.dsl.expressions.base import ExecutionContext, Expr from cudf_polars.dsl.expressions.literal import Literal, LiteralColumn from cudf_polars.dsl.utils.reshape import broadcast -from cudf_polars.utils.versions import ( - POLARS_VERSION_LT_132, - POLARS_VERSION_LT_136, - POLARS_VERSION_LT_138, -) if TYPE_CHECKING: from typing import Self @@ -233,14 +231,11 @@ def _validate_input(self) -> None: "libcudf replace does not support empty strings" ) elif self.name is StringFunction.Name.ReplaceMany: - if POLARS_VERSION_LT_136: - (ascii_case_insensitive,) = self.options # pragma: no cover - else: - ascii_case_insensitive, leftmost = self.options - if leftmost: - raise NotImplementedError( - "leftmost=True not implemented for replace_many" - ) + ascii_case_insensitive, leftmost = self.options + if leftmost: + raise NotImplementedError( + "leftmost=True not implemented for replace_many" + ) if ascii_case_insensitive: raise NotImplementedError( "ascii_case_insensitive not implemented for replace_many" @@ -268,9 +263,8 @@ def _validate_input(self) -> None: (_, inclusive) = self.options if inclusive: raise NotImplementedError(f"{inclusive=} is not supported for split") - elif not POLARS_VERSION_LT_138 and self.name is StringFunction.Name.SplitRegex: + elif self.name is StringFunction.Name.SplitRegex: # See https://github.com/pola-rs/polars/pull/26060 - # SplitRegex introduced in polars>=1.38, but we don't support it yet raise NotImplementedError( "String split with regex (literal=False) is not supported." ) @@ -291,21 +285,6 @@ def _validate_input(self) -> None: raise NotImplementedError( "strip operations only support scalar patterns" ) - elif self.name is StringFunction.Name.ZFill: - if isinstance(self.children[1], Literal): - _, width = self.children - assert isinstance(width, Literal) - if ( - POLARS_VERSION_LT_132 - and width.value is not None - and width.value < 0 - ): # pragma: no cover - dtypestr = polars.dtype_str_repr(width.dtype.polars_type) - raise InvalidOperationError( - f"conversion from `{dtypestr}` to `u64` " - f"failed in column 'literal' for 1 out of " - f"1 values: [{width.value}]" - ) from None @staticmethod def _create_regex_program( @@ -436,24 +415,6 @@ def do_evaluate( else: col_width = self.children[1].evaluate(df, context=context) assert isinstance(col_width, Column) - all_gt_0 = plc.binaryop.binary_operation( - col_width.obj, - plc.Scalar.from_py( - 0, plc.DataType(plc.TypeId.INT64), stream=df.stream - ), - plc.binaryop.BinaryOperator.GREATER_EQUAL, - plc.DataType(plc.TypeId.BOOL8), - stream=df.stream, - ) - - if POLARS_VERSION_LT_132 and not plc.reduce.reduce( - all_gt_0, - plc.aggregation.all(), - plc.DataType(plc.TypeId.BOOL8), - stream=df.stream, - ).to_py(stream=df.stream): # pragma: no cover - raise InvalidOperationError("fill conversion failed.") - return Column( plc.strings.padding.zfill_by_widths( column.obj, col_width.obj, stream=df.stream @@ -964,21 +925,14 @@ def do_evaluate( dtype=self.dtype, ) elif self.name is StringFunction.Name.PadStart: - if POLARS_VERSION_LT_132: # pragma: no cover - (column,) = columns - width_arg, char = self.options - pad_width = cast(int, width_arg) - else: - (column, width_col) = columns - (char,) = self.options - # TODO: Maybe accept a string scalar in - # cudf::strings::pad to avoid DtoH transfer - # See https://github.com/rapidsai/cudf/issues/20202 - width_py = width_col.obj.to_scalar(stream=df.stream).to_py( - stream=df.stream - ) - assert width_py is not None - pad_width = int(width_py) + (column, width_col) = columns + (char,) = self.options + # TODO: Maybe accept a string scalar in + # cudf::strings::pad to avoid DtoH transfer + # See https://github.com/rapidsai/cudf/issues/20202 + width_py = width_col.obj.to_scalar(stream=df.stream).to_py(stream=df.stream) + assert width_py is not None + pad_width = int(width_py) return Column( plc.strings.padding.pad( @@ -991,20 +945,13 @@ def do_evaluate( dtype=self.dtype, ) elif self.name is StringFunction.Name.PadEnd: - if POLARS_VERSION_LT_132: # pragma: no cover - (column,) = columns - width_arg, char = self.options - pad_width = cast(int, width_arg) - else: - (column, width_col) = columns - (char,) = self.options - # TODO: Maybe accept a string scalar in - # cudf::strings::pad to avoid DtoH transfer - width_py = width_col.obj.to_scalar(stream=df.stream).to_py( - stream=df.stream - ) - assert width_py is not None - pad_width = int(width_py) + (column, width_col) = columns + (char,) = self.options + # TODO: Maybe accept a string scalar in + # cudf::strings::pad to avoid DtoH transfer + width_py = width_col.obj.to_scalar(stream=df.stream).to_py(stream=df.stream) + assert width_py is not None + pad_width = int(width_py) return Column( plc.strings.padding.pad( diff --git a/python/cudf_polars/cudf_polars/dsl/expressions/unary.py b/python/cudf_polars/cudf_polars/dsl/expressions/unary.py index 99ffb2185cf4..dbede44e30af 100644 --- a/python/cudf_polars/cudf_polars/dsl/expressions/unary.py +++ b/python/cudf_polars/cudf_polars/dsl/expressions/unary.py @@ -13,6 +13,7 @@ from cudf_polars.dsl.expressions.base import ExecutionContext, Expr from cudf_polars.dsl.expressions.literal import Literal from cudf_polars.utils import dtypes +from cudf_polars.utils.versions import POLARS_VERSION_LT_139 if TYPE_CHECKING: from cudf_polars.containers import DataFrame, DataType @@ -244,16 +245,22 @@ def do_evaluate( if maintain_order: column = column.sorted_like(values) return column - elif self.name == "set_sorted": # pragma: no cover - # TODO: LazyFrame.set_sorted is proper IR concept (ie. FunctionIR::Hint) - # and is is currently not implemented. We should reimplement it as a MapFunction. + elif self.name == "set_sorted": (column,) = (child.evaluate(df, context=context) for child in self.children) - (asc,) = self.options - order = ( - plc.types.Order.ASCENDING - if asc == "ascending" - else plc.types.Order.DESCENDING - ) + if POLARS_VERSION_LT_139: + (asc,) = self.options + order = ( + plc.types.Order.ASCENDING + if asc == "ascending" + else plc.types.Order.DESCENDING + ) + else: + descending, _nulls_last = self.options + order = ( + plc.types.Order.DESCENDING + if descending + else plc.types.Order.ASCENDING + ) null_order = plc.types.NullOrder.BEFORE if column.null_count > 0 and (n := column.size) > 1: # PERF: This invokes four stream synchronisations! diff --git a/python/cudf_polars/cudf_polars/dsl/ir.py b/python/cudf_polars/cudf_polars/dsl/ir.py index 0465244c4296..d0de726b4a17 100644 --- a/python/cudf_polars/cudf_polars/dsl/ir.py +++ b/python/cudf_polars/cudf_polars/dsl/ir.py @@ -50,13 +50,6 @@ get_new_cuda_stream, join_cuda_streams, ) -from cudf_polars.utils.versions import ( - POLARS_VERSION_LT_131, - POLARS_VERSION_LT_134, - POLARS_VERSION_LT_136, - POLARS_VERSION_LT_137, - POLARS_VERSION_LT_138, -) if TYPE_CHECKING: from collections.abc import Callable, Generator, Hashable, Iterable, Sequence @@ -523,16 +516,7 @@ def __init__( raise NotImplementedError( "Read from cloud storage" ) # pragma: no cover; no test yet - if ( - any(str(p).startswith("https:/") for p in self.paths) - and POLARS_VERSION_LT_131 - ): # pragma: no cover; polars passed us the wrong URI - # https://github.com/pola-rs/polars/issues/22766 - raise NotImplementedError("Read from https") - if any( - str(p).startswith("file:/" if POLARS_VERSION_LT_131 else "file://") - for p in self.paths - ): + if any(str(p).startswith("file://") for p in self.paths): raise NotImplementedError("Read from file URI") if self.typ == "csv": if any( @@ -1050,8 +1034,8 @@ def __init__( raise NotImplementedError( f"Compression type '{compression}' is not supported." ) - elif kind == ( - "Json" if POLARS_VERSION_LT_137 else "NDJson" + elif ( + kind == "NDJson" ): # pragma: no cover; options are validated on the polars side if not all( plc.io.json.is_supported_write_json(dtype.plc_type) @@ -1235,13 +1219,11 @@ def do_evaluate( """Write the dataframe to a file.""" target = plc.io.SinkInfo([path]) - if POLARS_VERSION_LT_136 and options.get("mkdir", False): - Path(path).parent.mkdir(parents=True, exist_ok=True) # pragma: no cover if kind == "Csv": cls._write_csv(target, options, df) elif kind == "Parquet": cls._write_parquet(target, parquet_options, options, df) - elif kind == ("Json" if POLARS_VERSION_LT_137 else "NDJson"): + elif kind == "NDJson": cls._write_json(target, df) return DataFrame([], stream=df.stream) @@ -1829,14 +1811,6 @@ def __init__( raise NotImplementedError("Nested list[struct] types not supported") for request in agg_requests: expr = request.value - if ( - POLARS_VERSION_LT_136 - and isinstance(expr, unary.UnaryFunction) - and expr.name == "value_counts" - ): - raise NotImplementedError( - "value_counts is not supported in groupby" - ) # pragma: no cover; Nested list[struct] types not supported if any( isinstance(child, unary.UnaryFunction) and child.name == "value_counts" for child in expr.children @@ -1989,19 +1963,15 @@ def _strip_predicate_casts(node: expr.Expr) -> expr.Expr: ): return child - if ( - not POLARS_VERSION_LT_134 - and isinstance(child, expr.ColRef) - and ( - ( - plc.traits.is_floating_point(src.plc_type) - and plc.traits.is_floating_point(dst.plc_type) - ) - or ( - plc.traits.is_integral(src.plc_type) - and plc.traits.is_integral(dst.plc_type) - and src.plc_type.id() == dst.plc_type.id() - ) + if isinstance(child, expr.ColRef) and ( + ( + plc.traits.is_floating_point(src.plc_type) + and plc.traits.is_floating_point(dst.plc_type) + ) + or ( + plc.traits.is_integral(src.plc_type) + and plc.traits.is_integral(dst.plc_type) + and src.plc_type.id() == dst.plc_type.id() ) ): return child @@ -3035,11 +3005,11 @@ def __init__(self, schema: Schema, name: str, options: Any, df: IR): f"Unhandled map function {self.name}" ) # pragma: no cover if self.name == "explode": - if POLARS_VERSION_LT_136 or len(self.options) == 1: + if len(self.options) == 1: (to_explode,) = self.options else: (to_explode, empty_as_null, keep_nulls) = self.options - if (POLARS_VERSION_LT_136 and not empty_as_null) or not keep_nulls: + if not keep_nulls: raise NotImplementedError( "Explode an empty or null list/array" ) # pragma: no cover @@ -3049,23 +3019,10 @@ def __init__(self, schema: Schema, name: str, options: Any, df: IR): # same sub-shapes raise NotImplementedError("Explode with more than one column") self.options = (tuple(to_explode),) - elif POLARS_VERSION_LT_131 and self.name == "rename": # pragma: no cover - # As of 1.31, polars validates renaming in the IR - old, new, strict = self.options - if len(new) != len(set(new)) or ( - set(new) & (set(df.schema.keys()) - set(old)) - ): - raise NotImplementedError( - "Duplicate new names in rename." - ) # pragma: no cover - self.options = (tuple(old), tuple(new), strict) elif self.name == "unpivot": indices, pivotees, variable_name, value_name = self.options value_name = "value" if value_name is None else value_name variable_name = "variable" if variable_name is None else variable_name - if POLARS_VERSION_LT_136 and len(pivotees) == 0: # pragma: no cover - index = frozenset(indices) - pivotees = [name for name in df.schema if name not in index] if not all( dtypes.can_cast(df.schema[p].plc_type, self.schema[value_name].plc_type) for p in pivotees @@ -3082,8 +3039,6 @@ def __init__(self, schema: Schema, name: str, options: Any, df: IR): ) elif self.name == "row_index": col_name, offset = options - if POLARS_VERSION_LT_138 and col_name in df.schema: # pragma: no cover - raise NotImplementedError("Duplicate row index name") self.options = (col_name, offset) elif self.name == "fast_count": # TODO: Remove this once all scan types support projections @@ -3132,11 +3087,6 @@ def do_evaluate( # No-op in our data model # Don't think this appears in a plan tree from python return df # pragma: no cover - elif POLARS_VERSION_LT_131 and name == "rename": # pragma: no cover - # final tag is "swapping" which is useful for the - # optimiser (it blocks some pushdown operations) - old, new, _ = options - return df.rename_columns(dict(zip(old, new, strict=True))) elif name == "explode": ((to_explode,),) = options index = df.column_names.index(to_explode) diff --git a/python/cudf_polars/cudf_polars/dsl/translate.py b/python/cudf_polars/cudf_polars/dsl/translate.py index 0adbf604e74f..6ba9d667d93f 100644 --- a/python/cudf_polars/cudf_polars/dsl/translate.py +++ b/python/cudf_polars/cudf_polars/dsl/translate.py @@ -33,13 +33,7 @@ from cudf_polars.typing import Schema from cudf_polars.utils import config, sorting from cudf_polars.utils.versions import ( - POLARS_VERSION_LT_131, - POLARS_VERSION_LT_132, - POLARS_VERSION_LT_133, - POLARS_VERSION_LT_134, - POLARS_VERSION_LT_136, - POLARS_VERSION_LT_138, - POLARS_VERSION_LT_1323, + POLARS_VERSION_LT_139, ) if TYPE_CHECKING: @@ -130,7 +124,7 @@ def translate_ir(self, *, n: int | None = None) -> ir.IR: # IR is versioned with major.minor, minor is bumped for backwards # compatible changes (e.g. adding new nodes), major is bumped for # incompatible changes (e.g. renaming nodes). - if (version := self.visitor.version()) >= (12, 1): + if (version := self.visitor.version()) >= (12, 2): e = NotImplementedError( f"No support for polars IR {version=}" ) # pragma: no cover; no such version for now. @@ -252,6 +246,17 @@ def __exit__(self, *args: Any) -> None: self.translator._expr_context = self._prev +def _is_dynamic_pred(visitor: Any, expr_ir: Any) -> bool: + try: + visitor.view_expression(expr_ir.node) + except Exception as e: + if str(e) == "dynamic_pred": + return True + raise + else: + return False + + @singledispatch def _translate_ir(node: Any, translator: Translator, schema: Schema) -> ir.IR: raise NotImplementedError( @@ -290,12 +295,11 @@ def _(node: plrs._ir_nodes.Scan, translator: Translator, schema: Schema) -> ir.I with_columns = file_options.with_columns row_index = file_options.row_index include_file_paths = file_options.include_file_paths - if not POLARS_VERSION_LT_131: - deletion_files = file_options.deletion_files # pragma: no cover - if deletion_files: # pragma: no cover - raise NotImplementedError( - "Iceberg format is not supported in cudf-polars. Furthermore, row-level deletions are not supported." - ) # pragma: no cover + deletion_files = file_options.deletion_files + if deletion_files: # pragma: no cover + raise NotImplementedError( + "Iceberg format is not supported in cudf-polars. Furthermore, row-level deletions are not supported." + ) # pragma: no cover config_options = translator.config_options parquet_options = config_options.parquet_options @@ -332,19 +336,19 @@ def _(node: plrs._ir_nodes.Scan, translator: Translator, schema: Schema) -> ir.I n_rows, row_index, include_file_paths, - translate_named_expr(translator, n=node.predicate, schema=schema) - if node.predicate is not None - else None, + ( + None + if node.predicate is None + or _is_dynamic_pred(translator.visitor, node.predicate) + else translate_named_expr(translator, n=node.predicate, schema=schema) + ), parquet_options, ) @_translate_ir.register def _(node: plrs._ir_nodes.Cache, translator: Translator, schema: Schema) -> ir.IR: - if POLARS_VERSION_LT_1323: # pragma: no cover - refcount = node.cache_hits - else: - refcount = None + refcount = None # Make sure Cache nodes with the same id_ # are actually the same object. @@ -548,6 +552,8 @@ def _(node: plrs._ir_nodes.Slice, translator: Translator, schema: Schema) -> ir. def _(node: plrs._ir_nodes.Filter, translator: Translator, schema: Schema) -> ir.IR: with set_node(translator.visitor, node.input): inp = translator.translate_ir(n=None) + if _is_dynamic_pred(translator.visitor, node.predicate): + return inp mask = translate_named_expr(translator, n=node.predicate, schema=inp.schema) return ir.Filter(schema, mask, inp) @@ -608,9 +614,7 @@ def _(node: plrs._ir_nodes.Sink, translator: Translator, schema: Schema) -> ir.I payload = json.loads(node.payload) try: file = payload["File"] - sink_kind_options = file[ - "file_type" if POLARS_VERSION_LT_136 else "file_format" - ] + sink_kind_options = file["file_format"] except KeyError as err: # pragma: no cover raise NotImplementedError("Unsupported payload structure") from err if isinstance(sink_kind_options, dict): @@ -622,19 +626,13 @@ def _(node: plrs._ir_nodes.Sink, translator: Translator, schema: Schema) -> ir.I "Unsupported sink options structure" ) # pragma: no cover - if POLARS_VERSION_LT_138: # pragma: no cover - sink_options = file.get("sink_options", {}) - cloud_options = file.get("cloud_options") - options = format_options.copy() - options.update(sink_options) - else: - unified_args = file.get("unified_sink_args", {}) - cloud_options = unified_args.get("cloud_options") - options = {} if sink_kind == "NDJson" else format_options.copy() + unified_args = file.get("unified_sink_args", {}) + cloud_options = unified_args.get("cloud_options") + options = {} if sink_kind == "NDJson" else format_options.copy() - for k, v in unified_args.items(): - if k in {"mkdir", "maintain_order", "sync_on_close"}: - options[k] = v + for k, v in unified_args.items(): + if k in {"mkdir", "maintain_order", "sync_on_close"}: + options[k] = v if sink_kind in ("Csv", "NDJson"): compression = format_options.get("compression") @@ -643,12 +641,7 @@ def _(node: plrs._ir_nodes.Sink, translator: Translator, schema: Schema) -> ir.I f"{sink_kind} compression ('{compression}') is not supported." ) - if POLARS_VERSION_LT_132: # pragma: no cover - path = file["target"] - elif POLARS_VERSION_LT_138: # pragma: no cover - path = file["target"]["Local"] - else: - path = file["target"]["inner"] + path = file["target"]["inner"] return ir.Sink( schema=schema, @@ -796,9 +789,7 @@ def _( if name in needs_cast: return expr.Cast(dtype, True, result_expr) # noqa: FBT003 return result_expr - elif not POLARS_VERSION_LT_131 and isinstance( - name, plrs._expr_nodes.StructFunction - ): + elif isinstance(name, plrs._expr_nodes.StructFunction): return expr.StructFunction( dtype, expr.StructFunction.Name.from_polars(name), @@ -808,37 +799,17 @@ def _( elif isinstance(name, str): children = (translator.translate_expr(n=n, schema=schema) for n in node.input) if name == "log" or ( - not POLARS_VERSION_LT_133 - and name == "l" + name == "l" and isinstance(options[0], str) and "".join((name, *options)) == "log" ): - if POLARS_VERSION_LT_133: # pragma: no cover - (base,) = options - (child,) = children - return expr.BinOp( - dtype, - plc.binaryop.BinaryOperator.LOG_BASE, - child, - expr.Literal(dtype, base), - ) - else: - (child, base) = children - res = expr.BinOp( - dtype, - plc.binaryop.BinaryOperator.LOG_BASE, - child, - expr.Literal(dtype, base.value), - ) - return ( - res - if not POLARS_VERSION_LT_134 - else expr.Cast( - DataType(pl.Float64()), - True, # noqa: FBT003 - res, - ) - ) + (child, base) = children + return expr.BinOp( + dtype, + plc.binaryop.BinaryOperator.LOG_BASE, + child, + expr.Literal(dtype, base.value), # type: ignore[attr-defined] + ) elif name == "pow": return expr.BinOp(dtype, plc.binaryop.BinaryOperator.POW, *children) return expr.UnaryFunction(dtype, name, options, *children) @@ -854,55 +825,7 @@ def _( dtype: DataType, schema: Schema, ) -> expr.Expr: - if isinstance( - node.options, plrs._expr_nodes.RollingGroupOptions - ): # pragma: no cover; polars gives Aexpr::Rolling node now - # TODO: As of polars 1.36.0, rolling is represented as a Rolling expression node - # but is currently not implemented in the python node visitor. Once we support it, - # we should move this branch to separate translator dispatch function for the new node. - - # pl.col("a").rolling(...) - with set_expr_context(translator, ExecutionContext.ROLLING): - agg = translator.translate_expr(n=node.function, schema=schema) - name_generator = unique_names(schema) - aggs, named_post_agg = decompose_single_agg( - expr.NamedExpr(next(name_generator), agg), - name_generator, - is_top=True, - context=ExecutionContext.ROLLING, - ) - named_aggs = [agg for agg, _ in aggs] - orderby = node.options.index_column - orderby_dtype = schema[orderby].plc_type - if plc.traits.is_integral(orderby_dtype): - # Integer orderby column is cast in implementation to int64 in polars - orderby_dtype = plc.DataType(plc.TypeId.INT64) - closed_window = node.options.closed_window - if isinstance(named_post_agg.value, expr.Col): - (named_agg,) = named_aggs - return expr.RollingWindow( - named_agg.value.dtype, - orderby_dtype, - node.options.offset, - node.options.period, - closed_window, - orderby, - named_agg.value, - ) - replacements: dict[expr.Expr, expr.Expr] = { - expr.Col(agg.value.dtype, agg.name): expr.RollingWindow( - agg.value.dtype, - orderby_dtype, - node.options.offset, - node.options.period, - closed_window, - orderby, - agg.value, - ) - for agg in named_aggs - } - return replace([named_post_agg.value], replacements)[0] - elif isinstance(node.options, plrs._expr_nodes.WindowMapping): + if isinstance(node.options, plrs._expr_nodes.WindowMapping): # pl.col("a").over(...) with set_expr_context(translator, ExecutionContext.WINDOW): agg = translator.translate_expr(n=node.function, schema=schema) @@ -959,6 +882,56 @@ def _( assert_never(node.options) +if not POLARS_VERSION_LT_139: # type: ignore[misc] + + @_translate_expr.register + def _( + node: plrs._expr_nodes.Rolling, + translator: Translator, + dtype: DataType, + schema: Schema, + ) -> expr.Expr: + with set_expr_context(translator, ExecutionContext.ROLLING): + agg_expr = translator.translate_expr(n=node.function, schema=schema) + orderby = translator.visitor.view_expression(node.index_column).name + orderby_dtype = schema[orderby].plc_type + if plc.traits.is_integral(orderby_dtype): + orderby_dtype = plc.DataType(plc.TypeId.INT64) + name_generator = unique_names(schema) + aggs, named_post_agg = decompose_single_agg( + expr.NamedExpr(next(name_generator), agg_expr), + name_generator, + is_top=True, + context=ExecutionContext.ROLLING, + ) + named_aggs = [a for a, _ in aggs] + closed_window = node.closed_window + if isinstance(named_post_agg.value, expr.Col): + (named_agg,) = named_aggs + return expr.RollingWindow( + named_agg.value.dtype, + orderby_dtype, + node.offset, + node.period, + closed_window, + orderby, + named_agg.value, + ) + replacements: dict[expr.Expr, expr.Expr] = { + expr.Col(a.value.dtype, a.name): expr.RollingWindow( + a.value.dtype, + orderby_dtype, + node.offset, + node.period, + closed_window, + orderby, + a.value, + ) + for a in named_aggs + } + return replace([named_post_agg.value], replacements)[0] + + @_translate_expr.register def _( node: plrs._expr_nodes.Literal, @@ -1140,12 +1113,6 @@ def _( ) -> expr.Expr: left = translator.translate_expr(n=node.left, schema=schema) right = translator.translate_expr(n=node.right, schema=schema) - if ( - POLARS_VERSION_LT_133 - and plc.traits.is_boolean(dtype.plc_type) - and node.op == plrs._expr_nodes.Operator.TrueDivide - ): - dtype = DataType(pl.Float64()) # pragma: no cover if node.op == plrs._expr_nodes.Operator.TrueDivide and ( plc.traits.is_fixed_point(left.dtype.plc_type) or plc.traits.is_fixed_point(right.dtype.plc_type) @@ -1163,8 +1130,7 @@ def _( ) if ( - not POLARS_VERSION_LT_134 - and node.op == plrs._expr_nodes.Operator.Multiply + node.op == plrs._expr_nodes.Operator.Multiply and plc.traits.is_fixed_point(left.dtype.plc_type) and plc.traits.is_fixed_point(right.dtype.plc_type) ): diff --git a/python/cudf_polars/cudf_polars/dsl/utils/aggregations.py b/python/cudf_polars/cudf_polars/dsl/utils/aggregations.py index 437d844881c5..4f96f30b6175 100644 --- a/python/cudf_polars/cudf_polars/dsl/utils/aggregations.py +++ b/python/cudf_polars/cudf_polars/dsl/utils/aggregations.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. # SPDX-License-Identifier: Apache-2.0 """Utilities for rewriting aggregations.""" @@ -17,7 +17,6 @@ from cudf_polars.containers import DataType from cudf_polars.dsl import expr, ir from cudf_polars.dsl.expressions.base import ExecutionContext -from cudf_polars.utils.versions import POLARS_VERSION_LT_134, POLARS_VERSION_LT_1323 if TYPE_CHECKING: from collections.abc import Callable, Generator, Iterable, Sequence @@ -191,7 +190,7 @@ def decompose_single_agg( # mean/median on decimal: Polars returns float -> pre-cast decimal_unsupported = False if plc.traits.is_fixed_point(child_dtype): - cast_for_quantile = is_quantile and not POLARS_VERSION_LT_134 + cast_for_quantile = is_quantile cast_for_mean_or_median = ( agg.name in {"mean", "median"} ) and plc.traits.is_floating_point(agg.dtype.plc_type) @@ -205,8 +204,6 @@ def decompose_single_agg( child, ) child_dtype = child.dtype.plc_type - elif is_quantile and POLARS_VERSION_LT_134: # pragma: no cover - decimal_unsupported = True is_group_quantile_supported = plc.traits.is_integral( child_dtype @@ -241,46 +238,14 @@ def decompose_single_agg( ) else expr.Col(agg.dtype, name) ) - # Polars semantics for sum differ by context: + # Polars semantics for sum: # - GROUPBY: sum(all-null group) => 0; sum(empty group) => 0 (fill-null) - # - ROLLING: sum(all-null window) => null; sum(empty window) => 0 (fill only if empty) # # Must post-process because libcudf returns null for both empty and all-null windows/groups - if not POLARS_VERSION_LT_1323 or context in { - ExecutionContext.GROUPBY, - ExecutionContext.WINDOW, - }: - # GROUPBY: always fill top-level nulls with 0 - return [(named_expr, True)], expr.NamedExpr( - name, replace_nulls(col, 0, is_top=is_top) - ) - else: # pragma: no cover - # ROLLING: - # Add a second rolling agg to compute the window size, then only - # replace nulls with 0 when the window size is 0 (ie. empty window). - win_len_name = next(name_generator) - win_len = expr.NamedExpr( - win_len_name, - expr.Len(DataType(pl.Int32())), - ) - - win_len_col = expr.Col(DataType(pl.Int32()), win_len_name) - win_len_filled = replace_nulls(win_len_col, 0, is_top=True) - - is_empty = expr.BinOp( - DataType(pl.Boolean()), - plc.binaryop.BinaryOperator.EQUAL, - win_len_filled, - expr.Literal(DataType(pl.Int32()), 0), - ) - - # If empty -> fill 0; else keep libcudf's semantics for all-null windows. - filled = replace_nulls(col, 0, is_top=is_top) - post_ternary_expr = expr.Ternary(agg.dtype, is_empty, filled, col) - - return [(named_expr, True), (win_len, True)], expr.NamedExpr( - name, post_ternary_expr - ) + # Always fill top-level nulls with 0 + return [(named_expr, True)], expr.NamedExpr( + name, replace_nulls(col, 0, is_top=is_top) + ) elif agg.name in {"mean", "median", "quantile", "std", "var"}: post_agg_col: expr.Expr = expr.Col( DataType(pl.Float64()), name diff --git a/python/cudf_polars/cudf_polars/experimental/benchmarks/asserts.py b/python/cudf_polars/cudf_polars/experimental/benchmarks/asserts.py index 3f3741e84a5b..7414dbf796e4 100644 --- a/python/cudf_polars/cudf_polars/experimental/benchmarks/asserts.py +++ b/python/cudf_polars/cudf_polars/experimental/benchmarks/asserts.py @@ -10,8 +10,6 @@ import polars as pl import polars.testing -from cudf_polars.utils.versions import POLARS_VERSION_LT_1323 - if TYPE_CHECKING: from typing import Any @@ -163,10 +161,7 @@ def assert_tpch_result_equal( "categorical_as_str": categorical_as_str, } - if POLARS_VERSION_LT_1323: # pragma: no cover - tol_kwargs = {"rtol": rel_tol, "atol": abs_tol} - else: - tol_kwargs = {"rel_tol": rel_tol, "abs_tol": abs_tol} + tol_kwargs = {"rel_tol": rel_tol, "abs_tol": abs_tol} polars_kwargs.update(tol_kwargs) if left.columns != right.columns: diff --git a/python/cudf_polars/cudf_polars/experimental/io.py b/python/cudf_polars/cudf_polars/experimental/io.py index 2b34358810d7..9486eb6b9772 100644 --- a/python/cudf_polars/cudf_polars/experimental/io.py +++ b/python/cudf_polars/cudf_polars/experimental/io.py @@ -35,7 +35,6 @@ ) from cudf_polars.experimental.dispatch import generate_ir_tasks, lower_ir_node from cudf_polars.utils.cuda_stream import get_cuda_stream -from cudf_polars.utils.versions import POLARS_VERSION_LT_137 if TYPE_CHECKING: from collections.abc import Hashable, MutableMapping @@ -515,7 +514,7 @@ def _sink_to_file( # Path.open returns IO[Any] but SinkInfo needs more specific IO types sink = plc.io.types.SinkInfo([f]) # type: ignore[arg-type] Sink._write_csv(sink, use_options, df) - elif kind == "Json" if POLARS_VERSION_LT_137 else "NDJson": + elif kind == "NDJson": mode = "wb" if writer_state is None else "ab" with Path.open(Path(path), mode) as f: # Path.open returns IO[Any] but SinkInfo needs more specific IO types diff --git a/python/cudf_polars/cudf_polars/testing/asserts.py b/python/cudf_polars/cudf_polars/testing/asserts.py index 413205ec6e51..5a0d06864fff 100644 --- a/python/cudf_polars/cudf_polars/testing/asserts.py +++ b/python/cudf_polars/cudf_polars/testing/asserts.py @@ -14,7 +14,6 @@ from cudf_polars.dsl.translate import Translator from cudf_polars.utils.config import ConfigOptions, StreamingFallbackMode -from cudf_polars.utils.versions import POLARS_VERSION_LT_1323 if TYPE_CHECKING: from cudf_polars.typing import OptimizationArgs @@ -137,10 +136,7 @@ def assert_gpu_result_equal( } tol_kwargs: dict[str, float] - if POLARS_VERSION_LT_1323: # pragma: no cover - tol_kwargs = {"rtol": rtol, "atol": atol} - else: - tol_kwargs = {"rel_tol": rtol, "abs_tol": atol} + tol_kwargs = {"rel_tol": rtol, "abs_tol": atol} # the type checker errors with: # Argument 4 to "assert_frame_equal" has incompatible type "**dict[str, float]"; expected "bool" [arg-type] diff --git a/python/cudf_polars/cudf_polars/utils/versions.py b/python/cudf_polars/cudf_polars/utils/versions.py index e6be1027b0ca..92d0b4eec203 100644 --- a/python/cudf_polars/cudf_polars/utils/versions.py +++ b/python/cudf_polars/cudf_polars/utils/versions.py @@ -11,17 +11,8 @@ from polars import __version__ POLARS_VERSION = parse(__version__) -POLARS_LOWER_BOUND = parse("1.30") -POLARS_VERSION_LT_131 = POLARS_VERSION < parse("1.31") -POLARS_VERSION_LT_132 = POLARS_VERSION < parse("1.32") -POLARS_VERSION_LT_1321 = POLARS_VERSION < parse("1.32.1") -POLARS_VERSION_LT_1323 = POLARS_VERSION < parse("1.32.3") -POLARS_VERSION_LT_133 = POLARS_VERSION < parse("1.33.0") -POLARS_VERSION_LT_134 = POLARS_VERSION < parse("1.34.0") -POLARS_VERSION_LT_135 = POLARS_VERSION < parse("1.35.0") -POLARS_VERSION_LT_136 = POLARS_VERSION < parse("1.36.0") -POLARS_VERSION_LT_137 = POLARS_VERSION < parse("1.37.0") -POLARS_VERSION_LT_138 = POLARS_VERSION < parse("1.38.0") +POLARS_LOWER_BOUND = parse("1.38") +POLARS_VERSION_LT_139 = POLARS_VERSION < parse("1.39.0") def _ensure_polars_version() -> None: diff --git a/python/cudf_polars/pyproject.toml b/python/cudf_polars/pyproject.toml index 00941147c631..1fc2cbeadad2 100644 --- a/python/cudf_polars/pyproject.toml +++ b/python/cudf_polars/pyproject.toml @@ -24,7 +24,7 @@ dependencies = [ "cuda-python>=13.0.1,<14.0", "nvidia-ml-py>=12", "packaging", - "polars>=1.30,<1.39", + "polars>=1.38,<1.40", "pylibcudf==26.6.*,>=0.0.0a0", "typing_extensions>=4.0.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/cudf_polars/tests/dsl/test_serialization.py b/python/cudf_polars/tests/dsl/test_serialization.py index a496a1e6a870..8184092a51a8 100644 --- a/python/cudf_polars/tests/dsl/test_serialization.py +++ b/python/cudf_polars/tests/dsl/test_serialization.py @@ -12,21 +12,11 @@ from cudf_polars.dsl.expressions.boolean import BooleanFunction from cudf_polars.dsl.expressions.datetime import TemporalFunction from cudf_polars.dsl.expressions.string import StringFunction -from cudf_polars.utils.versions import ( - POLARS_VERSION_LT_131, - POLARS_VERSION_LT_132, - POLARS_VERSION_LT_138, - POLARS_VERSION_LT_1321, -) - -if not POLARS_VERSION_LT_131: - from cudf_polars.dsl.expressions.struct import StructFunction +from cudf_polars.dsl.expressions.struct import StructFunction @pytest.fixture( - params=[BooleanFunction, StringFunction, TemporalFunction] - if POLARS_VERSION_LT_131 - else [BooleanFunction, StringFunction, TemporalFunction, StructFunction] + params=[BooleanFunction, StringFunction, TemporalFunction, StructFunction] ) def function(request): return request.param @@ -53,32 +43,18 @@ def test_from_polars_all_names(function): # Check names advertised by polars are the same as we advertise polars_names_set = set(polars_names) cudf_polars_names_set = set(function.Name.__members__) - if not POLARS_VERSION_LT_132 and function == StructFunction: + if function == StructFunction: cudf_polars_names_set = cudf_polars_names_set - { "FieldByIndex", "MultipleFields", } - if POLARS_VERSION_LT_1321 and function == TemporalFunction: - cudf_polars_names_set = cudf_polars_names_set - { - "DaysInMonth", - } - if POLARS_VERSION_LT_132 and function == BooleanFunction: - cudf_polars_names_set = cudf_polars_names_set - {"IsClose"} - if POLARS_VERSION_LT_138 and function == StringFunction: - cudf_polars_names_set = cudf_polars_names_set - {"SplitRegex"} assert polars_names_set == cudf_polars_names_set names = function.Name - if not POLARS_VERSION_LT_132 and function == StructFunction: + if function == StructFunction: names = set(names) - { StructFunction.Name.FieldByIndex, StructFunction.Name.MultipleFields, } - if POLARS_VERSION_LT_1321 and function == TemporalFunction: - names = set(names) - {TemporalFunction.Name.DaysInMonth} - if POLARS_VERSION_LT_132 and function == BooleanFunction: - names = set(names) - {BooleanFunction.Name.IsClose} - if POLARS_VERSION_LT_138 and function == StringFunction: - names = set(names) - {StringFunction.Name.SplitRegex} for name in names: attr = getattr(polars_function, name.name) assert function.Name.from_polars(attr) == name diff --git a/python/cudf_polars/tests/experimental/test_rolling.py b/python/cudf_polars/tests/experimental/test_rolling.py index 7fa03bee2bd3..398c83817017 100644 --- a/python/cudf_polars/tests/experimental/test_rolling.py +++ b/python/cudf_polars/tests/experimental/test_rolling.py @@ -11,7 +11,7 @@ from cudf_polars.experimental.rapidsmpf.frontend.spmd import SPMDEngine from cudf_polars.testing.asserts import assert_gpu_result_equal -from cudf_polars.utils.versions import POLARS_VERSION_LT_136 +from cudf_polars.utils.versions import POLARS_VERSION_LT_139 if TYPE_CHECKING: from collections.abc import Generator @@ -34,11 +34,11 @@ def engine( yield engine -def test_rolling_datetime(request, engine): - if not POLARS_VERSION_LT_136: - request.applymarker( - pytest.mark.xfail(reason="See https://github.com/pola-rs/polars/pull/25117") - ) +@pytest.mark.xfail( + condition=POLARS_VERSION_LT_139, + reason="AExpr::Rolling Python API requires polars >= 1.39", +) +def test_rolling_datetime(engine): dates = [ "2020-01-01 13:45:48", "2020-01-01 16:42:13", diff --git a/python/cudf_polars/tests/experimental/test_select.py b/python/cudf_polars/tests/experimental/test_select.py index 115ad0483d4d..0c6495e3d1f2 100644 --- a/python/cudf_polars/tests/experimental/test_select.py +++ b/python/cudf_polars/tests/experimental/test_select.py @@ -14,11 +14,6 @@ from cudf_polars.experimental.rapidsmpf.frontend.spmd import SPMDEngine from cudf_polars.testing.asserts import ( assert_gpu_result_equal, - assert_ir_translation_raises, -) -from cudf_polars.utils.versions import ( - POLARS_VERSION_LT_132, - POLARS_VERSION_LT_134, ) if TYPE_CHECKING: @@ -100,14 +95,11 @@ def test_select_reduce_fallback(df, fallback_mode, engine): def test_select_fill_null_with_strategy(df, engine): q = df.select(pl.col("a").forward_fill()) - if POLARS_VERSION_LT_132: - assert_ir_translation_raises(q, NotImplementedError) - else: - with pytest.warns( - UserWarning, - match="fill_null with strategy other than 'zero' or 'one' is not supported for multiple partitions", - ): - assert_gpu_result_equal(q, engine=engine) + with pytest.warns( + UserWarning, + match="fill_null with strategy other than 'zero' or 'one' is not supported for multiple partitions", + ): + assert_gpu_result_equal(q, engine=engine) @pytest.mark.parametrize( @@ -186,13 +178,13 @@ def test_select_with_empty_partitions(engine): q = df.select(pl.col("b").sum() / Decimal("7.00")) # Polars pre their decimal overhaul: https://github.com/pola-rs/polars/issues/19784 # returned a different precision and scale, so we skip dtype check - assert_gpu_result_equal(q, engine=engine, check_dtypes=not POLARS_VERSION_LT_134) + assert_gpu_result_equal(q, engine=engine, check_dtypes=True) def test_select_mean_with_decimals(engine): df = pl.LazyFrame({"d": [Decimal("1.23")] * 4}) q = df.select(pl.mean("d")) - assert_gpu_result_equal(q, engine=engine, check_dtypes=not POLARS_VERSION_LT_134) + assert_gpu_result_equal(q, engine=engine, check_dtypes=True) def test_select_with_len(engine): diff --git a/python/cudf_polars/tests/expressions/test_agg.py b/python/cudf_polars/tests/expressions/test_agg.py index e10e1969f9b6..4b9c9c66e2d9 100644 --- a/python/cudf_polars/tests/expressions/test_agg.py +++ b/python/cudf_polars/tests/expressions/test_agg.py @@ -14,11 +14,6 @@ assert_gpu_result_equal, assert_ir_translation_raises, ) -from cudf_polars.utils.versions import ( - POLARS_VERSION_LT_134, - POLARS_VERSION_LT_135, - POLARS_VERSION_LT_136, -) @pytest.fixture( @@ -59,7 +54,7 @@ def is_sorted(request): @pytest.fixture def xfail_if_sorted_gt_135(is_sorted, request): # See https://github.com/rapidsai/cudf/pull/20791#issuecomment-3750528419 - if is_sorted and not POLARS_VERSION_LT_135: + if is_sorted: request.applymarker( pytest.mark.xfail(reason="See https://github.com/pola-rs/polars/pull/24981") ) @@ -219,10 +214,6 @@ def test_decimal_quantile(decimal_df, interp): assert_gpu_result_equal(q) -@pytest.mark.skipif( - POLARS_VERSION_LT_134, - reason="std/var on decimal not supported before polars 1.34", -) def test_decimal_std_var(decimal_df): q = decimal_df.select( std=pl.col("a").std(), @@ -232,12 +223,7 @@ def test_decimal_std_var(decimal_df): def test_invalid_agg(request): - request.applymarker( - pytest.mark.xfail( - condition=not POLARS_VERSION_LT_136, - reason="polars raises now", - ) - ) + request.applymarker(pytest.mark.xfail(reason="polars raises now")) df = pl.LazyFrame({"s": pl.Series(["a", "b", "c"], dtype=pl.String())}) q = df.select(pl.col("s").sum()) assert_ir_translation_raises(q, NotImplementedError) diff --git a/python/cudf_polars/tests/expressions/test_booleanfunction.py b/python/cudf_polars/tests/expressions/test_booleanfunction.py index 5344068e886a..bc9791806fb5 100644 --- a/python/cudf_polars/tests/expressions/test_booleanfunction.py +++ b/python/cudf_polars/tests/expressions/test_booleanfunction.py @@ -12,7 +12,6 @@ assert_gpu_result_equal, assert_ir_translation_raises, ) -from cudf_polars.utils.versions import POLARS_VERSION_LT_132 if TYPE_CHECKING: from collections.abc import Callable @@ -266,12 +265,7 @@ def test_is_in_shape_mismatch_raises(needles, haystack): assert_ir_translation_raises(q, NotImplementedError) -def test_boolean_is_close(request): - request.applymarker( - pytest.mark.xfail( - condition=POLARS_VERSION_LT_132, reason="Not supported until polars 1.32" - ) - ) +def test_boolean_is_close(): ldf = pl.LazyFrame({"a": [1.0, 1.2, 1.4, 1.45, 1.6]}) q = ldf.select(pl.col("a").is_close(1.4, abs_tol=0.1)) diff --git a/python/cudf_polars/tests/expressions/test_casting.py b/python/cudf_polars/tests/expressions/test_casting.py index 9fedf3e3658c..23ec8c4daf9c 100644 --- a/python/cudf_polars/tests/expressions/test_casting.py +++ b/python/cudf_polars/tests/expressions/test_casting.py @@ -13,7 +13,6 @@ assert_gpu_result_equal, assert_ir_translation_raises, ) -from cudf_polars.utils.versions import POLARS_VERSION_LT_132 _supported_dtypes = [(pl.Int8(), pl.Int64())] @@ -101,4 +100,4 @@ def test_float_to_decimal_rounding(): } ) q = df.select(pl.col("foo") / pl.col("bar")) - assert_gpu_result_equal(q, check_dtypes=not POLARS_VERSION_LT_132) + assert_gpu_result_equal(q, check_dtypes=True) diff --git a/python/cudf_polars/tests/expressions/test_numeric_binops.py b/python/cudf_polars/tests/expressions/test_numeric_binops.py index e499583d28fe..487011e48515 100644 --- a/python/cudf_polars/tests/expressions/test_numeric_binops.py +++ b/python/cudf_polars/tests/expressions/test_numeric_binops.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations @@ -11,7 +11,6 @@ from cudf_polars.testing.asserts import ( assert_gpu_result_equal, ) -from cudf_polars.utils.versions import POLARS_VERSION_LT_132 dtypes = [ pl.Int8, @@ -120,7 +119,7 @@ def test_true_div_with_decimals(): schema={"foo": pl.Decimal(15, 2), "bar": pl.Decimal(15, 2)}, ) q = df.select(pl.col("bar") / pl.col("foo")) - assert_gpu_result_equal(q, check_dtypes=not POLARS_VERSION_LT_132) + assert_gpu_result_equal(q, check_dtypes=True) def test_multiply_with_decimals(): @@ -133,4 +132,4 @@ def test_multiply_with_decimals(): ) q = df.select(pl.col("x") * pl.col("y")) - assert_gpu_result_equal(q, check_dtypes=not POLARS_VERSION_LT_132) + assert_gpu_result_equal(q, check_dtypes=True) diff --git a/python/cudf_polars/tests/expressions/test_numeric_unaryops.py b/python/cudf_polars/tests/expressions/test_numeric_unaryops.py index 3cfbf0906c93..4b0e9f114ce5 100644 --- a/python/cudf_polars/tests/expressions/test_numeric_unaryops.py +++ b/python/cudf_polars/tests/expressions/test_numeric_unaryops.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations @@ -13,7 +13,6 @@ assert_gpu_result_equal, assert_ir_translation_raises, ) -from cudf_polars.utils.versions import POLARS_VERSION_LT_132 if TYPE_CHECKING: from cudf_polars.typing import RankMethod, RoundMethod @@ -127,9 +126,6 @@ def test_null_count(): def test_rank_supported( request, ldf: pl.LazyFrame, method: RankMethod, *, descending: bool ): - request.applymarker( - pytest.mark.xfail(condition=POLARS_VERSION_LT_132, reason="rank unsupported") - ) expr = pl.col("a").rank(method=method, descending=descending) q = ldf.select(expr) assert_gpu_result_equal(q) @@ -141,10 +137,6 @@ def test_rank_supported( def test_rank_methods_with_nulls_or_ties( request, ldf: pl.LazyFrame, method: RankMethod, *, descending: bool, test: str ) -> None: - request.applymarker( - pytest.mark.xfail(condition=POLARS_VERSION_LT_132, reason="rank unsupported") - ) - base = pl.col("a") if test == "with_nulls": expr = pl.when((base % 2) == 0).then(None).otherwise(base) diff --git a/python/cudf_polars/tests/expressions/test_rolling.py b/python/cudf_polars/tests/expressions/test_rolling.py index 4b40178e8bfd..49b5730ef998 100644 --- a/python/cudf_polars/tests/expressions/test_rolling.py +++ b/python/cudf_polars/tests/expressions/test_rolling.py @@ -13,7 +13,7 @@ assert_gpu_result_equal, assert_ir_translation_raises, ) -from cudf_polars.utils.versions import POLARS_VERSION_LT_132, POLARS_VERSION_LT_136 +from cudf_polars.utils.versions import POLARS_VERSION_LT_139 if TYPE_CHECKING: from cudf_polars.typing import RankMethod @@ -32,14 +32,12 @@ def df(): ) +@pytest.mark.xfail( + condition=POLARS_VERSION_LT_139, + reason="AExpr::Rolling Python API requires polars >= 1.39", +) @pytest.mark.parametrize("time_unit", ["ns", "us", "ms"]) -def test_rolling_datetime(request, time_unit): - request.applymarker( - pytest.mark.xfail( - condition=not POLARS_VERSION_LT_136, - reason="Polars translates this to AExpr::Rolling which is NotImplemented", - ) - ) +def test_rolling_datetime(time_unit): dates = [ "2020-01-01 13:45:48", "2020-01-01 16:42:13", @@ -62,11 +60,11 @@ def test_rolling_datetime(request, time_unit): assert_gpu_result_equal(q) -def test_rolling_date(request): - if not POLARS_VERSION_LT_136: - request.applymarker( - pytest.mark.xfail(reason="See https://github.com/pola-rs/polars/pull/25117") - ) +@pytest.mark.xfail( + condition=POLARS_VERSION_LT_139, + reason="AExpr::Rolling Python API requires polars >= 1.39", +) +def test_rolling_date(): dates = [ "2020-01-01", "2020-01-01", @@ -87,12 +85,12 @@ def test_rolling_date(request): assert_gpu_result_equal(q) +@pytest.mark.xfail( + condition=POLARS_VERSION_LT_139, + reason="AExpr::Rolling Python API requires polars >= 1.39", +) @pytest.mark.parametrize("dtype", [pl.Int32, pl.UInt32, pl.Int64, pl.UInt64]) -def test_rolling_integral_orderby(request, dtype): - if not POLARS_VERSION_LT_136: - request.applymarker( - pytest.mark.xfail(reason="See https://github.com/pola-rs/polars/pull/25117") - ) +def test_rolling_integral_orderby(dtype): df = pl.LazyFrame( { "orderby": pl.Series([1, 4, 8, 10, 12, 13, 14, 22], dtype=dtype), @@ -106,14 +104,14 @@ def test_rolling_integral_orderby(request, dtype): assert_gpu_result_equal(q) -@pytest.mark.skipif( - POLARS_VERSION_LT_136, - reason="Rolling expression node only exists in polars >= 1.36", +@pytest.mark.xfail( + condition=POLARS_VERSION_LT_139, + reason="AExpr::Rolling Python API requires polars >= 1.39", ) -def test_rolling_translation_raises(): +def test_rolling_agg_first(): df = pl.LazyFrame({"a": [1, 2, 3], "b": [1, 2, 3]}) q = df.with_columns(pl.col("a").sum().rolling("b", period="2i")) - assert_ir_translation_raises(q, NotImplementedError) + assert_gpu_result_equal(q) def test_rolling_collect_list_raises(): @@ -129,11 +127,11 @@ def test_rolling_collect_list_raises(): ) -def test_unsorted_raises(request): - if not POLARS_VERSION_LT_136: - request.applymarker( - pytest.mark.xfail(reason="See https://github.com/pola-rs/polars/pull/25117") - ) +@pytest.mark.xfail( + condition=POLARS_VERSION_LT_139, + reason="AExpr::Rolling Python API requires polars >= 1.39", +) +def test_unsorted_raises(): df = pl.LazyFrame({"orderby": [1, 2, 4, 2], "values": [1, 2, 3, 4]}) q = df.select(pl.col("values").sum().rolling("orderby", period="2i")) with pytest.raises(pl.exceptions.InvalidOperationError): @@ -145,11 +143,11 @@ def test_unsorted_raises(request): q.collect(engine=pl.GPUEngine(raise_on_fail=True)) -def test_orderby_nulls_raises_computeerror(request): - if not POLARS_VERSION_LT_136: - request.applymarker( - pytest.mark.xfail(reason="See https://github.com/pola-rs/polars/pull/25117") - ) +@pytest.mark.xfail( + condition=POLARS_VERSION_LT_139, + reason="AExpr::Rolling Python API requires polars >= 1.39", +) +def test_orderby_nulls_raises_computeerror(): df = pl.LazyFrame({"orderby": [1, 2, 4, None], "values": [1, 2, 3, 4]}) q = df.select(pl.col("values").sum().rolling("orderby", period="2i")) with pytest.raises(pl.exceptions.InvalidOperationError): @@ -160,23 +158,18 @@ def test_orderby_nulls_raises_computeerror(request): q.collect(engine=pl.GPUEngine(raise_on_fail=True)) -def test_invalid_duration_spec_raises_in_translation(request): - if not POLARS_VERSION_LT_136: - request.applymarker( - pytest.mark.xfail(reason="See https://github.com/pola-rs/polars/pull/25117") - ) +@pytest.mark.xfail( + condition=POLARS_VERSION_LT_139, + reason="AExpr::Rolling Python API requires polars >= 1.39", +) +def test_invalid_duration_spec_raises_in_translation(): df = pl.LazyFrame({"orderby": [1, 2, 4, 5], "values": [1, 2, 3, 4]}) q = df.select(pl.col("values").sum().rolling("orderby", period="3d")) assert_ir_translation_raises(q, pl.exceptions.InvalidOperationError) def test_rolling_inside_groupby_raises(request): - request.applymarker( - pytest.mark.xfail( - condition=not POLARS_VERSION_LT_136, - reason="not supported as of polars 1.36", - ) - ) + request.applymarker(pytest.mark.xfail(reason="not supported")) df = pl.LazyFrame( {"keys": [1, 1, 1, 2], "orderby": [1, 2, 4, 2], "values": [1, 2, 3, 4]} ) @@ -188,11 +181,11 @@ def test_rolling_inside_groupby_raises(request): assert_ir_translation_raises(q, NotImplementedError) -def test_rolling_sum_all_null_window_returns_null(request): - if not POLARS_VERSION_LT_136: - request.applymarker( - pytest.mark.xfail(reason="See https://github.com/pola-rs/polars/pull/25117") - ) +@pytest.mark.xfail( + condition=POLARS_VERSION_LT_139, + reason="AExpr::Rolling Python API requires polars >= 1.39", +) +def test_rolling_sum_all_null_window_returns_null(): df = pl.LazyFrame( { "orderby": [1, 2, 3, 4, 5, 6], @@ -321,9 +314,6 @@ def test_rank_over( descending: bool, order_by: None | list[str | pl.Expr], ) -> None: - request.applymarker( - pytest.mark.xfail(condition=POLARS_VERSION_LT_132, reason="rank unsupported") - ) q = df.select( pl.col("x") .rank(method=method, descending=descending) @@ -343,9 +333,6 @@ def test_rank_over_with_ties( descending: bool, order_by: None | list[str | pl.Expr], ) -> None: - request.applymarker( - pytest.mark.xfail(condition=POLARS_VERSION_LT_132, reason="rank unsupported") - ) q = df.select( pl.when(pl.col("g") == 2) .then(pl.lit(4)) @@ -367,9 +354,6 @@ def test_rank_over_with_null_values( descending: bool, order_by: None | list[str | pl.Expr], ) -> None: - request.applymarker( - pytest.mark.xfail(condition=POLARS_VERSION_LT_132, reason="rank unsupported") - ) q = df.select( pl.when((pl.col("x") % 2) == 0) .then(None) @@ -391,9 +375,6 @@ def test_rank_over_with_null_group_keys( descending: bool, order_by: None | list[str | pl.Expr], ) -> None: - request.applymarker( - pytest.mark.xfail(condition=POLARS_VERSION_LT_132, reason="rank unsupported") - ) q = df.select( pl.col("x") .rank(method=method, descending=descending) @@ -431,10 +412,7 @@ def test_fill_over( group_key, order_by=order_by ) ) - if POLARS_VERSION_LT_132: - assert_ir_translation_raises(q, NotImplementedError) - else: - assert_gpu_result_equal(q) + assert_gpu_result_equal(q) def test_fill_null_with_mean_over_unsupported(df: pl.LazyFrame) -> None: diff --git a/python/cudf_polars/tests/expressions/test_sort.py b/python/cudf_polars/tests/expressions/test_sort.py index b7c650dbb9cf..94120c69e245 100644 --- a/python/cudf_polars/tests/expressions/test_sort.py +++ b/python/cudf_polars/tests/expressions/test_sort.py @@ -8,12 +8,7 @@ import polars as pl -import pylibcudf as plc - -from cudf_polars import Translator -from cudf_polars.dsl.ir import IRExecutionContext from cudf_polars.testing.asserts import assert_gpu_result_equal -from cudf_polars.utils.versions import POLARS_VERSION_LT_135, POLARS_VERSION_LT_136 @pytest.mark.parametrize("descending", [False, True]) @@ -62,13 +57,6 @@ def test_sort_by_expression(descending, nulls_last, maintain_order): @pytest.mark.parametrize("nulls_last", [False, True]) @pytest.mark.parametrize("with_nulls", ["no_nulls", "nulls"]) def test_setsorted(request, descending, nulls_last, with_nulls): - if not POLARS_VERSION_LT_135 and POLARS_VERSION_LT_136: - request.applymarker( - pytest.mark.xfail( - reason="See https://github.com/pola-rs/polars/pull/24981, " - "fixed in https://github.com/pola-rs/polars/pull/25250" - ) - ) values = sorted([1, 2, 3, 4, 5, 6, -2], reverse=descending) if with_nulls == "nulls": values[-1 if nulls_last else 0] = None @@ -78,28 +66,6 @@ def test_setsorted(request, descending, nulls_last, with_nulls): assert_gpu_result_equal(q) - if POLARS_VERSION_LT_135: - translator = Translator(q._ldf.visit(), pl.GPUEngine()) - - df = translator.translate_ir().evaluate( - cache={}, - timer=None, - context=IRExecutionContext.from_config_options(translator.config_options), - ) - - a = df.column_map["a"] - - assert a.is_sorted == plc.types.Sorted.YES - null_order = ( - plc.types.NullOrder.AFTER - if (descending ^ nulls_last) and with_nulls == "nulls" - else plc.types.NullOrder.BEFORE - ) - assert a.null_order == null_order - assert a.order == ( - plc.types.Order.DESCENDING if descending else plc.types.Order.ASCENDING - ) - def test_sort_concat_filtered_to_empty(): df = pl.LazyFrame({"a": [1, 2, 3]}) diff --git a/python/cudf_polars/tests/expressions/test_stringfunction.py b/python/cudf_polars/tests/expressions/test_stringfunction.py index 6b3d204b60a2..0548a4fb2e1f 100644 --- a/python/cudf_polars/tests/expressions/test_stringfunction.py +++ b/python/cudf_polars/tests/expressions/test_stringfunction.py @@ -14,13 +14,6 @@ assert_gpu_result_equal, assert_ir_translation_raises, ) -from cudf_polars.utils.versions import ( - POLARS_VERSION_LT_131, - POLARS_VERSION_LT_132, - POLARS_VERSION_LT_133, - POLARS_VERSION_LT_136, - POLARS_VERSION_LT_138, -) @pytest.fixture @@ -352,8 +345,6 @@ def test_replace_many(ldf, target, repl): _need_support_for_implode_agg = isinstance(repl, list) if _need_support_for_implode_agg: assert_gpu_result_equal(q) - elif POLARS_VERSION_LT_131: - assert_ir_translation_raises(q, NotImplementedError) else: # Polars 1.31 now gives us replacement argument as a list assert_gpu_result_equal(q) @@ -376,7 +367,6 @@ def test_replace_many_ascii_case(ldf): assert_ir_translation_raises(q, NotImplementedError) -@pytest.mark.skipif(POLARS_VERSION_LT_136, reason="leftmost arg added in 1.36") def test_replace_many_leftmost(ldf): q = ldf.select(pl.col("a").str.replace_many(["a", "b"], "x", leftmost=True)) @@ -555,31 +545,16 @@ def test_string_zfill(fill, input_strings): q = ldf.select(pl.col("a").str.zfill(fill)) if fill is not None and fill < 0: - cudf_except = ( - pl.exceptions.InvalidOperationError - if not POLARS_VERSION_LT_132 - else pl.exceptions.ComputeError - ) assert_collect_raises( q, polars_except=pl.exceptions.InvalidOperationError, - cudf_except=cudf_except, + cudf_except=pl.exceptions.InvalidOperationError, ) else: assert_gpu_result_equal(q) -@pytest.mark.parametrize( - "fill", - [ - 5 - if not POLARS_VERSION_LT_131 - else pytest.param(5, marks=pytest.mark.xfail(reason="fixed in Polars 1.30")), - 999 - if not POLARS_VERSION_LT_131 - else pytest.param(999, marks=pytest.mark.xfail(reason="fixed in Polars 1.30")), - ], -) +@pytest.mark.parametrize("fill", [5, 999]) def test_string_zfill_pl_129(fill): ldf = pl.LazyFrame({"a": ["-1", "+2"]}) q = ldf.select(pl.col("a").str.zfill(fill)) @@ -592,12 +567,8 @@ def test_string_zfill_pl_129(fill): 0, 1, 2, - 5 - if not POLARS_VERSION_LT_131 - else pytest.param(5, marks=pytest.mark.xfail(reason="fixed in Polars 1.30")), - 999 - if not POLARS_VERSION_LT_131 - else pytest.param(999, marks=pytest.mark.xfail(reason="fixed in Polars 1.30")), + 5, + 999, -1, pytest.param(None, marks=pytest.mark.xfail(reason="None dtype")), ], @@ -611,13 +582,10 @@ def test_string_zfill_column(fill): ).lazy() q = ldf.select(pl.col("input_strings").str.zfill(pl.col("fill"))) if fill is not None and fill < 0: - cudf_except = ( - pl.exceptions.InvalidOperationError if POLARS_VERSION_LT_132 else () - ) assert_collect_raises( q, polars_except=pl.exceptions.InvalidOperationError, - cudf_except=cudf_except, + cudf_except=(), ) else: assert_gpu_result_equal(q) @@ -808,10 +776,6 @@ def test_json_decode(ldf_jsonlike): q = ldf_jsonlike.select(pl.col("a").str.json_decode(pl.Struct({"a": pl.String()}))) assert_gpu_result_equal(q) - if POLARS_VERSION_LT_133: - q = ldf_jsonlike.select(pl.col("a").str.json_decode(None)) - assert_ir_translation_raises(q, NotImplementedError) - @pytest.mark.parametrize("dtype", [pl.Int64(), pl.Float64()]) def test_json_decode_numeric_types(dtype): @@ -910,9 +874,6 @@ def test_concat_str_with_boolean(): assert_gpu_result_equal(q) -@pytest.mark.skipif( - POLARS_VERSION_LT_138, reason="Split with literal parameter added in 1.38" -) def test_split_regex_not_supported(): lf = pl.LazyFrame({"a": ["foo1bar", "baz456boo", "abc321"]}) diff --git a/python/cudf_polars/tests/expressions/test_struct.py b/python/cudf_polars/tests/expressions/test_struct.py index 2e4686b9079b..fd425ed203ad 100644 --- a/python/cudf_polars/tests/expressions/test_struct.py +++ b/python/cudf_polars/tests/expressions/test_struct.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations @@ -10,7 +10,6 @@ assert_gpu_result_equal, assert_ir_translation_raises, ) -from cudf_polars.utils.versions import POLARS_VERSION_LT_131 @pytest.fixture @@ -20,47 +19,23 @@ def ldf(): ) -def test_field_getitem(request, ldf): - request.applymarker( - pytest.mark.xfail( - condition=POLARS_VERSION_LT_131, - reason="not supported until polars 1.31", - ) - ) +def test_field_getitem(ldf): q = ldf.select(pl.col("a").struct[0]) assert_gpu_result_equal(q) @pytest.mark.parametrize("fields", [("b",), ("b", "d"), ("^b.*|f.*$",)]) -def test_field(request, ldf, fields): - request.applymarker( - pytest.mark.xfail( - condition=POLARS_VERSION_LT_131, - reason="not supported until polars 1.31", - ) - ) +def test_field(ldf, fields): q = ldf.select(pl.col("a").struct.field(*fields)) assert_gpu_result_equal(q) -def test_unnest(request, ldf): - request.applymarker( - pytest.mark.xfail( - condition=POLARS_VERSION_LT_131, - reason="not supported until polars 1.31", - ) - ) +def test_unnest(ldf): q = ldf.select(pl.col("a").struct.unnest()) assert_gpu_result_equal(q) -def test_json_encode(request, ldf): - request.applymarker( - pytest.mark.xfail( - condition=POLARS_VERSION_LT_131, - reason="not supported until polars 1.31", - ) - ) +def test_json_encode(ldf): q = ldf.select(pl.col("a").struct.json_encode()) assert_gpu_result_equal(q) @@ -69,13 +44,7 @@ def test_json_encode(request, ldf): assert_gpu_result_equal(q) -def test_rename_fields(request, ldf): - request.applymarker( - pytest.mark.xfail( - condition=POLARS_VERSION_LT_131, - reason="not supported until polars 1.31", - ) - ) +def test_rename_fields(ldf): q = ldf.select(pl.col("a").struct.rename_fields(["1", "2", "3"]).struct.unnest()) assert_gpu_result_equal(q) @@ -92,13 +61,7 @@ def test_with_fields(ldf): [pl.col("a").name.prefix_fields, pl.col("a").name.suffix_fields], ids=lambda x: x.__name__, ) -def test_prefix_suffix_fields(request, ldf, expr): - request.applymarker( - pytest.mark.xfail( - condition=POLARS_VERSION_LT_131, - reason="not supported until polars 1.31", - ) - ) +def test_prefix_suffix_fields(ldf, expr): q = ldf.select(expr("foo").struct.unnest()) assert_gpu_result_equal(q) diff --git a/python/cudf_polars/tests/test_cache.py b/python/cudf_polars/tests/test_cache.py index b6b5ec5fb3d9..62f4100649a1 100644 --- a/python/cudf_polars/tests/test_cache.py +++ b/python/cudf_polars/tests/test_cache.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations @@ -11,16 +11,10 @@ from cudf_polars.dsl.ir import IRExecutionContext from cudf_polars.dsl.traversal import traversal from cudf_polars.testing.asserts import assert_gpu_result_equal -from cudf_polars.utils.versions import POLARS_VERSION_LT_1323 def test_cache(request): - request.applymarker( - pytest.mark.xfail( - condition=not POLARS_VERSION_LT_1323, - reason="python no longer manages cache hits", - ) - ) + request.applymarker(pytest.mark.xfail(reason="python no longer manages cache hits")) df1 = pl.LazyFrame( { "a": [1, 2, 3, 4, 5, 6, 7], diff --git a/python/cudf_polars/tests/test_dataframescan.py b/python/cudf_polars/tests/test_dataframescan.py index 2986c58b5fef..6054de0b438a 100644 --- a/python/cudf_polars/tests/test_dataframescan.py +++ b/python/cudf_polars/tests/test_dataframescan.py @@ -13,7 +13,6 @@ assert_gpu_result_equal, assert_ir_translation_raises, ) -from cudf_polars.utils.versions import POLARS_VERSION_LT_138 @pytest.mark.parametrize( @@ -82,10 +81,6 @@ def test_dataframescan_with_decimals(): assert_gpu_result_equal(q) -@pytest.mark.skipif( - POLARS_VERSION_LT_138, - reason="height parameter added in Polars 1.38", -) def test_dataframescan_zero_width_with_rows(request, using_rapidsmpf): request.applymarker( pytest.mark.xfail( diff --git a/python/cudf_polars/tests/test_drop_nulls.py b/python/cudf_polars/tests/test_drop_nulls.py index 0fe9b963eddb..8ea42ce675c0 100644 --- a/python/cudf_polars/tests/test_drop_nulls.py +++ b/python/cudf_polars/tests/test_drop_nulls.py @@ -10,7 +10,6 @@ assert_gpu_result_equal, assert_ir_translation_raises, ) -from cudf_polars.utils.versions import POLARS_VERSION_LT_132 @pytest.fixture( @@ -57,10 +56,7 @@ def test_fill_null_with_string(): ) def test_fill_null_with_strategy(null_data, strategy): q = null_data.select(pl.col("a").fill_null(strategy=strategy)) - if POLARS_VERSION_LT_132: - assert_ir_translation_raises(q, NotImplementedError) - else: - assert_gpu_result_equal(q) + assert_gpu_result_equal(q) @pytest.mark.parametrize("strategy", ["zero", "one"]) @@ -68,10 +64,7 @@ def test_fill_null_with_strategy_bool(strategy): q = pl.LazyFrame({"a": [True, None, False]}).select( pl.col("a").fill_null(strategy=strategy) ) - if POLARS_VERSION_LT_132: - assert_ir_translation_raises(q, NotImplementedError) - else: - assert_gpu_result_equal(q) + assert_gpu_result_equal(q) @pytest.mark.parametrize("strategy", ["forward", "backward"]) @@ -81,7 +74,4 @@ def test_fill_null_with_limit(null_data, strategy, limit): if limit != 0: assert_ir_translation_raises(q, NotImplementedError) else: - if POLARS_VERSION_LT_132: - assert_ir_translation_raises(q, NotImplementedError) - else: - assert_gpu_result_equal(q) + assert_gpu_result_equal(q) diff --git a/python/cudf_polars/tests/test_groupby.py b/python/cudf_polars/tests/test_groupby.py index bad81b6f8265..8e5b952de8e5 100644 --- a/python/cudf_polars/tests/test_groupby.py +++ b/python/cudf_polars/tests/test_groupby.py @@ -15,11 +15,6 @@ assert_gpu_result_equal, assert_ir_translation_raises, ) -from cudf_polars.utils.versions import ( - POLARS_VERSION_LT_132, - POLARS_VERSION_LT_136, - POLARS_VERSION_LT_1321, -) @pytest.fixture @@ -48,11 +43,9 @@ def df(): ], } ) - if not POLARS_VERSION_LT_132: - lf = lf.with_columns( - pl.col("float").cast(pl.Decimal(precision=9, scale=2)).alias("decimal") - ) - return lf + return lf.with_columns( + pl.col("float").cast(pl.Decimal(precision=9, scale=2)).alias("decimal") + ) @pytest.fixture( @@ -106,8 +99,7 @@ def keys(request): # polars gives us precision=None, which we # do not supprt -if not POLARS_VERSION_LT_132: - _EXPRS.append([pl.col("decimal").median()]) +_EXPRS.append([pl.col("decimal").median()]) @pytest.fixture( @@ -272,7 +264,6 @@ def test_groupby_nan_minmax_raises(op): pytest.param( pl.Series("value", [[4, 5, 6]], dtype=pl.List(pl.Int32)), marks=pytest.mark.xfail( - condition=not POLARS_VERSION_LT_1321, reason="https://github.com/rapidsai/cudf/issues/19610", ), ), @@ -421,7 +412,7 @@ def test_groupby_aggs_keep_unsupported_as_null( ) -> None: request.applymarker( pytest.mark.xfail( - condition="sum" in str(agg_expr) and not POLARS_VERSION_LT_136, + condition="sum" in str(agg_expr), reason="polars raises now", ) ) diff --git a/python/cudf_polars/tests/test_join.py b/python/cudf_polars/tests/test_join.py index 404fca50087f..658576448475 100644 --- a/python/cudf_polars/tests/test_join.py +++ b/python/cudf_polars/tests/test_join.py @@ -18,7 +18,6 @@ assert_gpu_result_equal, get_default_engine, ) -from cudf_polars.utils.versions import POLARS_VERSION_LT_132 @pytest.fixture(params=[False, True], ids=["nulls_not_equal", "nulls_equal"]) @@ -176,10 +175,7 @@ def test_join_where(left, right, conditions, zlice): assert_gpu_result_equal(q_len) -def test_cross_join_empty_right_table(request): - request.applymarker( - pytest.mark.xfail(condition=POLARS_VERSION_LT_132, reason="nested loop join") - ) +def test_cross_join_empty_right_table(): a = pl.LazyFrame({"a": [1, 2, 3], "x": [7, 2, 1]}) b = pl.LazyFrame({"b": [2, 2, 2], "x": [7, 1, 3]}) @@ -290,16 +286,7 @@ def test_join_maintain_order_with_slice(left, right, maintain_order, how, zlice) (pl.Decimal(15, 2), pl.Float64), ], ) -def test_cross_join_filter_with_decimals(request, expr, left_dtype, right_dtype): - request.applymarker( - pytest.mark.xfail( - POLARS_VERSION_LT_132 - and isinstance(left_dtype, pl.Decimal) - and isinstance(right_dtype, pl.Decimal) - and "==" in repr(expr), - reason="Hash Inner Join between i128 and i128", - ) - ) +def test_cross_join_filter_with_decimals(expr, left_dtype, right_dtype): left = pl.LazyFrame( { "foo": [Decimal("1.00"), Decimal("2.50"), Decimal("3.00")], diff --git a/python/cudf_polars/tests/test_mapfunction.py b/python/cudf_polars/tests/test_mapfunction.py index 1590bb00ef1d..05f78f73d715 100644 --- a/python/cudf_polars/tests/test_mapfunction.py +++ b/python/cudf_polars/tests/test_mapfunction.py @@ -13,7 +13,6 @@ assert_gpu_result_equal, assert_ir_translation_raises, ) -from cudf_polars.utils.versions import POLARS_VERSION_LT_131, POLARS_VERSION_LT_135 def test_explode_multiple_raises(): @@ -43,12 +42,9 @@ def test_rename_duplicate_raises(mapping): q = df.rename(mapping) - if POLARS_VERSION_LT_131: + # Now raises before translation + with pytest.raises(pl.exceptions.DuplicateError, match="is duplicate"): assert_ir_translation_raises(q, NotImplementedError) - else: - # Now raises before translation - with pytest.raises(pl.exceptions.DuplicateError, match="is duplicate"): - assert_ir_translation_raises(q, NotImplementedError) @pytest.mark.parametrize( @@ -116,12 +112,7 @@ def test_unique_hash(): def test_set_sorted_then_inner_join(request): - request.applymarker( - pytest.mark.xfail( - condition=not POLARS_VERSION_LT_135, - reason="HintIR not supported", - ) - ) + request.applymarker(pytest.mark.xfail(reason="HintIR not supported")) df = pl.LazyFrame({"a": [1, 2, 3, 4, 5]}) q = df.set_sorted("a").join( @@ -131,8 +122,8 @@ def test_set_sorted_then_inner_join(request): def test_explode_single_legacy_options(): - # Cover the branch: POLARS_VERSION_LT_136 or len(self.options) == 1 - # On polars >= 1.36 this branch is only reachable by direct construction + # Cover the branch: len(self.options) == 1 + # This branch is only reachable by direct construction # with 1-element options (the old pre-1.36 format). df = pl.DataFrame({"a": [[1, 2], [3, 4]]}) child = DataFrameScan({"a": DataType(pl.List(pl.Int64()))}, df._df, None) diff --git a/python/cudf_polars/tests/test_rolling.py b/python/cudf_polars/tests/test_rolling.py index a9a0617ff719..7d69f09a9689 100644 --- a/python/cudf_polars/tests/test_rolling.py +++ b/python/cudf_polars/tests/test_rolling.py @@ -13,7 +13,6 @@ assert_gpu_result_equal, assert_ir_translation_raises, ) -from cudf_polars.utils.versions import POLARS_VERSION_LT_136 @pytest.fixture(params=[False, True], ids=["nosort", "sort"]) @@ -161,12 +160,7 @@ def test_orderby_nulls_raises_computeerror(): def test_rolling_nested_raises(request): - request.applymarker( - pytest.mark.xfail( - condition=not POLARS_VERSION_LT_136, - reason="polars raises now", - ) - ) + request.applymarker(pytest.mark.xfail(reason="polars raises now")) q = ( pl.LazyFrame( { diff --git a/python/cudf_polars/tests/test_scan.py b/python/cudf_polars/tests/test_scan.py index 66f74b9bce12..76a6b27bb3cf 100644 --- a/python/cudf_polars/tests/test_scan.py +++ b/python/cudf_polars/tests/test_scan.py @@ -19,9 +19,7 @@ ) from cudf_polars.testing.io import make_partitioned_source from cudf_polars.utils.versions import ( - POLARS_VERSION_LT_131, - POLARS_VERSION_LT_135, - POLARS_VERSION_LT_138, + POLARS_VERSION_LT_139, ) if TYPE_CHECKING: @@ -510,12 +508,6 @@ def test_scan_from_file_uri(tmp_path: Path) -> None: def test_scan_parquet_remote( request, tmp_path: Path, df: pl.DataFrame, httpserver: HTTPServer, *, chunked: bool ) -> None: - request.applymarker( - pytest.mark.xfail( - condition=POLARS_VERSION_LT_131, - reason="remote IO not supported", - ) - ) path = tmp_path / "foo.parquet" df.write_parquet(path) bytes_ = path.read_bytes() @@ -580,8 +572,8 @@ def test_scan_ndjson_remote( ) -> None: request.applymarker( pytest.mark.xfail( - condition=POLARS_VERSION_LT_131, - reason="remote IO not supported", + condition=not POLARS_VERSION_LT_139, + reason="polars 1.39+ ndjson remote reader requires range request support", ) ) path = tmp_path / "foo.jsonl" @@ -642,23 +634,14 @@ def test_scan_csv_blank_line(tmp_path): def test_hits_scan_row_index_duplicate(request, tmp_path): - request.applymarker( - pytest.mark.xfail( - condition=not POLARS_VERSION_LT_138, - reason="polars fails ahead of time", - ) - ) + request.applymarker(pytest.mark.xfail(reason="polars fails ahead of time")) pl.DataFrame({"col": [1, 2, 3]}).write_parquet(tmp_path / "a.parquet") q = pl.scan_parquet(tmp_path / "*.parquet", row_index_name="index").with_row_index( "index" ) - if POLARS_VERSION_LT_135: - # Did not raise before - assert_gpu_result_equal(q) - else: - assert_ir_translation_raises(q, NotImplementedError) + assert_ir_translation_raises(q, NotImplementedError) @pytest.mark.parametrize("compression", ["gzip", "zlib", "zstd"]) @@ -697,10 +680,6 @@ def test_scan_tiny_file_not_compressed(tmp_path): assert_gpu_result_equal(q) -@pytest.mark.skipif( - POLARS_VERSION_LT_138, - reason="height parameter added in Polars 1.38", -) @pytest.mark.parametrize("engine", [None, NO_CHUNK_ENGINE]) def test_scan_parquet_zero_width_with_limit(tmp_path, engine, request, using_rapidsmpf): request.applymarker( diff --git a/python/cudf_polars/tests/test_select.py b/python/cudf_polars/tests/test_select.py index 7f85e233de34..49ccbbe83541 100644 --- a/python/cudf_polars/tests/test_select.py +++ b/python/cudf_polars/tests/test_select.py @@ -12,7 +12,6 @@ assert_gpu_result_equal, assert_ir_translation_raises, ) -from cudf_polars.utils.versions import POLARS_VERSION_LT_134 def test_select(): @@ -51,7 +50,7 @@ def test_select_decimal_precision_none_result_max_precision(): gpu_result = query.collect(engine="gpu") # See github.com/pola-rs/polars/issues/19784 # for context on the decimal changes. - assert cpu_result.schema["a"].precision is None if POLARS_VERSION_LT_134 else 38 + assert cpu_result.schema["a"].precision == 38 assert gpu_result.schema["a"].precision == 38 diff --git a/python/cudf_polars/tests/test_sink.py b/python/cudf_polars/tests/test_sink.py index 022d1a9962cb..b98f6a1e7976 100644 --- a/python/cudf_polars/tests/test_sink.py +++ b/python/cudf_polars/tests/test_sink.py @@ -11,7 +11,6 @@ assert_sink_ir_translation_raises, assert_sink_result_equal, ) -from cudf_polars.utils.versions import POLARS_VERSION_LT_138 @pytest.fixture(scope="module") @@ -164,10 +163,6 @@ def test_sink_in_memory_executor(df, tmp_path, file_type): @pytest.mark.parametrize("compression", ["gzip", "zstd"]) @pytest.mark.parametrize("file_type", ["csv", "ndjson"]) -@pytest.mark.skipif( - POLARS_VERSION_LT_138, - reason="compression parameter added in Polars 1.38", -) def test_sink_compression_raises(df, tmp_path, compression, file_type): path = tmp_path / f"out.{file_type}" assert_sink_ir_translation_raises( diff --git a/python/cudf_polars/tests/test_window_functions.py b/python/cudf_polars/tests/test_window_functions.py index e7f9181f833c..0451f060c252 100644 --- a/python/cudf_polars/tests/test_window_functions.py +++ b/python/cudf_polars/tests/test_window_functions.py @@ -13,7 +13,7 @@ assert_gpu_result_equal, assert_ir_translation_raises, ) -from cudf_polars.utils.versions import POLARS_VERSION_LT_132, POLARS_VERSION_LT_136 +from cudf_polars.utils.versions import POLARS_VERSION_LT_139 @pytest.fixture @@ -118,19 +118,19 @@ def test_over_mapping_strategy(df: pl.LazyFrame, mapping_strategy: str): ) ] ) - if not POLARS_VERSION_LT_132 and mapping_strategy == "group_to_rows": + if mapping_strategy == "group_to_rows": assert_gpu_result_equal(q) else: assert_ir_translation_raises(q, NotImplementedError) +@pytest.mark.xfail( + condition=POLARS_VERSION_LT_139, + reason="AExpr::Rolling Python API requires polars >= 1.39", +) @pytest.mark.parametrize("period", ["2d", "3d"]) -def test_rolling(request, df: pl.LazyFrame, agg_expr, period: str): +def test_rolling(df: pl.LazyFrame, agg_expr, period: str): """Test rolling window functions over time series.""" - if not POLARS_VERSION_LT_136: - request.applymarker( - pytest.mark.xfail(reason="See https://github.com/pola-rs/polars/pull/25117") - ) window_expr = agg_expr.rolling(period=period, index_column="date") result_name = f"{agg_expr!s}_rolling_{period}" window_expr = window_expr.alias(result_name) @@ -151,13 +151,13 @@ def test_rolling_unsupported(df: pl.LazyFrame, unsupported_agg_expr): assert_ir_translation_raises(query, NotImplementedError) +@pytest.mark.xfail( + condition=POLARS_VERSION_LT_139, + reason="AExpr::Rolling Python API requires polars >= 1.39", +) @pytest.mark.parametrize("closed", ["left", "right", "both", "none"]) -def test_rolling_closed(request, df: pl.LazyFrame, closed: str): +def test_rolling_closed(df: pl.LazyFrame, closed: str): """Test rolling window functions with different closed parameters.""" - if not POLARS_VERSION_LT_136: - request.applymarker( - pytest.mark.xfail(reason="See https://github.com/pola-rs/polars/pull/25117") - ) # ignore is for polars' ClosedInterval, which isn't publicly exported. # https://github.com/pola-rs/polars/issues/17420 query = df.with_columns( From 714fefe23232cf8c4752e91c3f755366bf727cc0 Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Tue, 7 Apr 2026 15:38:00 -0700 Subject: [PATCH 03/44] remove unnecessary loop --- python/cudf_polars/cudf_polars/dsl/ir.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/python/cudf_polars/cudf_polars/dsl/ir.py b/python/cudf_polars/cudf_polars/dsl/ir.py index bea3f1685419..181cac363377 100644 --- a/python/cudf_polars/cudf_polars/dsl/ir.py +++ b/python/cudf_polars/cudf_polars/dsl/ir.py @@ -32,7 +32,7 @@ import cudf_polars.dsl.expr as expr from cudf_polars.containers import Column, DataFrame, DataType from cudf_polars.containers.dataframe import NamedColumn -from cudf_polars.dsl.expressions import rolling, unary +from cudf_polars.dsl.expressions import rolling from cudf_polars.dsl.expressions.base import ExecutionContext from cudf_polars.dsl.nodebase import Node from cudf_polars.dsl.to_ast import to_ast, to_parquet_filter @@ -1809,13 +1809,6 @@ def __init__( for dtype in schema.values(): if _has_struct_in_list(dtype.polars_type): raise NotImplementedError("Nested list[struct] types not supported") - for request in agg_requests: - expr = request.value - if any( - isinstance(child, unary.UnaryFunction) and child.name == "value_counts" - for child in expr.children - ): - raise NotImplementedError("value_counts is not supported in groupby") self.agg_requests = tuple(agg_requests) self.maintain_order = maintain_order self.zlice = zlice From fa3c810c04b7c7dd5cf4b921b12b2b0e801116e7 Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Tue, 7 Apr 2026 16:07:57 -0700 Subject: [PATCH 04/44] bump minimum version --- .../all_cuda-129_arch-aarch64.yaml | 2 +- .../all_cuda-129_arch-x86_64.yaml | 2 +- .../all_cuda-131_arch-aarch64.yaml | 2 +- .../all_cuda-131_arch-x86_64.yaml | 2 +- conda/recipes/cudf-polars/recipe.yaml | 2 +- dependencies.yaml | 2 +- .../cudf_polars/containers/dataframe.py | 3 - .../cudf_polars/dsl/expressions/unary.py | 19 +--- python/cudf_polars/cudf_polars/dsl/ir.py | 3 - .../cudf_polars/cudf_polars/dsl/translate.py | 95 +++++++++---------- .../cudf_polars/cudf_polars/utils/versions.py | 3 +- python/cudf_polars/pyproject.toml | 2 +- .../tests/experimental/test_rolling.py | 5 - .../tests/expressions/test_rolling.py | 33 ------- python/cudf_polars/tests/test_scan.py | 4 - .../tests/test_window_functions.py | 9 -- 16 files changed, 57 insertions(+), 131 deletions(-) diff --git a/conda/environments/all_cuda-129_arch-aarch64.yaml b/conda/environments/all_cuda-129_arch-aarch64.yaml index 365804533ddf..bcbd20315a55 100644 --- a/conda/environments/all_cuda-129_arch-aarch64.yaml +++ b/conda/environments/all_cuda-129_arch-aarch64.yaml @@ -67,7 +67,7 @@ dependencies: - packaging - pandas>=2.0,<2.4.0 - pandoc -- polars>=1.38,<1.40 +- polars>=1.39,<1.40 - pre-commit - pyarrow>=19.0.0 - pydata-sphinx-theme>=0.15.4 diff --git a/conda/environments/all_cuda-129_arch-x86_64.yaml b/conda/environments/all_cuda-129_arch-x86_64.yaml index 12dbf0c87431..bc138f1644e4 100644 --- a/conda/environments/all_cuda-129_arch-x86_64.yaml +++ b/conda/environments/all_cuda-129_arch-x86_64.yaml @@ -67,7 +67,7 @@ dependencies: - packaging - pandas>=2.0,<2.4.0 - pandoc -- polars>=1.38,<1.40 +- polars>=1.39,<1.40 - pre-commit - pyarrow>=19.0.0 - pydata-sphinx-theme>=0.15.4 diff --git a/conda/environments/all_cuda-131_arch-aarch64.yaml b/conda/environments/all_cuda-131_arch-aarch64.yaml index c67c1eb758ae..73f87a0ff3fd 100644 --- a/conda/environments/all_cuda-131_arch-aarch64.yaml +++ b/conda/environments/all_cuda-131_arch-aarch64.yaml @@ -67,7 +67,7 @@ dependencies: - packaging - pandas>=2.0,<2.4.0 - pandoc -- polars>=1.38,<1.40 +- polars>=1.39,<1.40 - pre-commit - pyarrow>=19.0.0 - pydata-sphinx-theme>=0.15.4 diff --git a/conda/environments/all_cuda-131_arch-x86_64.yaml b/conda/environments/all_cuda-131_arch-x86_64.yaml index 233797432f04..9240ad3da2e5 100644 --- a/conda/environments/all_cuda-131_arch-x86_64.yaml +++ b/conda/environments/all_cuda-131_arch-x86_64.yaml @@ -67,7 +67,7 @@ dependencies: - packaging - pandas>=2.0,<2.4.0 - pandoc -- polars>=1.38,<1.40 +- polars>=1.39,<1.40 - pre-commit - pyarrow>=19.0.0 - pydata-sphinx-theme>=0.15.4 diff --git a/conda/recipes/cudf-polars/recipe.yaml b/conda/recipes/cudf-polars/recipe.yaml index e46ce5376231..f28e98fda738 100644 --- a/conda/recipes/cudf-polars/recipe.yaml +++ b/conda/recipes/cudf-polars/recipe.yaml @@ -36,7 +36,7 @@ requirements: - nvidia-ml-py>=12 - python - pylibcudf =${{ version }} - - polars>=1.38,<1.40 + - polars>=1.39,<1.40 - packaging - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }} - if: cuda_major == "12" diff --git a/dependencies.yaml b/dependencies.yaml index 1ff0001f72a3..b097e135636c 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -817,7 +817,7 @@ dependencies: # 'nvidia-ml-py' provides the 'pynvml' module - &nvidia_ml_py nvidia-ml-py>=12 - packaging - - polars>=1.38,<1.40 + - polars>=1.39,<1.40 - typing_extensions>=4.0.0 run_cudf_polars_experimental: common: diff --git a/python/cudf_polars/cudf_polars/containers/dataframe.py b/python/cudf_polars/cudf_polars/containers/dataframe.py index 977449a971e8..aa15a186fff7 100644 --- a/python/cudf_polars/cudf_polars/containers/dataframe.py +++ b/python/cudf_polars/cudf_polars/containers/dataframe.py @@ -117,9 +117,6 @@ def copy(self) -> Self: def to_polars(self) -> pl.DataFrame: """Convert to a polars DataFrame.""" - if self._num_rows_override is not None and len(self.column_map) == 0: - return pl.DataFrame(height=self._num_rows_override) - # If the arrow table has empty names, from_arrow produces # column_$i. But here we know there is only one such column # (by construction) and it should have an empty name. diff --git a/python/cudf_polars/cudf_polars/dsl/expressions/unary.py b/python/cudf_polars/cudf_polars/dsl/expressions/unary.py index f61512d68039..635029c6341d 100644 --- a/python/cudf_polars/cudf_polars/dsl/expressions/unary.py +++ b/python/cudf_polars/cudf_polars/dsl/expressions/unary.py @@ -13,7 +13,6 @@ from cudf_polars.dsl.expressions.base import ExecutionContext, Expr from cudf_polars.dsl.expressions.literal import Literal from cudf_polars.utils import dtypes -from cudf_polars.utils.versions import POLARS_VERSION_LT_139 if TYPE_CHECKING: from cudf_polars.containers import DataFrame, DataType @@ -243,20 +242,10 @@ def do_evaluate( return column elif self.name == "set_sorted": (column,) = (child.evaluate(df, context=context) for child in self.children) - if POLARS_VERSION_LT_139: - (asc,) = self.options - order = ( - plc.types.Order.ASCENDING - if asc == "ascending" - else plc.types.Order.DESCENDING - ) - else: - descending, _nulls_last = self.options - order = ( - plc.types.Order.DESCENDING - if descending - else plc.types.Order.ASCENDING - ) + descending, _nulls_last = self.options + order = ( + plc.types.Order.DESCENDING if descending else plc.types.Order.ASCENDING + ) null_order = plc.types.NullOrder.BEFORE if column.null_count > 0 and (n := column.size) > 1: # PERF: This invokes four stream synchronisations! diff --git a/python/cudf_polars/cudf_polars/dsl/ir.py b/python/cudf_polars/cudf_polars/dsl/ir.py index 181cac363377..1c84692481f9 100644 --- a/python/cudf_polars/cudf_polars/dsl/ir.py +++ b/python/cudf_polars/cudf_polars/dsl/ir.py @@ -1417,11 +1417,8 @@ def do_evaluate( if projection is not None: df = df.select(projection) - # Zero-width dataframes lose their row count when converted through - # pylibcudf. See https://github.com/rapidsai/cudf/issues/21428 if len(schema) == 0: return DataFrame([], stream=context.get_cuda_stream(), num_rows=height) - df = DataFrame.from_polars(df, stream=context.get_cuda_stream()) assert all( c.obj.type() == dtype.plc_type diff --git a/python/cudf_polars/cudf_polars/dsl/translate.py b/python/cudf_polars/cudf_polars/dsl/translate.py index 6ba9d667d93f..372aadecc74e 100644 --- a/python/cudf_polars/cudf_polars/dsl/translate.py +++ b/python/cudf_polars/cudf_polars/dsl/translate.py @@ -32,9 +32,6 @@ from cudf_polars.dsl.utils.rolling import rewrite_rolling from cudf_polars.typing import Schema from cudf_polars.utils import config, sorting -from cudf_polars.utils.versions import ( - POLARS_VERSION_LT_139, -) if TYPE_CHECKING: from polars import GPUEngine @@ -882,54 +879,52 @@ def _( assert_never(node.options) -if not POLARS_VERSION_LT_139: # type: ignore[misc] - - @_translate_expr.register - def _( - node: plrs._expr_nodes.Rolling, - translator: Translator, - dtype: DataType, - schema: Schema, - ) -> expr.Expr: - with set_expr_context(translator, ExecutionContext.ROLLING): - agg_expr = translator.translate_expr(n=node.function, schema=schema) - orderby = translator.visitor.view_expression(node.index_column).name - orderby_dtype = schema[orderby].plc_type - if plc.traits.is_integral(orderby_dtype): - orderby_dtype = plc.DataType(plc.TypeId.INT64) - name_generator = unique_names(schema) - aggs, named_post_agg = decompose_single_agg( - expr.NamedExpr(next(name_generator), agg_expr), - name_generator, - is_top=True, - context=ExecutionContext.ROLLING, +@_translate_expr.register +def _( + node: plrs._expr_nodes.Rolling, + translator: Translator, + dtype: DataType, + schema: Schema, +) -> expr.Expr: + with set_expr_context(translator, ExecutionContext.ROLLING): + agg_expr = translator.translate_expr(n=node.function, schema=schema) + orderby = translator.visitor.view_expression(node.index_column).name + orderby_dtype = schema[orderby].plc_type + if plc.traits.is_integral(orderby_dtype): + orderby_dtype = plc.DataType(plc.TypeId.INT64) + name_generator = unique_names(schema) + aggs, named_post_agg = decompose_single_agg( + expr.NamedExpr(next(name_generator), agg_expr), + name_generator, + is_top=True, + context=ExecutionContext.ROLLING, + ) + named_aggs = [a for a, _ in aggs] + closed_window = node.closed_window + if isinstance(named_post_agg.value, expr.Col): + (named_agg,) = named_aggs + return expr.RollingWindow( + named_agg.value.dtype, + orderby_dtype, + node.offset, + node.period, + closed_window, + orderby, + named_agg.value, ) - named_aggs = [a for a, _ in aggs] - closed_window = node.closed_window - if isinstance(named_post_agg.value, expr.Col): - (named_agg,) = named_aggs - return expr.RollingWindow( - named_agg.value.dtype, - orderby_dtype, - node.offset, - node.period, - closed_window, - orderby, - named_agg.value, - ) - replacements: dict[expr.Expr, expr.Expr] = { - expr.Col(a.value.dtype, a.name): expr.RollingWindow( - a.value.dtype, - orderby_dtype, - node.offset, - node.period, - closed_window, - orderby, - a.value, - ) - for a in named_aggs - } - return replace([named_post_agg.value], replacements)[0] + replacements: dict[expr.Expr, expr.Expr] = { + expr.Col(a.value.dtype, a.name): expr.RollingWindow( + a.value.dtype, + orderby_dtype, + node.offset, + node.period, + closed_window, + orderby, + a.value, + ) + for a in named_aggs + } + return replace([named_post_agg.value], replacements)[0] @_translate_expr.register diff --git a/python/cudf_polars/cudf_polars/utils/versions.py b/python/cudf_polars/cudf_polars/utils/versions.py index 92d0b4eec203..c32124f2d967 100644 --- a/python/cudf_polars/cudf_polars/utils/versions.py +++ b/python/cudf_polars/cudf_polars/utils/versions.py @@ -11,8 +11,7 @@ from polars import __version__ POLARS_VERSION = parse(__version__) -POLARS_LOWER_BOUND = parse("1.38") -POLARS_VERSION_LT_139 = POLARS_VERSION < parse("1.39.0") +POLARS_LOWER_BOUND = parse("1.39") def _ensure_polars_version() -> None: diff --git a/python/cudf_polars/pyproject.toml b/python/cudf_polars/pyproject.toml index 1fc2cbeadad2..0b415224db83 100644 --- a/python/cudf_polars/pyproject.toml +++ b/python/cudf_polars/pyproject.toml @@ -24,7 +24,7 @@ dependencies = [ "cuda-python>=13.0.1,<14.0", "nvidia-ml-py>=12", "packaging", - "polars>=1.38,<1.40", + "polars>=1.39,<1.40", "pylibcudf==26.6.*,>=0.0.0a0", "typing_extensions>=4.0.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/cudf_polars/tests/experimental/test_rolling.py b/python/cudf_polars/tests/experimental/test_rolling.py index 398c83817017..ffc0e5ee3414 100644 --- a/python/cudf_polars/tests/experimental/test_rolling.py +++ b/python/cudf_polars/tests/experimental/test_rolling.py @@ -11,7 +11,6 @@ from cudf_polars.experimental.rapidsmpf.frontend.spmd import SPMDEngine from cudf_polars.testing.asserts import assert_gpu_result_equal -from cudf_polars.utils.versions import POLARS_VERSION_LT_139 if TYPE_CHECKING: from collections.abc import Generator @@ -34,10 +33,6 @@ def engine( yield engine -@pytest.mark.xfail( - condition=POLARS_VERSION_LT_139, - reason="AExpr::Rolling Python API requires polars >= 1.39", -) def test_rolling_datetime(engine): dates = [ "2020-01-01 13:45:48", diff --git a/python/cudf_polars/tests/expressions/test_rolling.py b/python/cudf_polars/tests/expressions/test_rolling.py index 49b5730ef998..c4d7eb01dd69 100644 --- a/python/cudf_polars/tests/expressions/test_rolling.py +++ b/python/cudf_polars/tests/expressions/test_rolling.py @@ -13,7 +13,6 @@ assert_gpu_result_equal, assert_ir_translation_raises, ) -from cudf_polars.utils.versions import POLARS_VERSION_LT_139 if TYPE_CHECKING: from cudf_polars.typing import RankMethod @@ -32,10 +31,6 @@ def df(): ) -@pytest.mark.xfail( - condition=POLARS_VERSION_LT_139, - reason="AExpr::Rolling Python API requires polars >= 1.39", -) @pytest.mark.parametrize("time_unit", ["ns", "us", "ms"]) def test_rolling_datetime(time_unit): dates = [ @@ -60,10 +55,6 @@ def test_rolling_datetime(time_unit): assert_gpu_result_equal(q) -@pytest.mark.xfail( - condition=POLARS_VERSION_LT_139, - reason="AExpr::Rolling Python API requires polars >= 1.39", -) def test_rolling_date(): dates = [ "2020-01-01", @@ -85,10 +76,6 @@ def test_rolling_date(): assert_gpu_result_equal(q) -@pytest.mark.xfail( - condition=POLARS_VERSION_LT_139, - reason="AExpr::Rolling Python API requires polars >= 1.39", -) @pytest.mark.parametrize("dtype", [pl.Int32, pl.UInt32, pl.Int64, pl.UInt64]) def test_rolling_integral_orderby(dtype): df = pl.LazyFrame( @@ -104,10 +91,6 @@ def test_rolling_integral_orderby(dtype): assert_gpu_result_equal(q) -@pytest.mark.xfail( - condition=POLARS_VERSION_LT_139, - reason="AExpr::Rolling Python API requires polars >= 1.39", -) def test_rolling_agg_first(): df = pl.LazyFrame({"a": [1, 2, 3], "b": [1, 2, 3]}) q = df.with_columns(pl.col("a").sum().rolling("b", period="2i")) @@ -127,10 +110,6 @@ def test_rolling_collect_list_raises(): ) -@pytest.mark.xfail( - condition=POLARS_VERSION_LT_139, - reason="AExpr::Rolling Python API requires polars >= 1.39", -) def test_unsorted_raises(): df = pl.LazyFrame({"orderby": [1, 2, 4, 2], "values": [1, 2, 3, 4]}) q = df.select(pl.col("values").sum().rolling("orderby", period="2i")) @@ -143,10 +122,6 @@ def test_unsorted_raises(): q.collect(engine=pl.GPUEngine(raise_on_fail=True)) -@pytest.mark.xfail( - condition=POLARS_VERSION_LT_139, - reason="AExpr::Rolling Python API requires polars >= 1.39", -) def test_orderby_nulls_raises_computeerror(): df = pl.LazyFrame({"orderby": [1, 2, 4, None], "values": [1, 2, 3, 4]}) q = df.select(pl.col("values").sum().rolling("orderby", period="2i")) @@ -158,10 +133,6 @@ def test_orderby_nulls_raises_computeerror(): q.collect(engine=pl.GPUEngine(raise_on_fail=True)) -@pytest.mark.xfail( - condition=POLARS_VERSION_LT_139, - reason="AExpr::Rolling Python API requires polars >= 1.39", -) def test_invalid_duration_spec_raises_in_translation(): df = pl.LazyFrame({"orderby": [1, 2, 4, 5], "values": [1, 2, 3, 4]}) q = df.select(pl.col("values").sum().rolling("orderby", period="3d")) @@ -181,10 +152,6 @@ def test_rolling_inside_groupby_raises(request): assert_ir_translation_raises(q, NotImplementedError) -@pytest.mark.xfail( - condition=POLARS_VERSION_LT_139, - reason="AExpr::Rolling Python API requires polars >= 1.39", -) def test_rolling_sum_all_null_window_returns_null(): df = pl.LazyFrame( { diff --git a/python/cudf_polars/tests/test_scan.py b/python/cudf_polars/tests/test_scan.py index 76a6b27bb3cf..a57f68043bf2 100644 --- a/python/cudf_polars/tests/test_scan.py +++ b/python/cudf_polars/tests/test_scan.py @@ -18,9 +18,6 @@ assert_ir_translation_raises, ) from cudf_polars.testing.io import make_partitioned_source -from cudf_polars.utils.versions import ( - POLARS_VERSION_LT_139, -) if TYPE_CHECKING: from pathlib import Path @@ -572,7 +569,6 @@ def test_scan_ndjson_remote( ) -> None: request.applymarker( pytest.mark.xfail( - condition=not POLARS_VERSION_LT_139, reason="polars 1.39+ ndjson remote reader requires range request support", ) ) diff --git a/python/cudf_polars/tests/test_window_functions.py b/python/cudf_polars/tests/test_window_functions.py index 0451f060c252..488a65a83c91 100644 --- a/python/cudf_polars/tests/test_window_functions.py +++ b/python/cudf_polars/tests/test_window_functions.py @@ -13,7 +13,6 @@ assert_gpu_result_equal, assert_ir_translation_raises, ) -from cudf_polars.utils.versions import POLARS_VERSION_LT_139 @pytest.fixture @@ -124,10 +123,6 @@ def test_over_mapping_strategy(df: pl.LazyFrame, mapping_strategy: str): assert_ir_translation_raises(q, NotImplementedError) -@pytest.mark.xfail( - condition=POLARS_VERSION_LT_139, - reason="AExpr::Rolling Python API requires polars >= 1.39", -) @pytest.mark.parametrize("period", ["2d", "3d"]) def test_rolling(df: pl.LazyFrame, agg_expr, period: str): """Test rolling window functions over time series.""" @@ -151,10 +146,6 @@ def test_rolling_unsupported(df: pl.LazyFrame, unsupported_agg_expr): assert_ir_translation_raises(query, NotImplementedError) -@pytest.mark.xfail( - condition=POLARS_VERSION_LT_139, - reason="AExpr::Rolling Python API requires polars >= 1.39", -) @pytest.mark.parametrize("closed", ["left", "right", "both", "none"]) def test_rolling_closed(df: pl.LazyFrame, closed: str): """Test rolling window functions with different closed parameters.""" From a84384d7ab3c6a1226f4af9208b7438c887d4b42 Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Wed, 8 Apr 2026 06:24:36 -0700 Subject: [PATCH 05/44] add workaround for epty df with nonzero rows --- python/cudf_polars/cudf_polars/containers/dataframe.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/cudf_polars/cudf_polars/containers/dataframe.py b/python/cudf_polars/cudf_polars/containers/dataframe.py index aa15a186fff7..d01382b22c2a 100644 --- a/python/cudf_polars/cudf_polars/containers/dataframe.py +++ b/python/cudf_polars/cudf_polars/containers/dataframe.py @@ -117,6 +117,8 @@ def copy(self) -> Self: def to_polars(self) -> pl.DataFrame: """Convert to a polars DataFrame.""" + if self._num_rows_override is not None and len(self.column_map) == 0: + return pl.DataFrame(height=self._num_rows_override) # If the arrow table has empty names, from_arrow produces # column_$i. But here we know there is only one such column # (by construction) and it should have an empty name. From f9181e80ba1255fa87a9abb9bd1a0c67c2fd52db Mon Sep 17 00:00:00 2001 From: Matthew Murray Date: Mon, 13 Apr 2026 09:57:23 -0700 Subject: [PATCH 06/44] broaden version set --- .../all_cuda-129_arch-aarch64.yaml | 2 +- .../all_cuda-129_arch-x86_64.yaml | 2 +- .../all_cuda-131_arch-aarch64.yaml | 2 +- .../all_cuda-131_arch-x86_64.yaml | 2 +- conda/recipes/cudf-polars/recipe.yaml | 2 +- dependencies.yaml | 2 +- .../cudf_polars/containers/dataframe.py | 3 + .../dsl/expressions/aggregation.py | 11 +++ .../cudf_polars/dsl/expressions/string.py | 17 ++-- .../cudf_polars/dsl/expressions/unary.py | 5 +- python/cudf_polars/cudf_polars/dsl/ir.py | 27 +++--- .../cudf_polars/cudf_polars/dsl/translate.py | 89 ++++++++++++++++--- .../cudf_polars/experimental/io.py | 3 +- .../cudf_polars/cudf_polars/utils/versions.py | 8 +- python/cudf_polars/pyproject.toml | 2 +- .../tests/dsl/test_serialization.py | 7 ++ .../tests/experimental/test_rolling.py | 5 ++ .../cudf_polars/tests/expressions/test_agg.py | 10 ++- .../tests/expressions/test_rolling.py | 21 ++++- .../tests/expressions/test_sort.py | 7 ++ .../tests/expressions/test_stringfunction.py | 9 ++ .../cudf_polars/tests/test_dataframescan.py | 4 + python/cudf_polars/tests/test_groupby.py | 3 +- python/cudf_polars/tests/test_mapfunction.py | 8 +- python/cudf_polars/tests/test_rolling.py | 4 +- python/cudf_polars/tests/test_scan.py | 15 +++- python/cudf_polars/tests/test_sink.py | 4 + .../tests/test_window_functions.py | 9 ++ 28 files changed, 237 insertions(+), 46 deletions(-) diff --git a/conda/environments/all_cuda-129_arch-aarch64.yaml b/conda/environments/all_cuda-129_arch-aarch64.yaml index bcbd20315a55..cac4a6a95723 100644 --- a/conda/environments/all_cuda-129_arch-aarch64.yaml +++ b/conda/environments/all_cuda-129_arch-aarch64.yaml @@ -67,7 +67,7 @@ dependencies: - packaging - pandas>=2.0,<2.4.0 - pandoc -- polars>=1.39,<1.40 +- polars>=1.35,<1.40 - pre-commit - pyarrow>=19.0.0 - pydata-sphinx-theme>=0.15.4 diff --git a/conda/environments/all_cuda-129_arch-x86_64.yaml b/conda/environments/all_cuda-129_arch-x86_64.yaml index bc138f1644e4..27ba93a8d486 100644 --- a/conda/environments/all_cuda-129_arch-x86_64.yaml +++ b/conda/environments/all_cuda-129_arch-x86_64.yaml @@ -67,7 +67,7 @@ dependencies: - packaging - pandas>=2.0,<2.4.0 - pandoc -- polars>=1.39,<1.40 +- polars>=1.35,<1.40 - pre-commit - pyarrow>=19.0.0 - pydata-sphinx-theme>=0.15.4 diff --git a/conda/environments/all_cuda-131_arch-aarch64.yaml b/conda/environments/all_cuda-131_arch-aarch64.yaml index 73f87a0ff3fd..a89efc6ae3ef 100644 --- a/conda/environments/all_cuda-131_arch-aarch64.yaml +++ b/conda/environments/all_cuda-131_arch-aarch64.yaml @@ -67,7 +67,7 @@ dependencies: - packaging - pandas>=2.0,<2.4.0 - pandoc -- polars>=1.39,<1.40 +- polars>=1.35,<1.40 - pre-commit - pyarrow>=19.0.0 - pydata-sphinx-theme>=0.15.4 diff --git a/conda/environments/all_cuda-131_arch-x86_64.yaml b/conda/environments/all_cuda-131_arch-x86_64.yaml index 9240ad3da2e5..1d42a9de97aa 100644 --- a/conda/environments/all_cuda-131_arch-x86_64.yaml +++ b/conda/environments/all_cuda-131_arch-x86_64.yaml @@ -67,7 +67,7 @@ dependencies: - packaging - pandas>=2.0,<2.4.0 - pandoc -- polars>=1.39,<1.40 +- polars>=1.35,<1.40 - pre-commit - pyarrow>=19.0.0 - pydata-sphinx-theme>=0.15.4 diff --git a/conda/recipes/cudf-polars/recipe.yaml b/conda/recipes/cudf-polars/recipe.yaml index f28e98fda738..405983591f44 100644 --- a/conda/recipes/cudf-polars/recipe.yaml +++ b/conda/recipes/cudf-polars/recipe.yaml @@ -36,7 +36,7 @@ requirements: - nvidia-ml-py>=12 - python - pylibcudf =${{ version }} - - polars>=1.39,<1.40 + - polars>=1.35,<1.40 - packaging - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }} - if: cuda_major == "12" diff --git a/dependencies.yaml b/dependencies.yaml index b097e135636c..aefbad017814 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -817,7 +817,7 @@ dependencies: # 'nvidia-ml-py' provides the 'pynvml' module - &nvidia_ml_py nvidia-ml-py>=12 - packaging - - polars>=1.39,<1.40 + - polars>=1.35,<1.40 - typing_extensions>=4.0.0 run_cudf_polars_experimental: common: diff --git a/python/cudf_polars/cudf_polars/containers/dataframe.py b/python/cudf_polars/cudf_polars/containers/dataframe.py index d01382b22c2a..e7c2408d6421 100644 --- a/python/cudf_polars/cudf_polars/containers/dataframe.py +++ b/python/cudf_polars/cudf_polars/containers/dataframe.py @@ -14,6 +14,7 @@ from cudf_polars.containers import Column, DataType from cudf_polars.utils import conversion +from cudf_polars.utils.versions import POLARS_VERSION_LT_138 if TYPE_CHECKING: from collections.abc import Iterable, Mapping, Sequence, Set @@ -118,6 +119,8 @@ def copy(self) -> Self: def to_polars(self) -> pl.DataFrame: """Convert to a polars DataFrame.""" if self._num_rows_override is not None and len(self.column_map) == 0: + if POLARS_VERSION_LT_138: # pragma: no cover + return pl.DataFrame() return pl.DataFrame(height=self._num_rows_override) # If the arrow table has empty names, from_arrow produces # column_$i. But here we know there is only one such column diff --git a/python/cudf_polars/cudf_polars/dsl/expressions/aggregation.py b/python/cudf_polars/cudf_polars/dsl/expressions/aggregation.py index 238bf853fbb6..cca9a1562c91 100644 --- a/python/cudf_polars/cudf_polars/dsl/expressions/aggregation.py +++ b/python/cudf_polars/cudf_polars/dsl/expressions/aggregation.py @@ -15,6 +15,7 @@ from cudf_polars.containers import Column from cudf_polars.dsl.expressions.base import ExecutionContext, Expr from cudf_polars.dsl.expressions.literal import Literal +from cudf_polars.utils.versions import POLARS_VERSION_LT_136 if TYPE_CHECKING: from rmm.pylibrmm.stream import Stream @@ -94,6 +95,16 @@ def __init__( raise NotImplementedError( f"Unreachable, {name=} is incorrectly listed in _SUPPORTED" ) # pragma: no cover + if ( + POLARS_VERSION_LT_136 + and context == ExecutionContext.FRAME + and req is not None + and not plc.aggregation.is_valid_aggregation(dtype.plc_type, req) + ): # pragma: no cover; polars may raise ahead of time + # TODO: Check which cases polars raises vs returns all-NULL column. + # For the all-NULL column cases, we could build it using Column.all_null_like + # at evaluation time. + raise NotImplementedError(f"Invalid aggregation {req} with dtype {dtype}") self.request = req op = getattr(self, f"_{name}", None) if op is None: diff --git a/python/cudf_polars/cudf_polars/dsl/expressions/string.py b/python/cudf_polars/cudf_polars/dsl/expressions/string.py index 33de97eac9f0..be8dae72fdbe 100644 --- a/python/cudf_polars/cudf_polars/dsl/expressions/string.py +++ b/python/cudf_polars/cudf_polars/dsl/expressions/string.py @@ -24,6 +24,7 @@ from cudf_polars.dsl.expressions.base import ExecutionContext, Expr from cudf_polars.dsl.expressions.literal import Literal, LiteralColumn from cudf_polars.dsl.utils.reshape import broadcast +from cudf_polars.utils.versions import POLARS_VERSION_LT_136, POLARS_VERSION_LT_138 if TYPE_CHECKING: from typing import Self @@ -231,11 +232,14 @@ def _validate_input(self) -> None: "libcudf replace does not support empty strings" ) elif self.name is StringFunction.Name.ReplaceMany: - ascii_case_insensitive, leftmost = self.options - if leftmost: - raise NotImplementedError( - "leftmost=True not implemented for replace_many" - ) + if POLARS_VERSION_LT_136: + (ascii_case_insensitive,) = self.options # pragma: no cover + else: + ascii_case_insensitive, leftmost = self.options + if leftmost: + raise NotImplementedError( + "leftmost=True not implemented for replace_many" + ) if ascii_case_insensitive: raise NotImplementedError( "ascii_case_insensitive not implemented for replace_many" @@ -263,8 +267,9 @@ def _validate_input(self) -> None: (_, inclusive) = self.options if inclusive: raise NotImplementedError(f"{inclusive=} is not supported for split") - elif self.name is StringFunction.Name.SplitRegex: + elif not POLARS_VERSION_LT_138 and self.name is StringFunction.Name.SplitRegex: # See https://github.com/pola-rs/polars/pull/26060 + # SplitRegex introduced in polars>=1.38, but we don't support it yet raise NotImplementedError( "String split with regex (literal=False) is not supported." ) diff --git a/python/cudf_polars/cudf_polars/dsl/expressions/unary.py b/python/cudf_polars/cudf_polars/dsl/expressions/unary.py index 635029c6341d..de03a4623a43 100644 --- a/python/cudf_polars/cudf_polars/dsl/expressions/unary.py +++ b/python/cudf_polars/cudf_polars/dsl/expressions/unary.py @@ -242,7 +242,10 @@ def do_evaluate( return column elif self.name == "set_sorted": (column,) = (child.evaluate(df, context=context) for child in self.children) - descending, _nulls_last = self.options + if isinstance(self.options[0], str): + descending = self.options[0] == "descending" # pragma: no cover + else: + descending, _ = self.options order = ( plc.types.Order.DESCENDING if descending else plc.types.Order.ASCENDING ) diff --git a/python/cudf_polars/cudf_polars/dsl/ir.py b/python/cudf_polars/cudf_polars/dsl/ir.py index 1c84692481f9..27ef664d7331 100644 --- a/python/cudf_polars/cudf_polars/dsl/ir.py +++ b/python/cudf_polars/cudf_polars/dsl/ir.py @@ -50,6 +50,11 @@ get_new_cuda_stream, join_cuda_streams, ) +from cudf_polars.utils.versions import ( + POLARS_VERSION_LT_136, + POLARS_VERSION_LT_137, + POLARS_VERSION_LT_138, +) if TYPE_CHECKING: from collections.abc import Callable, Generator, Hashable, Iterable, Sequence @@ -1034,8 +1039,8 @@ def __init__( raise NotImplementedError( f"Compression type '{compression}' is not supported." ) - elif ( - kind == "NDJson" + elif kind == ( + "Json" if POLARS_VERSION_LT_137 else "NDJson" ): # pragma: no cover; options are validated on the polars side if not all( plc.io.json.is_supported_write_json(dtype.plc_type) @@ -1219,11 +1224,13 @@ def do_evaluate( """Write the dataframe to a file.""" target = plc.io.SinkInfo([path]) + if POLARS_VERSION_LT_136 and options.get("mkdir", False): + Path(path).parent.mkdir(parents=True, exist_ok=True) # pragma: no cover if kind == "Csv": cls._write_csv(target, options, df) elif kind == "Parquet": cls._write_parquet(target, parquet_options, options, df) - elif kind == "NDJson": + elif kind == ("Json" if POLARS_VERSION_LT_137 else "NDJson"): cls._write_json(target, df) return DataFrame([], stream=df.stream) @@ -2991,7 +2998,6 @@ class MapFunction(IR): "unpivot", "row_index", "fast_count", - "hint_sorted", ] ) @@ -3007,11 +3013,11 @@ def __init__(self, schema: Schema, name: str, options: Any, df: IR): f"Unhandled map function {self.name}" ) # pragma: no cover if self.name == "explode": - if len(self.options) == 1: + if POLARS_VERSION_LT_136 or len(self.options) == 1: (to_explode,) = self.options else: (to_explode, empty_as_null, keep_nulls) = self.options - if not keep_nulls: + if (POLARS_VERSION_LT_136 and not empty_as_null) or not keep_nulls: raise NotImplementedError( "Explode an empty or null list/array" ) # pragma: no cover @@ -3025,6 +3031,9 @@ def __init__(self, schema: Schema, name: str, options: Any, df: IR): indices, pivotees, variable_name, value_name = self.options value_name = "value" if value_name is None else value_name variable_name = "variable" if variable_name is None else variable_name + if POLARS_VERSION_LT_136 and len(pivotees) == 0: # pragma: no cover + index = frozenset(indices) + pivotees = [name for name in df.schema if name not in index] if not all( dtypes.can_cast(df.schema[p].plc_type, self.schema[value_name].plc_type) for p in pivotees @@ -3041,6 +3050,8 @@ def __init__(self, schema: Schema, name: str, options: Any, df: IR): ) elif self.name == "row_index": col_name, offset = options + if POLARS_VERSION_LT_138 and col_name in df.schema: # pragma: no cover + raise NotImplementedError("Duplicate row index name") self.options = (col_name, offset) elif self.name == "fast_count": # TODO: Remove this once all scan types support projections @@ -3051,10 +3062,6 @@ def __init__(self, schema: Schema, name: str, options: Any, df: IR): raise NotImplementedError( "Fast count unsupported for CSV scans" ) # pragma: no cover - elif ( - self.name == "hint_sorted" - ): # pragma: no cover; polars prunes hints in some cases - raise NotImplementedError("Hint sorted unsupported") self._non_child_args = (schema, name, self.options) def get_hashable(self) -> Hashable: diff --git a/python/cudf_polars/cudf_polars/dsl/translate.py b/python/cudf_polars/cudf_polars/dsl/translate.py index 372aadecc74e..b5661b94fffb 100644 --- a/python/cudf_polars/cudf_polars/dsl/translate.py +++ b/python/cudf_polars/cudf_polars/dsl/translate.py @@ -32,6 +32,10 @@ from cudf_polars.dsl.utils.rolling import rewrite_rolling from cudf_polars.typing import Schema from cudf_polars.utils import config, sorting +from cudf_polars.utils.versions import ( + POLARS_VERSION_LT_136, + POLARS_VERSION_LT_138, +) if TYPE_CHECKING: from polars import GPUEngine @@ -611,7 +615,9 @@ def _(node: plrs._ir_nodes.Sink, translator: Translator, schema: Schema) -> ir.I payload = json.loads(node.payload) try: file = payload["File"] - sink_kind_options = file["file_format"] + sink_kind_options = file[ + "file_type" if POLARS_VERSION_LT_136 else "file_format" + ] except KeyError as err: # pragma: no cover raise NotImplementedError("Unsupported payload structure") from err if isinstance(sink_kind_options, dict): @@ -623,22 +629,31 @@ def _(node: plrs._ir_nodes.Sink, translator: Translator, schema: Schema) -> ir.I "Unsupported sink options structure" ) # pragma: no cover - unified_args = file.get("unified_sink_args", {}) - cloud_options = unified_args.get("cloud_options") - options = {} if sink_kind == "NDJson" else format_options.copy() + if POLARS_VERSION_LT_138: # pragma: no cover + sink_options = file.get("sink_options", {}) + cloud_options = file.get("cloud_options") + options = format_options.copy() + options.update(sink_options) + else: + unified_args = file.get("unified_sink_args", {}) + cloud_options = unified_args.get("cloud_options") + options = {} if sink_kind == "NDJson" else format_options.copy() - for k, v in unified_args.items(): - if k in {"mkdir", "maintain_order", "sync_on_close"}: - options[k] = v + for k, v in unified_args.items(): + if k in {"mkdir", "maintain_order", "sync_on_close"}: + options[k] = v - if sink_kind in ("Csv", "NDJson"): + if sink_kind in ("Csv", "NDJson", "Json"): compression = format_options.get("compression") if compression and compression != "Uncompressed": raise NotImplementedError( f"{sink_kind} compression ('{compression}') is not supported." ) - path = file["target"]["inner"] + if POLARS_VERSION_LT_138: # pragma: no cover + path = file["target"]["Local"] + else: + path = file["target"]["inner"] return ir.Sink( schema=schema, @@ -822,7 +837,51 @@ def _( dtype: DataType, schema: Schema, ) -> expr.Expr: - if isinstance(node.options, plrs._expr_nodes.WindowMapping): + if hasattr(plrs._expr_nodes, "RollingGroupOptions") and isinstance( + node.options, plrs._expr_nodes.RollingGroupOptions + ): # pragma: no cover; polars >=1.36 uses AExpr::Rolling now + # pl.col("a").rolling(...) in polars <1.36 + with set_expr_context(translator, ExecutionContext.ROLLING): + agg = translator.translate_expr(n=node.function, schema=schema) + name_generator = unique_names(schema) + aggs, named_post_agg = decompose_single_agg( + expr.NamedExpr(next(name_generator), agg), + name_generator, + is_top=True, + context=ExecutionContext.ROLLING, + ) + named_aggs = [agg for agg, _ in aggs] + orderby = node.options.index_column + orderby_dtype = schema[orderby].plc_type + if plc.traits.is_integral(orderby_dtype): + # Integer orderby column is cast in implementation to int64 in polars + orderby_dtype = plc.DataType(plc.TypeId.INT64) + closed_window = node.options.closed_window + if isinstance(named_post_agg.value, expr.Col): + (named_agg,) = named_aggs + return expr.RollingWindow( + named_agg.value.dtype, + orderby_dtype, + node.options.offset, + node.options.period, + closed_window, + orderby, + named_agg.value, + ) + replacements: dict[expr.Expr, expr.Expr] = { + expr.Col(agg.value.dtype, agg.name): expr.RollingWindow( + agg.value.dtype, + orderby_dtype, + node.options.offset, + node.options.period, + closed_window, + orderby, + agg.value, + ) + for agg in named_aggs + } + return replace([named_post_agg.value], replacements)[0] + elif isinstance(node.options, plrs._expr_nodes.WindowMapping): # pl.col("a").over(...) with set_expr_context(translator, ExecutionContext.WINDOW): agg = translator.translate_expr(n=node.function, schema=schema) @@ -879,8 +938,7 @@ def _( assert_never(node.options) -@_translate_expr.register -def _( +def _translate_rolling( node: plrs._expr_nodes.Rolling, translator: Translator, dtype: DataType, @@ -927,6 +985,13 @@ def _( return replace([named_post_agg.value], replacements)[0] +# Rolling was added in polars 1.39. Use explicit registration (not annotation-based) +# so that Python 3.14's stricter get_type_hints() resolution does not fail when +# running on older polars versions that lack this node type. +if hasattr(plrs._expr_nodes, "Rolling"): + _translate_expr.register(plrs._expr_nodes.Rolling)(_translate_rolling) + + @_translate_expr.register def _( node: plrs._expr_nodes.Literal, diff --git a/python/cudf_polars/cudf_polars/experimental/io.py b/python/cudf_polars/cudf_polars/experimental/io.py index 9486eb6b9772..ec04f09883cc 100644 --- a/python/cudf_polars/cudf_polars/experimental/io.py +++ b/python/cudf_polars/cudf_polars/experimental/io.py @@ -35,6 +35,7 @@ ) from cudf_polars.experimental.dispatch import generate_ir_tasks, lower_ir_node from cudf_polars.utils.cuda_stream import get_cuda_stream +from cudf_polars.utils.versions import POLARS_VERSION_LT_137 if TYPE_CHECKING: from collections.abc import Hashable, MutableMapping @@ -514,7 +515,7 @@ def _sink_to_file( # Path.open returns IO[Any] but SinkInfo needs more specific IO types sink = plc.io.types.SinkInfo([f]) # type: ignore[arg-type] Sink._write_csv(sink, use_options, df) - elif kind == "NDJson": + elif kind == ("Json" if POLARS_VERSION_LT_137 else "NDJson"): mode = "wb" if writer_state is None else "ab" with Path.open(Path(path), mode) as f: # Path.open returns IO[Any] but SinkInfo needs more specific IO types diff --git a/python/cudf_polars/cudf_polars/utils/versions.py b/python/cudf_polars/cudf_polars/utils/versions.py index c32124f2d967..13ff49e4b786 100644 --- a/python/cudf_polars/cudf_polars/utils/versions.py +++ b/python/cudf_polars/cudf_polars/utils/versions.py @@ -11,7 +11,13 @@ from polars import __version__ POLARS_VERSION = parse(__version__) -POLARS_LOWER_BOUND = parse("1.39") +POLARS_LOWER_BOUND = parse("1.35") + +POLARS_VERSION_LT_135 = POLARS_VERSION < parse("1.35") +POLARS_VERSION_LT_136 = POLARS_VERSION < parse("1.36") +POLARS_VERSION_LT_137 = POLARS_VERSION < parse("1.37") +POLARS_VERSION_LT_138 = POLARS_VERSION < parse("1.38") +POLARS_VERSION_LT_139 = POLARS_VERSION < parse("1.39") def _ensure_polars_version() -> None: diff --git a/python/cudf_polars/pyproject.toml b/python/cudf_polars/pyproject.toml index 0b415224db83..23bf81faebce 100644 --- a/python/cudf_polars/pyproject.toml +++ b/python/cudf_polars/pyproject.toml @@ -24,7 +24,7 @@ dependencies = [ "cuda-python>=13.0.1,<14.0", "nvidia-ml-py>=12", "packaging", - "polars>=1.39,<1.40", + "polars>=1.35,<1.40", "pylibcudf==26.6.*,>=0.0.0a0", "typing_extensions>=4.0.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/cudf_polars/tests/dsl/test_serialization.py b/python/cudf_polars/tests/dsl/test_serialization.py index 8184092a51a8..8683361f25c2 100644 --- a/python/cudf_polars/tests/dsl/test_serialization.py +++ b/python/cudf_polars/tests/dsl/test_serialization.py @@ -13,6 +13,7 @@ from cudf_polars.dsl.expressions.datetime import TemporalFunction from cudf_polars.dsl.expressions.string import StringFunction from cudf_polars.dsl.expressions.struct import StructFunction +from cudf_polars.utils.versions import POLARS_VERSION_LT_138 @pytest.fixture( @@ -48,6 +49,9 @@ def test_from_polars_all_names(function): "FieldByIndex", "MultipleFields", } + if function == StringFunction and POLARS_VERSION_LT_138: + # SplitRegex was added in polars 1.38 + cudf_polars_names_set = cudf_polars_names_set - {"SplitRegex"} assert polars_names_set == cudf_polars_names_set names = function.Name if function == StructFunction: @@ -55,6 +59,9 @@ def test_from_polars_all_names(function): StructFunction.Name.FieldByIndex, StructFunction.Name.MultipleFields, } + if function == StringFunction and POLARS_VERSION_LT_138: + # SplitRegex was added in polars 1.38 + names = set(names) - {StringFunction.Name.SplitRegex} for name in names: attr = getattr(polars_function, name.name) assert function.Name.from_polars(attr) == name diff --git a/python/cudf_polars/tests/experimental/test_rolling.py b/python/cudf_polars/tests/experimental/test_rolling.py index ffc0e5ee3414..0a61cef85281 100644 --- a/python/cudf_polars/tests/experimental/test_rolling.py +++ b/python/cudf_polars/tests/experimental/test_rolling.py @@ -11,6 +11,7 @@ from cudf_polars.experimental.rapidsmpf.frontend.spmd import SPMDEngine from cudf_polars.testing.asserts import assert_gpu_result_equal +from cudf_polars.utils.versions import POLARS_VERSION_LT_136, POLARS_VERSION_LT_139 if TYPE_CHECKING: from collections.abc import Generator @@ -33,6 +34,10 @@ def engine( yield engine +@pytest.mark.skipif( + not POLARS_VERSION_LT_136 and POLARS_VERSION_LT_139, + reason="Rolling window expressions are not accessible in polars 1.36-1.38", +) def test_rolling_datetime(engine): dates = [ "2020-01-01 13:45:48", diff --git a/python/cudf_polars/tests/expressions/test_agg.py b/python/cudf_polars/tests/expressions/test_agg.py index 4b9c9c66e2d9..0b33908af5be 100644 --- a/python/cudf_polars/tests/expressions/test_agg.py +++ b/python/cudf_polars/tests/expressions/test_agg.py @@ -14,6 +14,7 @@ assert_gpu_result_equal, assert_ir_translation_raises, ) +from cudf_polars.utils.versions import POLARS_VERSION_LT_135, POLARS_VERSION_LT_136 @pytest.fixture( @@ -53,10 +54,12 @@ def is_sorted(request): @pytest.fixture def xfail_if_sorted_gt_135(is_sorted, request): - # See https://github.com/rapidsai/cudf/pull/20791#issuecomment-3750528419 if is_sorted: request.applymarker( - pytest.mark.xfail(reason="See https://github.com/pola-rs/polars/pull/24981") + pytest.mark.xfail( + condition=not POLARS_VERSION_LT_135, + reason="HintIR not supported", + ) ) @@ -223,7 +226,8 @@ def test_decimal_std_var(decimal_df): def test_invalid_agg(request): - request.applymarker(pytest.mark.xfail(reason="polars raises now")) + if not POLARS_VERSION_LT_136: + request.applymarker(pytest.mark.xfail(reason="polars raises now")) df = pl.LazyFrame({"s": pl.Series(["a", "b", "c"], dtype=pl.String())}) q = df.select(pl.col("s").sum()) assert_ir_translation_raises(q, NotImplementedError) diff --git a/python/cudf_polars/tests/expressions/test_rolling.py b/python/cudf_polars/tests/expressions/test_rolling.py index c4d7eb01dd69..6d8c0e6585c5 100644 --- a/python/cudf_polars/tests/expressions/test_rolling.py +++ b/python/cudf_polars/tests/expressions/test_rolling.py @@ -13,11 +13,21 @@ assert_gpu_result_equal, assert_ir_translation_raises, ) +from cudf_polars.utils.versions import POLARS_VERSION_LT_136, POLARS_VERSION_LT_139 if TYPE_CHECKING: from cudf_polars.typing import RankMethod +# In polars 1.36-1.38, rolling window expressions (pl.col(...).rolling(...)) +# are represented as an opaque Rust node that is not accessible via view_expression(). +# Support requires polars <1.36 (via Window+RollingGroupOptions) or >=1.39 (via Rolling). +skip_rolling_expr_136_to_138 = pytest.mark.skipif( + not POLARS_VERSION_LT_136 and POLARS_VERSION_LT_139, + reason="Rolling window expressions are not accessible in polars 1.36-1.38", +) + + @pytest.fixture def df(): return pl.LazyFrame( @@ -31,6 +41,7 @@ def df(): ) +@skip_rolling_expr_136_to_138 @pytest.mark.parametrize("time_unit", ["ns", "us", "ms"]) def test_rolling_datetime(time_unit): dates = [ @@ -55,6 +66,7 @@ def test_rolling_datetime(time_unit): assert_gpu_result_equal(q) +@skip_rolling_expr_136_to_138 def test_rolling_date(): dates = [ "2020-01-01", @@ -76,6 +88,7 @@ def test_rolling_date(): assert_gpu_result_equal(q) +@skip_rolling_expr_136_to_138 @pytest.mark.parametrize("dtype", [pl.Int32, pl.UInt32, pl.Int64, pl.UInt64]) def test_rolling_integral_orderby(dtype): df = pl.LazyFrame( @@ -91,6 +104,7 @@ def test_rolling_integral_orderby(dtype): assert_gpu_result_equal(q) +@skip_rolling_expr_136_to_138 def test_rolling_agg_first(): df = pl.LazyFrame({"a": [1, 2, 3], "b": [1, 2, 3]}) q = df.with_columns(pl.col("a").sum().rolling("b", period="2i")) @@ -110,6 +124,7 @@ def test_rolling_collect_list_raises(): ) +@skip_rolling_expr_136_to_138 def test_unsorted_raises(): df = pl.LazyFrame({"orderby": [1, 2, 4, 2], "values": [1, 2, 3, 4]}) q = df.select(pl.col("values").sum().rolling("orderby", period="2i")) @@ -122,6 +137,7 @@ def test_unsorted_raises(): q.collect(engine=pl.GPUEngine(raise_on_fail=True)) +@skip_rolling_expr_136_to_138 def test_orderby_nulls_raises_computeerror(): df = pl.LazyFrame({"orderby": [1, 2, 4, None], "values": [1, 2, 3, 4]}) q = df.select(pl.col("values").sum().rolling("orderby", period="2i")) @@ -133,6 +149,7 @@ def test_orderby_nulls_raises_computeerror(): q.collect(engine=pl.GPUEngine(raise_on_fail=True)) +@skip_rolling_expr_136_to_138 def test_invalid_duration_spec_raises_in_translation(): df = pl.LazyFrame({"orderby": [1, 2, 4, 5], "values": [1, 2, 3, 4]}) q = df.select(pl.col("values").sum().rolling("orderby", period="3d")) @@ -140,7 +157,8 @@ def test_invalid_duration_spec_raises_in_translation(): def test_rolling_inside_groupby_raises(request): - request.applymarker(pytest.mark.xfail(reason="not supported")) + if not POLARS_VERSION_LT_136: + request.applymarker(pytest.mark.xfail(reason="not supported")) df = pl.LazyFrame( {"keys": [1, 1, 1, 2], "orderby": [1, 2, 4, 2], "values": [1, 2, 3, 4]} ) @@ -152,6 +170,7 @@ def test_rolling_inside_groupby_raises(request): assert_ir_translation_raises(q, NotImplementedError) +@skip_rolling_expr_136_to_138 def test_rolling_sum_all_null_window_returns_null(): df = pl.LazyFrame( { diff --git a/python/cudf_polars/tests/expressions/test_sort.py b/python/cudf_polars/tests/expressions/test_sort.py index 94120c69e245..6763b6593f8a 100644 --- a/python/cudf_polars/tests/expressions/test_sort.py +++ b/python/cudf_polars/tests/expressions/test_sort.py @@ -9,6 +9,7 @@ import polars as pl from cudf_polars.testing.asserts import assert_gpu_result_equal +from cudf_polars.utils.versions import POLARS_VERSION_LT_135, POLARS_VERSION_LT_136 @pytest.mark.parametrize("descending", [False, True]) @@ -57,6 +58,12 @@ def test_sort_by_expression(descending, nulls_last, maintain_order): @pytest.mark.parametrize("nulls_last", [False, True]) @pytest.mark.parametrize("with_nulls", ["no_nulls", "nulls"]) def test_setsorted(request, descending, nulls_last, with_nulls): + request.applymarker( + pytest.mark.xfail( + condition=not POLARS_VERSION_LT_135 and POLARS_VERSION_LT_136, + reason="HintIR not supported", + ) + ) values = sorted([1, 2, 3, 4, 5, 6, -2], reverse=descending) if with_nulls == "nulls": values[-1 if nulls_last else 0] = None diff --git a/python/cudf_polars/tests/expressions/test_stringfunction.py b/python/cudf_polars/tests/expressions/test_stringfunction.py index 0548a4fb2e1f..41243a23917c 100644 --- a/python/cudf_polars/tests/expressions/test_stringfunction.py +++ b/python/cudf_polars/tests/expressions/test_stringfunction.py @@ -14,6 +14,7 @@ assert_gpu_result_equal, assert_ir_translation_raises, ) +from cudf_polars.utils.versions import POLARS_VERSION_LT_136, POLARS_VERSION_LT_138 @pytest.fixture @@ -367,6 +368,10 @@ def test_replace_many_ascii_case(ldf): assert_ir_translation_raises(q, NotImplementedError) +@pytest.mark.skipif( + POLARS_VERSION_LT_136, + reason="leftmost parameter requires polars >= 1.36", +) def test_replace_many_leftmost(ldf): q = ldf.select(pl.col("a").str.replace_many(["a", "b"], "x", leftmost=True)) @@ -874,6 +879,10 @@ def test_concat_str_with_boolean(): assert_gpu_result_equal(q) +@pytest.mark.skipif( + POLARS_VERSION_LT_138, + reason="str.split(literal=False) / SplitRegex requires polars >= 1.38", +) def test_split_regex_not_supported(): lf = pl.LazyFrame({"a": ["foo1bar", "baz456boo", "abc321"]}) diff --git a/python/cudf_polars/tests/test_dataframescan.py b/python/cudf_polars/tests/test_dataframescan.py index 6054de0b438a..1c7141d90389 100644 --- a/python/cudf_polars/tests/test_dataframescan.py +++ b/python/cudf_polars/tests/test_dataframescan.py @@ -13,6 +13,7 @@ assert_gpu_result_equal, assert_ir_translation_raises, ) +from cudf_polars.utils.versions import POLARS_VERSION_LT_138 @pytest.mark.parametrize( @@ -81,6 +82,9 @@ def test_dataframescan_with_decimals(): assert_gpu_result_equal(q) +@pytest.mark.skipif( + POLARS_VERSION_LT_138, reason="pl.LazyFrame(height=...) requires polars >= 1.38" +) def test_dataframescan_zero_width_with_rows(request, using_rapidsmpf): request.applymarker( pytest.mark.xfail( diff --git a/python/cudf_polars/tests/test_groupby.py b/python/cudf_polars/tests/test_groupby.py index 01f44eb288e6..342017bc225e 100644 --- a/python/cudf_polars/tests/test_groupby.py +++ b/python/cudf_polars/tests/test_groupby.py @@ -15,6 +15,7 @@ assert_gpu_result_equal, assert_ir_translation_raises, ) +from cudf_polars.utils.versions import POLARS_VERSION_LT_136 @pytest.fixture @@ -384,7 +385,7 @@ def test_groupby_aggs_keep_unsupported_as_null( ) -> None: request.applymarker( pytest.mark.xfail( - condition="sum" in str(agg_expr), + condition=not POLARS_VERSION_LT_136 and "sum" in str(agg_expr), reason="polars raises now", ) ) diff --git a/python/cudf_polars/tests/test_mapfunction.py b/python/cudf_polars/tests/test_mapfunction.py index 05f78f73d715..da72b75c70f5 100644 --- a/python/cudf_polars/tests/test_mapfunction.py +++ b/python/cudf_polars/tests/test_mapfunction.py @@ -13,6 +13,7 @@ assert_gpu_result_equal, assert_ir_translation_raises, ) +from cudf_polars.utils.versions import POLARS_VERSION_LT_135 def test_explode_multiple_raises(): @@ -112,7 +113,12 @@ def test_unique_hash(): def test_set_sorted_then_inner_join(request): - request.applymarker(pytest.mark.xfail(reason="HintIR not supported")) + request.applymarker( + pytest.mark.xfail( + condition=not POLARS_VERSION_LT_135, + reason="HintIR not supported", + ) + ) df = pl.LazyFrame({"a": [1, 2, 3, 4, 5]}) q = df.set_sorted("a").join( diff --git a/python/cudf_polars/tests/test_rolling.py b/python/cudf_polars/tests/test_rolling.py index 7d69f09a9689..1ad20185a22c 100644 --- a/python/cudf_polars/tests/test_rolling.py +++ b/python/cudf_polars/tests/test_rolling.py @@ -13,6 +13,7 @@ assert_gpu_result_equal, assert_ir_translation_raises, ) +from cudf_polars.utils.versions import POLARS_VERSION_LT_136 @pytest.fixture(params=[False, True], ids=["nosort", "sort"]) @@ -160,7 +161,8 @@ def test_orderby_nulls_raises_computeerror(): def test_rolling_nested_raises(request): - request.applymarker(pytest.mark.xfail(reason="polars raises now")) + if not POLARS_VERSION_LT_136: + request.applymarker(pytest.mark.xfail(reason="polars raises now")) q = ( pl.LazyFrame( { diff --git a/python/cudf_polars/tests/test_scan.py b/python/cudf_polars/tests/test_scan.py index a57f68043bf2..387b583aef1e 100644 --- a/python/cudf_polars/tests/test_scan.py +++ b/python/cudf_polars/tests/test_scan.py @@ -18,6 +18,10 @@ assert_ir_translation_raises, ) from cudf_polars.testing.io import make_partitioned_source +from cudf_polars.utils.versions import ( + POLARS_VERSION_LT_138, + POLARS_VERSION_LT_139, +) if TYPE_CHECKING: from pathlib import Path @@ -569,6 +573,7 @@ def test_scan_ndjson_remote( ) -> None: request.applymarker( pytest.mark.xfail( + condition=not POLARS_VERSION_LT_139, reason="polars 1.39+ ndjson remote reader requires range request support", ) ) @@ -630,7 +635,12 @@ def test_scan_csv_blank_line(tmp_path): def test_hits_scan_row_index_duplicate(request, tmp_path): - request.applymarker(pytest.mark.xfail(reason="polars fails ahead of time")) + request.applymarker( + pytest.mark.xfail( + condition=not POLARS_VERSION_LT_138, + reason="polars >= 1.38 raises duplicate row_index name ahead of time", + ) + ) pl.DataFrame({"col": [1, 2, 3]}).write_parquet(tmp_path / "a.parquet") q = pl.scan_parquet(tmp_path / "*.parquet", row_index_name="index").with_row_index( @@ -676,6 +686,9 @@ def test_scan_tiny_file_not_compressed(tmp_path): assert_gpu_result_equal(q) +@pytest.mark.skipif( + POLARS_VERSION_LT_138, reason="pl.LazyFrame(height=...) requires polars >= 1.38" +) @pytest.mark.parametrize("engine", [None, NO_CHUNK_ENGINE]) def test_scan_parquet_zero_width_with_limit(tmp_path, engine, request, using_rapidsmpf): request.applymarker( diff --git a/python/cudf_polars/tests/test_sink.py b/python/cudf_polars/tests/test_sink.py index b98f6a1e7976..a5175c996db2 100644 --- a/python/cudf_polars/tests/test_sink.py +++ b/python/cudf_polars/tests/test_sink.py @@ -11,6 +11,7 @@ assert_sink_ir_translation_raises, assert_sink_result_equal, ) +from cudf_polars.utils.versions import POLARS_VERSION_LT_138 @pytest.fixture(scope="module") @@ -161,6 +162,9 @@ def test_sink_in_memory_executor(df, tmp_path, file_type): ) +@pytest.mark.skipif( + POLARS_VERSION_LT_138, reason="sink compression kwarg requires polars >= 1.38" +) @pytest.mark.parametrize("compression", ["gzip", "zstd"]) @pytest.mark.parametrize("file_type", ["csv", "ndjson"]) def test_sink_compression_raises(df, tmp_path, compression, file_type): diff --git a/python/cudf_polars/tests/test_window_functions.py b/python/cudf_polars/tests/test_window_functions.py index 488a65a83c91..b5a685427d24 100644 --- a/python/cudf_polars/tests/test_window_functions.py +++ b/python/cudf_polars/tests/test_window_functions.py @@ -13,6 +13,7 @@ assert_gpu_result_equal, assert_ir_translation_raises, ) +from cudf_polars.utils.versions import POLARS_VERSION_LT_136, POLARS_VERSION_LT_139 @pytest.fixture @@ -123,6 +124,10 @@ def test_over_mapping_strategy(df: pl.LazyFrame, mapping_strategy: str): assert_ir_translation_raises(q, NotImplementedError) +@pytest.mark.skipif( + not POLARS_VERSION_LT_136 and POLARS_VERSION_LT_139, + reason="Rolling window expressions are not accessible in polars 1.36-1.38", +) @pytest.mark.parametrize("period", ["2d", "3d"]) def test_rolling(df: pl.LazyFrame, agg_expr, period: str): """Test rolling window functions over time series.""" @@ -146,6 +151,10 @@ def test_rolling_unsupported(df: pl.LazyFrame, unsupported_agg_expr): assert_ir_translation_raises(query, NotImplementedError) +@pytest.mark.skipif( + not POLARS_VERSION_LT_136 and POLARS_VERSION_LT_139, + reason="Rolling window expressions are not accessible in polars 1.36-1.38", +) @pytest.mark.parametrize("closed", ["left", "right", "both", "none"]) def test_rolling_closed(df: pl.LazyFrame, closed: str): """Test rolling window functions with different closed parameters.""" From 8892885a0c5be0c24a58b7137ee47b47c1a16f5c Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Mon, 11 May 2026 18:12:20 +0000 Subject: [PATCH 07/44] Bump dependnecy lower bounds to 1.35 --- .pre-commit-config.yaml | 2 +- conda/environments/all_cuda-129_arch-aarch64.yaml | 2 +- conda/environments/all_cuda-129_arch-x86_64.yaml | 2 +- conda/environments/all_cuda-131_arch-aarch64.yaml | 2 +- conda/environments/all_cuda-131_arch-x86_64.yaml | 2 +- dependencies.yaml | 2 +- python/cudf_polars/pyproject.toml | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6d29a2153cc3..6d122ef7030d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -35,7 +35,7 @@ repos: hooks: - id: mypy additional_dependencies: [ - "polars>=1.30,<1.39", + "polars>=1.35,<1.39", "numpy", pyarrow-stubs, "pyarrow<24.0.0", # https://github.com/rapidsai/cudf/issues/22229 diff --git a/conda/environments/all_cuda-129_arch-aarch64.yaml b/conda/environments/all_cuda-129_arch-aarch64.yaml index 3bf006400bce..3b802eaf654e 100644 --- a/conda/environments/all_cuda-129_arch-aarch64.yaml +++ b/conda/environments/all_cuda-129_arch-aarch64.yaml @@ -67,7 +67,7 @@ dependencies: - packaging - pandas>=2.0,<2.4.0 - pandoc -- polars>=1.30,<1.39 +- polars>=1.35,<1.39 - pre-commit - pyarrow>=19.0.0,<24 - pydata-sphinx-theme>=0.15.4 diff --git a/conda/environments/all_cuda-129_arch-x86_64.yaml b/conda/environments/all_cuda-129_arch-x86_64.yaml index d206963fc362..0424ce5276fc 100644 --- a/conda/environments/all_cuda-129_arch-x86_64.yaml +++ b/conda/environments/all_cuda-129_arch-x86_64.yaml @@ -67,7 +67,7 @@ dependencies: - packaging - pandas>=2.0,<2.4.0 - pandoc -- polars>=1.30,<1.39 +- polars>=1.35,<1.39 - pre-commit - pyarrow>=19.0.0,<24 - pydata-sphinx-theme>=0.15.4 diff --git a/conda/environments/all_cuda-131_arch-aarch64.yaml b/conda/environments/all_cuda-131_arch-aarch64.yaml index c73cea89f296..b7c222a8e3b4 100644 --- a/conda/environments/all_cuda-131_arch-aarch64.yaml +++ b/conda/environments/all_cuda-131_arch-aarch64.yaml @@ -67,7 +67,7 @@ dependencies: - packaging - pandas>=2.0,<2.4.0 - pandoc -- polars>=1.30,<1.39 +- polars>=1.35,<1.39 - pre-commit - pyarrow>=19.0.0,<24 - pydata-sphinx-theme>=0.15.4 diff --git a/conda/environments/all_cuda-131_arch-x86_64.yaml b/conda/environments/all_cuda-131_arch-x86_64.yaml index f4d888666905..a00c9349179b 100644 --- a/conda/environments/all_cuda-131_arch-x86_64.yaml +++ b/conda/environments/all_cuda-131_arch-x86_64.yaml @@ -67,7 +67,7 @@ dependencies: - packaging - pandas>=2.0,<2.4.0 - pandoc -- polars>=1.30,<1.39 +- polars>=1.35,<1.39 - pre-commit - pyarrow>=19.0.0,<24 - pydata-sphinx-theme>=0.15.4 diff --git a/dependencies.yaml b/dependencies.yaml index 3890e1f3135f..369320f86629 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -819,7 +819,7 @@ dependencies: # 'nvidia-ml-py' provides the 'pynvml' module - &nvidia_ml_py nvidia-ml-py>=12 - packaging - - polars>=1.30,<1.39 + - polars>=1.35,<1.39 - typing_extensions>=4.0.0 run_cudf_polars_experimental: common: diff --git a/python/cudf_polars/pyproject.toml b/python/cudf_polars/pyproject.toml index 7703cad7dad4..7c21055d0b4c 100644 --- a/python/cudf_polars/pyproject.toml +++ b/python/cudf_polars/pyproject.toml @@ -24,7 +24,7 @@ dependencies = [ "cuda-python>=13.0.1,<14.0", "nvidia-ml-py>=12", "packaging", - "polars>=1.30,<1.39", + "polars>=1.35,<1.39", "pylibcudf==26.6.*,>=0.0.0a0", "rapidsmpf==26.6.*,>=0.0.0a0", "typing_extensions>=4.0.0", From a6c4404aa19f2266937f8452f5b94fa5cc20c571 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Mon, 11 May 2026 18:22:33 +0000 Subject: [PATCH 08/44] Remove 1.31 workarounds --- python/cudf_polars/cudf_polars/dsl/ir.py | 27 +-------- .../cudf_polars/cudf_polars/dsl/translate.py | 16 ++---- .../cudf_polars/cudf_polars/utils/versions.py | 1 - .../tests/dsl/test_serialization.py | 9 +-- .../tests/expressions/test_stringfunction.py | 27 ++------- .../tests/expressions/test_struct.py | 57 +++---------------- python/cudf_polars/tests/test_mapfunction.py | 8 +-- python/cudf_polars/tests/test_scan.py | 16 +----- 8 files changed, 23 insertions(+), 138 deletions(-) diff --git a/python/cudf_polars/cudf_polars/dsl/ir.py b/python/cudf_polars/cudf_polars/dsl/ir.py index 1c48f70bb114..3331877a9a3a 100644 --- a/python/cudf_polars/cudf_polars/dsl/ir.py +++ b/python/cudf_polars/cudf_polars/dsl/ir.py @@ -49,7 +49,6 @@ join_cuda_streams, ) from cudf_polars.utils.versions import ( - POLARS_VERSION_LT_131, POLARS_VERSION_LT_134, POLARS_VERSION_LT_136, POLARS_VERSION_LT_137, @@ -505,16 +504,7 @@ def __init__( raise NotImplementedError( "Read from cloud storage" ) # pragma: no cover; no test yet - if ( - any(str(p).startswith("https:/") for p in self.paths) - and POLARS_VERSION_LT_131 - ): # pragma: no cover; polars passed us the wrong URI - # https://github.com/pola-rs/polars/issues/22766 - raise NotImplementedError("Read from https") - if any( - str(p).startswith("file:/" if POLARS_VERSION_LT_131 else "file://") - for p in self.paths - ): + if any(str(p).startswith("file://") for p in self.paths): raise NotImplementedError("Read from file URI") if self.typ == "csv": if any( @@ -3040,16 +3030,6 @@ def __init__(self, schema: Schema, name: str, options: Any, df: IR): # same sub-shapes raise NotImplementedError("Explode with more than one column") self.options = (tuple(to_explode),) - elif POLARS_VERSION_LT_131 and self.name == "rename": # pragma: no cover - # As of 1.31, polars validates renaming in the IR - old, new, strict = self.options - if len(new) != len(set(new)) or ( - set(new) & (set(df.schema.keys()) - set(old)) - ): - raise NotImplementedError( - "Duplicate new names in rename." - ) # pragma: no cover - self.options = (tuple(old), tuple(new), strict) elif self.name == "unpivot": indices, pivotees, variable_name, value_name = self.options value_name = "value" if value_name is None else value_name @@ -3123,11 +3103,6 @@ def do_evaluate( # No-op in our data model # Don't think this appears in a plan tree from python return df # pragma: no cover - elif POLARS_VERSION_LT_131 and name == "rename": # pragma: no cover - # final tag is "swapping" which is useful for the - # optimiser (it blocks some pushdown operations) - old, new, _ = options - return df.rename_columns(dict(zip(old, new, strict=True))) elif name == "explode": ((to_explode,),) = options index = df.column_names.index(to_explode) diff --git a/python/cudf_polars/cudf_polars/dsl/translate.py b/python/cudf_polars/cudf_polars/dsl/translate.py index 760d0541615d..28004159a3d9 100644 --- a/python/cudf_polars/cudf_polars/dsl/translate.py +++ b/python/cudf_polars/cudf_polars/dsl/translate.py @@ -33,7 +33,6 @@ from cudf_polars.typing import Schema from cudf_polars.utils import config, sorting from cudf_polars.utils.versions import ( - POLARS_VERSION_LT_131, POLARS_VERSION_LT_132, POLARS_VERSION_LT_133, POLARS_VERSION_LT_134, @@ -290,12 +289,11 @@ def _(node: plrs._ir_nodes.Scan, translator: Translator, schema: Schema) -> ir.I with_columns = file_options.with_columns row_index = file_options.row_index include_file_paths = file_options.include_file_paths - if not POLARS_VERSION_LT_131: - deletion_files = file_options.deletion_files # pragma: no cover - if deletion_files: # pragma: no cover - raise NotImplementedError( - "Iceberg format is not supported in cudf-polars. Furthermore, row-level deletions are not supported." - ) # pragma: no cover + deletion_files = file_options.deletion_files + if deletion_files: # pragma: no cover + raise NotImplementedError( + "Iceberg format is not supported in cudf-polars. Furthermore, row-level deletions are not supported." + ) # pragma: no cover config_options = translator.config_options parquet_options = config_options.parquet_options @@ -796,9 +794,7 @@ def _( if name in needs_cast: return expr.Cast(dtype, True, result_expr) # noqa: FBT003 return result_expr - elif not POLARS_VERSION_LT_131 and isinstance( - name, plrs._expr_nodes.StructFunction - ): + elif isinstance(name, plrs._expr_nodes.StructFunction): return expr.StructFunction( dtype, expr.StructFunction.Name.from_polars(name), diff --git a/python/cudf_polars/cudf_polars/utils/versions.py b/python/cudf_polars/cudf_polars/utils/versions.py index e6be1027b0ca..0a15d57c0483 100644 --- a/python/cudf_polars/cudf_polars/utils/versions.py +++ b/python/cudf_polars/cudf_polars/utils/versions.py @@ -12,7 +12,6 @@ POLARS_VERSION = parse(__version__) POLARS_LOWER_BOUND = parse("1.30") -POLARS_VERSION_LT_131 = POLARS_VERSION < parse("1.31") POLARS_VERSION_LT_132 = POLARS_VERSION < parse("1.32") POLARS_VERSION_LT_1321 = POLARS_VERSION < parse("1.32.1") POLARS_VERSION_LT_1323 = POLARS_VERSION < parse("1.32.3") diff --git a/python/cudf_polars/tests/dsl/test_serialization.py b/python/cudf_polars/tests/dsl/test_serialization.py index a496a1e6a870..5608dac96662 100644 --- a/python/cudf_polars/tests/dsl/test_serialization.py +++ b/python/cudf_polars/tests/dsl/test_serialization.py @@ -12,21 +12,16 @@ from cudf_polars.dsl.expressions.boolean import BooleanFunction from cudf_polars.dsl.expressions.datetime import TemporalFunction from cudf_polars.dsl.expressions.string import StringFunction +from cudf_polars.dsl.expressions.struct import StructFunction from cudf_polars.utils.versions import ( - POLARS_VERSION_LT_131, POLARS_VERSION_LT_132, POLARS_VERSION_LT_138, POLARS_VERSION_LT_1321, ) -if not POLARS_VERSION_LT_131: - from cudf_polars.dsl.expressions.struct import StructFunction - @pytest.fixture( - params=[BooleanFunction, StringFunction, TemporalFunction] - if POLARS_VERSION_LT_131 - else [BooleanFunction, StringFunction, TemporalFunction, StructFunction] + params=[BooleanFunction, StringFunction, TemporalFunction, StructFunction] ) def function(request): return request.param diff --git a/python/cudf_polars/tests/expressions/test_stringfunction.py b/python/cudf_polars/tests/expressions/test_stringfunction.py index f363015b028d..9e0c51be96a4 100644 --- a/python/cudf_polars/tests/expressions/test_stringfunction.py +++ b/python/cudf_polars/tests/expressions/test_stringfunction.py @@ -16,7 +16,6 @@ ) from cudf_polars.testing.engine_utils import is_streaming_engine from cudf_polars.utils.versions import ( - POLARS_VERSION_LT_131, POLARS_VERSION_LT_132, POLARS_VERSION_LT_133, POLARS_VERSION_LT_136, @@ -352,14 +351,7 @@ def test_replace_re(ldf): ) def test_replace_many(engine: pl.GPUEngine, ldf, target, repl): q = ldf.select(pl.col("a").str.replace_many(target, repl)) - _need_support_for_implode_agg = isinstance(repl, list) - if _need_support_for_implode_agg: - assert_gpu_result_equal(q, engine=engine) - elif POLARS_VERSION_LT_131: - assert_ir_translation_raises(q, NotImplementedError) - else: - # Polars 1.31 now gives us replacement argument as a list - assert_gpu_result_equal(q, engine=engine) + assert_gpu_result_equal(q, engine=engine) @pytest.mark.parametrize( @@ -574,14 +566,7 @@ def test_string_zfill(engine: pl.GPUEngine, fill, input_strings): @pytest.mark.parametrize( "fill", - [ - 5 - if not POLARS_VERSION_LT_131 - else pytest.param(5, marks=pytest.mark.xfail(reason="fixed in Polars 1.30")), - 999 - if not POLARS_VERSION_LT_131 - else pytest.param(999, marks=pytest.mark.xfail(reason="fixed in Polars 1.30")), - ], + [5, 999], ) def test_string_zfill_pl_129(engine: pl.GPUEngine, fill): ldf = pl.LazyFrame({"a": ["-1", "+2"]}) @@ -595,12 +580,8 @@ def test_string_zfill_pl_129(engine: pl.GPUEngine, fill): 0, 1, 2, - 5 - if not POLARS_VERSION_LT_131 - else pytest.param(5, marks=pytest.mark.xfail(reason="fixed in Polars 1.30")), - 999 - if not POLARS_VERSION_LT_131 - else pytest.param(999, marks=pytest.mark.xfail(reason="fixed in Polars 1.30")), + 5, + 999, -1, pytest.param(None, marks=pytest.mark.xfail(reason="None dtype")), ], diff --git a/python/cudf_polars/tests/expressions/test_struct.py b/python/cudf_polars/tests/expressions/test_struct.py index 5fa58bf9e842..2e20824586a3 100644 --- a/python/cudf_polars/tests/expressions/test_struct.py +++ b/python/cudf_polars/tests/expressions/test_struct.py @@ -10,7 +10,6 @@ assert_gpu_result_equal, assert_ir_translation_raises, ) -from cudf_polars.utils.versions import POLARS_VERSION_LT_131 @pytest.fixture @@ -20,47 +19,23 @@ def ldf(): ) -def test_field_getitem(engine: pl.GPUEngine, request, ldf): - request.applymarker( - pytest.mark.xfail( - condition=POLARS_VERSION_LT_131, - reason="not supported until polars 1.31", - ) - ) +def test_field_getitem(engine: pl.GPUEngine, ldf): q = ldf.select(pl.col("a").struct[0]) assert_gpu_result_equal(q, engine=engine) @pytest.mark.parametrize("fields", [("b",), ("b", "d"), ("^b.*|f.*$",)]) -def test_field(engine: pl.GPUEngine, request, ldf, fields): - request.applymarker( - pytest.mark.xfail( - condition=POLARS_VERSION_LT_131, - reason="not supported until polars 1.31", - ) - ) +def test_field(engine: pl.GPUEngine, ldf, fields): q = ldf.select(pl.col("a").struct.field(*fields)) assert_gpu_result_equal(q, engine=engine) -def test_unnest(engine: pl.GPUEngine, request, ldf): - request.applymarker( - pytest.mark.xfail( - condition=POLARS_VERSION_LT_131, - reason="not supported until polars 1.31", - ) - ) +def test_unnest(engine: pl.GPUEngine, ldf): q = ldf.select(pl.col("a").struct.unnest()) assert_gpu_result_equal(q, engine=engine) -def test_json_encode(engine: pl.GPUEngine, request, ldf): - request.applymarker( - pytest.mark.xfail( - condition=POLARS_VERSION_LT_131, - reason="not supported until polars 1.31", - ) - ) +def test_json_encode(engine: pl.GPUEngine, ldf): q = ldf.select(pl.col("a").struct.json_encode()) assert_gpu_result_equal(q, engine=engine) @@ -69,16 +44,10 @@ def test_json_encode(engine: pl.GPUEngine, request, ldf): assert_gpu_result_equal(q, engine=engine) -def test_json_encode_empty(engine: pl.GPUEngine, request): +def test_json_encode_empty(engine: pl.GPUEngine): # ``write_json`` emits no lines for a zero-row input, so the # ``from_iterable_of_py(buff.split())`` round-trip cannot infer a # dtype. The expression short-circuits to an empty string column. - request.applymarker( - pytest.mark.xfail( - condition=POLARS_VERSION_LT_131, - reason="not supported until polars 1.31", - ) - ) ldf = pl.LazyFrame( {"a": pl.Series([], dtype=pl.Struct({"b": pl.String, "d": pl.String}))} ) @@ -86,13 +55,7 @@ def test_json_encode_empty(engine: pl.GPUEngine, request): assert_gpu_result_equal(q, engine=engine) -def test_rename_fields(engine: pl.GPUEngine, request, ldf): - request.applymarker( - pytest.mark.xfail( - condition=POLARS_VERSION_LT_131, - reason="not supported until polars 1.31", - ) - ) +def test_rename_fields(engine: pl.GPUEngine, ldf): q = ldf.select(pl.col("a").struct.rename_fields(["1", "2", "3"]).struct.unnest()) assert_gpu_result_equal(q, engine=engine) @@ -109,13 +72,7 @@ def test_with_fields(ldf): [pl.col("a").name.prefix_fields, pl.col("a").name.suffix_fields], ids=lambda x: x.__name__, ) -def test_prefix_suffix_fields(engine: pl.GPUEngine, request, ldf, expr): - request.applymarker( - pytest.mark.xfail( - condition=POLARS_VERSION_LT_131, - reason="not supported until polars 1.31", - ) - ) +def test_prefix_suffix_fields(engine: pl.GPUEngine, ldf, expr): q = ldf.select(expr("foo").struct.unnest()) assert_gpu_result_equal(q, engine=engine) diff --git a/python/cudf_polars/tests/test_mapfunction.py b/python/cudf_polars/tests/test_mapfunction.py index b369a06bf35e..f4934fbc5f6e 100644 --- a/python/cudf_polars/tests/test_mapfunction.py +++ b/python/cudf_polars/tests/test_mapfunction.py @@ -14,7 +14,7 @@ assert_ir_translation_raises, ) from cudf_polars.testing.engine_utils import get_blocksize_mode -from cudf_polars.utils.versions import POLARS_VERSION_LT_131, POLARS_VERSION_LT_135 +from cudf_polars.utils.versions import POLARS_VERSION_LT_135 def test_explode_multiple_raises(): @@ -44,12 +44,8 @@ def test_rename_duplicate_raises(mapping): q = df.rename(mapping) - if POLARS_VERSION_LT_131: + with pytest.raises(pl.exceptions.DuplicateError, match="is duplicate"): assert_ir_translation_raises(q, NotImplementedError) - else: - # Now raises before translation - with pytest.raises(pl.exceptions.DuplicateError, match="is duplicate"): - assert_ir_translation_raises(q, NotImplementedError) @pytest.mark.parametrize( diff --git a/python/cudf_polars/tests/test_scan.py b/python/cudf_polars/tests/test_scan.py index a655efbe422a..894e0b61b638 100644 --- a/python/cudf_polars/tests/test_scan.py +++ b/python/cudf_polars/tests/test_scan.py @@ -20,7 +20,6 @@ from cudf_polars.testing.engine_utils import is_streaming_engine from cudf_polars.testing.io import make_partitioned_source from cudf_polars.utils.versions import ( - POLARS_VERSION_LT_131, POLARS_VERSION_LT_135, POLARS_VERSION_LT_138, ) @@ -516,14 +515,8 @@ def test_scan_from_file_uri(tmp_path: Path) -> None: @pytest.mark.parametrize("chunked", [False, True]) def test_scan_parquet_remote( - request, tmp_path: Path, df: pl.DataFrame, httpserver: HTTPServer, *, chunked: bool + tmp_path: Path, df: pl.DataFrame, httpserver: HTTPServer, *, chunked: bool ) -> None: - request.applymarker( - pytest.mark.xfail( - condition=POLARS_VERSION_LT_131, - reason="remote IO not supported", - ) - ) path = tmp_path / "foo.parquet" df.write_parquet(path) bytes_ = path.read_bytes() @@ -582,17 +575,10 @@ def get_handler(req: Request) -> Response: def test_scan_ndjson_remote( engine: pl.GPUEngine, - request: pytest.FixtureRequest, tmp_path: Path, df: pl.DataFrame, httpserver: HTTPServer, ) -> None: - request.applymarker( - pytest.mark.xfail( - condition=POLARS_VERSION_LT_131, - reason="remote IO not supported", - ) - ) path = tmp_path / "foo.jsonl" df.write_ndjson(path) bytes_ = path.read_bytes() From 1143dbc36e988ef33164972939aa769ff259cf6b Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Mon, 11 May 2026 18:43:12 +0000 Subject: [PATCH 09/44] Remove 1.32 workarounds --- .../cudf_polars/dsl/expressions/string.py | 82 ++++--------------- .../cudf_polars/cudf_polars/dsl/translate.py | 13 +-- .../cudf_polars/dsl/utils/aggregations.py | 40 +-------- .../experimental/benchmarks/asserts.py | 7 +- .../cudf_polars/testing/asserts.py | 7 +- .../cudf_polars/cudf_polars/utils/versions.py | 3 - .../tests/dsl/test_serialization.py | 16 +--- .../tests/experimental/test_select.py | 17 ++-- .../tests/expressions/test_booleanfunction.py | 8 +- .../tests/expressions/test_casting.py | 3 +- .../tests/expressions/test_numeric_binops.py | 5 +- .../expressions/test_numeric_unaryops.py | 10 --- .../tests/expressions/test_rolling.py | 23 +----- .../tests/expressions/test_stringfunction.py | 13 +-- python/cudf_polars/tests/test_cache.py | 10 +-- python/cudf_polars/tests/test_drop_nulls.py | 16 +--- python/cudf_polars/tests/test_groupby.py | 20 ++--- python/cudf_polars/tests/test_join.py | 17 +--- .../tests/test_window_functions.py | 4 +- 19 files changed, 58 insertions(+), 256 deletions(-) diff --git a/python/cudf_polars/cudf_polars/dsl/expressions/string.py b/python/cudf_polars/cudf_polars/dsl/expressions/string.py index 9c5214dc57cb..48973153f56b 100644 --- a/python/cudf_polars/cudf_polars/dsl/expressions/string.py +++ b/python/cudf_polars/cudf_polars/dsl/expressions/string.py @@ -12,7 +12,7 @@ from enum import IntEnum, auto from typing import TYPE_CHECKING, Any, ClassVar, cast -from polars import Struct as pl_Struct, polars # type: ignore[attr-defined] +from polars import Struct as pl_Struct from polars.exceptions import InvalidOperationError import pylibcudf as plc @@ -23,7 +23,6 @@ from cudf_polars.dsl.utils.reshape import broadcast from cudf_polars.utils.dtypes import make_empty_column from cudf_polars.utils.versions import ( - POLARS_VERSION_LT_132, POLARS_VERSION_LT_136, POLARS_VERSION_LT_138, ) @@ -31,6 +30,8 @@ if TYPE_CHECKING: from typing import Self + from polars import polars # type: ignore[attr-defined] + from cudf_polars.containers import DataFrame, DataType __all__ = ["StringFunction"] @@ -292,21 +293,6 @@ def _validate_input(self) -> None: raise NotImplementedError( "strip operations only support scalar patterns" ) - elif self.name is StringFunction.Name.ZFill: - if isinstance(self.children[1], Literal): - _, width = self.children - assert isinstance(width, Literal) - if ( - POLARS_VERSION_LT_132 - and width.value is not None - and width.value < 0 - ): # pragma: no cover - dtypestr = polars.dtype_str_repr(width.dtype.polars_type) - raise InvalidOperationError( - f"conversion from `{dtypestr}` to `u64` " - f"failed in column 'literal' for 1 out of " - f"1 values: [{width.value}]" - ) from None @staticmethod def _create_regex_program( @@ -437,24 +423,6 @@ def do_evaluate( else: col_width = self.children[1].evaluate(df, context=context) assert isinstance(col_width, Column) - all_gt_0 = plc.binaryop.binary_operation( - col_width.obj, - plc.Scalar.from_py( - 0, plc.DataType(plc.TypeId.INT64), stream=df.stream - ), - plc.binaryop.BinaryOperator.GREATER_EQUAL, - plc.DataType(plc.TypeId.BOOL8), - stream=df.stream, - ) - - if POLARS_VERSION_LT_132 and not plc.reduce.reduce( - all_gt_0, - plc.aggregation.all(), - plc.DataType(plc.TypeId.BOOL8), - stream=df.stream, - ).to_py(stream=df.stream): # pragma: no cover - raise InvalidOperationError("fill conversion failed.") - return Column( plc.strings.padding.zfill_by_widths( column.obj, col_width.obj, stream=df.stream @@ -981,21 +949,14 @@ def do_evaluate( dtype=self.dtype, ) elif self.name is StringFunction.Name.PadStart: - if POLARS_VERSION_LT_132: # pragma: no cover - (column,) = columns - width_arg, char = self.options - pad_width = cast(int, width_arg) - else: - (column, width_col) = columns - (char,) = self.options - # TODO: Maybe accept a string scalar in - # cudf::strings::pad to avoid DtoH transfer - # See https://github.com/rapidsai/cudf/issues/20202 - width_py = width_col.obj.to_scalar(stream=df.stream).to_py( - stream=df.stream - ) - assert width_py is not None - pad_width = int(width_py) + (column, width_col) = columns + (char,) = self.options + # TODO: Maybe accept a string scalar in + # cudf::strings::pad to avoid DtoH transfer + # See https://github.com/rapidsai/cudf/issues/20202 + width_py = width_col.obj.to_scalar(stream=df.stream).to_py(stream=df.stream) + assert width_py is not None + pad_width = int(width_py) return Column( plc.strings.padding.pad( @@ -1008,20 +969,13 @@ def do_evaluate( dtype=self.dtype, ) elif self.name is StringFunction.Name.PadEnd: - if POLARS_VERSION_LT_132: # pragma: no cover - (column,) = columns - width_arg, char = self.options - pad_width = cast(int, width_arg) - else: - (column, width_col) = columns - (char,) = self.options - # TODO: Maybe accept a string scalar in - # cudf::strings::pad to avoid DtoH transfer - width_py = width_col.obj.to_scalar(stream=df.stream).to_py( - stream=df.stream - ) - assert width_py is not None - pad_width = int(width_py) + (column, width_col) = columns + (char,) = self.options + # TODO: Maybe accept a string scalar in + # cudf::strings::pad to avoid DtoH transfer + width_py = width_col.obj.to_scalar(stream=df.stream).to_py(stream=df.stream) + assert width_py is not None + pad_width = int(width_py) return Column( plc.strings.padding.pad( diff --git a/python/cudf_polars/cudf_polars/dsl/translate.py b/python/cudf_polars/cudf_polars/dsl/translate.py index 28004159a3d9..a7fbcffa7a09 100644 --- a/python/cudf_polars/cudf_polars/dsl/translate.py +++ b/python/cudf_polars/cudf_polars/dsl/translate.py @@ -33,12 +33,10 @@ from cudf_polars.typing import Schema from cudf_polars.utils import config, sorting from cudf_polars.utils.versions import ( - POLARS_VERSION_LT_132, POLARS_VERSION_LT_133, POLARS_VERSION_LT_134, POLARS_VERSION_LT_136, POLARS_VERSION_LT_138, - POLARS_VERSION_LT_1323, ) if TYPE_CHECKING: @@ -339,18 +337,13 @@ def _(node: plrs._ir_nodes.Scan, translator: Translator, schema: Schema) -> ir.I @_translate_ir.register def _(node: plrs._ir_nodes.Cache, translator: Translator, schema: Schema) -> ir.IR: - if POLARS_VERSION_LT_1323: # pragma: no cover - refcount = node.cache_hits - else: - refcount = None - # Make sure Cache nodes with the same id_ # are actually the same object. if node.id_ not in translator._cache_nodes: translator._cache_nodes[node.id_] = ir.Cache( schema, node.id_, - refcount, + None, translator.translate_ir(n=node.input), ) return translator._cache_nodes[node.id_] @@ -641,9 +634,7 @@ def _(node: plrs._ir_nodes.Sink, translator: Translator, schema: Schema) -> ir.I f"{sink_kind} compression ('{compression}') is not supported." ) - if POLARS_VERSION_LT_132: # pragma: no cover - path = file["target"] - elif POLARS_VERSION_LT_138: # pragma: no cover + if POLARS_VERSION_LT_138: # pragma: no cover path = file["target"]["Local"] else: path = file["target"]["inner"] diff --git a/python/cudf_polars/cudf_polars/dsl/utils/aggregations.py b/python/cudf_polars/cudf_polars/dsl/utils/aggregations.py index 0aab66e0acab..f8b7e0170dc6 100644 --- a/python/cudf_polars/cudf_polars/dsl/utils/aggregations.py +++ b/python/cudf_polars/cudf_polars/dsl/utils/aggregations.py @@ -17,7 +17,7 @@ from cudf_polars.containers import DataType from cudf_polars.dsl import expr, ir from cudf_polars.dsl.expressions.base import ExecutionContext -from cudf_polars.utils.versions import POLARS_VERSION_LT_134, POLARS_VERSION_LT_1323 +from cudf_polars.utils.versions import POLARS_VERSION_LT_134 if TYPE_CHECKING: from collections.abc import Callable, Generator, Iterable, Sequence @@ -256,41 +256,9 @@ def decompose_single_agg( # - ROLLING: sum(all-null window) => null; sum(empty window) => 0 (fill only if empty) # # Must post-process because libcudf returns null for both empty and all-null windows/groups - if not POLARS_VERSION_LT_1323 or context in { - ExecutionContext.GROUPBY, - ExecutionContext.WINDOW, - }: - # GROUPBY: always fill top-level nulls with 0 - return [(named_expr, True)], expr.NamedExpr( - name, replace_nulls(col, 0, is_top=is_top) - ) - else: # pragma: no cover - # ROLLING: - # Add a second rolling agg to compute the window size, then only - # replace nulls with 0 when the window size is 0 (ie. empty window). - win_len_name = next(name_generator) - win_len = expr.NamedExpr( - win_len_name, - expr.Len(DataType(pl.Int32())), - ) - - win_len_col = expr.Col(DataType(pl.Int32()), win_len_name) - win_len_filled = replace_nulls(win_len_col, 0, is_top=True) - - is_empty = expr.BinOp( - DataType(pl.Boolean()), - plc.binaryop.BinaryOperator.EQUAL, - win_len_filled, - expr.Literal(DataType(pl.Int32()), 0), - ) - - # If empty -> fill 0; else keep libcudf's semantics for all-null windows. - filled = replace_nulls(col, 0, is_top=is_top) - post_ternary_expr = expr.Ternary(agg.dtype, is_empty, filled, col) - - return [(named_expr, True), (win_len, True)], expr.NamedExpr( - name, post_ternary_expr - ) + return [(named_expr, True)], expr.NamedExpr( + name, replace_nulls(col, 0, is_top=is_top) + ) elif agg.name in {"mean", "median", "quantile", "std", "var"}: post_agg_col: expr.Expr = expr.Col( DataType(pl.Float64()), name diff --git a/python/cudf_polars/cudf_polars/experimental/benchmarks/asserts.py b/python/cudf_polars/cudf_polars/experimental/benchmarks/asserts.py index e8c80e480cd5..fe96c9ac04e8 100644 --- a/python/cudf_polars/cudf_polars/experimental/benchmarks/asserts.py +++ b/python/cudf_polars/cudf_polars/experimental/benchmarks/asserts.py @@ -10,8 +10,6 @@ import polars as pl import polars.testing -from cudf_polars.utils.versions import POLARS_VERSION_LT_1323 - if TYPE_CHECKING: from typing import Any @@ -163,10 +161,7 @@ def assert_tpch_result_equal( "categorical_as_str": categorical_as_str, } - if POLARS_VERSION_LT_1323: # pragma: no cover - tol_kwargs = {"rtol": rel_tol, "atol": abs_tol} - else: - tol_kwargs = {"rel_tol": rel_tol, "abs_tol": abs_tol} + tol_kwargs = {"rel_tol": rel_tol, "abs_tol": abs_tol} polars_kwargs.update(tol_kwargs) if left.columns != right.columns: diff --git a/python/cudf_polars/cudf_polars/testing/asserts.py b/python/cudf_polars/cudf_polars/testing/asserts.py index 5611f8c3e704..f37b7f646f7e 100644 --- a/python/cudf_polars/cudf_polars/testing/asserts.py +++ b/python/cudf_polars/cudf_polars/testing/asserts.py @@ -14,7 +14,6 @@ from cudf_polars.dsl.translate import Translator from cudf_polars.utils.config import ConfigOptions -from cudf_polars.utils.versions import POLARS_VERSION_LT_1323 if TYPE_CHECKING: from cudf_polars.typing import CollectKwargs @@ -127,11 +126,7 @@ def assert_gpu_result_equal( "categorical_as_str": categorical_as_str, } - tol_kwargs: dict[str, float] - if POLARS_VERSION_LT_1323: # pragma: no cover - tol_kwargs = {"rtol": rtol, "atol": atol} - else: - tol_kwargs = {"rel_tol": rtol, "abs_tol": atol} + tol_kwargs: dict[str, float] = {"rel_tol": rtol, "abs_tol": atol} # the type checker errors with: # Argument 4 to "assert_frame_equal" has incompatible type "**dict[str, float]"; expected "bool" [arg-type] diff --git a/python/cudf_polars/cudf_polars/utils/versions.py b/python/cudf_polars/cudf_polars/utils/versions.py index 0a15d57c0483..1a415961c2f0 100644 --- a/python/cudf_polars/cudf_polars/utils/versions.py +++ b/python/cudf_polars/cudf_polars/utils/versions.py @@ -12,9 +12,6 @@ POLARS_VERSION = parse(__version__) POLARS_LOWER_BOUND = parse("1.30") -POLARS_VERSION_LT_132 = POLARS_VERSION < parse("1.32") -POLARS_VERSION_LT_1321 = POLARS_VERSION < parse("1.32.1") -POLARS_VERSION_LT_1323 = POLARS_VERSION < parse("1.32.3") POLARS_VERSION_LT_133 = POLARS_VERSION < parse("1.33.0") POLARS_VERSION_LT_134 = POLARS_VERSION < parse("1.34.0") POLARS_VERSION_LT_135 = POLARS_VERSION < parse("1.35.0") diff --git a/python/cudf_polars/tests/dsl/test_serialization.py b/python/cudf_polars/tests/dsl/test_serialization.py index 5608dac96662..cea9771fca1c 100644 --- a/python/cudf_polars/tests/dsl/test_serialization.py +++ b/python/cudf_polars/tests/dsl/test_serialization.py @@ -14,9 +14,7 @@ from cudf_polars.dsl.expressions.string import StringFunction from cudf_polars.dsl.expressions.struct import StructFunction from cudf_polars.utils.versions import ( - POLARS_VERSION_LT_132, POLARS_VERSION_LT_138, - POLARS_VERSION_LT_1321, ) @@ -48,30 +46,20 @@ def test_from_polars_all_names(function): # Check names advertised by polars are the same as we advertise polars_names_set = set(polars_names) cudf_polars_names_set = set(function.Name.__members__) - if not POLARS_VERSION_LT_132 and function == StructFunction: + if function == StructFunction: cudf_polars_names_set = cudf_polars_names_set - { "FieldByIndex", "MultipleFields", } - if POLARS_VERSION_LT_1321 and function == TemporalFunction: - cudf_polars_names_set = cudf_polars_names_set - { - "DaysInMonth", - } - if POLARS_VERSION_LT_132 and function == BooleanFunction: - cudf_polars_names_set = cudf_polars_names_set - {"IsClose"} if POLARS_VERSION_LT_138 and function == StringFunction: cudf_polars_names_set = cudf_polars_names_set - {"SplitRegex"} assert polars_names_set == cudf_polars_names_set names = function.Name - if not POLARS_VERSION_LT_132 and function == StructFunction: + if function == StructFunction: names = set(names) - { StructFunction.Name.FieldByIndex, StructFunction.Name.MultipleFields, } - if POLARS_VERSION_LT_1321 and function == TemporalFunction: - names = set(names) - {TemporalFunction.Name.DaysInMonth} - if POLARS_VERSION_LT_132 and function == BooleanFunction: - names = set(names) - {BooleanFunction.Name.IsClose} if POLARS_VERSION_LT_138 and function == StringFunction: names = set(names) - {StringFunction.Name.SplitRegex} for name in names: diff --git a/python/cudf_polars/tests/experimental/test_select.py b/python/cudf_polars/tests/experimental/test_select.py index cef9f0f66cf0..51cdac43c48f 100644 --- a/python/cudf_polars/tests/experimental/test_select.py +++ b/python/cudf_polars/tests/experimental/test_select.py @@ -20,11 +20,9 @@ from cudf_polars.experimental.select import _inline_hstack_false, _sub_expr from cudf_polars.testing.asserts import ( assert_gpu_result_equal, - assert_ir_translation_raises, ) from cudf_polars.testing.engine_utils import warns_on_spmd from cudf_polars.utils.versions import ( - POLARS_VERSION_LT_132, POLARS_VERSION_LT_134, ) @@ -91,15 +89,12 @@ def test_select_fill_null_with_strategy(df, streaming_engine_factory): ) q = df.select(pl.col("a").forward_fill()) - if POLARS_VERSION_LT_132: - assert_ir_translation_raises(q, NotImplementedError) - else: - with warns_on_spmd( - engine, - UserWarning, - match="fill_null with strategy other than 'zero' or 'one' is not supported for multiple partitions", - ): - assert_gpu_result_equal(q, engine=engine) + with warns_on_spmd( + engine, + UserWarning, + match="fill_null with strategy other than 'zero' or 'one' is not supported for multiple partitions", + ): + assert_gpu_result_equal(q, engine=engine) @pytest.mark.parametrize( diff --git a/python/cudf_polars/tests/expressions/test_booleanfunction.py b/python/cudf_polars/tests/expressions/test_booleanfunction.py index 4f49a44de219..94463c62ca77 100644 --- a/python/cudf_polars/tests/expressions/test_booleanfunction.py +++ b/python/cudf_polars/tests/expressions/test_booleanfunction.py @@ -13,7 +13,6 @@ assert_ir_translation_raises, ) from cudf_polars.testing.engine_utils import is_streaming_engine -from cudf_polars.utils.versions import POLARS_VERSION_LT_132 if TYPE_CHECKING: from collections.abc import Callable @@ -269,12 +268,7 @@ def test_is_in_shape_mismatch_raises(needles, haystack): assert_ir_translation_raises(q, NotImplementedError) -def test_boolean_is_close(request): - request.applymarker( - pytest.mark.xfail( - condition=POLARS_VERSION_LT_132, reason="Not supported until polars 1.32" - ) - ) +def test_boolean_is_close(): ldf = pl.LazyFrame({"a": [1.0, 1.2, 1.4, 1.45, 1.6]}) q = ldf.select(pl.col("a").is_close(1.4, abs_tol=0.1)) diff --git a/python/cudf_polars/tests/expressions/test_casting.py b/python/cudf_polars/tests/expressions/test_casting.py index e8cd8c1fcddd..6a0c472f7f20 100644 --- a/python/cudf_polars/tests/expressions/test_casting.py +++ b/python/cudf_polars/tests/expressions/test_casting.py @@ -13,7 +13,6 @@ assert_gpu_result_equal, assert_ir_translation_raises, ) -from cudf_polars.utils.versions import POLARS_VERSION_LT_132 _supported_dtypes = [(pl.Int8(), pl.Int64())] @@ -101,4 +100,4 @@ def test_float_to_decimal_rounding(engine: pl.GPUEngine): } ) q = df.select(pl.col("foo") / pl.col("bar")) - assert_gpu_result_equal(q, engine=engine, check_dtypes=not POLARS_VERSION_LT_132) + assert_gpu_result_equal(q, engine=engine) diff --git a/python/cudf_polars/tests/expressions/test_numeric_binops.py b/python/cudf_polars/tests/expressions/test_numeric_binops.py index 32c203423dc3..54f47d63e13c 100644 --- a/python/cudf_polars/tests/expressions/test_numeric_binops.py +++ b/python/cudf_polars/tests/expressions/test_numeric_binops.py @@ -11,7 +11,6 @@ from cudf_polars.testing.asserts import ( assert_gpu_result_equal, ) -from cudf_polars.utils.versions import POLARS_VERSION_LT_132 dtypes = [ pl.Int8, @@ -120,7 +119,7 @@ def test_true_div_with_decimals(engine: pl.GPUEngine): schema={"foo": pl.Decimal(15, 2), "bar": pl.Decimal(15, 2)}, ) q = df.select(pl.col("bar") / pl.col("foo")) - assert_gpu_result_equal(q, engine=engine, check_dtypes=not POLARS_VERSION_LT_132) + assert_gpu_result_equal(q, engine=engine) def test_multiply_with_decimals(engine: pl.GPUEngine): @@ -133,7 +132,7 @@ def test_multiply_with_decimals(engine: pl.GPUEngine): ) q = df.select(pl.col("x") * pl.col("y")) - assert_gpu_result_equal(q, engine=engine, check_dtypes=not POLARS_VERSION_LT_132) + assert_gpu_result_equal(q, engine=engine) def test_sum_decimal_widens_precision(request) -> None: diff --git a/python/cudf_polars/tests/expressions/test_numeric_unaryops.py b/python/cudf_polars/tests/expressions/test_numeric_unaryops.py index 8286c5340fef..0e7f8db194cd 100644 --- a/python/cudf_polars/tests/expressions/test_numeric_unaryops.py +++ b/python/cudf_polars/tests/expressions/test_numeric_unaryops.py @@ -13,7 +13,6 @@ assert_gpu_result_equal, assert_ir_translation_raises, ) -from cudf_polars.utils.versions import POLARS_VERSION_LT_132 if TYPE_CHECKING: from cudf_polars.typing import RankMethod, RoundMethod @@ -126,15 +125,11 @@ def test_null_count(engine: pl.GPUEngine): @pytest.mark.parametrize("descending", [False, True]) def test_rank_supported( engine: pl.GPUEngine, - request, ldf: pl.LazyFrame, method: RankMethod, *, descending: bool, ): - request.applymarker( - pytest.mark.xfail(condition=POLARS_VERSION_LT_132, reason="rank unsupported") - ) expr = pl.col("a").rank(method=method, descending=descending) q = ldf.select(expr) assert_gpu_result_equal(q, engine=engine) @@ -145,17 +140,12 @@ def test_rank_supported( @pytest.mark.parametrize("test", ["with_nulls", "with_ties"]) def test_rank_methods_with_nulls_or_ties( engine: pl.GPUEngine, - request, ldf: pl.LazyFrame, method: RankMethod, *, descending: bool, test: str, ) -> None: - request.applymarker( - pytest.mark.xfail(condition=POLARS_VERSION_LT_132, reason="rank unsupported") - ) - base = pl.col("a") if test == "with_nulls": expr = pl.when((base % 2) == 0).then(None).otherwise(base) diff --git a/python/cudf_polars/tests/expressions/test_rolling.py b/python/cudf_polars/tests/expressions/test_rolling.py index 776196753b68..500520566078 100644 --- a/python/cudf_polars/tests/expressions/test_rolling.py +++ b/python/cudf_polars/tests/expressions/test_rolling.py @@ -13,7 +13,7 @@ assert_gpu_result_equal, assert_ir_translation_raises, ) -from cudf_polars.utils.versions import POLARS_VERSION_LT_132, POLARS_VERSION_LT_136 +from cudf_polars.utils.versions import POLARS_VERSION_LT_136 if TYPE_CHECKING: from cudf_polars.typing import RankMethod @@ -327,9 +327,7 @@ def test_rank_over( descending: bool, order_by: None | list[str | pl.Expr], ) -> None: - request.applymarker( - pytest.mark.xfail(condition=POLARS_VERSION_LT_132, reason="rank unsupported") - ) + request.applymarker() q = df.select( pl.col("x") .rank(method=method, descending=descending) @@ -343,16 +341,12 @@ def test_rank_over( @pytest.mark.parametrize("order_by", [None, ["g2", pl.col("x2") * 2]]) def test_rank_over_with_ties( engine: pl.GPUEngine, - request, df: pl.LazyFrame, method: RankMethod, *, descending: bool, order_by: None | list[str | pl.Expr], ) -> None: - request.applymarker( - pytest.mark.xfail(condition=POLARS_VERSION_LT_132, reason="rank unsupported") - ) q = df.select( pl.when(pl.col("g") == 2) .then(pl.lit(4)) @@ -368,16 +362,12 @@ def test_rank_over_with_ties( @pytest.mark.parametrize("order_by", [None, ["g2", pl.col("x2") * 2]]) def test_rank_over_with_null_values( engine: pl.GPUEngine, - request, df: pl.LazyFrame, method: RankMethod, *, descending: bool, order_by: None | list[str | pl.Expr], ) -> None: - request.applymarker( - pytest.mark.xfail(condition=POLARS_VERSION_LT_132, reason="rank unsupported") - ) q = df.select( pl.when((pl.col("x") % 2) == 0) .then(None) @@ -393,16 +383,12 @@ def test_rank_over_with_null_values( @pytest.mark.parametrize("order_by", [None, ["g2", pl.col("x2") * 2]]) def test_rank_over_with_null_group_keys( engine: pl.GPUEngine, - request, df: pl.LazyFrame, method: RankMethod, *, descending: bool, order_by: None | list[str | pl.Expr], ) -> None: - request.applymarker( - pytest.mark.xfail(condition=POLARS_VERSION_LT_132, reason="rank unsupported") - ) q = df.select( pl.col("x") .rank(method=method, descending=descending) @@ -441,10 +427,7 @@ def test_fill_over( group_key, order_by=order_by ) ) - if POLARS_VERSION_LT_132: - assert_ir_translation_raises(q, NotImplementedError) - else: - assert_gpu_result_equal(q, engine=engine) + assert_gpu_result_equal(q, engine=engine) def test_fill_null_with_mean_over_unsupported(df: pl.LazyFrame) -> None: diff --git a/python/cudf_polars/tests/expressions/test_stringfunction.py b/python/cudf_polars/tests/expressions/test_stringfunction.py index 9e0c51be96a4..b61595dec351 100644 --- a/python/cudf_polars/tests/expressions/test_stringfunction.py +++ b/python/cudf_polars/tests/expressions/test_stringfunction.py @@ -16,7 +16,6 @@ ) from cudf_polars.testing.engine_utils import is_streaming_engine from cudf_polars.utils.versions import ( - POLARS_VERSION_LT_132, POLARS_VERSION_LT_133, POLARS_VERSION_LT_136, POLARS_VERSION_LT_138, @@ -550,15 +549,10 @@ def test_string_zfill(engine: pl.GPUEngine, fill, input_strings): q = ldf.select(pl.col("a").str.zfill(fill)) if fill is not None and fill < 0: - cudf_except = ( - pl.exceptions.InvalidOperationError - if not POLARS_VERSION_LT_132 - else pl.exceptions.ComputeError - ) assert_collect_raises( q, polars_except=pl.exceptions.InvalidOperationError, - cudf_except=cudf_except, + cudf_except=pl.exceptions.InvalidOperationError, ) else: assert_gpu_result_equal(q, engine=engine) @@ -595,13 +589,10 @@ def test_string_zfill_column(engine: pl.GPUEngine, fill): ).lazy() q = ldf.select(pl.col("input_strings").str.zfill(pl.col("fill"))) if fill is not None and fill < 0: - cudf_except = ( - pl.exceptions.InvalidOperationError if POLARS_VERSION_LT_132 else () - ) assert_collect_raises( q, polars_except=pl.exceptions.InvalidOperationError, - cudf_except=cudf_except, + cudf_except=(), ) else: assert_gpu_result_equal(q, engine=engine) diff --git a/python/cudf_polars/tests/test_cache.py b/python/cudf_polars/tests/test_cache.py index 0a13ecb82f37..1bfb0a2b646c 100644 --- a/python/cudf_polars/tests/test_cache.py +++ b/python/cudf_polars/tests/test_cache.py @@ -11,16 +11,10 @@ from cudf_polars.dsl.ir import IRExecutionContext from cudf_polars.dsl.traversal import traversal from cudf_polars.testing.asserts import assert_gpu_result_equal -from cudf_polars.utils.versions import POLARS_VERSION_LT_1323 -def test_cache(engine: pl.GPUEngine, request): - request.applymarker( - pytest.mark.xfail( - condition=not POLARS_VERSION_LT_1323, - reason="python no longer manages cache hits", - ) - ) +@pytest.mark.xfail(reason="python no longer manages cache hits") +def test_cache(engine: pl.GPUEngine): df1 = pl.LazyFrame( { "a": [1, 2, 3, 4, 5, 6, 7], diff --git a/python/cudf_polars/tests/test_drop_nulls.py b/python/cudf_polars/tests/test_drop_nulls.py index b2f1a2098536..c7b974eb91dc 100644 --- a/python/cudf_polars/tests/test_drop_nulls.py +++ b/python/cudf_polars/tests/test_drop_nulls.py @@ -10,7 +10,6 @@ assert_gpu_result_equal, assert_ir_translation_raises, ) -from cudf_polars.utils.versions import POLARS_VERSION_LT_132 @pytest.fixture( @@ -57,10 +56,7 @@ def test_fill_null_with_string(engine: pl.GPUEngine): ) def test_fill_null_with_strategy(engine: pl.GPUEngine, null_data, strategy): q = null_data.select(pl.col("a").fill_null(strategy=strategy)) - if POLARS_VERSION_LT_132: - assert_ir_translation_raises(q, NotImplementedError) - else: - assert_gpu_result_equal(q, engine=engine) + assert_gpu_result_equal(q, engine=engine) @pytest.mark.parametrize("strategy", ["zero", "one"]) @@ -68,10 +64,7 @@ def test_fill_null_with_strategy_bool(engine: pl.GPUEngine, strategy): q = pl.LazyFrame({"a": [True, None, False]}).select( pl.col("a").fill_null(strategy=strategy) ) - if POLARS_VERSION_LT_132: - assert_ir_translation_raises(q, NotImplementedError) - else: - assert_gpu_result_equal(q, engine=engine) + assert_gpu_result_equal(q, engine=engine) @pytest.mark.parametrize("strategy", ["forward", "backward"]) @@ -81,7 +74,4 @@ def test_fill_null_with_limit(engine: pl.GPUEngine, null_data, strategy, limit): if limit != 0: assert_ir_translation_raises(q, NotImplementedError) else: - if POLARS_VERSION_LT_132: - assert_ir_translation_raises(q, NotImplementedError) - else: - assert_gpu_result_equal(q, engine=engine) + assert_gpu_result_equal(q, engine=engine) diff --git a/python/cudf_polars/tests/test_groupby.py b/python/cudf_polars/tests/test_groupby.py index f14160a1043e..cb0ece3de2f8 100644 --- a/python/cudf_polars/tests/test_groupby.py +++ b/python/cudf_polars/tests/test_groupby.py @@ -17,10 +17,8 @@ ) from cudf_polars.testing.engine_utils import get_blocksize_mode, is_streaming_engine from cudf_polars.utils.versions import ( - POLARS_VERSION_LT_132, POLARS_VERSION_LT_134, POLARS_VERSION_LT_136, - POLARS_VERSION_LT_1321, ) @@ -50,11 +48,9 @@ def df(): ], } ) - if not POLARS_VERSION_LT_132: - lf = lf.with_columns( - pl.col("float").cast(pl.Decimal(precision=9, scale=2)).alias("decimal") - ) - return lf + return lf.with_columns( + pl.col("float").cast(pl.Decimal(precision=9, scale=2)).alias("decimal") + ) @pytest.fixture( @@ -104,13 +100,11 @@ def keys(request): pl.col("datetime").max(), pl.col("datetime").max().dt.is_leap_year().alias("leapyear"), ], + # polars gives us precision=None, which we + # do not support + [pl.col("decimal").median()], ] -# polars gives us precision=None, which we -# do not supprt -if not POLARS_VERSION_LT_132: - _EXPRS.append([pl.col("decimal").median()]) - @pytest.fixture( params=_EXPRS, @@ -252,7 +246,6 @@ def test_groupby_nan_minmax_raises(op): pytest.param( pl.Series("value", [[4, 5, 6]], dtype=pl.List(pl.Int32)), marks=pytest.mark.xfail( - condition=not POLARS_VERSION_LT_1321, reason="https://github.com/rapidsai/cudf/issues/19610", ), ), @@ -423,7 +416,6 @@ def test_groupby_aggs_keep_unsupported_as_null( request.applymarker( pytest.mark.xfail( condition="quantile" in str(agg_expr) - and not POLARS_VERSION_LT_132 and POLARS_VERSION_LT_134 and is_streaming_engine(engine), reason="Decimal precision mismatch (9 vs 38)", diff --git a/python/cudf_polars/tests/test_join.py b/python/cudf_polars/tests/test_join.py index a192645602bf..80554005de0e 100644 --- a/python/cudf_polars/tests/test_join.py +++ b/python/cudf_polars/tests/test_join.py @@ -16,7 +16,6 @@ from cudf_polars.dsl.ir import ConditionalJoin from cudf_polars.testing.asserts import assert_gpu_result_equal from cudf_polars.testing.engine_utils import is_streaming_engine -from cudf_polars.utils.versions import POLARS_VERSION_LT_132 @pytest.fixture(params=[False, True], ids=["nulls_not_equal", "nulls_equal"]) @@ -181,10 +180,7 @@ def test_join_where(engine: pl.GPUEngine, left, right, conditions, zlice): assert_gpu_result_equal(q_len, engine=engine) -def test_cross_join_empty_right_table(engine: pl.GPUEngine, request): - request.applymarker( - pytest.mark.xfail(condition=POLARS_VERSION_LT_132, reason="nested loop join") - ) +def test_cross_join_empty_right_table(engine: pl.GPUEngine): a = pl.LazyFrame({"a": [1, 2, 3], "x": [7, 2, 1]}) b = pl.LazyFrame({"b": [2, 2, 2], "x": [7, 1, 3]}) @@ -312,17 +308,8 @@ def test_join_maintain_order_with_slice( "ConditionalJoin not supported for multiple partitions" ) def test_cross_join_filter_with_decimals( - engine: pl.GPUEngine, request, expr, left_dtype, right_dtype + engine: pl.GPUEngine, expr, left_dtype, right_dtype ): - request.applymarker( - pytest.mark.xfail( - POLARS_VERSION_LT_132 - and isinstance(left_dtype, pl.Decimal) - and isinstance(right_dtype, pl.Decimal) - and "==" in repr(expr), - reason="Hash Inner Join between i128 and i128", - ) - ) left = pl.LazyFrame( { "foo": [Decimal("1.00"), Decimal("2.50"), Decimal("3.00")], diff --git a/python/cudf_polars/tests/test_window_functions.py b/python/cudf_polars/tests/test_window_functions.py index cf7f9181652c..2dbb06fc271d 100644 --- a/python/cudf_polars/tests/test_window_functions.py +++ b/python/cudf_polars/tests/test_window_functions.py @@ -13,7 +13,7 @@ assert_gpu_result_equal, assert_ir_translation_raises, ) -from cudf_polars.utils.versions import POLARS_VERSION_LT_132, POLARS_VERSION_LT_136 +from cudf_polars.utils.versions import POLARS_VERSION_LT_136 @pytest.fixture @@ -120,7 +120,7 @@ def test_over_mapping_strategy( ) ] ) - if not POLARS_VERSION_LT_132 and mapping_strategy == "group_to_rows": + if mapping_strategy == "group_to_rows": assert_gpu_result_equal(q, engine=engine) else: assert_ir_translation_raises(q, NotImplementedError) From 97be3577bba8b60cce62e48279388d6c0e6ff02d Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Mon, 11 May 2026 18:49:58 +0000 Subject: [PATCH 10/44] Remove 1.33 workarounds --- .../cudf_polars/cudf_polars/dsl/translate.py | 53 +++++++------------ .../cudf_polars/cudf_polars/utils/versions.py | 1 - .../tests/expressions/test_stringfunction.py | 5 -- 3 files changed, 18 insertions(+), 41 deletions(-) diff --git a/python/cudf_polars/cudf_polars/dsl/translate.py b/python/cudf_polars/cudf_polars/dsl/translate.py index a7fbcffa7a09..2ee0a14f0ced 100644 --- a/python/cudf_polars/cudf_polars/dsl/translate.py +++ b/python/cudf_polars/cudf_polars/dsl/translate.py @@ -33,7 +33,6 @@ from cudf_polars.typing import Schema from cudf_polars.utils import config, sorting from cudf_polars.utils.versions import ( - POLARS_VERSION_LT_133, POLARS_VERSION_LT_134, POLARS_VERSION_LT_136, POLARS_VERSION_LT_138, @@ -794,38 +793,28 @@ def _( ) elif isinstance(name, str): children = (translator.translate_expr(n=n, schema=schema) for n in node.input) - if name == "log" or ( - not POLARS_VERSION_LT_133 - and name == "l" + if ( + name == "l" and isinstance(options[0], str) and "".join((name, *options)) == "log" ): - if POLARS_VERSION_LT_133: # pragma: no cover - (base,) = options - (child,) = children - return expr.BinOp( - dtype, - plc.binaryop.BinaryOperator.LOG_BASE, - child, - expr.Literal(dtype, base), - ) - else: - (child, base) = children - res = expr.BinOp( - dtype, - plc.binaryop.BinaryOperator.LOG_BASE, - child, - expr.Literal(dtype, base.value), - ) - return ( - res - if not POLARS_VERSION_LT_134 - else expr.Cast( - DataType(pl.Float64()), - True, # noqa: FBT003 - res, - ) + (child, base) = children + assert isinstance(base, expr.NamedExpr) + res = expr.BinOp( + dtype, + plc.binaryop.BinaryOperator.LOG_BASE, + child, + expr.Literal(dtype, base.value), + ) + return ( + res + if not POLARS_VERSION_LT_134 + else expr.Cast( + DataType(pl.Float64()), + True, # noqa: FBT003 + res, ) + ) elif name == "pow": return expr.BinOp(dtype, plc.binaryop.BinaryOperator.POW, *children) return expr.UnaryFunction(dtype, name, options, *children) @@ -1130,12 +1119,6 @@ def _( ) -> expr.Expr: left = translator.translate_expr(n=node.left, schema=schema) right = translator.translate_expr(n=node.right, schema=schema) - if ( - POLARS_VERSION_LT_133 - and plc.traits.is_boolean(dtype.plc_type) - and node.op == plrs._expr_nodes.Operator.TrueDivide - ): - dtype = DataType(pl.Float64()) # pragma: no cover if node.op == plrs._expr_nodes.Operator.TrueDivide and ( plc.traits.is_fixed_point(left.dtype.plc_type) or plc.traits.is_fixed_point(right.dtype.plc_type) diff --git a/python/cudf_polars/cudf_polars/utils/versions.py b/python/cudf_polars/cudf_polars/utils/versions.py index 1a415961c2f0..adfbabc0c6bf 100644 --- a/python/cudf_polars/cudf_polars/utils/versions.py +++ b/python/cudf_polars/cudf_polars/utils/versions.py @@ -12,7 +12,6 @@ POLARS_VERSION = parse(__version__) POLARS_LOWER_BOUND = parse("1.30") -POLARS_VERSION_LT_133 = POLARS_VERSION < parse("1.33.0") POLARS_VERSION_LT_134 = POLARS_VERSION < parse("1.34.0") POLARS_VERSION_LT_135 = POLARS_VERSION < parse("1.35.0") POLARS_VERSION_LT_136 = POLARS_VERSION < parse("1.36.0") diff --git a/python/cudf_polars/tests/expressions/test_stringfunction.py b/python/cudf_polars/tests/expressions/test_stringfunction.py index b61595dec351..f834ba62acb6 100644 --- a/python/cudf_polars/tests/expressions/test_stringfunction.py +++ b/python/cudf_polars/tests/expressions/test_stringfunction.py @@ -16,7 +16,6 @@ ) from cudf_polars.testing.engine_utils import is_streaming_engine from cudf_polars.utils.versions import ( - POLARS_VERSION_LT_133, POLARS_VERSION_LT_136, POLARS_VERSION_LT_138, ) @@ -792,10 +791,6 @@ def test_json_decode(engine: pl.GPUEngine, ldf_jsonlike): q = ldf_jsonlike.select(pl.col("a").str.json_decode(pl.Struct({"a": pl.String()}))) assert_gpu_result_equal(q, engine=engine) - if POLARS_VERSION_LT_133: - q = ldf_jsonlike.select(pl.col("a").str.json_decode(None)) # type: ignore[arg-type] - assert_ir_translation_raises(q, NotImplementedError) - def test_json_decode_empty(engine: pl.GPUEngine): # libcudf's ``read_json_from_string_column`` raises From 93470c7b1855e1ced1b96cc745467fba375c00e3 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Mon, 11 May 2026 18:57:57 +0000 Subject: [PATCH 11/44] Remove 1.34 workarounds --- .../cudf_polars/containers/datatype.py | 6 +---- python/cudf_polars/cudf_polars/dsl/ir.py | 23 ++++++++----------- .../cudf_polars/cudf_polars/dsl/translate.py | 15 ++---------- .../cudf_polars/dsl/utils/aggregations.py | 6 +---- .../cudf_polars/cudf_polars/utils/versions.py | 1 - .../tests/experimental/test_select.py | 7 ++---- .../cudf_polars/tests/expressions/test_agg.py | 5 ---- python/cudf_polars/tests/test_groupby.py | 9 -------- python/cudf_polars/tests/test_select.py | 3 +-- 9 files changed, 16 insertions(+), 59 deletions(-) diff --git a/python/cudf_polars/cudf_polars/containers/datatype.py b/python/cudf_polars/cudf_polars/containers/datatype.py index 23795ff3be93..c9da8a977ad2 100644 --- a/python/cudf_polars/cudf_polars/containers/datatype.py +++ b/python/cudf_polars/cudf_polars/containers/datatype.py @@ -45,13 +45,9 @@ def _dtype_to_header(dtype: pl.DataType) -> DataTypeHeader: if name in SCALAR_NAME_TO_POLARS_TYPE_MAP: return {"kind": "scalar", "name": name} if isinstance(dtype, pl.Decimal): - # TODO: Add version guard once we support polars 1.34 - # Also keep in mind the typing change in polars: - # https://github.com/pola-rs/polars/pull/25227 - precision = dtype.precision if dtype.precision is not None else 38 return { "kind": "decimal", - "precision": precision, + "precision": dtype.precision, "scale": dtype.scale, } if isinstance(dtype, pl.Datetime): diff --git a/python/cudf_polars/cudf_polars/dsl/ir.py b/python/cudf_polars/cudf_polars/dsl/ir.py index 3331877a9a3a..8537fea9faed 100644 --- a/python/cudf_polars/cudf_polars/dsl/ir.py +++ b/python/cudf_polars/cudf_polars/dsl/ir.py @@ -49,7 +49,6 @@ join_cuda_streams, ) from cudf_polars.utils.versions import ( - POLARS_VERSION_LT_134, POLARS_VERSION_LT_136, POLARS_VERSION_LT_137, POLARS_VERSION_LT_138, @@ -1970,19 +1969,15 @@ def _strip_predicate_casts(node: expr.Expr) -> expr.Expr: ): return child - if ( - not POLARS_VERSION_LT_134 - and isinstance(child, expr.ColRef) - and ( - ( - plc.traits.is_floating_point(src.plc_type) - and plc.traits.is_floating_point(dst.plc_type) - ) - or ( - plc.traits.is_integral(src.plc_type) - and plc.traits.is_integral(dst.plc_type) - and src.plc_type.id() == dst.plc_type.id() - ) + if isinstance(child, expr.ColRef) and ( + ( + plc.traits.is_floating_point(src.plc_type) + and plc.traits.is_floating_point(dst.plc_type) + ) + or ( + plc.traits.is_integral(src.plc_type) + and plc.traits.is_integral(dst.plc_type) + and src.plc_type.id() == dst.plc_type.id() ) ): return child diff --git a/python/cudf_polars/cudf_polars/dsl/translate.py b/python/cudf_polars/cudf_polars/dsl/translate.py index 2ee0a14f0ced..57e320c0c0a6 100644 --- a/python/cudf_polars/cudf_polars/dsl/translate.py +++ b/python/cudf_polars/cudf_polars/dsl/translate.py @@ -33,7 +33,6 @@ from cudf_polars.typing import Schema from cudf_polars.utils import config, sorting from cudf_polars.utils.versions import ( - POLARS_VERSION_LT_134, POLARS_VERSION_LT_136, POLARS_VERSION_LT_138, ) @@ -800,21 +799,12 @@ def _( ): (child, base) = children assert isinstance(base, expr.NamedExpr) - res = expr.BinOp( + return expr.BinOp( dtype, plc.binaryop.BinaryOperator.LOG_BASE, child, expr.Literal(dtype, base.value), ) - return ( - res - if not POLARS_VERSION_LT_134 - else expr.Cast( - DataType(pl.Float64()), - True, # noqa: FBT003 - res, - ) - ) elif name == "pow": return expr.BinOp(dtype, plc.binaryop.BinaryOperator.POW, *children) return expr.UnaryFunction(dtype, name, options, *children) @@ -1136,8 +1126,7 @@ def _( ) if ( - not POLARS_VERSION_LT_134 - and node.op == plrs._expr_nodes.Operator.Multiply + node.op == plrs._expr_nodes.Operator.Multiply and plc.traits.is_fixed_point(left.dtype.plc_type) and plc.traits.is_fixed_point(right.dtype.plc_type) ): diff --git a/python/cudf_polars/cudf_polars/dsl/utils/aggregations.py b/python/cudf_polars/cudf_polars/dsl/utils/aggregations.py index f8b7e0170dc6..0ba59e4828a1 100644 --- a/python/cudf_polars/cudf_polars/dsl/utils/aggregations.py +++ b/python/cudf_polars/cudf_polars/dsl/utils/aggregations.py @@ -17,7 +17,6 @@ from cudf_polars.containers import DataType from cudf_polars.dsl import expr, ir from cudf_polars.dsl.expressions.base import ExecutionContext -from cudf_polars.utils.versions import POLARS_VERSION_LT_134 if TYPE_CHECKING: from collections.abc import Callable, Generator, Iterable, Sequence @@ -201,7 +200,7 @@ def decompose_single_agg( # mean/median on decimal: Polars returns float -> pre-cast decimal_unsupported = False if plc.traits.is_fixed_point(child_dtype): - cast_for_quantile = is_quantile and not POLARS_VERSION_LT_134 + cast_for_quantile = is_quantile cast_for_mean_or_median = ( agg.name in {"mean", "median"} ) and plc.traits.is_floating_point(agg.dtype.plc_type) @@ -215,9 +214,6 @@ def decompose_single_agg( child, ) child_dtype = child.dtype.plc_type - elif is_quantile and POLARS_VERSION_LT_134: # pragma: no cover - decimal_unsupported = True - is_group_quantile_supported = plc.traits.is_integral( child_dtype ) or plc.traits.is_floating_point(child_dtype) diff --git a/python/cudf_polars/cudf_polars/utils/versions.py b/python/cudf_polars/cudf_polars/utils/versions.py index adfbabc0c6bf..52b6041c4493 100644 --- a/python/cudf_polars/cudf_polars/utils/versions.py +++ b/python/cudf_polars/cudf_polars/utils/versions.py @@ -12,7 +12,6 @@ POLARS_VERSION = parse(__version__) POLARS_LOWER_BOUND = parse("1.30") -POLARS_VERSION_LT_134 = POLARS_VERSION < parse("1.34.0") POLARS_VERSION_LT_135 = POLARS_VERSION < parse("1.35.0") POLARS_VERSION_LT_136 = POLARS_VERSION < parse("1.36.0") POLARS_VERSION_LT_137 = POLARS_VERSION < parse("1.37.0") diff --git a/python/cudf_polars/tests/experimental/test_select.py b/python/cudf_polars/tests/experimental/test_select.py index 51cdac43c48f..188e9e392d02 100644 --- a/python/cudf_polars/tests/experimental/test_select.py +++ b/python/cudf_polars/tests/experimental/test_select.py @@ -22,9 +22,6 @@ assert_gpu_result_equal, ) from cudf_polars.testing.engine_utils import warns_on_spmd -from cudf_polars.utils.versions import ( - POLARS_VERSION_LT_134, -) @pytest.fixture @@ -174,13 +171,13 @@ def test_select_with_empty_partitions(engine): q = df.select(pl.col("b").sum() / Decimal("7.00")) # Polars pre their decimal overhaul: https://github.com/pola-rs/polars/issues/19784 # returned a different precision and scale, so we skip dtype check - assert_gpu_result_equal(q, engine=engine, check_dtypes=not POLARS_VERSION_LT_134) + assert_gpu_result_equal(q, engine=engine) def test_select_mean_with_decimals(engine): df = pl.LazyFrame({"d": [Decimal("1.23")] * 4}) q = df.select(pl.mean("d")) - assert_gpu_result_equal(q, engine=engine, check_dtypes=not POLARS_VERSION_LT_134) + assert_gpu_result_equal(q, engine=engine) def test_select_with_len(streaming_engine_factory): diff --git a/python/cudf_polars/tests/expressions/test_agg.py b/python/cudf_polars/tests/expressions/test_agg.py index e465a87f0aa0..179d138c2933 100644 --- a/python/cudf_polars/tests/expressions/test_agg.py +++ b/python/cudf_polars/tests/expressions/test_agg.py @@ -15,7 +15,6 @@ assert_ir_translation_raises, ) from cudf_polars.utils.versions import ( - POLARS_VERSION_LT_134, POLARS_VERSION_LT_135, POLARS_VERSION_LT_136, ) @@ -213,10 +212,6 @@ def test_decimal_quantile(engine: pl.GPUEngine, decimal_df, interp): assert_gpu_result_equal(q, engine=engine) -@pytest.mark.skipif( - POLARS_VERSION_LT_134, - reason="std/var on decimal not supported before polars 1.34", -) def test_decimal_std_var(engine: pl.GPUEngine, decimal_df): q = decimal_df.select( std=pl.col("a").std(), diff --git a/python/cudf_polars/tests/test_groupby.py b/python/cudf_polars/tests/test_groupby.py index cb0ece3de2f8..c649a0997c50 100644 --- a/python/cudf_polars/tests/test_groupby.py +++ b/python/cudf_polars/tests/test_groupby.py @@ -17,7 +17,6 @@ ) from cudf_polars.testing.engine_utils import get_blocksize_mode, is_streaming_engine from cudf_polars.utils.versions import ( - POLARS_VERSION_LT_134, POLARS_VERSION_LT_136, ) @@ -413,14 +412,6 @@ def test_groupby_aggs_keep_unsupported_as_null( reason="polars raises now", ) ) - request.applymarker( - pytest.mark.xfail( - condition="quantile" in str(agg_expr) - and POLARS_VERSION_LT_134 - and is_streaming_engine(engine), - reason="Decimal precision mismatch (9 vs 38)", - ) - ) lf = df.filter(pl.col("datetime") == date(2004, 12, 1)) q = lf.group_by("datetime").agg(agg_expr) assert_gpu_result_equal(q, engine=engine) diff --git a/python/cudf_polars/tests/test_select.py b/python/cudf_polars/tests/test_select.py index 16ae1cf4384c..eeacec4d726b 100644 --- a/python/cudf_polars/tests/test_select.py +++ b/python/cudf_polars/tests/test_select.py @@ -12,7 +12,6 @@ assert_gpu_result_equal, assert_ir_translation_raises, ) -from cudf_polars.utils.versions import POLARS_VERSION_LT_134 def test_select(engine: pl.GPUEngine): @@ -51,7 +50,7 @@ def test_select_decimal_precision_none_result_max_precision(): gpu_result = query.collect(engine="gpu") # See github.com/pola-rs/polars/issues/19784 # for context on the decimal changes. - assert cpu_result.schema["a"].precision is None if POLARS_VERSION_LT_134 else 38 + assert cpu_result.schema["a"].precision == 38 assert gpu_result.schema["a"].precision == 38 From 7abf5b12ffdd0396e9c6cab727ecf63844876098 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Mon, 11 May 2026 19:03:25 +0000 Subject: [PATCH 12/44] Remove 1.35 workarounds --- .../cudf_polars/cudf_polars/utils/versions.py | 3 +- .../tests/experimental/test_explain.py | 4 --- .../cudf_polars/tests/expressions/test_agg.py | 9 +++--- .../tests/expressions/test_sort.py | 30 ++----------------- python/cudf_polars/tests/test_mapfunction.py | 17 ++--------- python/cudf_polars/tests/test_scan.py | 7 +---- 6 files changed, 10 insertions(+), 60 deletions(-) diff --git a/python/cudf_polars/cudf_polars/utils/versions.py b/python/cudf_polars/cudf_polars/utils/versions.py index 52b6041c4493..4b8e94f479bc 100644 --- a/python/cudf_polars/cudf_polars/utils/versions.py +++ b/python/cudf_polars/cudf_polars/utils/versions.py @@ -11,8 +11,7 @@ from polars import __version__ POLARS_VERSION = parse(__version__) -POLARS_LOWER_BOUND = parse("1.30") -POLARS_VERSION_LT_135 = POLARS_VERSION < parse("1.35.0") +POLARS_LOWER_BOUND = parse("1.35") POLARS_VERSION_LT_136 = POLARS_VERSION < parse("1.36.0") POLARS_VERSION_LT_137 = POLARS_VERSION < parse("1.37.0") POLARS_VERSION_LT_138 = POLARS_VERSION < parse("1.38.0") diff --git a/python/cudf_polars/tests/experimental/test_explain.py b/python/cudf_polars/tests/experimental/test_explain.py index 7f19e3187783..1d2abb72c8cc 100644 --- a/python/cudf_polars/tests/experimental/test_explain.py +++ b/python/cudf_polars/tests/experimental/test_explain.py @@ -445,10 +445,6 @@ def _gb(df): def test_serialize_query(): - # this test is sensitive to the polars version. - # we get a different query plan for polars < 1.35.0. - pytest.importorskip("polars", minversion="1.35.0") - left = pl.LazyFrame({"a": ["a", "b", "a"], "b": [1, 2, 3]}) right = pl.LazyFrame({"a": ["a", "b", "c"], "c": [4, 5, 6]}) diff --git a/python/cudf_polars/tests/expressions/test_agg.py b/python/cudf_polars/tests/expressions/test_agg.py index 179d138c2933..436c1c8b03f1 100644 --- a/python/cudf_polars/tests/expressions/test_agg.py +++ b/python/cudf_polars/tests/expressions/test_agg.py @@ -15,7 +15,6 @@ assert_ir_translation_raises, ) from cudf_polars.utils.versions import ( - POLARS_VERSION_LT_135, POLARS_VERSION_LT_136, ) @@ -56,9 +55,9 @@ def is_sorted(request): @pytest.fixture -def xfail_if_sorted_gt_135(is_sorted, request): +def xfail_if_sorted(is_sorted, request): # See https://github.com/rapidsai/cudf/pull/20791#issuecomment-3750528419 - if is_sorted and not POLARS_VERSION_LT_135: + if is_sorted: request.applymarker( pytest.mark.xfail(reason="See https://github.com/pola-rs/polars/pull/24981") ) @@ -97,7 +96,7 @@ def decimal_df() -> pl.LazyFrame: ) -def test_agg(engine: pl.GPUEngine, df, agg, xfail_if_sorted_gt_135): +def test_agg(engine: pl.GPUEngine, df, agg, xfail_if_sorted): expr = getattr(pl.col("a"), agg)() q = df.select(expr) assert_gpu_result_equal(q, engine=engine, check_exact=False) @@ -124,7 +123,7 @@ def test_cum_agg_reverse_unsupported(cum_agg): @pytest.mark.parametrize("q", [0.5, pl.lit(0.5)]) @pytest.mark.parametrize("interp", ["nearest", "higher", "lower", "midpoint", "linear"]) -def test_quantile(engine: pl.GPUEngine, df, q, interp, xfail_if_sorted_gt_135): +def test_quantile(engine: pl.GPUEngine, df, q, interp, xfail_if_sorted): expr = pl.col("a").quantile(q, interp) q = df.select(expr) assert_gpu_result_equal(q, engine=engine, check_exact=False) diff --git a/python/cudf_polars/tests/expressions/test_sort.py b/python/cudf_polars/tests/expressions/test_sort.py index df66b324db52..3f2f0dccc66d 100644 --- a/python/cudf_polars/tests/expressions/test_sort.py +++ b/python/cudf_polars/tests/expressions/test_sort.py @@ -8,12 +8,8 @@ import polars as pl -import pylibcudf as plc - -from cudf_polars import Translator -from cudf_polars.dsl.ir import IRExecutionContext from cudf_polars.testing.asserts import assert_gpu_result_equal -from cudf_polars.utils.versions import POLARS_VERSION_LT_135, POLARS_VERSION_LT_136 +from cudf_polars.utils.versions import POLARS_VERSION_LT_136 @pytest.mark.parametrize("descending", [False, True]) @@ -64,7 +60,7 @@ def test_sort_by_expression( @pytest.mark.parametrize("nulls_last", [False, True]) @pytest.mark.parametrize("with_nulls", ["no_nulls", "nulls"]) def test_setsorted(engine: pl.GPUEngine, request, descending, nulls_last, with_nulls): - if not POLARS_VERSION_LT_135 and POLARS_VERSION_LT_136: + if POLARS_VERSION_LT_136: request.applymarker( pytest.mark.xfail( reason="See https://github.com/pola-rs/polars/pull/24981, " @@ -81,28 +77,6 @@ def test_setsorted(engine: pl.GPUEngine, request, descending, nulls_last, with_n assert_gpu_result_equal(q, engine=engine) - if POLARS_VERSION_LT_135: - translator = Translator(q._ldf.visit(), pl.GPUEngine()) - - df = translator.translate_ir().evaluate( - cache={}, - timer=None, - context=IRExecutionContext(), - ) - - a = df.column_map["a"] - - assert a.is_sorted == plc.types.Sorted.YES - null_order = ( - plc.types.NullOrder.AFTER - if (descending ^ nulls_last) and with_nulls == "nulls" - else plc.types.NullOrder.BEFORE - ) - assert a.null_order == null_order - assert a.order == ( - plc.types.Order.DESCENDING if descending else plc.types.Order.ASCENDING - ) - def test_sort_concat_filtered_to_empty(engine: pl.GPUEngine): df = pl.LazyFrame({"a": [1, 2, 3]}) diff --git a/python/cudf_polars/tests/test_mapfunction.py b/python/cudf_polars/tests/test_mapfunction.py index f4934fbc5f6e..94883f133923 100644 --- a/python/cudf_polars/tests/test_mapfunction.py +++ b/python/cudf_polars/tests/test_mapfunction.py @@ -13,8 +13,6 @@ assert_gpu_result_equal, assert_ir_translation_raises, ) -from cudf_polars.testing.engine_utils import get_blocksize_mode -from cudf_polars.utils.versions import POLARS_VERSION_LT_135 def test_explode_multiple_raises(): @@ -112,19 +110,8 @@ def test_unique_hash(): assert hash(ir_a) != hash(ir_b) -def test_set_sorted_then_inner_join(engine: pl.GPUEngine, request): - request.applymarker( - pytest.mark.xfail( - condition=not POLARS_VERSION_LT_135, - reason="HintIR not supported", - ) - ) - request.applymarker( - pytest.mark.xfail( - condition=get_blocksize_mode(engine) == "small" and POLARS_VERSION_LT_135, - reason="set_sorted join result order differs in polars < 1.35", - ) - ) +@pytest.mark.xfail(reason="HintIR not supported") +def test_set_sorted_then_inner_join(engine: pl.GPUEngine): df = pl.LazyFrame({"a": [1, 2, 3, 4, 5]}) q = df.set_sorted("a").join( diff --git a/python/cudf_polars/tests/test_scan.py b/python/cudf_polars/tests/test_scan.py index 894e0b61b638..1875de2f7627 100644 --- a/python/cudf_polars/tests/test_scan.py +++ b/python/cudf_polars/tests/test_scan.py @@ -20,7 +20,6 @@ from cudf_polars.testing.engine_utils import is_streaming_engine from cudf_polars.testing.io import make_partitioned_source from cudf_polars.utils.versions import ( - POLARS_VERSION_LT_135, POLARS_VERSION_LT_138, ) @@ -651,11 +650,7 @@ def test_hits_scan_row_index_duplicate(engine: pl.GPUEngine, request, tmp_path): "index" ) - if POLARS_VERSION_LT_135: - # Did not raise before - assert_gpu_result_equal(q, engine=engine) - else: - assert_ir_translation_raises(q, NotImplementedError) + assert_ir_translation_raises(q, NotImplementedError) @pytest.mark.parametrize("compression", ["gzip", "zlib", "zstd"]) From 7defd7cdcc6ef4f922631d1743354f0c25eea724 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Mon, 11 May 2026 19:06:32 +0000 Subject: [PATCH 13/44] Update conda recipe --- conda/recipes/cudf-polars/recipe.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/recipes/cudf-polars/recipe.yaml b/conda/recipes/cudf-polars/recipe.yaml index a4a9441187c1..3a48d7fc4af6 100644 --- a/conda/recipes/cudf-polars/recipe.yaml +++ b/conda/recipes/cudf-polars/recipe.yaml @@ -37,7 +37,7 @@ requirements: - python - pylibcudf =${{ version }} - rapidsmpf =${{ minor_version }} - - polars>=1.30,<1.39 + - polars>=1.35,<1.39 - packaging - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }} - if: cuda_major == "12" From 2b7babcb091e1145a87df18129b77502d3c1d57a Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Mon, 11 May 2026 21:26:01 +0000 Subject: [PATCH 14/44] Fix empty marker in test_rolling, fix log translation --- python/cudf_polars/cudf_polars/dsl/translate.py | 4 ++-- python/cudf_polars/tests/expressions/test_rolling.py | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/python/cudf_polars/cudf_polars/dsl/translate.py b/python/cudf_polars/cudf_polars/dsl/translate.py index 57e320c0c0a6..47afd9b7e58e 100644 --- a/python/cudf_polars/cudf_polars/dsl/translate.py +++ b/python/cudf_polars/cudf_polars/dsl/translate.py @@ -792,13 +792,13 @@ def _( ) elif isinstance(name, str): children = (translator.translate_expr(n=n, schema=schema) for n in node.input) - if ( + if name == "log" or ( name == "l" and isinstance(options[0], str) and "".join((name, *options)) == "log" ): (child, base) = children - assert isinstance(base, expr.NamedExpr) + assert isinstance(base, expr.Literal) return expr.BinOp( dtype, plc.binaryop.BinaryOperator.LOG_BASE, diff --git a/python/cudf_polars/tests/expressions/test_rolling.py b/python/cudf_polars/tests/expressions/test_rolling.py index 500520566078..6c00f66347df 100644 --- a/python/cudf_polars/tests/expressions/test_rolling.py +++ b/python/cudf_polars/tests/expressions/test_rolling.py @@ -320,14 +320,12 @@ def test_over_broadcast_input_row_group_indices_aligned(engine: pl.GPUEngine): @pytest.mark.parametrize("order_by", [None, ["g2", pl.col("x2") * 2]]) def test_rank_over( engine: pl.GPUEngine, - request, df: pl.LazyFrame, method: RankMethod, *, descending: bool, order_by: None | list[str | pl.Expr], ) -> None: - request.applymarker() q = df.select( pl.col("x") .rank(method=method, descending=descending) From e59bc97336a207937468e1b4c0bc0c74248440a5 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Mon, 11 May 2026 21:45:34 +0000 Subject: [PATCH 15/44] Shorten timeout to 15 --- ci/test_wheel_cudf_polars.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/test_wheel_cudf_polars.sh b/ci/test_wheel_cudf_polars.sh index 1ae3544e642e..da8a1f1c7070 100755 --- a/ci/test_wheel_cudf_polars.sh +++ b/ci/test_wheel_cudf_polars.sh @@ -71,7 +71,7 @@ for version in "${VERSIONS[@]}"; do COVERAGE_ARGS=(--no-cov) fi - timeout 1h ./ci/run_cudf_polars_pytests.sh \ + timeout 15m ./ci/run_cudf_polars_pytests.sh \ "${COVERAGE_ARGS[@]}" \ --numprocesses=8 \ --dist=worksteal \ From 72009dfae5d1e624a30c778553656ed7a7f13a08 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Tue, 12 May 2026 00:48:25 +0000 Subject: [PATCH 16/44] Reduce diff changes, support for 1.35 --- .../cudf_polars/dsl/expressions/rolling.py | 20 +- python/cudf_polars/cudf_polars/dsl/ir.py | 20 +- .../cudf_polars/cudf_polars/testing/plugin.py | 282 ------------------ python/cudf_polars/tests/test_groupby.py | 2 +- python/cudf_polars/tests/test_mapfunction.py | 4 +- python/cudf_polars/tests/test_rolling.py | 8 +- 6 files changed, 42 insertions(+), 294 deletions(-) delete mode 100644 python/cudf_polars/cudf_polars/testing/plugin.py diff --git a/python/cudf_polars/cudf_polars/dsl/expressions/rolling.py b/python/cudf_polars/cudf_polars/dsl/expressions/rolling.py index fe3df51653c1..98532df2a878 100644 --- a/python/cudf_polars/cudf_polars/dsl/expressions/rolling.py +++ b/python/cudf_polars/cudf_polars/dsl/expressions/rolling.py @@ -22,6 +22,7 @@ offsets_to_windows, range_window_bounds, ) +from cudf_polars.utils.versions import POLARS_VERSION_LT_136 if TYPE_CHECKING: from collections.abc import Sequence @@ -78,9 +79,14 @@ def to_request( elif isinstance(value, expr.Agg): child = value.children[0] col = child.evaluate(df, context=ExecutionContext.ROLLING) - if value.name == "var": - # Polars rolling var returns null when nvalues <= ddof; libcudf returns NaN. - # Setting min_periods = ddof+1 causes libcudf to emit null instead of NaN. + if POLARS_VERSION_LT_136 and value.name == "var": # pragma: no cover + # Polars variance produces null if nvalues <= ddof + # libcudf produces NaN. However, we can get the polars + # behaviour by setting the minimum window size to ddof + + # 1. + # + # We still need this check, polars is not hitting it because + # of https://github.com/pola-rs/polars/pull/25117 min_periods = value.options + 1 else: col = value.evaluate( @@ -90,7 +96,7 @@ def to_request( return plc.rolling.RollingRequest(col.obj, min_periods, value.agg_request) -class RollingWindow(Expr): +class RollingWindow(Expr): # pragma: no cover; polars >1.36 uses AExpr::Rolling now __slots__ = ( "closed_window", "following_ordinal", @@ -140,10 +146,12 @@ def __init__( raise NotImplementedError( "Incorrect handling of empty groups for list collection" ) - if not plc.rolling.is_valid_rolling_aggregation( + if POLARS_VERSION_LT_136 and not plc.rolling.is_valid_rolling_aggregation( agg.dtype.plc_type, agg.agg_request ): - raise NotImplementedError(f"Unsupported rolling aggregation {agg}") + raise NotImplementedError( + f"Unsupported rolling aggregation {agg}" + ) # pragma: no cover; polars may raise ahead of time def do_evaluate( # noqa: D102 self, df: DataFrame, *, context: ExecutionContext = ExecutionContext.FRAME diff --git a/python/cudf_polars/cudf_polars/dsl/ir.py b/python/cudf_polars/cudf_polars/dsl/ir.py index 30ad33508555..8537fea9faed 100644 --- a/python/cudf_polars/cudf_polars/dsl/ir.py +++ b/python/cudf_polars/cudf_polars/dsl/ir.py @@ -32,7 +32,7 @@ import cudf_polars.dsl.expr as expr from cudf_polars.containers import Column, DataFrame, DataType from cudf_polars.containers.dataframe import NamedColumn -from cudf_polars.dsl.expressions import rolling +from cudf_polars.dsl.expressions import rolling, unary from cudf_polars.dsl.expressions.base import ExecutionContext from cudf_polars.dsl.nodebase import Node from cudf_polars.dsl.to_ast import to_ast, to_parquet_filter @@ -1408,8 +1408,11 @@ def do_evaluate( if projection is not None: df = df.select(projection) + # Zero-width dataframes lose their row count when converted through + # pylibcudf. See https://github.com/rapidsai/cudf/issues/21428 if len(schema) == 0: return DataFrame([], stream=context.get_cuda_stream(), num_rows=height) + df = DataFrame.from_polars(df, stream=context.get_cuda_stream()) assert all( c.obj.type() == dtype.plc_type @@ -1797,6 +1800,16 @@ def __init__( for dtype in schema.values(): if _has_struct_in_list(dtype.polars_type): raise NotImplementedError("Nested list[struct] types not supported") + for request in agg_requests: + expr = request.value + if ( + POLARS_VERSION_LT_136 + and isinstance(expr, unary.UnaryFunction) + and expr.name == "value_counts" + ): + raise NotImplementedError( + "value_counts is not supported in groupby" + ) # pragma: no cover; Nested list[struct] types not supported self.agg_requests = tuple(agg_requests) self.maintain_order = maintain_order self.zlice = zlice @@ -2982,6 +2995,7 @@ class MapFunction(IR): "unpivot", "row_index", "fast_count", + "hint_sorted", ] ) @@ -3046,6 +3060,10 @@ def __init__(self, schema: Schema, name: str, options: Any, df: IR): raise NotImplementedError( "Fast count unsupported for CSV scans" ) # pragma: no cover + elif ( + self.name == "hint_sorted" + ): # pragma: no cover; polars prunes hints in some cases + raise NotImplementedError("Hint sorted unsupported") self._non_child_args = (schema, name, self.options) def get_hashable(self) -> Hashable: diff --git a/python/cudf_polars/cudf_polars/testing/plugin.py b/python/cudf_polars/cudf_polars/testing/plugin.py deleted file mode 100644 index 6e2968cccc68..000000000000 --- a/python/cudf_polars/cudf_polars/testing/plugin.py +++ /dev/null @@ -1,282 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. -# SPDX-License-Identifier: Apache-2.0 - -"""Plugin for running polars test suite setting GPU engine as default.""" - -from __future__ import annotations - -from functools import partialmethod -from typing import TYPE_CHECKING - -import pytest - -import polars - -from cudf_polars.utils.config import StreamingFallbackMode - -if TYPE_CHECKING: - from collections.abc import Mapping - from typing import Any - - -def pytest_addoption(parser: pytest.Parser) -> None: - """Add plugin-specific options.""" - group = parser.getgroup( - "cudf-polars", "Plugin to set GPU as default engine for polars tests" - ) - group.addoption( - "--cudf-polars-no-fallback", - action="store_true", - help="Turn off fallback to CPU when running tests (default use fallback)", - ) - group.addoption( - "--executor", - action="store", - default="in-memory", - choices=("in-memory", "streaming"), - help="Executor to use for GPUEngine.", - ) - group.addoption( - "--blocksize-mode", - action="store", - default="default", - choices=("small", "default"), - help=( - "Blocksize to use for 'streaming' executor. Set to 'small' " - "to run most tests with multiple partitions." - ), - ) - - -def pytest_configure(config: pytest.Config) -> None: - """Enable use of this module as a pytest plugin to enable GPU collection.""" - no_fallback = config.getoption("--cudf-polars-no-fallback") - executor = config.getoption("--executor") - blocksize_mode = config.getoption("--blocksize-mode") - if no_fallback: - collect = polars.LazyFrame.collect - engine = polars.GPUEngine(raise_on_fail=no_fallback) - # https://github.com/python/mypy/issues/2427 - polars.LazyFrame.collect = partialmethod(collect, engine=engine) # type: ignore[method-assign, assignment] - elif executor == "in-memory": - collect = polars.LazyFrame.collect - engine = polars.GPUEngine(executor=executor) - polars.LazyFrame.collect = partialmethod(collect, engine=engine) # type: ignore[method-assign, assignment] - elif executor == "streaming" and blocksize_mode == "small": - executor_options: dict[str, Any] = {} - executor_options["max_rows_per_partition"] = 4 - executor_options["target_partition_size"] = 10 - # We expect many tests to fall back, so silence the warnings - executor_options["fallback_mode"] = StreamingFallbackMode.SILENT - collect = polars.LazyFrame.collect - engine = polars.GPUEngine(executor=executor, executor_options=executor_options) - polars.LazyFrame.collect = partialmethod(collect, engine=engine) # type: ignore[method-assign, assignment] - else: - # run with streaming executor and default blocksize - polars.Config.set_engine_affinity("gpu") - config.addinivalue_line( - "filterwarnings", - "ignore:.*GPU engine does not support streaming or background collection", - ) - config.addinivalue_line( - "filterwarnings", - "ignore:.*Query execution with GPU not possible", - ) - - -EXPECTED_FAILURES: Mapping[str, str | tuple[str, bool]] = { - "tests/unit/io/test_csv.py::test_read_csv_only_loads_selected_columns": "Memory usage won't be correct due to GPU", - "tests/unit/io/test_delta.py::test_scan_delta_version": "Need to expose hive partitioning", - "tests/unit/io/test_delta.py::test_scan_delta_relative": "Need to expose hive partitioning", - "tests/unit/io/test_delta.py::test_read_delta_version": "Need to expose hive partitioning", - "tests/unit/io/test_delta.py::test_scan_delta_schema_evolution_nested_struct_field_19915": "Need to expose hive partitioning", - "tests/unit/io/test_delta.py::test_scan_delta_nanosecond_timestamp": "polars generates the wrong schema: https://github.com/pola-rs/polars/issues/23949", - "tests/unit/io/test_delta.py::test_scan_delta_nanosecond_timestamp_nested": "polars generates the wrong schema: https://github.com/pola-rs/polars/issues/23949", - "tests/unit/io/test_lazy_count_star.py::test_count_parquet[small.parquet-4]": "Debug output on stderr doesn't match", - "tests/unit/io/test_lazy_count_star.py::test_count_parquet[foods*.parquet-54]": "Debug output on stderr doesn't match", - "tests/unit/io/test_lazy_parquet.py::test_parquet_is_in_statistics": "Debug output on stderr doesn't match", - "tests/unit/io/test_lazy_parquet.py::test_parquet_statistics": "Debug output on stderr doesn't match", - "tests/unit/io/test_partition.py::test_partition_to_memory[io_type0]": "partition sinks not yet supported in standard engine.", - "tests/unit/io/test_partition.py::test_partition_to_memory[io_type1]": "partition sinks not yet supported in standard engine.", - "tests/unit/io/test_partition.py::test_partition_to_memory[io_type2]": "partition sinks not yet supported in standard engine.", - "tests/unit/io/test_partition.py::test_partition_to_memory[io_type3]": "partition sinks not yet supported in standard engine.", - "tests/unit/io/test_partition.py::test_partition_to_memory_finish_callback[io_type1]": "partition sinks not yet supported in standard engine.", - "tests/unit/io/test_partition.py::test_partition_to_memory_finish_callback[io_type2]": "partition sinks not yet supported in standard engine.", - "tests/unit/io/test_partition.py::test_partition_to_memory_finish_callback[io_type3]": "partition sinks not yet supported in standard engine.", - "tests/unit/io/test_partition.py::test_partition_to_memory_sort_by[df1-a-io_type3]": "partition sinks not yet supported in standard engine.", - "tests/unit/io/test_partition.py::test_partition_to_memory_sort_by[df2-sorts2-io_type0]": "partition sinks not yet supported in standard engine.", - "tests/unit/io/test_partition.py::test_partition_to_memory_sort_by[df2-sorts2-io_type1]": "partition sinks not yet supported in standard engine.", - "tests/unit/io/test_partition.py::test_partition_to_memory_sort_by[df2-sorts2-io_type2]": "partition sinks not yet supported in standard engine.", - "tests/unit/io/test_partition.py::test_partition_to_memory_sort_by[df2-sorts2-io_type3]": "partition sinks not yet supported in standard engine.", - "tests/unit/io/test_partition.py::test_partition_to_memory_sort_by[df3-b-io_type0]": "partition sinks not yet supported in standard engine.", - "tests/unit/io/test_partition.py::test_partition_to_memory_sort_by[df3-b-io_type1]": "partition sinks not yet supported in standard engine.", - "tests/unit/io/test_partition.py::test_partition_to_memory_sort_by[df3-b-io_type2]": "partition sinks not yet supported in standard engine.", - "tests/unit/io/test_partition.py::test_partition_to_memory_sort_by[df3-b-io_type3]": "partition sinks not yet supported in standard engine.", - "tests/unit/io/test_partition.py::test_partition_to_memory_sort_by[df4-sorts4-io_type0]": "partition sinks not yet supported in standard engine.", - "tests/unit/io/test_partition.py::test_partition_to_memory_sort_by[df4-sorts4-io_type1]": "partition sinks not yet supported in standard engine.", - "tests/unit/io/test_partition.py::test_partition_to_memory_sort_by[df4-sorts4-io_type2]": "partition sinks not yet supported in standard engine.", - "tests/unit/io/test_partition.py::test_partition_to_memory_sort_by[df4-sorts4-io_type3]": "partition sinks not yet supported in standard engine.", - "tests/unit/io/test_partition.py::test_partition_to_memory_finish_callback[io_type0]": "partition sinks not yet supported in standard engine.", - "tests/unit/io/test_partition.py::test_partition_to_memory_sort_by[df0-a-io_type0]": "partition sinks not yet supported in standard engine.", - "tests/unit/io/test_partition.py::test_partition_to_memory_sort_by[df0-a-io_type1]": "partition sinks not yet supported in standard engine.", - "tests/unit/io/test_partition.py::test_partition_to_memory_sort_by[df0-a-io_type2]": "partition sinks not yet supported in standard engine.", - "tests/unit/io/test_partition.py::test_partition_to_memory_sort_by[df0-a-io_type3]": "partition sinks not yet supported in standard engine.", - "tests/unit/io/test_partition.py::test_partition_to_memory_sort_by[df1-a-io_type0]": "partition sinks not yet supported in standard engine.", - "tests/unit/io/test_partition.py::test_partition_to_memory_sort_by[df1-a-io_type1]": "partition sinks not yet supported in standard engine.", - "tests/unit/io/test_partition.py::test_partition_to_memory_sort_by[df1-a-io_type2]": "partition sinks not yet supported in standard engine.", - "tests/unit/io/test_sink.py::test_collect_all_lazy": "SinkMultiple not supported by InMemory CPU Engine, which we fallback to. See pola-rs/polars/pull/26537", - "tests/unit/io/test_lazy_parquet.py::test_scan_parquet_ignores_dtype_mismatch_for_non_projected_columns_19249[False-False]": "Needs some variant of cudf#16394", - "tests/unit/io/test_lazy_parquet.py::test_scan_parquet_ignores_dtype_mismatch_for_non_projected_columns_19249[True-False]": "Needs some variant of cudf#16394", - "tests/unit/io/test_lazy_parquet.py::test_parquet_unaligned_schema_read[False]": "Incomplete handling of projected reads with mismatching schemas, cudf#16394", - "tests/unit/io/test_parquet.py::test_read_parquet_only_loads_selected_columns_15098": "Memory usage won't be correct due to GPU", - "tests/unit/io/test_parquet.py::test_allow_missing_columns[projection0-False-none]": "Mismatching column read cudf#16394", - "tests/unit/io/test_parquet.py::test_allow_missing_columns[projection1-False-none]": "Mismatching column read cudf#16394", - "tests/unit/io/test_parquet.py::test_allow_missing_columns[projection0-False-prefiltered]": "Mismatching column read cudf#16394", - "tests/unit/io/test_parquet.py::test_allow_missing_columns[projection1-False-prefiltered]": "Mismatching column read cudf#16394", - "tests/unit/io/test_parquet.py::test_allow_missing_columns[projection0-False-row_groups]": "Mismatching column read cudf#16394", - "tests/unit/io/test_parquet.py::test_allow_missing_columns[projection1-False-row_groups]": "Mismatching column read cudf#16394", - "tests/unit/io/test_parquet.py::test_allow_missing_columns[projection0-False-columns]": "Mismatching column read cudf#16394", - "tests/unit/io/test_parquet.py::test_allow_missing_columns[projection1-False-columns]": "Mismatching column read cudf#16394", - "tests/unit/io/test_parquet.py::test_allow_missing_columns[projection0-True-none]": "Mismatching column read cudf#16394", - "tests/unit/io/test_parquet.py::test_allow_missing_columns[projection1-True-none]": "Mismatching column read cudf#16394", - "tests/unit/io/test_parquet.py::test_allow_missing_columns[projection0-True-prefiltered]": "Mismatching column read cudf#16394", - "tests/unit/io/test_parquet.py::test_allow_missing_columns[projection1-True-prefiltered]": "Mismatching column read cudf#16394", - "tests/unit/io/test_parquet.py::test_allow_missing_columns[projection0-True-row_groups]": "Mismatching column read cudf#16394", - "tests/unit/io/test_parquet.py::test_allow_missing_columns[projection1-True-row_groups]": "Mismatching column read cudf#16394", - "tests/unit/io/test_parquet.py::test_allow_missing_columns[projection0-True-columns]": "Mismatching column read cudf#16394", - "tests/unit/io/test_parquet.py::test_allow_missing_columns[projection1-True-columns]": "Mismatching column read cudf#16394", - "tests/unit/io/test_parquet.py::test_scan_parquet_filter_statistics_load_missing_column_21391": "Mismatching column read cudf#16394", - "tests/unit/io/test_parquet.py::test_binary_offset_roundtrip": "binary offset type unsupported", - "tests/unit/lazyframe/test_engine_selection.py::test_engine_import_error_raises[gpu]": "Expect this to pass because cudf-polars is installed", - "tests/unit/lazyframe/test_engine_selection.py::test_engine_import_error_raises[engine1]": "Expect this to pass because cudf-polars is installed", - "tests/unit/lazyframe/test_lazyframe.py::test_round[dtype2-123.55-1-123.6]": "libcudf HALF_EVEN rounding bug for Float64 with decimal_places > 0. See https://github.com/rapidsai/cudf/issues/21319", - "tests/unit/lazyframe/test_lazyframe.py::test_cast_frame": "Casting that raises not supported on GPU", - "tests/unit/lazyframe/test_lazyframe.py::test_lazy_cache_hit": "Debug output on stderr doesn't match", - "tests/unit/operations/aggregation/test_aggregations.py::test_binary_op_agg_context_no_simplify_expr_12423": "groupby-agg of just literals should not produce collect_list", - "tests/unit/operations/aggregation/test_aggregations.py::test_nan_inf_aggregation": "treatment of nans and nulls together is different in libcudf and polars in groupby-agg context", - "tests/unit/operations/test_abs.py::test_abs_duration": "Need to raise for unsupported uops on timelike values", - "tests/unit/operations/test_group_by.py::test_group_by_shorthand_quantile": "libcudf quantiles are round to nearest ties to even, polars quantiles are round to nearest ties away from zero", - "tests/unit/operations/test_group_by.py::test_group_by_mean_by_dtype[input10-expected10-Date-output_dtype10]": "Unsupported groupby-agg for a particular dtype", - "tests/unit/operations/test_group_by.py::test_group_by_mean_by_dtype[input11-expected11-input_dtype11-output_dtype11]": "Unsupported groupby-agg for a particular dtype", - "tests/unit/operations/test_group_by.py::test_group_by_mean_by_dtype[input12-expected12-input_dtype12-output_dtype12]": "Unsupported groupby-agg for a particular dtype", - "tests/unit/operations/test_group_by.py::test_group_by_mean_by_dtype[input13-expected13-input_dtype13-output_dtype13]": "Unsupported groupby-agg for a particular dtype", - "tests/unit/operations/test_group_by.py::test_group_by_median_by_dtype[input14-expected14-input_dtype14-output_dtype14]": "Unsupported groupby-agg for a particular dtype", - "tests/unit/operations/test_group_by.py::test_group_by_median_by_dtype[input15-expected15-input_dtype15-output_dtype15]": "Unsupported groupby-agg for a particular dtype", - "tests/unit/operations/test_group_by.py::test_group_by_median_by_dtype[input16-expected16-input_dtype16-output_dtype16]": "Unsupported groupby-agg for a particular dtype", - "tests/unit/operations/test_group_by.py::test_group_by_binary_agg_with_literal": "Incorrect broadcasting of literals in groupby-agg", - "tests/unit/operations/test_group_by.py::test_group_by_lit_series": "Incorrect broadcasting of literals in groupby-agg", - "tests/unit/operations/test_group_by.py::test_group_by_series_lit_22103[False]": "Incorrect broadcasting of literals in groupby-agg", - "tests/unit/operations/test_group_by.py::test_group_by_series_lit_22103[True]": "Incorrect broadcasting of literals in groupby-agg", - "tests/unit/operations/test_join.py::test_cross_join_slice_pushdown": "Need to implement slice pushdown for cross joins", - # We match the behavior of the polars[cpu] streaming engine (it makes doesn't make any ordering guarantees either when maintain_order is none). - # But this test does because the test is run with the polars[cpu] in-memory engine, which still preserves the order of the left dataframe - # when maintain order is none. - "tests/unit/operations/test_join.py::test_join_preserve_order_left": "polars[gpu] makes no ordering guarantees when maintain_order is none", - # TODO: As of polars 1.34, the column names for left and right came in unaligned, which causes the dtypes to mismatch when calling plc.replace.replace_nulls - # Need to investigate what changed in polars - "tests/unit/operations/test_join.py::test_join_coalesce_column_order_23177": "Misaligned left/right column names left and right tables in join op", - "tests/unit/operations/namespaces/string/test_pad.py::test_str_zfill_unicode_not_respected": "polars doesn't add zeros for unicode characters.", - "tests/unit/sql/test_cast.py::test_cast_errors[values0-values::uint8-conversion from `f64` to `u64` failed]": "Casting that raises not supported on GPU", - "tests/unit/sql/test_cast.py::test_cast_errors[values1-values::uint4-conversion from `i64` to `u32` failed]": "Casting that raises not supported on GPU", - "tests/unit/sql/test_cast.py::test_cast_errors[values2-values::int1-conversion from `i64` to `i8` failed]": "Casting that raises not supported on GPU", - "tests/unit/sql/test_cast.py::test_cast_errors[values5-values::int4-conversion from `str` to `i32` failed]": "Cast raises, but error user receives is wrong", - "tests/unit/lazyframe/test_predicates.py::test_predicate_pushdown_split_pushable": "Casting that raises not supported on GPU", - "tests/unit/sql/test_miscellaneous.py::test_read_csv": "Incorrect handling of missing_is_null in read_csv", - "tests/unit/lazyframe/test_cse.py::test_cse_predicate_self_join[False]": "Debug output on stderr doesn't match", - "tests/unit/io/test_scan_row_deletion.py::test_scan_row_deletion_skips_file_with_all_rows_deleted": "The test intentionally corrupts the parquet file, so we cannot read the row count from the header.", - "tests/unit/io/test_multiscan.py::test_multiscan_row_index[scan_csv-write_csv-csv]": "Debug output on stderr doesn't match", - "tests/unit/io/test_lazy_parquet.py::test_parquet_schema_arg[True-columns]": "allow_missing_columns argument in read_parquet not translated in IR", - "tests/unit/io/test_lazy_parquet.py::test_parquet_schema_arg[True-row_groups]": "allow_missing_columns argument in read_parquet not translated in IR", - "tests/unit/io/test_lazy_parquet.py::test_parquet_schema_arg[True-prefiltered]": "allow_missing_columns argument in read_parquet not translated in IR", - "tests/unit/io/test_lazy_parquet.py::test_parquet_schema_arg[True-none]": "allow_missing_columns argument in read_parquet not translated in IR", - "tests/unit/io/test_lazy_parquet.py::test_parquet_schema_arg[False-columns]": "allow_missing_columns argument in read_parquet not translated in IR", - "tests/unit/io/test_lazy_parquet.py::test_parquet_schema_arg[False-row_groups]": "allow_missing_columns argument in read_parquet not translated in IR", - "tests/unit/io/test_lazy_parquet.py::test_parquet_schema_arg[False-prefiltered]": "allow_missing_columns argument in read_parquet not translated in IR", - "tests/unit/io/test_lazy_parquet.py::test_parquet_schema_arg[False-none]": "allow_missing_columns argument in read_parquet not translated in IR", - "tests/unit/io/test_multiscan.py::test_multiscan_row_index[scan_csv-write_csv]": "CSV multiscan with row_index and no row limit is not yet supported.", - "tests/unit/operations/namespaces/test_binary.py::test_binary_compounded_literal_aggstate_24460": "List literal loses nesting in gather: cudf#19610", - "tests/unit/operations/test_slice.py::test_schema_gather_get_on_literal_24101[lit1-0-False]": "List literal loses nesting in gather: cudf#19610", - "tests/unit/operations/test_slice.py::test_schema_gather_get_on_literal_24101[lit1-idx1-False]": "List literal loses nesting in gather: cudf#19610", - "tests/unit/operations/test_slice.py::test_schema_gather_get_on_literal_24101[lit1-idx2-False]": "List literal loses nesting in gather: cudf#19610", - "tests/unit/operations/test_slice.py::test_schema_head_tail_on_literal_24102[lit1-1-False]": "List literal loses nesting in head/tail: cudf#19610", - "tests/unit/operations/test_slice.py::test_schema_head_tail_on_literal_24102[lit1-len1-False]": "List literal loses nesting in head/tail: cudf#19610", - "tests/unit/operations/test_slice.py::test_schema_slice_on_literal_23999[lit1-0-0-False]": "List literal loses nesting in slice: cudf#19610", - "tests/unit/operations/test_slice.py::test_schema_slice_on_literal_23999[lit1-0-len1-False]": "List literal loses nesting in slice: cudf#19610", - "tests/unit/operations/test_slice.py::test_schema_slice_on_literal_23999[lit1-offset1-0-False]": "List literal loses nesting in slice: cudf#19610", - "tests/unit/operations/test_slice.py::test_schema_slice_on_literal_23999[lit1-offset1-len1-False]": "List literal loses nesting in slice: cudf#19610", -} - - -TESTS_TO_SKIP: Mapping[str, str] = { - "tests/unit/operations/test_profile.py::test_profile_with_cse": "Shape assertion won't match", - # On Ubuntu 20.04, the tzdata package contains a bunch of symlinks - # for obsolete timezone names. However, the chrono_tz package that - # polars uses doesn't read /usr/share/zoneinfo, instead packaging - # the current zoneinfo database from IANA. Consequently, when this - # hypothesis-generated test runs and generates timezones from the - # available zoneinfo-reported timezones, we can get an error from - # polars that the requested timezone is unknown. - # Since this is random, just skip it, rather than xfailing. - "tests/unit/lazyframe/test_serde.py::test_lf_serde_roundtrip_binary": "chrono_tz doesn't have all tzdata symlink names", - # Tests performance difference of CPU engine - "tests/unit/operations/test_join.py::test_join_where_eager_perf_21145": "Tests performance bug in CPU engine", - "tests/unit/operations/namespaces/list/test_list.py::test_list_struct_field_perf": "Tests CPU Engine perf", - "tests/benchmark/test_with_columns.py::test_with_columns_quadratic_19503": "Tests performance bug in CPU engine", - # The test may segfault with the legacy streaming engine. We should - # remove this skip when all polars tests use the new streaming engine. - "tests/unit/streaming/test_streaming_group_by.py::test_streaming_group_by_literal[1]": "May segfault w/the legacy streaming engine", - # Fails in CI, but passes locally - "tests/unit/streaming/test_streaming.py::test_streaming_streamable_functions": "RuntimeError: polars_python::sql::PySQLContext is unsendable, but is being dropped on another thread", - # Remove when polars supports Pydantic V3 - "tests/unit/constructors/test_constructors.py::test_init_structured_objects": "pydantic deprecation warning", - "tests/unit/constructors/test_constructors.py::test_init_pydantic_2x": "pydantic deprecation warning", - "tests/unit/constructors/test_constructors.py::test_init_structured_objects_nested[_TestFooPD-_TestBarPD-_TestBazPD]": "pydantic deprecation warning", - "tests/unit/series/test_series.py::test_init_structured_objects": "pydantic deprecation warning", - "tests/unit/series/test_describe.py::test_series_describe_float": "https://github.com/rapidsai/cudf/issues/19324", - "tests/unit/series/test_describe.py::test_series_describe_int": "https://github.com/rapidsai/cudf/issues/19324", - "tests/unit/streaming/test_streaming.py::test_streaming_apply": "https://github.com/pola-rs/polars/issues/22558", - # New iceberg release causes this test to fail. We can remove in the next polars version bump: https://github.com/rapidsai/cudf/pull/19912 - "tests/unit/io/test_iceberg.py::test_fill_missing_fields_with_identity_partition_values[False]": "https://github.com/pola-rs/polars/pull/24456", - "tests/unit/operations/test_rolling.py::test_rolling_agg_bad_input_types[str]": "https://github.com/rapidsai/cudf/issues/20551", - "tests/unit/operations/test_group_by_dynamic.py::test_group_by_dynamic_agg_bad_input_types[str]": "TODO: Need to investigate why this fails in CI but passes locally. We should fallback to CPU for group_by_dynamic", - "tests/unit/expr/test_exprs.py::test_exp_log1p[Float16-Float16]": "Flaky test: Small floating-point precision differences in exp/log1p results", - # TODO: Investigate why these tests fail in CI but pass locally. - "tests/unit/io/test_delta.py::test_scan_delta_extract_table_statistics_df": "schemas mismatch: dtypes different", - "tests/unit/io/test_partition.py::test_sink_partitioned_no_columns_in_file_25535[scan_parquet-sink_parquet]": "Incorrect row count. Related to https://github.com/rapidsai/cudf/issues/21428", - "tests/unit/operations/test_group_by.py::test_unique_head_tail_26429[0]": "ZeroDivisionError: division by zero", -} - - -STREAMING_ONLY_EXPECTED_FAILURES: Mapping[str, str] = { - # Add tests that are expected to fail with the streaming executor -} - - -def pytest_collection_modifyitems( - session: pytest.Session, config: pytest.Config, items: list[pytest.Item] -) -> None: - """Mark known failing tests.""" - if config.getoption("--cudf-polars-no-fallback"): - # Don't xfail tests if running without fallback - return - for item in items: - if (reason := TESTS_TO_SKIP.get(item.nodeid, None)) is not None: - item.add_marker(pytest.mark.skip(reason=reason)) - elif ( - config.getoption("--executor") == "streaming" - and (s_reason := STREAMING_ONLY_EXPECTED_FAILURES.get(item.nodeid, None)) - is not None - ): - item.add_marker(pytest.mark.xfail(reason=s_reason)) - elif (entry := EXPECTED_FAILURES.get(item.nodeid, None)) is not None: - if isinstance(entry, tuple): - # the second entry in the tuple is the condition to xfail on - reason, condition = entry - item.add_marker( - pytest.mark.xfail( - condition=condition, - reason=reason, - ), - ) - else: - item.add_marker(pytest.mark.xfail(reason=entry)) diff --git a/python/cudf_polars/tests/test_groupby.py b/python/cudf_polars/tests/test_groupby.py index a8ccd1abf936..c649a0997c50 100644 --- a/python/cudf_polars/tests/test_groupby.py +++ b/python/cudf_polars/tests/test_groupby.py @@ -408,7 +408,7 @@ def test_groupby_aggs_keep_unsupported_as_null( ) -> None: request.applymarker( pytest.mark.xfail( - condition=not POLARS_VERSION_LT_136 and "sum" in str(agg_expr), + condition="sum" in str(agg_expr) and not POLARS_VERSION_LT_136, reason="polars raises now", ) ) diff --git a/python/cudf_polars/tests/test_mapfunction.py b/python/cudf_polars/tests/test_mapfunction.py index bbd88c3dfd36..94883f133923 100644 --- a/python/cudf_polars/tests/test_mapfunction.py +++ b/python/cudf_polars/tests/test_mapfunction.py @@ -121,8 +121,8 @@ def test_set_sorted_then_inner_join(engine: pl.GPUEngine): def test_explode_single_legacy_options(): - # Cover the branch: len(self.options) == 1 - # This branch is only reachable by direct construction + # Cover the branch: POLARS_VERSION_LT_136 or len(self.options) == 1 + # On polars >= 1.36 this branch is only reachable by direct construction # with 1-element options (the old pre-1.36 format). df = pl.DataFrame({"a": [[1, 2], [3, 4]]}) child = DataFrameScan({"a": DataType(pl.List(pl.Int64()))}, df._df, None) diff --git a/python/cudf_polars/tests/test_rolling.py b/python/cudf_polars/tests/test_rolling.py index 564b0aa4f967..40de49360f73 100644 --- a/python/cudf_polars/tests/test_rolling.py +++ b/python/cudf_polars/tests/test_rolling.py @@ -161,8 +161,12 @@ def test_orderby_nulls_raises_computeerror(engine_raise_on_fail: pl.GPUEngine): def test_rolling_nested_raises(request): - if not POLARS_VERSION_LT_136: - request.applymarker(pytest.mark.xfail(reason="polars raises now")) + request.applymarker( + pytest.mark.xfail( + condition=not POLARS_VERSION_LT_136, + reason="polars raises now", + ) + ) q = ( pl.LazyFrame( { From 1f72afde247b759d2dffea67e38c45a2e7f9fc55 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Tue, 12 May 2026 00:52:15 +0000 Subject: [PATCH 17/44] Reduce diff more --- .../cudf_polars/cudf_polars/containers/dataframe.py | 1 + .../cudf_polars/dsl/utils/aggregations.py | 3 ++- python/cudf_polars/tests/expressions/test_agg.py | 12 +++++++----- .../tests/expressions/test_stringfunction.py | 8 ++------ python/cudf_polars/tests/test_dataframescan.py | 3 ++- 5 files changed, 14 insertions(+), 13 deletions(-) diff --git a/python/cudf_polars/cudf_polars/containers/dataframe.py b/python/cudf_polars/cudf_polars/containers/dataframe.py index e7c2408d6421..d96e96cc997b 100644 --- a/python/cudf_polars/cudf_polars/containers/dataframe.py +++ b/python/cudf_polars/cudf_polars/containers/dataframe.py @@ -122,6 +122,7 @@ def to_polars(self) -> pl.DataFrame: if POLARS_VERSION_LT_138: # pragma: no cover return pl.DataFrame() return pl.DataFrame(height=self._num_rows_override) + # If the arrow table has empty names, from_arrow produces # column_$i. But here we know there is only one such column # (by construction) and it should have an empty name. diff --git a/python/cudf_polars/cudf_polars/dsl/utils/aggregations.py b/python/cudf_polars/cudf_polars/dsl/utils/aggregations.py index d2eadf41ec3d..0ba59e4828a1 100644 --- a/python/cudf_polars/cudf_polars/dsl/utils/aggregations.py +++ b/python/cudf_polars/cudf_polars/dsl/utils/aggregations.py @@ -247,8 +247,9 @@ def decompose_single_agg( ) else expr.Col(agg.dtype, name) ) - # Polars semantics for sum: + # Polars semantics for sum differ by context: # - GROUPBY: sum(all-null group) => 0; sum(empty group) => 0 (fill-null) + # - ROLLING: sum(all-null window) => null; sum(empty window) => 0 (fill only if empty) # # Must post-process because libcudf returns null for both empty and all-null windows/groups return [(named_expr, True)], expr.NamedExpr( diff --git a/python/cudf_polars/tests/expressions/test_agg.py b/python/cudf_polars/tests/expressions/test_agg.py index 7213ba6360c5..436c1c8b03f1 100644 --- a/python/cudf_polars/tests/expressions/test_agg.py +++ b/python/cudf_polars/tests/expressions/test_agg.py @@ -59,9 +59,7 @@ def xfail_if_sorted(is_sorted, request): # See https://github.com/rapidsai/cudf/pull/20791#issuecomment-3750528419 if is_sorted: request.applymarker( - pytest.mark.xfail( - reason="HintIR not supported", - ) + pytest.mark.xfail(reason="See https://github.com/pola-rs/polars/pull/24981") ) @@ -222,8 +220,12 @@ def test_decimal_std_var(engine: pl.GPUEngine, decimal_df): def test_invalid_agg(request): - if not POLARS_VERSION_LT_136: - request.applymarker(pytest.mark.xfail(reason="polars raises now")) + request.applymarker( + pytest.mark.xfail( + condition=not POLARS_VERSION_LT_136, + reason="polars raises now", + ) + ) df = pl.LazyFrame({"s": pl.Series(["a", "b", "c"], dtype=pl.String())}) q = df.select(pl.col("s").sum()) assert_ir_translation_raises(q, NotImplementedError) diff --git a/python/cudf_polars/tests/expressions/test_stringfunction.py b/python/cudf_polars/tests/expressions/test_stringfunction.py index a0df7917263e..f834ba62acb6 100644 --- a/python/cudf_polars/tests/expressions/test_stringfunction.py +++ b/python/cudf_polars/tests/expressions/test_stringfunction.py @@ -369,10 +369,7 @@ def test_replace_many_ascii_case(ldf): assert_ir_translation_raises(q, NotImplementedError) -@pytest.mark.skipif( - POLARS_VERSION_LT_136, - reason="leftmost parameter requires polars >= 1.36", -) +@pytest.mark.skipif(POLARS_VERSION_LT_136, reason="leftmost arg added in 1.36") def test_replace_many_leftmost(ldf): q = ldf.select(pl.col("a").str.replace_many(["a", "b"], "x", leftmost=True)) @@ -903,8 +900,7 @@ def test_concat_str_with_boolean(engine: pl.GPUEngine): @pytest.mark.skipif( - POLARS_VERSION_LT_138, - reason="str.split(literal=False) / SplitRegex requires polars >= 1.38", + POLARS_VERSION_LT_138, reason="Split with literal parameter added in 1.38" ) def test_split_regex_not_supported(): lf = pl.LazyFrame({"a": ["foo1bar", "baz456boo", "abc321"]}) diff --git a/python/cudf_polars/tests/test_dataframescan.py b/python/cudf_polars/tests/test_dataframescan.py index 2b6842a155bd..b85d37ce29a8 100644 --- a/python/cudf_polars/tests/test_dataframescan.py +++ b/python/cudf_polars/tests/test_dataframescan.py @@ -85,7 +85,8 @@ def test_dataframescan_with_decimals(engine: pl.GPUEngine): @pytest.mark.skipif( - POLARS_VERSION_LT_138, reason="pl.LazyFrame(height=...) requires polars >= 1.38" + POLARS_VERSION_LT_138, + reason="height parameter added in Polars 1.38", ) def test_dataframescan_zero_width_with_rows(engine: pl.GPUEngine, request): request.applymarker( From 0ef9dcbe4976c9583d21a689507515892ecb01d5 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Tue, 12 May 2026 00:53:52 +0000 Subject: [PATCH 18/44] One more reduce diff --- python/cudf_polars/tests/test_sink.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/python/cudf_polars/tests/test_sink.py b/python/cudf_polars/tests/test_sink.py index dea81d21fd38..d23559d21340 100644 --- a/python/cudf_polars/tests/test_sink.py +++ b/python/cudf_polars/tests/test_sink.py @@ -173,11 +173,12 @@ def test_sink_in_memory_executor(df, tmp_path, file_type): ) -@pytest.mark.skipif( - POLARS_VERSION_LT_138, reason="sink compression kwarg requires polars >= 1.38" -) @pytest.mark.parametrize("compression", ["gzip", "zstd"]) @pytest.mark.parametrize("file_type", ["csv", "ndjson"]) +@pytest.mark.skipif( + POLARS_VERSION_LT_138, + reason="compression parameter added in Polars 1.38", +) def test_sink_compression_raises(df, tmp_path, compression, file_type): path = tmp_path / f"out.{file_type}" assert_sink_ir_translation_raises( From d3801f76b1ee8e1712d595d3219e6cdec60c84f0 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Tue, 12 May 2026 23:47:16 +0000 Subject: [PATCH 19/44] More fixes to mostly xpassing rolling tests in 1.39 --- .../cudf_polars/dsl/expressions/rolling.py | 18 ++++---- .../tests/expressions/test_rolling.py | 45 ++++--------------- .../tests/test_window_functions.py | 2 +- 3 files changed, 19 insertions(+), 46 deletions(-) diff --git a/python/cudf_polars/cudf_polars/dsl/expressions/rolling.py b/python/cudf_polars/cudf_polars/dsl/expressions/rolling.py index 98532df2a878..c6e214ab7998 100644 --- a/python/cudf_polars/cudf_polars/dsl/expressions/rolling.py +++ b/python/cudf_polars/cudf_polars/dsl/expressions/rolling.py @@ -22,7 +22,7 @@ offsets_to_windows, range_window_bounds, ) -from cudf_polars.utils.versions import POLARS_VERSION_LT_136 +from cudf_polars.utils.versions import POLARS_VERSION_LT_136, POLARS_VERSION_LT_139 if TYPE_CHECKING: from collections.abc import Sequence @@ -79,14 +79,14 @@ def to_request( elif isinstance(value, expr.Agg): child = value.children[0] col = child.evaluate(df, context=ExecutionContext.ROLLING) - if POLARS_VERSION_LT_136 and value.name == "var": # pragma: no cover + if (POLARS_VERSION_LT_136 or not POLARS_VERSION_LT_139) and value.name == "var": # Polars variance produces null if nvalues <= ddof # libcudf produces NaN. However, we can get the polars # behaviour by setting the minimum window size to ddof + # 1. - # - # We still need this check, polars is not hitting it because - # of https://github.com/pola-rs/polars/pull/25117 + # In polars 1.36-1.38 this code path is not hit because + # rolling goes through the IR path (not RollingWindow). + # See https://github.com/pola-rs/polars/pull/25117 min_periods = value.options + 1 else: col = value.evaluate( @@ -146,12 +146,12 @@ def __init__( raise NotImplementedError( "Incorrect handling of empty groups for list collection" ) - if POLARS_VERSION_LT_136 and not plc.rolling.is_valid_rolling_aggregation( + if ( + POLARS_VERSION_LT_136 or not POLARS_VERSION_LT_139 + ) and not plc.rolling.is_valid_rolling_aggregation( agg.dtype.plc_type, agg.agg_request ): - raise NotImplementedError( - f"Unsupported rolling aggregation {agg}" - ) # pragma: no cover; polars may raise ahead of time + raise NotImplementedError(f"Unsupported rolling aggregation {agg}") def do_evaluate( # noqa: D102 self, df: DataFrame, *, context: ExecutionContext = ExecutionContext.FRAME diff --git a/python/cudf_polars/tests/expressions/test_rolling.py b/python/cudf_polars/tests/expressions/test_rolling.py index f0c1b3523cb8..a516d68d8b3a 100644 --- a/python/cudf_polars/tests/expressions/test_rolling.py +++ b/python/cudf_polars/tests/expressions/test_rolling.py @@ -24,7 +24,7 @@ # Support requires polars <1.36 (via Window+RollingGroupOptions) or >=1.39 (via Rolling). skip_rolling_expr_136_to_138 = pytest.mark.skipif( not POLARS_VERSION_LT_136 and POLARS_VERSION_LT_139, - reason="Rolling window expressions are not accessible in polars 1.36-1.38", + reason="Rolling window expressions are not accessible in polars 1.36-1.38, see https://github.com/pola-rs/polars/pull/25117", ) @@ -43,13 +43,7 @@ def df(): @skip_rolling_expr_136_to_138 @pytest.mark.parametrize("time_unit", ["ns", "us", "ms"]) -def test_rolling_datetime(engine: pl.GPUEngine, request, time_unit): - request.applymarker( - pytest.mark.xfail( - condition=not POLARS_VERSION_LT_136, - reason="Polars translates this to AExpr::Rolling which is NotImplemented", - ) - ) +def test_rolling_datetime(engine: pl.GPUEngine, time_unit): dates = [ "2020-01-01 13:45:48", "2020-01-01 16:42:13", @@ -73,11 +67,7 @@ def test_rolling_datetime(engine: pl.GPUEngine, request, time_unit): @skip_rolling_expr_136_to_138 -def test_rolling_date(engine: pl.GPUEngine, request): - if not POLARS_VERSION_LT_136: - request.applymarker( - pytest.mark.xfail(reason="See https://github.com/pola-rs/polars/pull/25117") - ) +def test_rolling_date(engine: pl.GPUEngine): dates = [ "2020-01-01", "2020-01-01", @@ -100,11 +90,7 @@ def test_rolling_date(engine: pl.GPUEngine, request): @skip_rolling_expr_136_to_138 @pytest.mark.parametrize("dtype", [pl.Int32, pl.UInt32, pl.Int64, pl.UInt64]) -def test_rolling_integral_orderby(engine: pl.GPUEngine, request, dtype): - if not POLARS_VERSION_LT_136: - request.applymarker( - pytest.mark.xfail(reason="See https://github.com/pola-rs/polars/pull/25117") - ) +def test_rolling_integral_orderby(engine: pl.GPUEngine, dtype): df = pl.LazyFrame( { "orderby": pl.Series([1, 4, 8, 10, 12, 13, 14, 22], dtype=dtype), @@ -139,11 +125,7 @@ def test_rolling_collect_list_raises(): @skip_rolling_expr_136_to_138 -def test_unsorted_raises(engine_raise_on_fail: pl.GPUEngine, request): - if not POLARS_VERSION_LT_136: - request.applymarker( - pytest.mark.xfail(reason="See https://github.com/pola-rs/polars/pull/25117") - ) +def test_unsorted_raises(engine_raise_on_fail: pl.GPUEngine): df = pl.LazyFrame({"orderby": [1, 2, 4, 2], "values": [1, 2, 3, 4]}) q = df.select(pl.col("values").sum().rolling("orderby", period="2i")) with pytest.raises(pl.exceptions.InvalidOperationError): @@ -156,11 +138,7 @@ def test_unsorted_raises(engine_raise_on_fail: pl.GPUEngine, request): @skip_rolling_expr_136_to_138 -def test_orderby_nulls_raises_computeerror(engine_raise_on_fail: pl.GPUEngine, request): - if not POLARS_VERSION_LT_136: - request.applymarker( - pytest.mark.xfail(reason="See https://github.com/pola-rs/polars/pull/25117") - ) +def test_orderby_nulls_raises_computeerror(engine_raise_on_fail: pl.GPUEngine): df = pl.LazyFrame({"orderby": [1, 2, 4, None], "values": [1, 2, 3, 4]}) q = df.select(pl.col("values").sum().rolling("orderby", period="2i")) with pytest.raises(pl.exceptions.InvalidOperationError): @@ -178,9 +156,8 @@ def test_invalid_duration_spec_raises_in_translation(): assert_ir_translation_raises(q, pl.exceptions.InvalidOperationError) -def test_rolling_inside_groupby_raises(request): - if not POLARS_VERSION_LT_136: - request.applymarker(pytest.mark.xfail(reason="not supported")) +@pytest.mark.xfail(condition=not POLARS_VERSION_LT_136, reason="not supported") +def test_rolling_inside_groupby_raises(): df = pl.LazyFrame( {"keys": [1, 1, 1, 2], "orderby": [1, 2, 4, 2], "values": [1, 2, 3, 4]} ) @@ -193,11 +170,7 @@ def test_rolling_inside_groupby_raises(request): @skip_rolling_expr_136_to_138 -def test_rolling_sum_all_null_window_returns_null(engine: pl.GPUEngine, request): - if not POLARS_VERSION_LT_136: - request.applymarker( - pytest.mark.xfail(reason="See https://github.com/pola-rs/polars/pull/25117") - ) +def test_rolling_sum_all_null_window_returns_null(engine: pl.GPUEngine): df = pl.LazyFrame( { "orderby": [1, 2, 3, 4, 5, 6], diff --git a/python/cudf_polars/tests/test_window_functions.py b/python/cudf_polars/tests/test_window_functions.py index 00cf21a6c75f..743aa3152e60 100644 --- a/python/cudf_polars/tests/test_window_functions.py +++ b/python/cudf_polars/tests/test_window_functions.py @@ -147,7 +147,7 @@ def test_rolling( def test_rolling_unsupported(df: pl.LazyFrame, unsupported_agg_expr): """Test rolling window functions over time series.""" window_expr = unsupported_agg_expr.rolling(period="2d", index_column="date") - result_name = f"{agg_expr!s}_rolling" + result_name = f"{unsupported_agg_expr!s}_rolling" window_expr = window_expr.alias(result_name) query = df.with_columns(window_expr) From 9c4762f4a240626694d21a9f6fa061fbb00172a8 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Wed, 13 May 2026 19:21:27 +0000 Subject: [PATCH 20/44] Pass engine to test_rolling_agg_first --- python/cudf_polars/tests/expressions/test_rolling.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/cudf_polars/tests/expressions/test_rolling.py b/python/cudf_polars/tests/expressions/test_rolling.py index a516d68d8b3a..89fb23e3da0d 100644 --- a/python/cudf_polars/tests/expressions/test_rolling.py +++ b/python/cudf_polars/tests/expressions/test_rolling.py @@ -105,10 +105,10 @@ def test_rolling_integral_orderby(engine: pl.GPUEngine, dtype): @skip_rolling_expr_136_to_138 -def test_rolling_agg_first(): +def test_rolling_agg_first(engine: pl.GPUEngine): df = pl.LazyFrame({"a": [1, 2, 3], "b": [1, 2, 3]}) q = df.with_columns(pl.col("a").sum().rolling("b", period="2i")) - assert_gpu_result_equal(q) + assert_gpu_result_equal(q, engine=engine) def test_rolling_collect_list_raises(): From 9ede121db15ba6da4c333dd1606b3d4f4a9d99db Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Wed, 13 May 2026 21:09:58 +0000 Subject: [PATCH 21/44] Add decimal to float casting for is_bewteen (pass test_decimal_dynamic_float_st) --- .../cudf_polars/cudf_polars/dsl/translate.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/python/cudf_polars/cudf_polars/dsl/translate.py b/python/cudf_polars/cudf_polars/dsl/translate.py index 1963d10a0ec3..d60ffce149d8 100644 --- a/python/cudf_polars/cudf_polars/dsl/translate.py +++ b/python/cudf_polars/cudf_polars/dsl/translate.py @@ -35,6 +35,7 @@ from cudf_polars.utils.versions import ( POLARS_VERSION_LT_136, POLARS_VERSION_LT_138, + POLARS_VERSION_LT_139, ) if TYPE_CHECKING: @@ -45,6 +46,29 @@ __all__ = ["Translator", "translate_named_expr"] +def _align_decimal_float_for_comparison( + *operands: expr.Expr, +) -> tuple[expr.Expr, ...]: + """ + Cast Decimal operands to Float64 when mixed with Float operands. + + libcudf does not produce correct results for Decimal vs Float + comparisons. This matches Polars' supertype resolution where + Decimal + Float yields Float64. + """ + has_decimal = any(plc.traits.is_fixed_point(op.dtype.plc_type) for op in operands) + has_float = any(plc.traits.is_floating_point(op.dtype.plc_type) for op in operands) + if has_decimal and has_float: + f64 = DataType(pl.Float64()) + return tuple( + expr.Cast(f64, False, op) # noqa: FBT003 + if plc.traits.is_fixed_point(op.dtype.plc_type) + else op + for op in operands + ) + return operands + + def _check_compression(data: bytes) -> str | None: # Vendored from Polars' SupportedCompression::check in # polars-io/src/utils/compression.rs @@ -764,6 +788,10 @@ def _( ) (closed,) = options lop, rop = expr.BooleanFunction._BETWEEN_OPS[closed] + if not POLARS_VERSION_LT_139: + # IsBetween type coercion does not + # insert casts for Decimal/Float comparisons. + column, lo, hi = _align_decimal_float_for_comparison(column, lo, hi) return expr.BinOp( dtype, plc.binaryop.BinaryOperator.LOGICAL_AND, From 5b10326fa11fbf5c26b02d7783c636e3b84d0409 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Wed, 13 May 2026 22:07:37 +0000 Subject: [PATCH 22/44] Fix py-polars/tests/unit/sql/test_window_functions.py failures by carving out code path for filling nulls for cummulative aggregations --- .../cudf_polars/dsl/expressions/rolling.py | 90 +++++++++++++++---- 1 file changed, 71 insertions(+), 19 deletions(-) diff --git a/python/cudf_polars/cudf_polars/dsl/expressions/rolling.py b/python/cudf_polars/cudf_polars/dsl/expressions/rolling.py index c6e214ab7998..b19bcc9b9ed8 100644 --- a/python/cudf_polars/cudf_polars/dsl/expressions/rolling.py +++ b/python/cudf_polars/cudf_polars/dsl/expressions/rolling.py @@ -9,7 +9,7 @@ from collections import defaultdict from dataclasses import dataclass from functools import singledispatchmethod -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, ClassVar import pylibcudf as plc @@ -25,7 +25,7 @@ from cudf_polars.utils.versions import POLARS_VERSION_LT_136, POLARS_VERSION_LT_139 if TYPE_CHECKING: - from collections.abc import Sequence + from collections.abc import Callable, Sequence from rmm.pylibrmm.stream import Stream @@ -372,6 +372,14 @@ def _( _, rank_tables = grouper.scan(rank_requests) return rank_out_names, rank_out_dtypes, rank_tables + _CUM_AGG_FUNCTIONS: ClassVar[ + dict[str, Callable[[], plc.aggregation.Aggregation]] + ] = { + "cum_sum": plc.aggregation.sum, + "cum_min": plc.aggregation.min, + "cum_max": plc.aggregation.max, + } + @_apply_unary_op.register def _( # type: ignore[no-untyped-def] self, @@ -380,28 +388,72 @@ def _( # type: ignore[no-untyped-def] _, ) -> tuple[list[str], list[DataType], list[plc.Table]]: named_exprs = op.named_exprs + local_grouper = op.local_grouper + assert isinstance(local_grouper, plc.groupby.GroupBy) + + plc_cols: list[plc.Column] = [] + for ne in named_exprs: + child = ne.value.children[0] + # When a cumulative agg (cum_sum/cum_min/cum_max) is wrapped + # in fill_null_with_strategy (Polars SQL OVER ORDER BY), + # the scan must run in ORDER BY order, not original row order. + if ( + isinstance(child, expr.UnaryFunction) + and (cum_agg_factory := self._CUM_AGG_FUNCTIONS.get(child.name)) + is not None + ): + inner_col = ( + child.children[0].evaluate(df, context=ExecutionContext.FRAME).obj + ) + if op.order_index is not None: + (inner_col,) = plc.copying.gather( + plc.Table([inner_col]), + op.order_index, + plc.copying.OutOfBoundsPolicy.NULLIFY, + stream=df.stream, + ).columns() + _, [scan_tbl] = local_grouper.scan( + [plc.groupby.GroupByRequest(inner_col, [cum_agg_factory()])], + stream=df.stream, + ) + (scan_col,) = scan_tbl.columns() + _, filled_tbl = local_grouper.replace_nulls( + plc.Table([scan_col]), [op.policy], stream=df.stream + ) + (filled_col,) = filled_tbl.columns() + plc_cols.append(filled_col) + else: + plc_cols.append(child.evaluate(df, context=ExecutionContext.FRAME).obj) - plc_cols = [ - ne.value.children[0].evaluate(df, context=ExecutionContext.FRAME).obj - for ne in named_exprs + # Fill nulls for non-cumulative children (already appended raw) + non_cum_indices = [ + i + for i, ne in enumerate(named_exprs) + if not ( + isinstance(ne.value.children[0], expr.UnaryFunction) + and ne.value.children[0].name in self._CUM_AGG_FUNCTIONS + ) ] - if op.order_index is not None: - vals_tbl = plc.copying.gather( - plc.Table(plc_cols), - op.order_index, - plc.copying.OutOfBoundsPolicy.NULLIFY, + if non_cum_indices: + raw_cols = [plc_cols[i] for i in non_cum_indices] + if op.order_index is not None: + raw_tbl = plc.copying.gather( + plc.Table(raw_cols), + op.order_index, + plc.copying.OutOfBoundsPolicy.NULLIFY, + stream=df.stream, + ) + else: + raw_tbl = plc.Table(raw_cols) + _, filled_tbl = local_grouper.replace_nulls( + raw_tbl, + [op.policy] * len(raw_cols), stream=df.stream, ) - else: - vals_tbl = plc.Table(plc_cols) - local_grouper = op.local_grouper - assert isinstance(local_grouper, plc.groupby.GroupBy) - _, filled_tbl = local_grouper.replace_nulls( - vals_tbl, - [op.policy] * len(plc_cols), - ) + for j, col in zip(non_cum_indices, filled_tbl.columns(), strict=True): + plc_cols[j] = col - tables = [plc.Table([column]) for column in filled_tbl.columns()] + tables = [plc.Table([column]) for column in plc_cols] names = [ne.name for ne in named_exprs] dtypes = [ne.value.dtype for ne in named_exprs] return names, dtypes, tables From d54780f29db59cab7f441922a384eb5e680b0089 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Wed, 13 May 2026 22:32:24 +0000 Subject: [PATCH 23/44] xfail, but prepare for HConcat passing strict to validate num_rows for test_concat_horizontal_zero_width_height_mismatch_26876 --- python/cudf_polars/cudf_polars/dsl/ir.py | 18 ++++++++++++++---- .../cudf_polars/cudf_polars/dsl/translate.py | 5 +++++ .../cudf_polars/experimental/expressions.py | 1 + .../cudf_polars/experimental/select.py | 2 ++ .../cudf_polars/testing/inject_gpu_engine.py | 3 ++- python/cudf_polars/tests/test_hconcat.py | 2 +- 6 files changed, 25 insertions(+), 6 deletions(-) diff --git a/python/cudf_polars/cudf_polars/dsl/ir.py b/python/cudf_polars/cudf_polars/dsl/ir.py index f056d1ab6109..43c74b1fb5c2 100644 --- a/python/cudf_polars/cudf_polars/dsl/ir.py +++ b/python/cudf_polars/cudf_polars/dsl/ir.py @@ -3187,19 +3187,21 @@ def do_evaluate( class HConcat(IR): """Concatenate dataframes horizontally.""" - __slots__ = ("should_broadcast",) - _non_child = ("schema", "should_broadcast") - _n_non_child_args = 2 + __slots__ = ("should_broadcast", "strict") + _non_child = ("schema", "should_broadcast", "strict") + _n_non_child_args = 3 def __init__( self, schema: Schema, should_broadcast: bool, # noqa: FBT001 + strict: bool, # noqa: FBT001 *children: IR, ): self.schema = schema self.should_broadcast = should_broadcast - self._non_child_args = (schema, should_broadcast) + self.strict = strict + self._non_child_args = (schema, should_broadcast, strict) self.children = children @staticmethod @@ -3242,6 +3244,7 @@ def do_evaluate( cls, schema: Schema, should_broadcast: bool, # noqa: FBT001 + strict: bool, # noqa: FBT001 *dfs: DataFrame, context: IRExecutionContext, ) -> DataFrame: @@ -3259,6 +3262,13 @@ def do_evaluate( result = DataFrame(ordered, stream=stream) else: max_rows = max(df.num_rows for df in dfs) + if strict and any(df.num_rows != max_rows for df in dfs): + heights = [df.num_rows for df in dfs] + msg = ( + f"cannot concat DataFrames horizontally" + f" with strict=True: height mismatch {heights}" + ) + raise pl.exceptions.ShapeError(msg) # Horizontal concatenation extends shorter tables with nulls result = DataFrame( itertools.chain.from_iterable( diff --git a/python/cudf_polars/cudf_polars/dsl/translate.py b/python/cudf_polars/cudf_polars/dsl/translate.py index d60ffce149d8..c34e97190cd0 100644 --- a/python/cudf_polars/cudf_polars/dsl/translate.py +++ b/python/cudf_polars/cudf_polars/dsl/translate.py @@ -625,9 +625,14 @@ def _(node: plrs._ir_nodes.Union, translator: Translator, schema: Schema) -> ir. @_translate_ir.register def _(node: plrs._ir_nodes.HConcat, translator: Translator, schema: Schema) -> ir.IR: + if POLARS_VERSION_LT_139: + strict = False + else: + _, strict, _ = node.options return ir.HConcat( schema, False, # noqa: FBT003 + strict, *(translator.translate_ir(n=n) for n in node.inputs), ) diff --git a/python/cudf_polars/cudf_polars/experimental/expressions.py b/python/cudf_polars/cudf_polars/experimental/expressions.py index f578e3b0e071..df87cbe6db21 100644 --- a/python/cudf_polars/cudf_polars/experimental/expressions.py +++ b/python/cudf_polars/cudf_polars/experimental/expressions.py @@ -506,6 +506,7 @@ def _decompose( input_ir = HConcat( schema, True, # noqa: FBT003 + False, # noqa: FBT003 *unique_input_irs, ) partition_info[input_ir] = PartitionInfo(count=partition_count) diff --git a/python/cudf_polars/cudf_polars/experimental/select.py b/python/cudf_polars/cudf_polars/experimental/select.py index 9ab30f9be136..7385f9db162e 100644 --- a/python/cudf_polars/cudf_polars/experimental/select.py +++ b/python/cudf_polars/cudf_polars/experimental/select.py @@ -174,6 +174,7 @@ def decompose_select( new_ir = HConcat( select_ir.schema, True, # noqa: FBT003 + False, # noqa: FBT003 *selections, ) partition_info[new_ir] = PartitionInfo( @@ -348,6 +349,7 @@ def _fuse_simple_reductions( new_hconcat = HConcat( hconcat_schema, True, # noqa: FBT003 + False, # noqa: FBT003 *new_decomposed_select_irs, ) count = max(pi[c].count for c in new_decomposed_select_irs) diff --git a/python/cudf_polars/cudf_polars/testing/inject_gpu_engine.py b/python/cudf_polars/cudf_polars/testing/inject_gpu_engine.py index 7cfb62c414ee..e27455523ae8 100644 --- a/python/cudf_polars/cudf_polars/testing/inject_gpu_engine.py +++ b/python/cudf_polars/cudf_polars/testing/inject_gpu_engine.py @@ -370,12 +370,13 @@ def pytest_report_header(config: pytest.Config) -> str: # xfail for tests that produce different results than CPU Polars STREAMING_ENGINE_EXPECTED_FAILURES: Mapping[str, str] = { "tests/unit/functions/range/test_linear_space.py::test_linear_space_num_samples_expr": "https://github.com/rapidsai/cudf/issues/22072", + "tests/unit/functions/test_concat.py::test_concat_horizontal_zero_width_height_mismatch_26876": "https://github.com/rapidsai/cudf/issues/21644", + "tests/unit/interop/test_interop.py::test_0_width_df_roundtrip": "https://github.com/rapidsai/cudf/issues/21644", "tests/unit/lazyframe/test_projections.py::test_join_projection_pushdown_struct_field_as_key_24446": "https://github.com/rapidsai/cudf/issues/22105", "tests/unit/operations/test_slice.py::test_slice_pushdown_literal_projection_14349": "https://github.com/rapidsai/cudf/issues/22072", "tests/unit/operations/test_group_by.py::test_group_by_lit_series": "Incorrect broadcasting of literals in groupby-agg", "tests/unit/operations/test_group_by.py::test_group_by_series_partitioned": "https://github.com/rapidsai/cudf/issues/22072", "tests/unit/operations/test_group_by.py::test_partitioned_group_by_chunked": "https://github.com/rapidsai/cudf/issues/22072", - "tests/unit/interop/test_interop.py::test_0_width_df_roundtrip": "https://github.com/rapidsai/cudf/issues/21644", "tests/unit/operations/test_group_by.py::test_group_by_unique_parametric[n_unique-True-True]": "https://github.com/rapidsai/cudf/issues/21641", "tests/unit/operations/test_group_by.py::test_unique_head_tail_26429[1]": "https://github.com/rapidsai/cudf/issues/22075", "tests/unit/operations/test_group_by.py::test_unique_head_tail_26429[4]": "https://github.com/rapidsai/cudf/issues/22075", diff --git a/python/cudf_polars/tests/test_hconcat.py b/python/cudf_polars/tests/test_hconcat.py index 761b673a682a..04c80d99b4c2 100644 --- a/python/cudf_polars/tests/test_hconcat.py +++ b/python/cudf_polars/tests/test_hconcat.py @@ -42,7 +42,7 @@ def test_hconcat_should_broadcast(): child2 = DataFrameScan({"b": DataType(pl.Float64())}, df2._df, None) schema = {"a": DataType(pl.Int64()), "b": DataType(pl.Float64())} - node = HConcat(schema, True, child1, child2) # noqa: FBT003 + node = HConcat(schema, True, False, child1, child2) # noqa: FBT003 result = node.evaluate(cache={}, timer=None, context=context) polars_result = result.to_polars() From 4011aa46e82e2a1dd63c8435cdc3c8998db5551d Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Wed, 13 May 2026 22:54:52 +0000 Subject: [PATCH 24/44] Add fix for Polars test test_over_literal_cum_sum_26800 --- .../cudf_polars/dsl/expressions/rolling.py | 51 +++++++++++-------- 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/python/cudf_polars/cudf_polars/dsl/expressions/rolling.py b/python/cudf_polars/cudf_polars/dsl/expressions/rolling.py index b19bcc9b9ed8..629c5c637dce 100644 --- a/python/cudf_polars/cudf_polars/dsl/expressions/rolling.py +++ b/python/cudf_polars/cudf_polars/dsl/expressions/rolling.py @@ -471,37 +471,48 @@ def _( # type: ignore[no-untyped-def] requests: list[plc.groupby.GroupByRequest] = [] out_names: list[str] = [] out_dtypes: list[DataType] = [] - - # Instead of calling self._gather_columns, let's call plc.copying.gather directly - # since we need plc.Column objects, not cudf_polars Column objects + evaluated = [ + ne.value.children[0].evaluate(df, context=ExecutionContext.FRAME) + for ne in cum_named + ] + is_scalar = [c.obj.size() <= 1 for c in evaluated] + evaluated = broadcast(*evaluated, target_length=df.num_rows, stream=df.stream) if order_index is not None: - plc_cols = [ - ne.value.children[0].evaluate(df, context=ExecutionContext.FRAME).obj - for ne in cum_named - ] val_cols = plc.copying.gather( - plc.Table(plc_cols), + plc.Table([c.obj for c in evaluated]), order_index, plc.copying.OutOfBoundsPolicy.NULLIFY, stream=df.stream, ).columns() else: - val_cols = [ - ne.value.children[0].evaluate(df, context=ExecutionContext.FRAME).obj - for ne in cum_named - ] - agg = plc.aggregation.sum() - - for ne, val_col in zip(cum_named, val_cols, strict=True): - requests.append(plc.groupby.GroupByRequest(val_col, [agg])) + val_cols = [c.obj for c in evaluated] + + scan_requests: list[tuple[int, plc.Column]] = [] + for i, (is_slr, ne, val_col) in enumerate( + zip(is_scalar, cum_named, val_cols, strict=True) + ): out_names.append(ne.name) out_dtypes.append(ne.value.dtype) + if not is_slr: + scan_requests.append((i, val_col)) - local_grouper = op.local_grouper - assert isinstance(local_grouper, plc.groupby.GroupBy) - _, tables = local_grouper.scan(requests) + result_tables: list[plc.Table] = [plc.Table([c]) for c in val_cols] + + if scan_requests: + agg = plc.aggregation.sum() + requests.extend( + plc.groupby.GroupByRequest(val_col, [agg]) + for _, val_col in scan_requests + ) + + local_grouper = op.local_grouper + assert isinstance(local_grouper, plc.groupby.GroupBy) + _, scan_tables = local_grouper.scan(requests, stream=df.stream) + + for (idx, _), tbl in zip(scan_requests, scan_tables, strict=True): + result_tables[idx] = tbl - return out_names, out_dtypes, tables + return out_names, out_dtypes, result_tables def _reorder_to_input( self, From 3615f52f8157735965188a233d9122e8e78a4917 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Wed, 13 May 2026 23:12:14 +0000 Subject: [PATCH 25/44] Skip test_scan_metrics --- python/cudf_polars/cudf_polars/testing/inject_gpu_engine.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/python/cudf_polars/cudf_polars/testing/inject_gpu_engine.py b/python/cudf_polars/cudf_polars/testing/inject_gpu_engine.py index e27455523ae8..9937f2bdccff 100644 --- a/python/cudf_polars/cudf_polars/testing/inject_gpu_engine.py +++ b/python/cudf_polars/cudf_polars/testing/inject_gpu_engine.py @@ -302,6 +302,12 @@ def pytest_report_header(config: pytest.Config) -> str: # Short term adds in the aftermath of the rapidsmpf switch to get CI passing "tests/unit/io/test_lazy_parquet.py::test_scan_parquet_local_with_async": "Flaky, otherwise TBD", "tests/unit/operations/test_join.py::test_join_where_nested_expr_21066": "Flaky, otherwise TBD", + "tests/unit/io/test_scan.py::test_scan_metrics[True-parquet]": "Specific to IO metrics on Polars CPU", + "tests/unit/io/test_scan.py::test_scan_metrics[True-csv]": "Specific to IO metrics on Polars CPU", + "tests/unit/io/test_scan.py::test_scan_metrics[True-ndjson]": "Specific to IO metrics on Polars CPU", + "tests/unit/io/test_scan.py::test_scan_metrics[False-parquet]": "Specific to IO metrics on Polars CPU", + "tests/unit/io/test_scan.py::test_scan_metrics[False-csv]": "Specific to IO metrics on Polars CPU", + "tests/unit/io/test_scan.py::test_scan_metrics[False-ndjson]": "Specific to IO metrics on Polars CPU", } From a9161c5585f462a6c364f80d65fbee3f5584ee31 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Wed, 13 May 2026 23:13:48 +0000 Subject: [PATCH 26/44] Simplify EXPECTED_FAILURES --- .../cudf_polars/testing/inject_gpu_engine.py | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/python/cudf_polars/cudf_polars/testing/inject_gpu_engine.py b/python/cudf_polars/cudf_polars/testing/inject_gpu_engine.py index 9937f2bdccff..697c7bac27b7 100644 --- a/python/cudf_polars/cudf_polars/testing/inject_gpu_engine.py +++ b/python/cudf_polars/cudf_polars/testing/inject_gpu_engine.py @@ -135,8 +135,7 @@ def pytest_report_header(config: pytest.Config) -> str: return f"injected GPU engine: {cls.__module__}.{cls.__name__}" -# TODO: This is just Mapping[str, str]? -EXPECTED_FAILURES: Mapping[str, str | tuple[str, bool]] = { +EXPECTED_FAILURES: Mapping[str, str] = { "tests/unit/io/test_csv.py::test_read_csv_only_loads_selected_columns": "Memory usage won't be correct due to GPU", "tests/unit/io/test_delta.py::test_scan_delta_version": "Need to expose hive partitioning", "tests/unit/io/test_delta.py::test_scan_delta_relative": "Need to expose hive partitioning", @@ -475,15 +474,5 @@ def pytest_collection_modifyitems( is not None ): item.add_marker(pytest.mark.xfail(reason=s_reason)) - elif (entry := EXPECTED_FAILURES.get(item.nodeid, None)) is not None: - if isinstance(entry, tuple): - # the second entry in the tuple is the condition to xfail on - reason, condition = entry - item.add_marker( - pytest.mark.xfail( - condition=condition, - reason=reason, - ), - ) - else: - item.add_marker(pytest.mark.xfail(reason=entry)) + elif (reason := EXPECTED_FAILURES.get(item.nodeid, None)) is not None: + item.add_marker(pytest.mark.xfail(reason=reason)) From 948b3a8fd951b53bac529becb5fdf4ff0dd1cfa1 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Thu, 14 May 2026 22:56:14 +0000 Subject: [PATCH 27/44] Revert "Add fix for Polars test test_over_literal_cum_sum_26800" This reverts commit 4011aa46e82e2a1dd63c8435cdc3c8998db5551d. --- .../cudf_polars/dsl/expressions/rolling.py | 51 ++++++++----------- 1 file changed, 20 insertions(+), 31 deletions(-) diff --git a/python/cudf_polars/cudf_polars/dsl/expressions/rolling.py b/python/cudf_polars/cudf_polars/dsl/expressions/rolling.py index 629c5c637dce..b19bcc9b9ed8 100644 --- a/python/cudf_polars/cudf_polars/dsl/expressions/rolling.py +++ b/python/cudf_polars/cudf_polars/dsl/expressions/rolling.py @@ -471,48 +471,37 @@ def _( # type: ignore[no-untyped-def] requests: list[plc.groupby.GroupByRequest] = [] out_names: list[str] = [] out_dtypes: list[DataType] = [] - evaluated = [ - ne.value.children[0].evaluate(df, context=ExecutionContext.FRAME) - for ne in cum_named - ] - is_scalar = [c.obj.size() <= 1 for c in evaluated] - evaluated = broadcast(*evaluated, target_length=df.num_rows, stream=df.stream) + + # Instead of calling self._gather_columns, let's call plc.copying.gather directly + # since we need plc.Column objects, not cudf_polars Column objects if order_index is not None: + plc_cols = [ + ne.value.children[0].evaluate(df, context=ExecutionContext.FRAME).obj + for ne in cum_named + ] val_cols = plc.copying.gather( - plc.Table([c.obj for c in evaluated]), + plc.Table(plc_cols), order_index, plc.copying.OutOfBoundsPolicy.NULLIFY, stream=df.stream, ).columns() else: - val_cols = [c.obj for c in evaluated] - - scan_requests: list[tuple[int, plc.Column]] = [] - for i, (is_slr, ne, val_col) in enumerate( - zip(is_scalar, cum_named, val_cols, strict=True) - ): + val_cols = [ + ne.value.children[0].evaluate(df, context=ExecutionContext.FRAME).obj + for ne in cum_named + ] + agg = plc.aggregation.sum() + + for ne, val_col in zip(cum_named, val_cols, strict=True): + requests.append(plc.groupby.GroupByRequest(val_col, [agg])) out_names.append(ne.name) out_dtypes.append(ne.value.dtype) - if not is_slr: - scan_requests.append((i, val_col)) - - result_tables: list[plc.Table] = [plc.Table([c]) for c in val_cols] - if scan_requests: - agg = plc.aggregation.sum() - requests.extend( - plc.groupby.GroupByRequest(val_col, [agg]) - for _, val_col in scan_requests - ) - - local_grouper = op.local_grouper - assert isinstance(local_grouper, plc.groupby.GroupBy) - _, scan_tables = local_grouper.scan(requests, stream=df.stream) - - for (idx, _), tbl in zip(scan_requests, scan_tables, strict=True): - result_tables[idx] = tbl + local_grouper = op.local_grouper + assert isinstance(local_grouper, plc.groupby.GroupBy) + _, tables = local_grouper.scan(requests) - return out_names, out_dtypes, result_tables + return out_names, out_dtypes, tables def _reorder_to_input( self, From 387dd841b275ae8bae05be16ae775ae623fdd6be Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Thu, 14 May 2026 22:56:30 +0000 Subject: [PATCH 28/44] Revert "Fix py-polars/tests/unit/sql/test_window_functions.py failures by carving out code path for filling nulls for cummulative aggregations" This reverts commit 5b10326fa11fbf5c26b02d7783c636e3b84d0409. --- .../cudf_polars/dsl/expressions/rolling.py | 90 ++++--------------- 1 file changed, 19 insertions(+), 71 deletions(-) diff --git a/python/cudf_polars/cudf_polars/dsl/expressions/rolling.py b/python/cudf_polars/cudf_polars/dsl/expressions/rolling.py index b19bcc9b9ed8..c6e214ab7998 100644 --- a/python/cudf_polars/cudf_polars/dsl/expressions/rolling.py +++ b/python/cudf_polars/cudf_polars/dsl/expressions/rolling.py @@ -9,7 +9,7 @@ from collections import defaultdict from dataclasses import dataclass from functools import singledispatchmethod -from typing import TYPE_CHECKING, Any, ClassVar +from typing import TYPE_CHECKING, Any import pylibcudf as plc @@ -25,7 +25,7 @@ from cudf_polars.utils.versions import POLARS_VERSION_LT_136, POLARS_VERSION_LT_139 if TYPE_CHECKING: - from collections.abc import Callable, Sequence + from collections.abc import Sequence from rmm.pylibrmm.stream import Stream @@ -372,14 +372,6 @@ def _( _, rank_tables = grouper.scan(rank_requests) return rank_out_names, rank_out_dtypes, rank_tables - _CUM_AGG_FUNCTIONS: ClassVar[ - dict[str, Callable[[], plc.aggregation.Aggregation]] - ] = { - "cum_sum": plc.aggregation.sum, - "cum_min": plc.aggregation.min, - "cum_max": plc.aggregation.max, - } - @_apply_unary_op.register def _( # type: ignore[no-untyped-def] self, @@ -388,72 +380,28 @@ def _( # type: ignore[no-untyped-def] _, ) -> tuple[list[str], list[DataType], list[plc.Table]]: named_exprs = op.named_exprs - local_grouper = op.local_grouper - assert isinstance(local_grouper, plc.groupby.GroupBy) - - plc_cols: list[plc.Column] = [] - for ne in named_exprs: - child = ne.value.children[0] - # When a cumulative agg (cum_sum/cum_min/cum_max) is wrapped - # in fill_null_with_strategy (Polars SQL OVER ORDER BY), - # the scan must run in ORDER BY order, not original row order. - if ( - isinstance(child, expr.UnaryFunction) - and (cum_agg_factory := self._CUM_AGG_FUNCTIONS.get(child.name)) - is not None - ): - inner_col = ( - child.children[0].evaluate(df, context=ExecutionContext.FRAME).obj - ) - if op.order_index is not None: - (inner_col,) = plc.copying.gather( - plc.Table([inner_col]), - op.order_index, - plc.copying.OutOfBoundsPolicy.NULLIFY, - stream=df.stream, - ).columns() - _, [scan_tbl] = local_grouper.scan( - [plc.groupby.GroupByRequest(inner_col, [cum_agg_factory()])], - stream=df.stream, - ) - (scan_col,) = scan_tbl.columns() - _, filled_tbl = local_grouper.replace_nulls( - plc.Table([scan_col]), [op.policy], stream=df.stream - ) - (filled_col,) = filled_tbl.columns() - plc_cols.append(filled_col) - else: - plc_cols.append(child.evaluate(df, context=ExecutionContext.FRAME).obj) - # Fill nulls for non-cumulative children (already appended raw) - non_cum_indices = [ - i - for i, ne in enumerate(named_exprs) - if not ( - isinstance(ne.value.children[0], expr.UnaryFunction) - and ne.value.children[0].name in self._CUM_AGG_FUNCTIONS - ) + plc_cols = [ + ne.value.children[0].evaluate(df, context=ExecutionContext.FRAME).obj + for ne in named_exprs ] - if non_cum_indices: - raw_cols = [plc_cols[i] for i in non_cum_indices] - if op.order_index is not None: - raw_tbl = plc.copying.gather( - plc.Table(raw_cols), - op.order_index, - plc.copying.OutOfBoundsPolicy.NULLIFY, - stream=df.stream, - ) - else: - raw_tbl = plc.Table(raw_cols) - _, filled_tbl = local_grouper.replace_nulls( - raw_tbl, - [op.policy] * len(raw_cols), + if op.order_index is not None: + vals_tbl = plc.copying.gather( + plc.Table(plc_cols), + op.order_index, + plc.copying.OutOfBoundsPolicy.NULLIFY, stream=df.stream, ) - for j, col in zip(non_cum_indices, filled_tbl.columns(), strict=True): - plc_cols[j] = col + else: + vals_tbl = plc.Table(plc_cols) + local_grouper = op.local_grouper + assert isinstance(local_grouper, plc.groupby.GroupBy) + _, filled_tbl = local_grouper.replace_nulls( + vals_tbl, + [op.policy] * len(plc_cols), + ) - tables = [plc.Table([column]) for column in plc_cols] + tables = [plc.Table([column]) for column in filled_tbl.columns()] names = [ne.name for ne in named_exprs] dtypes = [ne.value.dtype for ne in named_exprs] return names, dtypes, tables From 5b6471b77654f4e4b9b3a4da4bf0361298b3ad47 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Thu, 14 May 2026 23:02:56 +0000 Subject: [PATCH 29/44] Skip Polars over test failures in 1.39 --- .../cudf_polars/cudf_polars/testing/inject_gpu_engine.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/python/cudf_polars/cudf_polars/testing/inject_gpu_engine.py b/python/cudf_polars/cudf_polars/testing/inject_gpu_engine.py index 697c7bac27b7..e86d236765dd 100644 --- a/python/cudf_polars/cudf_polars/testing/inject_gpu_engine.py +++ b/python/cudf_polars/cudf_polars/testing/inject_gpu_engine.py @@ -434,6 +434,7 @@ def pytest_report_header(config: pytest.Config) -> str: "tests/unit/operations/test_join.py::test_join_panic_on_binary_expr_5915": "https://github.com/rapidsai/cudf/issues/22105", "tests/unit/operations/test_join.py::test_join_rewrite_panic_23307": "https://github.com/rapidsai/cudf/issues/22085", "tests/unit/operations/test_join.py::test_semi_anti_join": "https://github.com/rapidsai/cudf/issues/22049", + "tests/unit/operations/test_window.py::test_over_literal_cum_sum_26800": "TODO: https://github.com/rapidsai/cudf/pull/22048#discussion_r3238041970", "tests/unit/sql/test_joins.py::test_cross_join_unnest_from_cte": "https://github.com/rapidsai/cudf/issues/22073", "tests/unit/sql/test_joins.py::test_join_anti_semi[SELECT * FROM tbl_a LEFT SEMI JOIN tbl_b USING (a)-expected2]": "https://github.com/rapidsai/cudf/issues/22105", "tests/unit/sql/test_joins.py::test_join_on_mixed_expression_conditions[df10-df20-df1.category = df2.category AND (df1.code * 2) = df2.code_doubled-df1.name, df1.code, df2.type-expected0-schema0]": "https://github.com/rapidsai/cudf/issues/22085 (or similar)", @@ -450,6 +451,13 @@ def pytest_report_header(config: pytest.Config) -> str: "tests/unit/sql/test_joins.py::test_join_on_reversed_constraint_order[df12-df22-(df1.a + df1.a) = df2.b-df2.b = (df1.a + df1.a)-expected2-schema2]": "https://github.com/rapidsai/cudf/issues/22105", "tests/unit/sql/test_joins.py::test_join_on_unqualified_expressions[df11-df21-x + y = sum-expected1-schema1]": "https://github.com/rapidsai/cudf/issues/22105", "tests/unit/sql/test_joins.py::test_join_on_unqualified_expressions[df12-df22-LENGTH(name) = len-expected2-schema2]": "https://github.com/rapidsai/cudf/issues/22105", + "tests/unit/sql/test_window_functions.py::test_over_with_cumulative_window_funcs": "TODO: https://github.com/rapidsai/cudf/pull/22048#discussion_r3238041970", + "tests/unit/sql/test_window_functions.py::test_over_with_order_by": "TODO: https://github.com/rapidsai/cudf/pull/22048#discussion_r3238041970", + "tests/unit/sql/test_window_functions.py::test_window_cumulative_agg_with_nulls": "TODO: https://github.com/rapidsai/cudf/pull/22048#discussion_r3238041970", + "tests/unit/sql/test_window_functions.py::test_window_function_order_by_multi": "TODO: https://github.com/rapidsai/cudf/pull/22048#discussion_r3238041970", + "tests/unit/sql/test_window_functions.py::test_window_frame_validation": "TODO: https://github.com/rapidsai/cudf/pull/22048#discussion_r3238041970", + "tests/unit/sql/test_window_functions.py::test_window_multiple_named_window": "TODO: https://github.com/rapidsai/cudf/pull/22048#discussion_r3238041970", + "tests/unit/sql/test_window_functions.py::test_window_named_window": "TODO: https://github.com/rapidsai/cudf/pull/22048#discussion_r3238041970", } From 7b50a4c216317251f435a5ed68548e24c012683a Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Thu, 14 May 2026 23:10:04 +0000 Subject: [PATCH 30/44] Replace test_concat_with_empty_dataframes_strict_25725 in expected failures --- python/cudf_polars/cudf_polars/testing/inject_gpu_engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/cudf_polars/cudf_polars/testing/inject_gpu_engine.py b/python/cudf_polars/cudf_polars/testing/inject_gpu_engine.py index e86d236765dd..734ef64a323f 100644 --- a/python/cudf_polars/cudf_polars/testing/inject_gpu_engine.py +++ b/python/cudf_polars/cudf_polars/testing/inject_gpu_engine.py @@ -255,7 +255,7 @@ def pytest_report_header(config: pytest.Config) -> str: "tests/unit/operations/test_slice.py::test_schema_slice_on_literal_23999[lit1-0-len1-False]": "List literal loses nesting in slice: cudf#19610", "tests/unit/operations/test_slice.py::test_schema_slice_on_literal_23999[lit1-offset1-0-False]": "List literal loses nesting in slice: cudf#19610", "tests/unit/operations/test_slice.py::test_schema_slice_on_literal_23999[lit1-offset1-len1-False]": "List literal loses nesting in slice: cudf#19610", - "tests/unit/functions/test_concat.py::test_concat_horizontally_strict": "polars doesnt hand us the hconcat options. Fixed in 1.39.", + "tests/unit/functions/test_concat.py::test_concat_with_empty_dataframes_strict_25725": "https://github.com/rapidsai/cudf/issues/21644", } From 2250f60b38f52aa65607947805aa517619e1b4c5 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Thu, 14 May 2026 23:12:45 +0000 Subject: [PATCH 31/44] Clarify test_scan_metrics skip reason --- .../cudf_polars/testing/inject_gpu_engine.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/python/cudf_polars/cudf_polars/testing/inject_gpu_engine.py b/python/cudf_polars/cudf_polars/testing/inject_gpu_engine.py index 734ef64a323f..961b1a5736ec 100644 --- a/python/cudf_polars/cudf_polars/testing/inject_gpu_engine.py +++ b/python/cudf_polars/cudf_polars/testing/inject_gpu_engine.py @@ -301,12 +301,12 @@ def pytest_report_header(config: pytest.Config) -> str: # Short term adds in the aftermath of the rapidsmpf switch to get CI passing "tests/unit/io/test_lazy_parquet.py::test_scan_parquet_local_with_async": "Flaky, otherwise TBD", "tests/unit/operations/test_join.py::test_join_where_nested_expr_21066": "Flaky, otherwise TBD", - "tests/unit/io/test_scan.py::test_scan_metrics[True-parquet]": "Specific to IO metrics on Polars CPU", - "tests/unit/io/test_scan.py::test_scan_metrics[True-csv]": "Specific to IO metrics on Polars CPU", - "tests/unit/io/test_scan.py::test_scan_metrics[True-ndjson]": "Specific to IO metrics on Polars CPU", - "tests/unit/io/test_scan.py::test_scan_metrics[False-parquet]": "Specific to IO metrics on Polars CPU", - "tests/unit/io/test_scan.py::test_scan_metrics[False-csv]": "Specific to IO metrics on Polars CPU", - "tests/unit/io/test_scan.py::test_scan_metrics[False-ndjson]": "Specific to IO metrics on Polars CPU", + "tests/unit/io/test_scan.py::test_scan_metrics[True-parquet]": "Checks to IO metric logs specific to Polars CPU", + "tests/unit/io/test_scan.py::test_scan_metrics[True-csv]": "Checks to IO metric logs specific to Polars CPU", + "tests/unit/io/test_scan.py::test_scan_metrics[True-ndjson]": "Checks to IO metric logs specific to Polars CPU", + "tests/unit/io/test_scan.py::test_scan_metrics[False-parquet]": "Checks to IO metric logs specific to Polars CPU", + "tests/unit/io/test_scan.py::test_scan_metrics[False-csv]": "Checks to IO metric logs specific to Polars CPU", + "tests/unit/io/test_scan.py::test_scan_metrics[False-ndjson]": "Checks to IO metric logs specific to Polars CPU", } From 27c31a38b48cf3db075cb6b28c0be98a20003a3d Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Fri, 15 May 2026 17:09:25 +0000 Subject: [PATCH 32/44] Also mark over test as failing for in-memory engine --- .../cudf_polars/cudf_polars/testing/inject_gpu_engine.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/python/cudf_polars/cudf_polars/testing/inject_gpu_engine.py b/python/cudf_polars/cudf_polars/testing/inject_gpu_engine.py index 961b1a5736ec..b4b48bee1b46 100644 --- a/python/cudf_polars/cudf_polars/testing/inject_gpu_engine.py +++ b/python/cudf_polars/cudf_polars/testing/inject_gpu_engine.py @@ -256,6 +256,14 @@ def pytest_report_header(config: pytest.Config) -> str: "tests/unit/operations/test_slice.py::test_schema_slice_on_literal_23999[lit1-offset1-0-False]": "List literal loses nesting in slice: cudf#19610", "tests/unit/operations/test_slice.py::test_schema_slice_on_literal_23999[lit1-offset1-len1-False]": "List literal loses nesting in slice: cudf#19610", "tests/unit/functions/test_concat.py::test_concat_with_empty_dataframes_strict_25725": "https://github.com/rapidsai/cudf/issues/21644", + "tests/unit/sql/test_window_functions.py::test_over_with_order_by": "TODO: https://github.com/rapidsai/cudf/pull/22048#discussion_r3238041970", + "tests/unit/sql/test_window_functions.py::test_over_with_cumulative_window_funcs": "TODO: https://github.com/rapidsai/cudf/pull/22048#discussion_r3238041970", + "tests/unit/sql/test_window_functions.py::test_window_function_order_by_multi": "TODO: https://github.com/rapidsai/cudf/pull/22048#discussion_r3238041970", + "tests/unit/sql/test_window_functions.py::test_window_cumulative_agg_with_nulls": "TODO: https://github.com/rapidsai/cudf/pull/22048#discussion_r3238041970", + "tests/unit/sql/test_window_functions.py::test_window_named_window": "TODO: https://github.com/rapidsai/cudf/pull/22048#discussion_r3238041970", + "tests/unit/sql/test_window_functions.py::test_window_multiple_named_windows": "TODO: https://github.com/rapidsai/cudf/pull/22048#discussion_r3238041970", + "tests/unit/sql/test_window_functions.py::test_window_frame_validation": "TODO: https://github.com/rapidsai/cudf/pull/22048#discussion_r3238041970", + "tests/unit/operations/test_window.py::test_over_literal_cum_sum_26800": "TODO: https://github.com/rapidsai/cudf/pull/22048#discussion_r3238041970", } From 463d172a7269f15d0a00b6ecead9194cb411ab6c Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Fri, 15 May 2026 17:13:42 +0000 Subject: [PATCH 33/44] Add is_between test for new decimal/float cast --- .../tests/expressions/test_booleanfunction.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/python/cudf_polars/tests/expressions/test_booleanfunction.py b/python/cudf_polars/tests/expressions/test_booleanfunction.py index 94463c62ca77..93a7d3ebeec6 100644 --- a/python/cudf_polars/tests/expressions/test_booleanfunction.py +++ b/python/cudf_polars/tests/expressions/test_booleanfunction.py @@ -165,6 +165,21 @@ def test_boolean_isbetween(engine: pl.GPUEngine, closed, bounds): assert_gpu_result_equal(q, engine=engine) +@pytest.mark.parametrize("closed", ["both", "left", "right", "none"]) +def test_boolean_isbetween_decimal_float(engine: pl.GPUEngine, closed): + df = pl.LazyFrame( + { + "a": pl.Series([1, 2, 3, 4], dtype=pl.Decimal(scale=2)), + "lo": pl.Series([0.5, 1.5, 2.5, 3.5], dtype=pl.Float64), + "hi": pl.Series([1.5, 2.5, 3.5, 4.5], dtype=pl.Float64), + } + ) + + q = df.select(pl.col("a").is_between(pl.col("lo"), pl.col("hi"), closed=closed)) + + assert_gpu_result_equal(q, engine=engine) + + @pytest.mark.parametrize( "expr", [pl.any_horizontal("*"), pl.all_horizontal("*")], ids=["any", "all"] ) From 13831f7344f3d4f8e28d812651ba8c193d4810e9 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Fri, 15 May 2026 10:15:44 -0700 Subject: [PATCH 34/44] Update python/cudf_polars/cudf_polars/dsl/translate.py Co-authored-by: Matthew Murray <41342305+Matt711@users.noreply.github.com> --- python/cudf_polars/cudf_polars/dsl/translate.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/cudf_polars/cudf_polars/dsl/translate.py b/python/cudf_polars/cudf_polars/dsl/translate.py index c34e97190cd0..b711a71271c3 100644 --- a/python/cudf_polars/cudf_polars/dsl/translate.py +++ b/python/cudf_polars/cudf_polars/dsl/translate.py @@ -872,7 +872,6 @@ def _( if hasattr(plrs._expr_nodes, "RollingGroupOptions") and isinstance( node.options, plrs._expr_nodes.RollingGroupOptions ): # pragma: no cover; polars >=1.36 uses AExpr::Rolling now - # pl.col("a").rolling(...) in polars <1.36 with set_expr_context(translator, ExecutionContext.ROLLING): agg = translator.translate_expr(n=node.function, schema=schema) name_generator = unique_names(schema) From 0d838112b3bd228e8d94eea764924db51dbd3249 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Fri, 15 May 2026 10:15:57 -0700 Subject: [PATCH 35/44] Update python/cudf_polars/cudf_polars/dsl/translate.py Co-authored-by: Matthew Murray <41342305+Matt711@users.noreply.github.com> --- python/cudf_polars/cudf_polars/dsl/translate.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/cudf_polars/cudf_polars/dsl/translate.py b/python/cudf_polars/cudf_polars/dsl/translate.py index b711a71271c3..e87cfa272c9b 100644 --- a/python/cudf_polars/cudf_polars/dsl/translate.py +++ b/python/cudf_polars/cudf_polars/dsl/translate.py @@ -978,6 +978,7 @@ def _translate_rolling( dtype: DataType, schema: Schema, ) -> expr.Expr: + # pl.col("a").rolling(...) with set_expr_context(translator, ExecutionContext.ROLLING): agg_expr = translator.translate_expr(n=node.function, schema=schema) orderby = translator.visitor.view_expression(node.index_column).name From f7e909e674c483808bf9957f8a0c0cce68ab6e44 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Fri, 15 May 2026 19:05:01 +0000 Subject: [PATCH 36/44] remove whitespace --- python/cudf_polars/cudf_polars/dsl/translate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/cudf_polars/cudf_polars/dsl/translate.py b/python/cudf_polars/cudf_polars/dsl/translate.py index e87cfa272c9b..a1defd023628 100644 --- a/python/cudf_polars/cudf_polars/dsl/translate.py +++ b/python/cudf_polars/cudf_polars/dsl/translate.py @@ -978,7 +978,7 @@ def _translate_rolling( dtype: DataType, schema: Schema, ) -> expr.Expr: - # pl.col("a").rolling(...) + # pl.col("a").rolling(...) with set_expr_context(translator, ExecutionContext.ROLLING): agg_expr = translator.translate_expr(n=node.function, schema=schema) orderby = translator.visitor.view_expression(node.index_column).name From 63550e8eaadd4053d873056e0ba9ecdea9fb0b43 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Fri, 15 May 2026 21:37:22 +0000 Subject: [PATCH 37/44] Update streaming fail list --- .../cudf_polars/testing/inject_gpu_engine.py | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/python/cudf_polars/cudf_polars/testing/inject_gpu_engine.py b/python/cudf_polars/cudf_polars/testing/inject_gpu_engine.py index b4b48bee1b46..f2a35c1ce81b 100644 --- a/python/cudf_polars/cudf_polars/testing/inject_gpu_engine.py +++ b/python/cudf_polars/cudf_polars/testing/inject_gpu_engine.py @@ -384,6 +384,7 @@ def pytest_report_header(config: pytest.Config) -> str: STREAMING_ENGINE_EXPECTED_FAILURES: Mapping[str, str] = { "tests/unit/functions/range/test_linear_space.py::test_linear_space_num_samples_expr": "https://github.com/rapidsai/cudf/issues/22072", "tests/unit/functions/test_concat.py::test_concat_horizontal_zero_width_height_mismatch_26876": "https://github.com/rapidsai/cudf/issues/21644", + "tests/unit/functions/test_concat.py::test_concat_horizontally_strict": "Correct polars.exceptions.ShapeError raised but it's in a ExceptionGroup", "tests/unit/interop/test_interop.py::test_0_width_df_roundtrip": "https://github.com/rapidsai/cudf/issues/21644", "tests/unit/lazyframe/test_projections.py::test_join_projection_pushdown_struct_field_as_key_24446": "https://github.com/rapidsai/cudf/issues/22105", "tests/unit/operations/test_slice.py::test_slice_pushdown_literal_projection_14349": "https://github.com/rapidsai/cudf/issues/22072", @@ -439,26 +440,8 @@ def pytest_report_header(config: pytest.Config) -> str: "tests/unit/operations/test_join.py::test_join_numeric_key_upcast_15338[False-dtypes43]": "https://github.com/rapidsai/cudf/issues/22085", "tests/unit/operations/test_join.py::test_join_numeric_key_upcast_15338[False-dtypes44]": "https://github.com/rapidsai/cudf/issues/22085", "tests/unit/operations/test_join.py::test_join_numeric_key_upcast_order": "https://github.com/rapidsai/cudf/issues/22085", - "tests/unit/operations/test_join.py::test_join_panic_on_binary_expr_5915": "https://github.com/rapidsai/cudf/issues/22105", - "tests/unit/operations/test_join.py::test_join_rewrite_panic_23307": "https://github.com/rapidsai/cudf/issues/22085", - "tests/unit/operations/test_join.py::test_semi_anti_join": "https://github.com/rapidsai/cudf/issues/22049", "tests/unit/operations/test_window.py::test_over_literal_cum_sum_26800": "TODO: https://github.com/rapidsai/cudf/pull/22048#discussion_r3238041970", "tests/unit/sql/test_joins.py::test_cross_join_unnest_from_cte": "https://github.com/rapidsai/cudf/issues/22073", - "tests/unit/sql/test_joins.py::test_join_anti_semi[SELECT * FROM tbl_a LEFT SEMI JOIN tbl_b USING (a)-expected2]": "https://github.com/rapidsai/cudf/issues/22105", - "tests/unit/sql/test_joins.py::test_join_on_mixed_expression_conditions[df10-df20-df1.category = df2.category AND (df1.code * 2) = df2.code_doubled-df1.name, df1.code, df2.type-expected0-schema0]": "https://github.com/rapidsai/cudf/issues/22085 (or similar)", - "tests/unit/sql/test_joins.py::test_join_on_mixed_expression_conditions[df11-df21-df1.id = df2.id AND LOWER(df1.name) = df2.match-df1.id, df1.name, df2.match-expected1-schema1]": "https://github.com/rapidsai/cudf/issues/22085 (or similar)", - "tests/unit/sql/test_joins.py::test_join_on_expression_conditions[LOWER(df1.text) = df2.text-2]": "https://github.com/rapidsai/cudf/issues/22105", - "tests/unit/sql/test_joins.py::test_join_on_expression_conditions[SUBSTR(df1.code, 1, 2) = SUBSTR(df2.code, 1, 2)-3]": "https://github.com/rapidsai/cudf/issues/22105", - "tests/unit/sql/test_joins.py::test_join_on_expression_conditions[LENGTH(df1.text) = LENGTH(df2.text)-5]": "https://github.com/rapidsai/cudf/issues/22105", - "tests/unit/sql/test_joins.py::test_join_on_expression_with_literals[df10-df20-df1.id = df2.id AND df1.multiplier * 5 = df2.base AND df1.category = 'A'-df1.id, df1.multiplier, df2.base-expected0-schema0]": "https://github.com/rapidsai/cudf/issues/22085 (or similar)", - "tests/unit/sql/test_joins.py::test_join_on_expression_with_literals[df11-df21-df1.id = df2.id AND (df1.value * 2) = df2.target AND df1.id = 2-df1.id, df1.value, df2.target-expected1-schema1]": "https://github.com/rapidsai/cudf/issues/22105", - "tests/unit/sql/test_joins.py::test_join_on_mixed_expression_conditions[df12-df22-df1.x * 2 = df2.a AND df1.y = df2.b-df1.x, df1.y, df2.a-expected2-schema2]": "https://github.com/rapidsai/cudf/issues/22105", - "tests/unit/sql/test_joins.py::test_join_on_nested_function_expressions[df10-df20-LOWER(TRIM(df1.text)) = df2.text-expected0]": "https://github.com/rapidsai/cudf/issues/22105", - "tests/unit/sql/test_joins.py::test_join_on_nested_function_expressions[df11-df21-LOWER(SUBSTR(df1.code,1,6)) = df2.code-expected1]": "https://github.com/rapidsai/cudf/issues/22105", - "tests/unit/sql/test_joins.py::test_join_on_nested_function_expressions[df12-df22-LENGTH(df1.name) = df2.len-expected2]": "https://github.com/rapidsai/cudf/issues/22105", - "tests/unit/sql/test_joins.py::test_join_on_reversed_constraint_order[df12-df22-(df1.a + df1.a) = df2.b-df2.b = (df1.a + df1.a)-expected2-schema2]": "https://github.com/rapidsai/cudf/issues/22105", - "tests/unit/sql/test_joins.py::test_join_on_unqualified_expressions[df11-df21-x + y = sum-expected1-schema1]": "https://github.com/rapidsai/cudf/issues/22105", - "tests/unit/sql/test_joins.py::test_join_on_unqualified_expressions[df12-df22-LENGTH(name) = len-expected2-schema2]": "https://github.com/rapidsai/cudf/issues/22105", "tests/unit/sql/test_window_functions.py::test_over_with_cumulative_window_funcs": "TODO: https://github.com/rapidsai/cudf/pull/22048#discussion_r3238041970", "tests/unit/sql/test_window_functions.py::test_over_with_order_by": "TODO: https://github.com/rapidsai/cudf/pull/22048#discussion_r3238041970", "tests/unit/sql/test_window_functions.py::test_window_cumulative_agg_with_nulls": "TODO: https://github.com/rapidsai/cudf/pull/22048#discussion_r3238041970", From 18d3ea90cd3e09e648f4b573d5be9ce1c8c5a78e Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Fri, 15 May 2026 21:54:58 +0000 Subject: [PATCH 38/44] Add coverage for cudf_polars/dsl/expressions/unary.py 206 4 98% 255-264 --- python/cudf_polars/tests/expressions/test_sort.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/python/cudf_polars/tests/expressions/test_sort.py b/python/cudf_polars/tests/expressions/test_sort.py index 3f2f0dccc66d..0fd0354308ef 100644 --- a/python/cudf_polars/tests/expressions/test_sort.py +++ b/python/cudf_polars/tests/expressions/test_sort.py @@ -78,6 +78,20 @@ def test_setsorted(engine: pl.GPUEngine, request, descending, nulls_last, with_n assert_gpu_result_equal(q, engine=engine) +@pytest.mark.parametrize("descending", [False, True]) +@pytest.mark.parametrize("nulls_last", [False, True]) +def test_setsorted_expr_with_nulls(engine: pl.GPUEngine, descending, nulls_last): + sorted_values = sorted([1, 2, 3, 4, 5], reverse=descending) + values: list[int | None] = [*sorted_values] + if nulls_last: + values.append(None) + else: + values.insert(0, None) + ldf = pl.LazyFrame({"a": values}) + q = ldf.select(pl.col("a").set_sorted(descending=descending)) + assert_gpu_result_equal(q, engine=engine) + + def test_sort_concat_filtered_to_empty(engine: pl.GPUEngine): df = pl.LazyFrame({"a": [1, 2, 3]}) q = pl.concat([df.filter(pl.col("a") == 0), df.filter(pl.col("a") == 4)]).sort("a") From 767735ee25ee859329a5e221e8d6ae3bb3c3e155 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Fri, 15 May 2026 22:00:55 +0000 Subject: [PATCH 39/44] Add coverage for cudf_polars/dsl/ir.py 1123 3 99% 3266-3271 --- python/cudf_polars/tests/test_hconcat.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/python/cudf_polars/tests/test_hconcat.py b/python/cudf_polars/tests/test_hconcat.py index 04c80d99b4c2..15c5b5e733c8 100644 --- a/python/cudf_polars/tests/test_hconcat.py +++ b/python/cudf_polars/tests/test_hconcat.py @@ -6,7 +6,7 @@ from cudf_polars.containers import DataType from cudf_polars.dsl.ir import DataFrameScan, Empty, HConcat, IRExecutionContext -from cudf_polars.testing.asserts import assert_gpu_result_equal +from cudf_polars.testing.asserts import assert_collect_raises, assert_gpu_result_equal def test_hconcat(engine: pl.GPUEngine): @@ -30,6 +30,17 @@ def test_hconcat_different_heights(engine: pl.GPUEngine): assert_gpu_result_equal(q, engine=engine) +def test_hconcat_strict_different_heights(): + left = pl.LazyFrame({"a": [1, 2, 3]}) + right = pl.LazyFrame({"b": [4, 5]}) + q = pl.concat([left, right], how="horizontal", strict=True) + assert_collect_raises( + q, + polars_except=pl.exceptions.ShapeError, + cudf_except=pl.exceptions.ShapeError, + ) + + def test_hconcat_should_broadcast(): # HConcat with should_broadcast=True is used by the streaming engine to # recombine decomposed expressions. Test it by constructing IR directly. From 05d1946a663e0277d7faa91eda5223fef3796508 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Fri, 15 May 2026 22:08:52 +0000 Subject: [PATCH 40/44] Add coverage for cudf_polars/dsl/to_ast.py 87 1 99% 168 --- python/cudf_polars/tests/test_parquet_filters.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/python/cudf_polars/tests/test_parquet_filters.py b/python/cudf_polars/tests/test_parquet_filters.py index 2009800ced53..d77a627fdb79 100644 --- a/python/cudf_polars/tests/test_parquet_filters.py +++ b/python/cudf_polars/tests/test_parquet_filters.py @@ -65,6 +65,17 @@ def test_scan_by_hand(expr, selection, pq_file, chunked): ) +def test_parquet_filter_ne_missing(tmp_path): + df = pl.DataFrame({"a": [1, 2, None, 3, None]}) + df.write_parquet(tmp_path / "df.parquet", row_group_size=3) + q = pl.scan_parquet(tmp_path / "df.parquet").filter( + pl.col("a").ne_missing(pl.lit(2, dtype=pl.Int64)) + ) + assert_gpu_result_equal( + q, engine=pl.GPUEngine(executor="in-memory", raise_on_fail=True) + ) + + def test_parquet_filter_boolean_column(engine: pl.GPUEngine, tmp_path): df = pl.DataFrame({"x": [1, 2, 3], "y": [True, False, True]}) df.write_parquet(tmp_path / "df.parquet") From 74b731e222138ac32d0f4b776baeeeb993ff8769 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Fri, 15 May 2026 22:19:41 +0000 Subject: [PATCH 41/44] Add no covers for the rest --- python/cudf_polars/cudf_polars/dsl/translate.py | 4 ++-- python/cudf_polars/tests/test_scan.py | 11 ++++------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/python/cudf_polars/cudf_polars/dsl/translate.py b/python/cudf_polars/cudf_polars/dsl/translate.py index a1defd023628..86396aae1391 100644 --- a/python/cudf_polars/cudf_polars/dsl/translate.py +++ b/python/cudf_polars/cudf_polars/dsl/translate.py @@ -277,7 +277,7 @@ def _is_dynamic_pred(visitor: Any, expr_ir: Any) -> bool: except Exception as e: if str(e) == "dynamic_pred": return True - raise + raise # pragma: no cover else: return False @@ -626,7 +626,7 @@ def _(node: plrs._ir_nodes.Union, translator: Translator, schema: Schema) -> ir. @_translate_ir.register def _(node: plrs._ir_nodes.HConcat, translator: Translator, schema: Schema) -> ir.IR: if POLARS_VERSION_LT_139: - strict = False + strict = False # pragma: no cover else: _, strict, _ = node.options return ir.HConcat( diff --git a/python/cudf_polars/tests/test_scan.py b/python/cudf_polars/tests/test_scan.py index c7bdead8cbdf..eb9c24c48af9 100644 --- a/python/cudf_polars/tests/test_scan.py +++ b/python/cudf_polars/tests/test_scan.py @@ -583,19 +583,16 @@ def get_handler(req: Request) -> Response: ) +@pytest.mark.xfail( + condition=not POLARS_VERSION_LT_139, + reason="polars 1.39+ ndjson remote reader requires range request support", +) def test_scan_ndjson_remote( engine: pl.GPUEngine, - request: pytest.FixtureRequest, tmp_path: Path, df: pl.DataFrame, httpserver: HTTPServer, ) -> None: - request.applymarker( - pytest.mark.xfail( - condition=not POLARS_VERSION_LT_139, - reason="polars 1.39+ ndjson remote reader requires range request support", - ) - ) path = tmp_path / "foo.jsonl" df.write_ndjson(path) bytes_ = path.read_bytes() From 445b5acb0820bc037ca239836b40655c8a469d43 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Fri, 15 May 2026 23:49:08 +0000 Subject: [PATCH 42/44] Add pragma no cover continue in _is_remote_uri (because test_scan_ndjson_remote is xfailed now) --- python/cudf_polars/cudf_polars/dsl/translate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/cudf_polars/cudf_polars/dsl/translate.py b/python/cudf_polars/cudf_polars/dsl/translate.py index 86396aae1391..c44df6090cf6 100644 --- a/python/cudf_polars/cudf_polars/dsl/translate.py +++ b/python/cudf_polars/cudf_polars/dsl/translate.py @@ -342,7 +342,7 @@ def _(node: plrs._ir_nodes.Scan, translator: Translator, schema: Schema) -> ir.I if typ in ("csv", "ndjson"): for p in paths: if plc.io.SourceInfo._is_remote_uri(p): - continue + continue # pragma: no cover data = _read_file_bytes(Path(p)) compression = _check_compression(data) if compression is not None: From 66fdb71b044c6be78808894de78e10fad5f2c38f Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Mon, 18 May 2026 16:53:45 +0000 Subject: [PATCH 43/44] skip test_hconcat_strict_different_heights for earlier Polars versions --- python/cudf_polars/tests/test_hconcat.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/cudf_polars/tests/test_hconcat.py b/python/cudf_polars/tests/test_hconcat.py index 15c5b5e733c8..298d63534f7c 100644 --- a/python/cudf_polars/tests/test_hconcat.py +++ b/python/cudf_polars/tests/test_hconcat.py @@ -2,11 +2,14 @@ # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations +import pytest + import polars as pl from cudf_polars.containers import DataType from cudf_polars.dsl.ir import DataFrameScan, Empty, HConcat, IRExecutionContext from cudf_polars.testing.asserts import assert_collect_raises, assert_gpu_result_equal +from cudf_polars.utils.versions import POLARS_VERSION_LT_139 def test_hconcat(engine: pl.GPUEngine): @@ -30,6 +33,7 @@ def test_hconcat_different_heights(engine: pl.GPUEngine): assert_gpu_result_equal(q, engine=engine) +@pytest.mark.skipif(POLARS_VERSION_LT_139, reason="strict keyword added in polars 1.39") def test_hconcat_strict_different_heights(): left = pl.LazyFrame({"a": [1, 2, 3]}) right = pl.LazyFrame({"b": [4, 5]}) From c0e23eb457469deb3f7ccce76782329ae66ba5b5 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Mon, 18 May 2026 17:01:14 +0000 Subject: [PATCH 44/44] Address coderabbit --- python/cudf_polars/cudf_polars/testing/inject_gpu_engine.py | 1 - python/cudf_polars/tests/expressions/test_rolling.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/python/cudf_polars/cudf_polars/testing/inject_gpu_engine.py b/python/cudf_polars/cudf_polars/testing/inject_gpu_engine.py index f2a35c1ce81b..446b424442a4 100644 --- a/python/cudf_polars/cudf_polars/testing/inject_gpu_engine.py +++ b/python/cudf_polars/cudf_polars/testing/inject_gpu_engine.py @@ -448,7 +448,6 @@ def pytest_report_header(config: pytest.Config) -> str: "tests/unit/sql/test_window_functions.py::test_window_function_order_by_multi": "TODO: https://github.com/rapidsai/cudf/pull/22048#discussion_r3238041970", "tests/unit/sql/test_window_functions.py::test_window_frame_validation": "TODO: https://github.com/rapidsai/cudf/pull/22048#discussion_r3238041970", "tests/unit/sql/test_window_functions.py::test_window_multiple_named_window": "TODO: https://github.com/rapidsai/cudf/pull/22048#discussion_r3238041970", - "tests/unit/sql/test_window_functions.py::test_window_named_window": "TODO: https://github.com/rapidsai/cudf/pull/22048#discussion_r3238041970", } diff --git a/python/cudf_polars/tests/expressions/test_rolling.py b/python/cudf_polars/tests/expressions/test_rolling.py index 89fb23e3da0d..f3384b2af730 100644 --- a/python/cudf_polars/tests/expressions/test_rolling.py +++ b/python/cudf_polars/tests/expressions/test_rolling.py @@ -105,7 +105,7 @@ def test_rolling_integral_orderby(engine: pl.GPUEngine, dtype): @skip_rolling_expr_136_to_138 -def test_rolling_agg_first(engine: pl.GPUEngine): +def test_rolling_agg_before_rolling(engine: pl.GPUEngine): df = pl.LazyFrame({"a": [1, 2, 3], "b": [1, 2, 3]}) q = df.with_columns(pl.col("a").sum().rolling("b", period="2i")) assert_gpu_result_equal(q, engine=engine)