From 4ac3be045e8b17b4bdcb575d305b3b01301ae3a5 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Wed, 17 Jun 2026 02:25:19 +0000 Subject: [PATCH 01/22] fix --- python/cudf/cudf/core/dataframe.py | 32 ++++- python/cudf/cudf/core/indexed_frame.py | 114 +++++++++++++++--- .../pandas/scripts/pandas-testing-plugin.py | 27 +---- 3 files changed, 126 insertions(+), 47 deletions(-) diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 62c164873ff4..b47b8ea1869f 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2018-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2018-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations @@ -2967,6 +2967,7 @@ def _set_columns_like(self, other: ColumnAccessor) -> None: def reindex( self, labels=None, + *, index=None, columns=None, axis=None, @@ -3079,6 +3080,11 @@ def reindex( if labels is None and index is None and columns is None: return self.copy(deep=copy) + if labels is not None and index is not None and columns is not None: + raise TypeError( + "Cannot specify all of 'labels', 'index', 'columns'." + ) + # pandas simply ignores the labels keyword if it is provided in # addition to index and columns, but it prohibits the axis arg. if (index is not None or columns is not None) and axis is not None: @@ -3280,7 +3286,18 @@ def set_index( and not isinstance(keys[0], (cudf.MultiIndex, pd.MultiIndex)) ): # Don't turn single level MultiIndex into an Index - idx = Index._from_column(data_to_add[0], name=names[0]) + freq = ( + getattr(keys[0], "freq", None) + if isinstance(keys[0], (cudf.Index, pd.Index)) + else None + ) + if freq is not None and data_to_add[0].dtype.kind == "M": + # Preserve the freq of a DatetimeIndex passed as the key. + idx = cudf.DatetimeIndex._from_column( + data_to_add[0], name=names[0], freq=freq + ) + else: + idx = Index._from_column(data_to_add[0], name=names[0]) else: idx = MultiIndex._from_data(dict(enumerate(data_to_add))) idx.names = names @@ -3663,7 +3680,16 @@ def _insert(self, loc, name, value, nan_as_null=None, ignore_index=True): self.index, how="right", sort=False ) - value = as_column(value, nan_as_null=nan_as_null) + if isinstance(value, (list, tuple)) and len(value) == 0: + # An empty list-like assigned as a DataFrame column has no + # inferable dtype and becomes float64, matching pandas (note + # pd.Series([]) is object, but DataFrame column assignment is + # float64). + value = as_column( + value, nan_as_null=nan_as_null, dtype=np.dtype(np.float64) + ) + else: + value = as_column(value, nan_as_null=nan_as_null) self._data.insert(name, value, loc=loc) @property diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index d999c245f5c2..657dcab35c22 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -3977,6 +3977,20 @@ def _reindex( dtypes = {} df = self + # Original column dtypes, captured before any index handling + # mutates ``df``; used to infer the dtype of brand-new columns. + orig_col_dtypes = [dtype for _, dtype in self._dtypes] + frame_common_dtype = ( + orig_col_dtypes[0] + if ( + orig_col_dtypes + and all(dt == orig_col_dtypes[0] for dt in orig_col_dtypes) + and isinstance(orig_col_dtypes[0], np.dtype) + ) + else None + ) + row_reindex = index is not None + rows_added = False if index is not None: if not df.index.is_unique: raise ValueError( @@ -4015,8 +4029,9 @@ def _reindex( index=df.index, ) diff = index.difference(df.index) + rows_added = len(diff) > 0 df = lhs.join(rhs, how="left", sort=True) - if fill_value is not NA and len(diff) > 0: + if fill_value is not NA and rows_added: df.loc[diff] = fill_value # double-argsort to map back from sorted to unsorted positions df = df.take(index.argsort(ascending=True).argsort()) @@ -4059,24 +4074,80 @@ def _reindex( multiindex = False rangeindex = False - cols = { - name: ( - df._data[name].copy(deep=deep) - if name in df._data - else ( - column_empty( - dtype=dtypes.get(name, np.dtype(np.float64)), - row_count=len(index), - ).fillna(fill_value) - if fill_value is not NA - else column_empty( - dtype=dtypes.get(name, np.dtype(np.float64)), - row_count=len(index), + def _new_column(name): + # Build a brand-new column produced by reindex (entirely + # missing / fill values), choosing a dtype that matches pandas + # for the common numeric cases. Non-numeric fills keep the + # legacy ``column_empty(...).fillna`` path, which raises for + # incompatible fills so cudf.pandas falls back to pandas. + if fill_value is NA: + # All-null new column: keep a homogeneous float dtype, + # else float64 (numpy integer/bool cannot hold NaN). + if name in dtypes: + target = dtypes[name] + elif ( + frame_common_dtype is not None + and frame_common_dtype.kind == "f" + ): + target = frame_common_dtype + else: + target = np.dtype(np.float64) + return column_empty(dtype=target, row_count=len(index)) + if name not in dtypes and is_scalar(fill_value): + scalar_col = as_column(fill_value, length=1) + if scalar_col.dtype.kind in "iuf": + # Numeric scalar fill: a row-reindex of a homogeneous + # numpy frame promotes the frame dtype against the fill + # value (uint8 + 10 -> uint8, uint8 + 300 -> int64); + # otherwise the new column's dtype is the fill value's. + if row_reindex and frame_common_dtype is not None: + if scalar_col.can_cast_safely(frame_common_dtype): + target = frame_common_dtype + else: + target = find_common_type( + [frame_common_dtype, scalar_col.dtype] + ) + else: + target = scalar_col.dtype + return as_column(fill_value, length=len(index)).astype( + target ) - ) - ) - for name in names - } + # Non-numeric / non-scalar fill: legacy behavior. + return column_empty( + dtype=dtypes.get(name, np.dtype(np.float64)), + row_count=len(index), + ).fillna(fill_value) + + # cudf cannot represent duplicate column names; pandas can. Raise + # so cudf.pandas falls back to pandas rather than silently + # collapsing duplicates. + names_list = list(names) + if len(names_list) != len(set(names_list)): + raise ValueError("Duplicate column names are not allowed") + + # pandas upcasts integer columns to float64 when default-NaN + # filling newly added rows on reindex (a numpy integer column + # cannot hold NA). Gated to preserve cudf's documented legacy + # behavior of keeping the integer dtype. + upcast_int_nulls = ( + rows_added + and fill_value is NA + and cudf.get_option("mode.pandas_compatible") + ) + + cols = {} + for name in names_list: + if name in df._data: + col = df._data[name].copy(deep=deep) + if ( + upcast_int_nulls + and col.dtype.kind in "iu" + and col.null_count + ): + col = col.astype(np.dtype(np.float64)) + else: + col = _new_column(name) + cols[name] = col result = self.__class__._from_data( data=ColumnAccessor( @@ -7292,6 +7363,13 @@ def _is_same_dtype(lhs_dtype, rhs_dtype): return True elif is_dtype_obj_string(lhs_dtype) and is_dtype_obj_string(rhs_dtype): return True + elif is_dtype_obj_numeric( + lhs_dtype, include_decimal=False + ) and is_dtype_obj_numeric(rhs_dtype, include_decimal=False): + # Numeric index labels are joinable across int/float widths + # (e.g. an int level reindexed against a float level), matching + # pandas instead of bailing to an all-null result. + return True else: return False diff --git a/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py b/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py index b4cda7f5bb48..9a8e3ea170cd 100644 --- a/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py +++ b/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 import json @@ -1435,37 +1435,12 @@ def pytest_unconfigure(config): "tests/frame/methods/test_quantile.py::TestQuantileExtensionDtype::test_quantile_ea_with_na[Int64-Series]": "TODO: Add a reason for failure", "tests/frame/methods/test_rank.py::TestRank::test_rank": "AssertionError: numpy array are different", "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex": "TODO: Add a reason for failure", - "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_axis_style_raises": "TODO: Add a reason for failure", - "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_empty[cat_idx4-CategoricalIndex]": "AssertionError: DataFrame are different", - "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_empty[cat_idx4-Index]": "AssertionError: DataFrame are different", - "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_empty[cat_idx5-CategoricalIndex]": "AssertionError: DataFrame are different", - "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_empty[cat_idx5-Index]": "AssertionError: DataFrame are different", "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_empty_frame[kwargs0]": "AssertionError: Attributes of DataFrame.iloc[:, 0] (column name='a') are different", "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_empty_frame[kwargs1]": "AssertionError: Attributes of DataFrame.iloc[:, 0] (column name='a') are different", "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_empty_frame[kwargs2]": "AssertionError: Attributes of DataFrame.iloc[:, 0] (column name='a') are different", "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_empty_frame[kwargs3]": "AssertionError: Attributes of DataFrame.iloc[:, 0] (column name='a') are different", "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_fill_value": "TODO: Add a reason for failure", "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_index_name_matches_multiindex_level": "AssertionError: Attributes of DataFrame.iloc[:, 0] (column name='value') are different", - "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_int": "TODO: Add a reason for failure", - "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_positional_raises": "TODO: Add a reason for failure", - "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_single_column_ea_index_and_columns[Float32]": "TODO: Add a reason for failure", - "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_single_column_ea_index_and_columns[Float64]": "TODO: Add a reason for failure", - "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_single_column_ea_index_and_columns[Int16]": "TODO: Add a reason for failure", - "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_single_column_ea_index_and_columns[Int32]": "TODO: Add a reason for failure", - "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_single_column_ea_index_and_columns[Int64]": "TODO: Add a reason for failure", - "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_single_column_ea_index_and_columns[Int8]": "TODO: Add a reason for failure", - "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_single_column_ea_index_and_columns[UInt16]": "TODO: Add a reason for failure", - "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_single_column_ea_index_and_columns[UInt32]": "TODO: Add a reason for failure", - "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_single_column_ea_index_and_columns[UInt64]": "TODO: Add a reason for failure", - "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_single_column_ea_index_and_columns[UInt8]": "TODO: Add a reason for failure", - "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_uint_dtypes_fill_value[uint16]": "TODO: Add a reason for failure", - "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_uint_dtypes_fill_value[uint32]": "TODO: Add a reason for failure", - "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_uint_dtypes_fill_value[uint64]": "TODO: Add a reason for failure", - "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_uint_dtypes_fill_value[uint8]": "TODO: Add a reason for failure", - "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_with_multi_index": "TODO: Add a reason for failure", - "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_without_upcasting": "AssertionError: assert np.False_", - "tests/frame/methods/test_reindex.py::TestReindexSetIndex::test_dti_set_index_reindex_freq_with_tz": "AssertionError: assert None == ", - "tests/frame/methods/test_reindex.py::TestReindexSetIndex::test_setitem_reset_index_dtypes": "TODO: Add a reason for failure", "tests/frame/methods/test_rename.py::TestRename::test_rename": "TODO: Add a reason for failure", "tests/frame/methods/test_rename.py::TestRename::test_rename_axis_style_raises": "TODO: Add a reason for failure", "tests/frame/methods/test_rename.py::TestRename::test_rename_inplace": "TODO: Add a reason for failure", From 083a96b27b562e5c342f64b0be93fdb0e76fb3ad Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Thu, 18 Jun 2026 18:38:55 +0000 Subject: [PATCH 02/22] fix --- python/cudf/cudf/core/indexed_frame.py | 5 +++++ python/cudf/cudf/core/series.py | 7 ++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index 657dcab35c22..28555799647d 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -4141,6 +4141,11 @@ def _new_column(name): col = df._data[name].copy(deep=deep) if ( upcast_int_nulls + # Only plain numpy integer columns cannot hold NA; + # nullable extension integers (masked ``IntX``, + # ``ArrowDtype``) natively represent NA, so pandas + # keeps their dtype rather than upcasting to float64. + and isinstance(col.dtype, np.dtype) and col.dtype.kind in "iu" and col.null_count ): diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index 012c6e53cbff..48a8c66dc67d 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -3243,7 +3243,12 @@ def value_counts( else None ) res = res[res.index.notna()] - res = res.reindex(self.dtype.categories).fillna(0) + # Fill missing categories with a 0 count directly via + # ``reindex`` (rather than ``reindex(...).fillna(0)``) so + # the integer count dtype is preserved: a default-NA + # reindex upcasts integer columns to float64 in + # pandas-compatible mode. + res = res.reindex(self.dtype.categories, fill_value=0) res.index = res.index.astype(self.dtype) if nan_count is not None: res = cudf.concat([res, nan_count]) From 0d9664b0150998c2b1f531c88f93ddc429361e4f Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Thu, 18 Jun 2026 15:52:49 -0500 Subject: [PATCH 03/22] Update pandas-testing-plugin.py --- python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py b/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py index c9254490a8bb..7f4b1da85283 100644 --- a/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py +++ b/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py @@ -1200,7 +1200,6 @@ def pytest_unconfigure(config): "tests/frame/indexing/test_getitem.py::TestGetitemCallable::test_loc_multiindex_columns_one_level": "TODO: Add a reason for failure", "tests/frame/indexing/test_indexing.py::TestDataFrameIndexing::test_fancy_getitem_slice_mixed": "TODO: Add a reason for failure", "tests/frame/indexing/test_indexing.py::TestDataFrameIndexing::test_getitem_setitem_boolean_misaligned": "TODO: Add a reason for failure", - "tests/frame/indexing/test_indexing.py::TestDataFrameIndexing::test_getitem_setitem_float_labels": "TODO: Add a reason for failure", "tests/frame/indexing/test_indexing.py::TestDataFrameIndexing::test_iloc_col_slice_view": "TODO: Add a reason for failure", "tests/frame/indexing/test_indexing.py::TestDataFrameIndexing::test_iloc_row_slice_view": "TODO: Add a reason for failure", "tests/frame/indexing/test_indexing.py::TestDataFrameIndexing::test_loc_bool_multiindex[True-bool-indexer1]": "AssertionError: Did not see expected warning of class 'PerformanceWarning'", @@ -1644,7 +1643,6 @@ def pytest_unconfigure(config): "tests/frame/test_constructors.py::TestDataFrameConstructors::test_1d_object_array_does_not_copy": "TODO: Add a reason for failure", "tests/frame/test_constructors.py::TestDataFrameConstructors::test_2d_object_array_does_not_copy": "TODO: Add a reason for failure", "tests/frame/test_constructors.py::TestDataFrameConstructors::test_construct_from_dict_ea_series": "AttributeError: 'ndarray' object has no attribute '_data'. Did you mean: 'data'?", - "tests/frame/test_constructors.py::TestDataFrameConstructors::test_construct_with_two_categoricalindex_series": "TODO: Add a reason for failure", "tests/frame/test_constructors.py::TestDataFrameConstructors::test_construction_nan_value_timedelta64_dtype": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", "tests/frame/test_constructors.py::TestDataFrameConstructors::test_constructor_dict": "TODO: Add a reason for failure", "tests/frame/test_constructors.py::TestDataFrameConstructors::test_constructor_dict_cast": "TODO: Add a reason for failure", @@ -3430,7 +3428,6 @@ def pytest_unconfigure(config): "tests/indexing/test_partial.py::TestEmptyFrameSetitemExpansion::test_empty_frame_setitem_index_name_retained": "AssertionError: DataFrame.index are different", "tests/indexing/test_partial.py::TestEmptyFrameSetitemExpansion::test_partial_set_empty_frame": "Failed: DID NOT RAISE ", "tests/indexing/test_partial.py::TestEmptyFrameSetitemExpansion::test_partial_set_empty_frame2": "TODO: Add a reason for failure", - "tests/indexing/test_partial.py::TestEmptyFrameSetitemExpansion::test_partial_set_empty_frame3": 'AssertionError: Column name="foo" are different', "tests/indexing/test_partial.py::TestEmptyFrameSetitemExpansion::test_partial_set_empty_frame5": "AssertionError: DataFrame.index are different", "tests/indexing/test_partial.py::TestEmptyFrameSetitemExpansion::test_partial_set_empty_frame_empty_consistencies": "AssertionError: Attributes of DataFrame.iloc[:, 1] (column name='y') are different", "tests/indexing/test_partial.py::TestEmptyFrameSetitemExpansion::test_partial_set_empty_frame_row": "AssertionError: Attributes of Series are different", @@ -4832,10 +4829,6 @@ def pytest_unconfigure(config): "tests/series/methods/test_info.py::test_info_series[False]": "TODO: Add a reason for failure", "tests/series/methods/test_info.py::test_info_series[True]": "TODO: Add a reason for failure", "tests/series/methods/test_info.py::test_info_show_counts_false": "assert ' Date: Thu, 25 Jun 2026 14:34:20 -0400 Subject: [PATCH 04/22] Add the predicate in physical plan explain output (#22984) Includes the predicate in the physical plan explain output for `ConditionalJoin`, `Filter`, and `Scan`. Purely an improvement; helps make studying plan easier. Authors: - Matthew Murray (https://github.com/Matt711) Approvers: - Matthew Roeschke (https://github.com/mroeschke) URL: https://github.com/rapidsai/cudf/pull/22984 --- .../cudf_polars/streaming/explain.py | 72 +++++++++++++++++-- .../tests/streaming/test_explain.py | 46 +++++++++++- 2 files changed, 111 insertions(+), 7 deletions(-) diff --git a/python/cudf_polars/cudf_polars/streaming/explain.py b/python/cudf_polars/cudf_polars/streaming/explain.py index 8287980277e0..f0680586d8fa 100644 --- a/python/cudf_polars/cudf_polars/streaming/explain.py +++ b/python/cudf_polars/cudf_polars/streaming/explain.py @@ -14,10 +14,15 @@ from itertools import groupby from typing import TYPE_CHECKING, Any, Self, TypeAlias -import cudf_polars.dsl.expressions.binaryop -import cudf_polars.dsl.expressions.literal -from cudf_polars.dsl.expressions.base import NamedExpr +import pylibcudf as plc + +from cudf_polars.dsl.expressions.base import Col, ColRef, Expr, NamedExpr +from cudf_polars.dsl.expressions.binaryop import BinOp +from cudf_polars.dsl.expressions.literal import Literal +from cudf_polars.dsl.expressions.ternary import Ternary +from cudf_polars.dsl.expressions.unary import Cast, UnaryFunction from cudf_polars.dsl.ir import ( + ConditionalJoin, Filter, GroupBy, HStack, @@ -257,6 +262,59 @@ def _(ir: Join, *, offset: str = "") -> str: return _repr_header(offset, f"JOIN {ir.options[0]} {left_on} {right_on}", ir.schema) +_BinaryOperator = plc.binaryop.BinaryOperator +_BINOP_SYMBOLS: dict[_BinaryOperator, str] = { + _BinaryOperator.EQUAL: "==", + _BinaryOperator.NOT_EQUAL: "!=", + _BinaryOperator.LESS: "<", + _BinaryOperator.LESS_EQUAL: "<=", + _BinaryOperator.GREATER: ">", + _BinaryOperator.GREATER_EQUAL: ">=", + _BinaryOperator.LOGICAL_AND: "&", + _BinaryOperator.NULL_LOGICAL_AND: "&", + _BinaryOperator.LOGICAL_OR: "|", + _BinaryOperator.NULL_LOGICAL_OR: "|", +} + + +def _predicate_to_str(expr: Expr) -> str: + match expr: + case Col(name=name): + return name + case ColRef(): + col = expr.children[0] + assert isinstance(col, Col) + return col.name + case Literal(value=value): + return repr(value) + case Cast(): + return _predicate_to_str(expr.children[0]) + case BinOp(op=op): + left, right = expr.children + sym = _BINOP_SYMBOLS.get(op, op.name) + return f"({_predicate_to_str(left)} {sym} {_predicate_to_str(right)})" + case UnaryFunction(name=name): + (child,) = expr.children + return f"{name}({_predicate_to_str(child)})" + case Ternary(): + when, then, otherwise = expr.children + return f"when({_predicate_to_str(when)}).then({_predicate_to_str(then)}).otherwise({_predicate_to_str(otherwise)})" + case _: + return type(expr).__name__ + + +@_repr_ir.register +def _(ir: ConditionalJoin, *, offset: str = "") -> str: + pred = _predicate_to_str(ir.predicate) + return _repr_header(offset, f"CONDITIONALJOIN {pred}", ir.schema) + + +@_repr_ir.register +def _(ir: Filter, *, offset: str = "") -> str: + pred = _predicate_to_str(ir.mask.value) + return _repr_header(offset, f"FILTER {pred}", ir.schema) + + @_repr_ir.register def _(ir: Sort, *, offset: str = "") -> str: by = tuple(ne.name for ne in ir.by) @@ -266,6 +324,8 @@ def _(ir: Sort, *, offset: str = "") -> str: @_repr_ir.register def _(ir: Scan, *, offset: str = "") -> str: label = f"SCAN {ir.typ.upper()}" + if ir.predicate is not None: + label += f" {_predicate_to_str(ir.predicate.value)}" return _repr_header(offset, label, ir.schema) @@ -344,11 +404,11 @@ def _serialize_expr(expr: Expr | NamedExpr) -> dict[str, Serializable]: match expr: case NamedExpr(name=name, value=value): return {"type": "NamedExpr", "name": name, "value": _serialize_expr(value)} - case cudf_polars.dsl.expressions.base.Col(name=name): + case Col(name=name): return {"type": "Col", "name": name} - case cudf_polars.dsl.expressions.literal.Literal(value=value): + case Literal(value=value): return {"type": "Literal", "value": _serialize_literal(value)} - case cudf_polars.dsl.expressions.binaryop.BinOp(): + case BinOp(): return { "op": expr.op.name, "left": _serialize_expr(expr.children[0]), diff --git a/python/cudf_polars/tests/streaming/test_explain.py b/python/cudf_polars/tests/streaming/test_explain.py index 5f4984d53123..41e0177dad10 100644 --- a/python/cudf_polars/tests/streaming/test_explain.py +++ b/python/cudf_polars/tests/streaming/test_explain.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations @@ -13,9 +13,15 @@ import polars as pl +import pylibcudf as plc + +from cudf_polars.containers import DataType +from cudf_polars.dsl.expressions.base import Col +from cudf_polars.dsl.expressions.binaryop import BinOp from cudf_polars.engine.options import StreamingOptions from cudf_polars.streaming.explain import ( _fmt_row_count, + _predicate_to_str, explain_query, serialize_query, ) @@ -658,6 +664,44 @@ def test_hstack_properties(): assert node.properties == {"columns": ["a", "b"]} +def test_predicate_to_str_col(): + col = Col(DataType(pl.Float64()), "c_acctbal") + assert _predicate_to_str(col) == "c_acctbal" + + +def test_predicate_to_str_binop(): + float_dtype = DataType(pl.Float64()) + bool_dtype = DataType(pl.Boolean()) + left = Col(float_dtype, "c_acctbal") + right = Col(float_dtype, "avg_acctbal") + expr = BinOp(bool_dtype, plc.binaryop.BinaryOperator.GREATER, left, right) + assert _predicate_to_str(expr) == "(c_acctbal > avg_acctbal)" + + +def test_predicate_to_str_nested_binop(): + float_dtype = DataType(pl.Float64()) + bool_dtype = DataType(pl.Boolean()) + a = Col(float_dtype, "a") + b = Col(float_dtype, "b") + c = Col(float_dtype, "c") + ab = BinOp(bool_dtype, plc.binaryop.BinaryOperator.GREATER, a, b) + abc = BinOp(bool_dtype, plc.binaryop.BinaryOperator.LOGICAL_AND, ab, c) + assert _predicate_to_str(abc) == "((a > b) & c)" + + +def test_explain_conditional_join_shows_predicate(): + customers = pl.LazyFrame({"c_val": [4.0, 5.0, 6.0]}) + avg = pl.LazyFrame({"avg_val": [3.5]}) + q = customers.join_where(avg, pl.col("c_val") > pl.col("avg_val")) + + engine = pl.GPUEngine(executor="streaming", raise_on_fail=True) + with pytest.warns(UserWarning, match="ConditionalJoin not supported"): + plan = explain_query(q, engine, physical=True) + + assert "CONDITIONALJOIN" in plan + assert "c_val > avg_val" in plan + + def test_explain_physical_plan(tmp_path, df): make_partitioned_source(df, tmp_path, fmt="parquet", n_files=5) From 2564a83f18c012e2ca277e08258b9b1272c23197 Mon Sep 17 00:00:00 2001 From: brandon-b-miller <53796099+brandon-b-miller@users.noreply.github.com> Date: Thu, 25 Jun 2026 20:30:39 -0500 Subject: [PATCH 05/22] Support `cudf-polars` `total_xxx` datetime extraction methods (#18171) Part of https://github.com/rapidsai/cudf/issues/16481 Since theres two consumers of this API now (pandas and polars), I am wondering if we might take another look at adding libcudf APIs here. xref https://github.com/rapidsai/cudf/issues/16802 and cc @bdice @mroeschke @galipremsagar . WIP Authors: - https://github.com/brandon-b-miller - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Matthew Murray (https://github.com/Matt711) - Matthew Roeschke (https://github.com/mroeschke) URL: https://github.com/rapidsai/cudf/pull/18171 --- .../cudf_polars/containers/column.py | 25 +++++++++- .../cudf_polars/dsl/expressions/datetime.py | 49 ++++++++++++++++++- .../tests/containers/test_column.py | 20 +++++++- .../tests/expressions/test_datetime_basic.py | 42 ++++++++++++++++ 4 files changed, 133 insertions(+), 3 deletions(-) diff --git a/python/cudf_polars/cudf_polars/containers/column.py b/python/cudf_polars/cudf_polars/containers/column.py index 9f8514dca77f..bca694352f6c 100644 --- a/python/cudf_polars/cudf_polars/containers/column.py +++ b/python/cudf_polars/cudf_polars/containers/column.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 """A column, with some properties.""" @@ -353,6 +353,29 @@ def astype(self, dtype: DataType, stream: Stream, *, strict: bool = True) -> Col dtype=dtype, name=self.name, ).sorted_like(self) + elif plc.traits.is_integral_not_bool(plc_dtype) and plc.traits.is_duration( + self.obj.type() + ): + # A duration is stored as an integer tick count, so casting to that + # integer type is a no-op reinterpret of the same bytes. Relabel the + # column instead of launching a cast kernel. + rep = plc.DataType( + plc.TypeId.INT32 + if self.obj.type().id() == plc.TypeId.DURATION_DAYS + else plc.TypeId.INT64 + ) + plc_col = plc.column.Column( + rep, + self.obj.size(), + self.obj.data(), + self.obj.null_mask(), + self.obj.null_count(), + self.obj.offset(), + self.obj.children(), + ) + if rep.id() != plc_dtype.id(): + plc_col = plc.unary.cast(plc_col, plc_dtype, stream=stream) + return Column(plc_col, dtype=dtype, name=self.name).sorted_like(self) elif plc.traits.is_floating_point( self.obj.type() ) and plc.traits.is_fixed_point(plc_dtype): diff --git a/python/cudf_polars/cudf_polars/dsl/expressions/datetime.py b/python/cudf_polars/cudf_polars/dsl/expressions/datetime.py index 4b6036e5c713..b6b582738188 100644 --- a/python/cudf_polars/cudf_polars/dsl/expressions/datetime.py +++ b/python/cudf_polars/cudf_polars/dsl/expressions/datetime.py @@ -28,6 +28,15 @@ __all__ = ["TemporalFunction"] +_unit_to_nanoseconds_conversion = { + plc.TypeId.DURATION_NANOSECONDS: 1, + plc.TypeId.DURATION_MICROSECONDS: 1_000, + plc.TypeId.DURATION_MILLISECONDS: 1_000_000, + plc.TypeId.DURATION_SECONDS: 1_000_000_000, + plc.TypeId.DURATION_DAYS: 86_400_000_000_000, +} + + class TemporalFunction(Expr): class Name(IntEnum): """Internal and picklable representation of polars' `TemporalFunction`.""" @@ -114,6 +123,16 @@ def from_polars(cls, obj: polars._expr_nodes.TemporalFunction) -> Self: "ns": plc.datetime.RoundingFrequency.NANOSECOND, } + # Number of nanoseconds represented by one unit of each ``total_*`` component. + _TOTAL_COMPONENT_NANOSECONDS: ClassVar[dict[Name, int]] = { + Name.TotalDays: 86_400_000_000_000, + Name.TotalHours: 3_600_000_000_000, + Name.TotalMinutes: 60_000_000_000, + Name.TotalSeconds: 1_000_000_000, + Name.TotalMilliseconds: 1_000_000, + Name.TotalMicroseconds: 1_000, + Name.TotalNanoseconds: 1, + } _valid_ops: ClassVar[set[Name]] = { *_COMPONENT_MAP.keys(), Name.IsLeapYear, @@ -126,6 +145,7 @@ def from_polars(cls, obj: polars._expr_nodes.TemporalFunction) -> Self: Name.TimeStamp, Name.CastTimeUnit, Name.Truncate, + *_TOTAL_COMPONENT_NANOSECONDS.keys(), } def __init__( @@ -159,6 +179,34 @@ def do_evaluate( ) -> Column: """Evaluate this expression given a dataframe for context.""" columns = [child.evaluate(df, context=context) for child in self.children] + if self.name in self._TOTAL_COMPONENT_NANOSECONDS: + (column,) = columns + source_ns = _unit_to_nanoseconds_conversion[column.obj.type().id()] + target_ns = self._TOTAL_COMPONENT_NANOSECONDS[self.name] + # Reinterpret the duration's integer tick count as int64. + casted = column.astype(self.dtype, stream=df.stream) + if source_ns >= target_ns: + # Coarser (or equal) storage unit: exact integer multiply. + op = plc.binaryop.BinaryOperator.MUL + factor = source_ns // target_ns + else: + # Finer storage unit: integer divide. libcudf (like polars) + # truncates toward zero for signed integer division. + op = plc.binaryop.BinaryOperator.DIV + factor = target_ns // source_ns + if factor == 1: + # Storage unit already matches the requested unit. + return casted + result = plc.binaryop.binary_operation( + casted.obj, + plc.Scalar.from_py( + factor, plc.DataType(plc.TypeId.INT64), stream=df.stream + ), + op, + self.dtype.plc_type, + stream=df.stream, + ) + return Column(result, dtype=self.dtype) if self.name is TemporalFunction.Name.TimeStamp: (column,) = columns (time_unit,) = self.options @@ -257,7 +305,6 @@ def do_evaluate( self.dtype.plc_type, stream=df.stream, ) - return Column(result, dtype=self.dtype) elif self.name is TemporalFunction.Name.MonthEnd: (column,) = columns diff --git a/python/cudf_polars/tests/containers/test_column.py b/python/cudf_polars/tests/containers/test_column.py index 674ae10edbab..19b249e2687e 100644 --- a/python/cudf_polars/tests/containers/test_column.py +++ b/python/cudf_polars/tests/containers/test_column.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations @@ -315,6 +315,24 @@ def test_astype_to_string(val, plc_tid, pl_type): assert result.dtype == target_dtype +def test_astype_duration_to_narrower_integer(): + stream = get_cuda_stream() + col = Column( + plc.unary.cast( + plc.Column.from_iterable_of_py( + [1, 2, -3], plc.DataType(plc.TypeId.INT64), stream=stream + ), + plc.DataType(plc.TypeId.DURATION_MICROSECONDS), + stream=stream, + ), + dtype=DataType(pl.Duration(time_unit="us")), + ) + target_dtype = DataType(pl.Int32()) + result = col.astype(target_dtype, stream=stream) + assert result.dtype == target_dtype + assert result.obj.type().id() == plc.TypeId.INT32 + + def test_astype_from_string_unsupported(): stream = get_cuda_stream() col = Column( diff --git a/python/cudf_polars/tests/expressions/test_datetime_basic.py b/python/cudf_polars/tests/expressions/test_datetime_basic.py index 01c876b57dde..26d4c01a4766 100644 --- a/python/cudf_polars/tests/expressions/test_datetime_basic.py +++ b/python/cudf_polars/tests/expressions/test_datetime_basic.py @@ -57,6 +57,16 @@ def test_datetime_dataframe_scan(engine: pl.GPUEngine, dtype): "nanosecond", ] +duration_extract_fields = [ + "total_seconds", + "total_milliseconds", + "total_microseconds", + "total_nanoseconds", + "total_days", + "total_hours", + "total_minutes", +] + @pytest.fixture( ids=datetime_extract_fields, @@ -171,6 +181,38 @@ def test_strftime_duration(engine: pl.GPUEngine, format): assert_ir_translation_raises(q, engine, NotImplementedError) +@pytest.mark.parametrize("field", duration_extract_fields) +@pytest.mark.parametrize( + "dtype", [pl.Duration("ms"), pl.Duration("us"), pl.Duration("ns")] +) +def test_duration_total_component_extract(engine: pl.GPUEngine, field, dtype): + ldf = pl.LazyFrame( + { + "durations": pl.Series( + [ + 0, + 1, + 15, + -1500, + 1000, + 1111, + 1500, + 11111, + -134234534, + 134234534, + # values beyond float64's exact-integer range to guard + # against precision loss in the unit conversion + 5857593848682946, + -5857593848682946, + ], + dtype=dtype, + ), + } + ) + q = ldf.select(getattr(pl.col("durations").dt, field)()) + assert_gpu_result_equal(q, engine=engine) + + @pytest.mark.parametrize( "dtype", [pl.Date(), pl.Datetime("ms"), pl.Datetime("us"), pl.Datetime("ns")] ) From 3c2eb2603045195f0dddb758514c4004a3096894 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Thu, 25 Jun 2026 21:44:18 -0500 Subject: [PATCH 06/22] Adjust verbosity of cudf-polars-polars-tests (#22980) We run upstream polars tests with cudf-polars using two of or engines: in-memory and SPMD. The recent CI failures running upstream polars tests have all been in the SPMD engine, while the in-memory engine has been passing fine. This change reduces the pytest verbosity for the in-memory run (making the logs easier to scan), and increases it for the SPMD run (hopefully helping with identifying the flaky tests). Authors: - Tom Augspurger (https://github.com/TomAugspurger) Approvers: - Gil Forsyth (https://github.com/gforsyth) - Matthew Murray (https://github.com/Matt711) URL: https://github.com/rapidsai/cudf/pull/22980 --- ci/run_cudf_polars_polars_tests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/run_cudf_polars_polars_tests.sh b/ci/run_cudf_polars_polars_tests.sh index 067f2ffc9f49..faccfadd48c6 100755 --- a/ci/run_cudf_polars_polars_tests.sh +++ b/ci/run_cudf_polars_polars_tests.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail @@ -61,7 +61,6 @@ DESELECTED_TESTS_STR=$(printf -- " --deselect %s" "${DESELECTED_TESTS[@]}") # shellcheck disable=SC2086 echo "Run polars tests with injected in-memory GPU engine" python -m pytest \ - -vv \ --import-mode=importlib \ --cache-clear \ -m "" \ @@ -84,6 +83,7 @@ CUDF_POLARS__EXECUTOR__FALLBACK_MODE=silent \ python -m pytest \ --import-mode=importlib \ --cache-clear \ + -v \ -m "" \ -p cudf_polars.testing.inject_gpu_engine \ -W ignore::ResourceWarning \ From e42c7c071e8a63c773226495146f20a1b926c6d3 Mon Sep 17 00:00:00 2001 From: Allen Xu Date: Fri, 26 Jun 2026 14:52:24 +0800 Subject: [PATCH 07/22] Omit Parquet min/max statistics for float/double columns containing NaN (#22818) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #22817. A `float`/`double` column containing a NaN mixed with non-NaN values was getting column-chunk min/max statistics computed from the non-NaN values only. The min/max reduce in `cpp/src/io/statistics/typed_statistics_chunk.cuh` uses `cuda::std::min/max` over `extrema_type::convert(val)` (which returns the value unchanged), and since `NaN < x` / `NaN > x` are both false a NaN never wins the reduce and is silently dropped — so the chunk ends up with a defined min/max that ignores the NaN. Per the Parquet convention adopted in parquet-mr (PARQUET-1246), a floating-point column that contains any NaN must omit min/max; otherwise a reader doing predicate pushdown (e.g. `col = NaN`, or a range predicate) uses the bogus bounds to skip the row group and silently drops valid rows. An all-NaN column was safe only incidentally — the merge's `has_minmax = (minimum_value <= maximum_value)` check flips it off because the reduce leaves the accumulator at inverted identity — but the mixed NaN + non-NaN case passes that check. Found via NVIDIA/spark-rapids#15004 (a GPU-written file read back by CPU Spark with a `= NaN` predicate-pushdown filter returned 0 rows instead of the matching NaN row). Affects any float/double leaf, top-level or nested in list/struct/map at any depth. ### Fix Track whether a NaN was seen during the float/double chunk reduce (`has_nan`), propagate it through `block_reduce` and the chunk merge, and force `has_minmax = false` for Parquet when it is set. ORC is unchanged: the guard is `if constexpr (IO == PARQUET)`, and the flag is only carried (never acted on) for ORC. Added `ParquetWriterTest.FloatingPointWithNaNStatsOmitted` (float / double / all-NaN must omit min/max; a no-NaN control still writes them). `ParquetWriterTest` (54 tests) and the `*Stats*` / `*ColumnIndex*` suites (45 tests) pass locally. Authors: - Allen Xu (https://github.com/wjxiz1992) Approvers: - Muhammad Haseeb (https://github.com/mhaseeb123) URL: https://github.com/rapidsai/cudf/pull/22818 --- cpp/src/io/functions.cpp | 2 +- .../parquet/experimental/deletion_vectors.cu | 2 +- cpp/src/io/parquet/reader_impl_helpers.cpp | 2 +- cpp/src/io/statistics/column_statistics.cuh | 8 +- cpp/src/io/statistics/statistics.cuh | 3 +- .../statistics_type_identification.cuh | 9 +- .../io/statistics/typed_statistics_chunk.cuh | 12 +- cpp/src/utilities/host_memory.cpp | 3 +- cpp/tests/io/io_test_utils.hpp | 2 +- cpp/tests/io/parquet_chunked_reader_test.cu | 2 +- cpp/tests/io/parquet_writer_test.cpp | 103 +++++++++++++++++- 11 files changed, 135 insertions(+), 13 deletions(-) diff --git a/cpp/src/io/functions.cpp b/cpp/src/io/functions.cpp index 2db331184395..55d1a750bc06 100644 --- a/cpp/src/io/functions.cpp +++ b/cpp/src/io/functions.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/io/parquet/experimental/deletion_vectors.cu b/cpp/src/io/parquet/experimental/deletion_vectors.cu index a04d1767a854..1d5ada49c69a 100644 --- a/cpp/src/io/parquet/experimental/deletion_vectors.cu +++ b/cpp/src/io/parquet/experimental/deletion_vectors.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/io/parquet/reader_impl_helpers.cpp b/cpp/src/io/parquet/reader_impl_helpers.cpp index a571d89ce4fc..667449fdb9ae 100644 --- a/cpp/src/io/parquet/reader_impl_helpers.cpp +++ b/cpp/src/io/parquet/reader_impl_helpers.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/src/io/statistics/column_statistics.cuh b/cpp/src/io/statistics/column_statistics.cuh index 81964cf2eabe..27d886f832d4 100644 --- a/cpp/src/io/statistics/column_statistics.cuh +++ b/cpp/src/io/statistics/column_statistics.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -202,6 +202,12 @@ struct merge_group_statistics_functor { chunk = block_reduce(chunk, storage); + // PARQUET-1246: if a float/double column contains any NaN, min/max must be omitted, + // else a reader doing NaN predicate pushdown skips the row group. spark-rapids#15004. + if constexpr (IO == detail::io_file_format::PARQUET) { + if (chunk.has_nan) { chunk.has_minmax = false; } + } + if (t == 0) { s.ck = get_untyped_chunk(chunk); } } diff --git a/cpp/src/io/statistics/statistics.cuh b/cpp/src/io/statistics/statistics.cuh index 57854d10f87b..1897cf272a2c 100644 --- a/cpp/src/io/statistics/statistics.cuh +++ b/cpp/src/io/statistics/statistics.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -88,6 +88,7 @@ struct statistics_chunk { statistics_val sum{}; //!< sum of chunk uint8_t has_minmax{}; //!< Nonzero if min_value and max_values are valid uint8_t has_sum{}; //!< Nonzero if sum is valid + uint8_t has_nan{}; //!< Nonzero if a NaN was seen (floating point only) }; struct statistics_group { diff --git a/cpp/src/io/statistics/statistics_type_identification.cuh b/cpp/src/io/statistics/statistics_type_identification.cuh index ed7812419afe..4324e9eb4ec8 100644 --- a/cpp/src/io/statistics/statistics_type_identification.cuh +++ b/cpp/src/io/statistics/statistics_type_identification.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -19,6 +19,9 @@ #include #include +#include +#include + #include namespace cudf { @@ -206,8 +209,8 @@ class aggregation_type { return val.size_bytes(); } else if constexpr (std::is_integral_v) { return val; - } else if constexpr (std::is_floating_point_v) { - return isnan(val) ? 0 : val; + } else if constexpr (cuda::std::is_floating_point_v) { + return cuda::std::isnan(val) ? 0 : val; } else if constexpr (cudf::is_fixed_point()) { return val.value(); } else if constexpr (cudf::is_duration()) { diff --git a/cpp/src/io/statistics/typed_statistics_chunk.cuh b/cpp/src/io/statistics/typed_statistics_chunk.cuh index a6125c42a78b..dd0748486ffe 100644 --- a/cpp/src/io/statistics/typed_statistics_chunk.cuh +++ b/cpp/src/io/statistics/typed_statistics_chunk.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2021-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -20,8 +20,10 @@ #include #include +#include #include #include +#include #include namespace cudf { @@ -112,6 +114,7 @@ struct typed_statistics_chunk { uint8_t has_minmax{false}; //!< Nonzero if min_value and max_values are valid uint8_t has_sum{false}; //!< Nonzero if sum is valid + uint8_t has_nan{false}; //!< Nonzero if a NaN was seen (floating point only) __device__ typed_statistics_chunk() : minimum_value(detail::minimum_identity()), @@ -123,6 +126,7 @@ struct typed_statistics_chunk { __device__ void reduce(T const& elem) { non_nulls++; + if constexpr (cuda::std::is_floating_point_v) { has_nan |= cuda::std::isnan(elem); } minimum_value = cuda::std::min(minimum_value, detail::extrema_type::convert(elem)); maximum_value = cuda::std::max(maximum_value, detail::extrema_type::convert(elem)); aggregate += detail::aggregation_type::convert(elem); @@ -138,6 +142,7 @@ struct typed_statistics_chunk { if (chunk.has_sum) { aggregate += union_member::get(chunk.sum); } non_nulls += chunk.non_nulls; null_count += chunk.null_count; + has_nan |= chunk.has_nan; } }; @@ -153,6 +158,7 @@ struct typed_statistics_chunk { uint8_t has_minmax{false}; //!< Nonzero if min_value and max_values are valid uint8_t has_sum{false}; //!< Nonzero if sum is valid + uint8_t has_nan{false}; //!< Nonzero if a NaN was seen (floating point only) __device__ typed_statistics_chunk() : minimum_value(detail::minimum_identity()), maximum_value(detail::maximum_identity()) @@ -162,6 +168,7 @@ struct typed_statistics_chunk { __device__ void reduce(T const& elem) { non_nulls++; + if constexpr (cuda::std::is_floating_point_v) { has_nan |= cuda::std::isnan(elem); } minimum_value = cuda::std::min(minimum_value, detail::extrema_type::convert(elem)); maximum_value = cuda::std::max(maximum_value, detail::extrema_type::convert(elem)); has_minmax = true; @@ -175,6 +182,7 @@ struct typed_statistics_chunk { } non_nulls += chunk.non_nulls; null_count += chunk.null_count; + has_nan |= chunk.has_nan; } }; @@ -209,6 +217,7 @@ __inline__ __device__ typed_statistics_chunk block_reduce( count_reduce(storage.template get()).Sum(output_chunk.null_count); __syncthreads(); output_chunk.has_minmax = __syncthreads_or(output_chunk.has_minmax); + output_chunk.has_nan = __syncthreads_or(output_chunk.has_nan); // FIXME : Is another syncthreads needed here? if constexpr (include_aggregate) { @@ -237,6 +246,7 @@ get_untyped_chunk(typed_statistics_chunk const& chunk) stat.non_nulls = chunk.non_nulls; stat.null_count = chunk.null_count; stat.has_minmax = chunk.has_minmax; + stat.has_nan = chunk.has_nan; stat.has_sum = [&]() { // invalidate the sum if overflow or underflow is possible if constexpr (std::is_floating_point_v or std::is_integral_v) { diff --git a/cpp/src/utilities/host_memory.cpp b/cpp/src/utilities/host_memory.cpp index 76bfca2ebcad..3d6e56471c50 100644 --- a/cpp/src/utilities/host_memory.cpp +++ b/cpp/src/utilities/host_memory.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -11,6 +11,7 @@ #include #include +#include #include #include #include diff --git a/cpp/tests/io/io_test_utils.hpp b/cpp/tests/io/io_test_utils.hpp index c51246353e5a..9fb6c64bb108 100644 --- a/cpp/tests/io/io_test_utils.hpp +++ b/cpp/tests/io/io_test_utils.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/tests/io/parquet_chunked_reader_test.cu b/cpp/tests/io/parquet_chunked_reader_test.cu index 3f8b8708d4c4..e36d14fd531c 100644 --- a/cpp/tests/io/parquet_chunked_reader_test.cu +++ b/cpp/tests/io/parquet_chunked_reader_test.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/cpp/tests/io/parquet_writer_test.cpp b/cpp/tests/io/parquet_writer_test.cpp index fd285daabe60..bcd79595049c 100644 --- a/cpp/tests/io/parquet_writer_test.cpp +++ b/cpp/tests/io/parquet_writer_test.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -853,6 +853,107 @@ TEST_F(ParquetWriterTest, Decimal128Stats) EXPECT_EQ(expected_max, stats.max_value); } +TEST_F(ParquetWriterTest, FloatingPointWithNaNStatsOmitted) +{ + // PARQUET-1246: a float/double column containing a NaN must not expose min/max, or a + // reader doing `= NaN` predicate pushdown skips the row group. NVIDIA/spark-rapids#15004. + auto constexpr nanf = std::numeric_limits::quiet_NaN(); + auto constexpr nand = std::numeric_limits::quiet_NaN(); + + column_wrapper col_f_nan{{1.0f, nanf, 3.0f, 2.0f}}; // NaN mixed with non-NaN + column_wrapper col_d_nan{{1.0, 2.0, nand, 4.0}}; // double variant + column_wrapper col_f_allnan{{nanf, nanf, nanf, nanf}}; // all NaN + column_wrapper col_f_nonan{{1.0f, 2.0f, 3.0f, 4.0f}}; // control: no NaN + column_wrapper col_f_nan_null{{1.0f, nanf, 3.0f, 5.0f}, + {true, true, true, false}}; // NaN alongside a null + + auto const expected = + table_view{{col_f_nan, col_d_nan, col_f_allnan, col_f_nonan, col_f_nan_null}}; + + auto const filepath = temp_env->get_temp_filepath("FloatingPointWithNaNStats.parquet"); + cudf::io::parquet_writer_options const out_opts = + cudf::io::parquet_writer_options::builder(cudf::io::sink_info{filepath}, expected); + cudf::io::write_parquet(out_opts); + + auto const source = cudf::io::datasource::create(filepath); + cudf::io::parquet::FileMetaData fmd; + read_footer(source, &fmd); + + auto const stats_f_nan = get_statistics(fmd.row_groups[0].columns[0]); + auto const stats_d_nan = get_statistics(fmd.row_groups[0].columns[1]); + auto const stats_f_allnan = get_statistics(fmd.row_groups[0].columns[2]); + auto const stats_f_nonan = get_statistics(fmd.row_groups[0].columns[3]); + auto const stats_f_nan_null = get_statistics(fmd.row_groups[0].columns[4]); + + // any column containing a NaN must not expose min/max + EXPECT_FALSE(stats_f_nan.min_value.has_value()); + EXPECT_FALSE(stats_f_nan.max_value.has_value()); + EXPECT_FALSE(stats_d_nan.min_value.has_value()); + EXPECT_FALSE(stats_d_nan.max_value.has_value()); + EXPECT_FALSE(stats_f_allnan.min_value.has_value()); + EXPECT_FALSE(stats_f_allnan.max_value.has_value()); + + // a column with no NaN is unaffected and still carries min/max + EXPECT_TRUE(stats_f_nonan.min_value.has_value()); + EXPECT_TRUE(stats_f_nonan.max_value.has_value()); + + // a null alongside the NaN does not interfere with NaN detection + EXPECT_FALSE(stats_f_nan_null.min_value.has_value()); + EXPECT_FALSE(stats_f_nan_null.max_value.has_value()); +} + +TEST_F(ParquetWriterTest, FloatingPointWithNaNStatsOmittedAcrossFragments) +{ + // A NaN in any page fragment must propagate through the fragment -> column-chunk statistics + // merge, so a multi-fragment column chunk with a single NaN still omits min/max. + // NVIDIA/spark-rapids#15004. + auto constexpr nanf = std::numeric_limits::quiet_NaN(); + auto constexpr num_rows = 20000; // > default 5000-row page fragment -> multiple fragments merged + std::vector data(num_rows); + for (int i = 0; i < num_rows; ++i) { + data[i] = static_cast(i); + } + data[num_rows / 2] = nanf; // a single NaN in a middle fragment + column_wrapper col(data.begin(), data.end()); + auto const expected = table_view{{col}}; + + auto const filepath = temp_env->get_temp_filepath("FloatingPointNaNStatsFragments.parquet"); + cudf::io::parquet_writer_options const out_opts = + cudf::io::parquet_writer_options::builder(cudf::io::sink_info{filepath}, expected); + cudf::io::write_parquet(out_opts); + + auto const source = cudf::io::datasource::create(filepath); + cudf::io::parquet::FileMetaData fmd; + read_footer(source, &fmd); + + ASSERT_EQ(fmd.row_groups.size(), 1); + auto const stats = get_statistics(fmd.row_groups[0].columns[0]); + EXPECT_FALSE(stats.min_value.has_value()); + EXPECT_FALSE(stats.max_value.has_value()); +} + +TEST_F(ParquetWriterTest, FloatingPointWithNaNStatsOmittedNested) +{ + // NaN detection must reach a float leaf nested in a LIST column (rapidsai/cudf#22817). + auto constexpr nanf = std::numeric_limits::quiet_NaN(); + cudf::test::lists_column_wrapper list_col{{1.0f, nanf, 3.0f}, {4.0f, 5.0f}}; + auto const expected = table_view{{list_col}}; + + auto const filepath = temp_env->get_temp_filepath("FloatingPointNaNStatsNested.parquet"); + cudf::io::parquet_writer_options const out_opts = + cudf::io::parquet_writer_options::builder(cudf::io::sink_info{filepath}, expected); + cudf::io::write_parquet(out_opts); + + auto const source = cudf::io::datasource::create(filepath); + cudf::io::parquet::FileMetaData fmd; + read_footer(source, &fmd); + + // the leaf float column (list element) contains a NaN -> min/max omitted + auto const stats = get_statistics(fmd.row_groups[0].columns[0]); + EXPECT_FALSE(stats.min_value.has_value()); + EXPECT_FALSE(stats.max_value.has_value()); +} + TEST_F(ParquetWriterTest, CheckColumnIndexTruncation) { std::array coldata{// in-range 7 bit. should truncate to "yyyyyyyz" From 12f9cda6f73cbed96009d48869ebc866f6246dc8 Mon Sep 17 00:00:00 2001 From: Matthew Murray <41342305+Matt711@users.noreply.github.com> Date: Fri, 26 Jun 2026 08:44:27 -0400 Subject: [PATCH 08/22] Add a I/O partition planning information to benchmark runner (#22945) Table: `--explain-partition-plan` ``` Partition Plan Summary +----+----------+--------+--------------+-------+------------------------+-----------+------------+ | Q | Table | Flavor | Factor | Files | Projected (bytes/file) | Size/task | Partitions | +----+----------+--------+--------------+-------+------------------------+-----------+------------+ | 1 | lineitem | SPLIT | 3 tasks/file | 60 | 4.35 GB | 1.45 GB | 180 | | 5 | region | FUSED | 1 file/task | 1 | 114 B | 114 B | 1 | | | nation | FUSED | 1 file/task | 1 | 329 B | 329 B | 1 | | | customer | FUSED | 1 file/task | 2 | 900 MB | 900 MB | 2 | | | orders | FUSED | 1 file/task | 15 | 1.38 GB | 1.38 GB | 15 | | | lineitem | SPLIT | 2 tasks/file | 60 | 2.71 GB | 1.36 GB | 120 | | | supplier | FUSED | 1 file/task | 1 | 120 MB | 120 MB | 1 | | 18 | orders | SPLIT | 2 tasks/file | 15 | 2.18 GB | 1.09 GB | 30 | | | lineitem | SPLIT | 2 tasks/file | 60 | 1.6 GB | 800.1 MB | 120 | | | customer | SPLIT | 2 tasks/file | 2 | 1.95 GB | 976.54 MB | 4 | | 21 | lineitem | FUSED | 1 file/task | 60 | 1.11 GB | 1.11 GB | 60 | | | lineitem | FUSED | 1 file/task | 60 | 1.42 GB | 1.42 GB | 60 | | | supplier | FUSED | 1 file/task | 1 | 300.41 MB | 300.41 MB | 1 | | | nation | FUSED | 1 file/task | 1 | 259 B | 259 B | 1 | | | orders | FUSED | 1 file/task | 15 | 1.3 GB | 1.3 GB | 15 | +----+----------+--------+--------------+-------+------------------------+-----------+------------+ ``` Explain output: `CUDF_POLARS__EXPLAIN__PARTITION_PLAN=1 and --explain` ``` SORT ('l_returnflag', 'l_linestatus') ('l_returnflag', 'l_linestatus', 'sum_qty', '...', 'avg_disc', 'count_order') [180] SELECT ('l_returnflag', 'l_linestatus', 'sum_qty', '...', 'avg_disc', 'count_order') [180] GROUPBY ('l_returnflag', 'l_linestatus') ('l_returnflag', 'l_linestatus', 'sum_qty', '...', 'avg_disc', '______________11') [180] HSTACK ('l_returnflag', 'l_linestatus', 'l_quantity', '...', 'l_tax', '__POLARS_CSER_0x6a4988e5b78df9ee') [180] PROJECTION ('l_returnflag', 'l_linestatus', 'l_quantity', 'l_extendedprice', 'l_discount', 'l_tax') [180] STREAMINGSCAN ('l_returnflag', 'l_linestatus', 'l_quantity', '...', 'l_tax', 'l_shipdate') [flavor=SPLIT_FILES factor=3 projected=4.35 GB] [180] ``` Authors: - Matthew Murray (https://github.com/Matt711) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) - Mads R. B. Kristensen (https://github.com/madsbk) URL: https://github.com/rapidsai/cudf/pull/22945 --- .../cudf_polars/streaming/benchmarks/utils.py | 27 ++- .../cudf_polars/streaming/explain.py | 200 +++++++++++++++++- .../cudf_polars/cudf_polars/streaming/io.py | 4 +- .../tests/streaming/test_explain.py | 170 +++++++++++++++ 4 files changed, 395 insertions(+), 6 deletions(-) diff --git a/python/cudf_polars/cudf_polars/streaming/benchmarks/utils.py b/python/cudf_polars/cudf_polars/streaming/benchmarks/utils.py index 8b83b31ed107..42e1722bbc89 100644 --- a/python/cudf_polars/cudf_polars/streaming/benchmarks/utils.py +++ b/python/cudf_polars/cudf_polars/streaming/benchmarks/utils.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 """Benchmark utilities for the RapidsMPF SPMD and Ray frontends.""" @@ -263,6 +263,7 @@ class QueryRunResult: plan: SerializablePlan | None iteration_failures: list[tuple[int, int]] validation_failed: bool + partition_plan_rows: list = dataclasses.field(default_factory=list) @dataclasses.dataclass @@ -991,6 +992,16 @@ def run_polars_query( plan = serialize_query(q, engine) + part_plan_rows = [] + if ( + getattr(args, "explain_partition_plan", False) + and engine is not None + and run_config.frontend in _STREAMING_FRONTENDS + ): + from cudf_polars.streaming.explain import collect_partition_plan + + part_plan_rows = collect_partition_plan(q, engine, q_id) + casts = benchmark.EXPECTED_CASTS.get(q_id, []) if numeric_type == "decimal": casts.extend(benchmark.EXPECTED_CASTS_DECIMAL.get(q_id, [])) @@ -1086,6 +1097,7 @@ def run_polars_query( plan=plan, iteration_failures=iteration_failures, validation_failed=validation_failed, + partition_plan_rows=part_plan_rows, ) @@ -1108,6 +1120,7 @@ def _run_query_loop( plans: dict[int, SerializablePlan] = {} validation_failures: list[int] = [] query_failures: list[tuple[int, int]] = [] + all_partition_plan_rows: list = [] for q_id in run_config.queries: try: @@ -1143,6 +1156,12 @@ def _run_query_loop( query_failures.extend(result.iteration_failures) if result.validation_failed: validation_failures.append(q_id) + all_partition_plan_rows.extend(result.partition_plan_rows) + + if all_partition_plan_rows and getattr(args, "explain_partition_plan", False): + from cudf_polars.streaming.explain import format_partition_plan_table + + print(format_partition_plan_table(all_partition_plan_rows), flush=True) return records, plans, validation_failures, query_failures @@ -1945,6 +1964,12 @@ def build_parser(num_queries: int = 22) -> argparse.ArgumentParser: help="Print an outline of the logical plan.", default=False, ) + parser.add_argument( + "--explain-partition-plan", + action=argparse.BooleanOptionalAction, + help="Print a combined partition plan summary table across all queries.", + default=False, + ) parser.add_argument( "--print-plans", action=argparse.BooleanOptionalAction, diff --git a/python/cudf_polars/cudf_polars/streaming/explain.py b/python/cudf_polars/cudf_polars/streaming/explain.py index f0680586d8fa..3f6a6dff2869 100644 --- a/python/cudf_polars/cudf_polars/streaming/explain.py +++ b/python/cudf_polars/cudf_polars/streaming/explain.py @@ -9,9 +9,11 @@ import dataclasses import datetime import functools +import os import os.path from collections.abc import Mapping, Sequence from itertools import groupby +from pathlib import Path from typing import TYPE_CHECKING, Any, Self, TypeAlias import pylibcudf as plc @@ -33,7 +35,8 @@ ) from cudf_polars.dsl.translate import Translator from cudf_polars.dsl.traversal import traversal -from cudf_polars.streaming.io import StreamingScan +from cudf_polars.streaming.base import IOPartitionFlavor +from cudf_polars.streaming.io import StreamingScan, scan_partition_plan from cudf_polars.streaming.parallel import lower_ir_graph from cudf_polars.streaming.shuffle import Shuffle from cudf_polars.streaming.statistics import ( @@ -51,6 +54,20 @@ from cudf_polars.streaming.base import PartitionInfo, StatsCollector +@dataclasses.dataclass +class PartitionPlanRow: + """One row of the partition plan summary table.""" + + query: int + table: str + flavor: IOPartitionFlavor + factor: int + files: int + projected_bytes: int + task_bytes: int + partitions: int + + Serializable: TypeAlias = ( str | int @@ -106,7 +123,7 @@ def explain_query( with cm: stats = collect_statistics(ir, config, executor) lowered_ir, partition_info = lower_ir_graph(ir, config, stats) - return _repr_ir_tree(lowered_ir, partition_info) + return _repr_ir_tree(lowered_ir, partition_info, stats=stats, config=config) else: if config.executor.name == "streaming": # Include row-count statistics for the logical plan @@ -117,6 +134,162 @@ def explain_query( return _repr_ir_tree(ir) +def collect_partition_plan( + q: pl.LazyFrame, + engine: pl.GPUEngine, + q_id: int, +) -> list[PartitionPlanRow]: + """ + Return one PartitionPlanRow per unique StreamingScan in the physical plan. + + Deduplicates scans that appear multiple times due to subquery structure. + """ + config = ConfigOptions.from_polars_engine(engine) + ir = Translator(q._ldf.visit(), engine).translate_ir() + + with concurrent.futures.ThreadPoolExecutor() as executor: + stats = collect_statistics(ir, config, executor) + lowered_ir, partition_info = lower_ir_graph(ir, config, stats) + + seen: set[tuple] = set() + rows: list[PartitionPlanRow] = [] + + for node in traversal([lowered_ir]): + if not isinstance(node, StreamingScan): + continue + base_scan = node.base_scan + + dedup_key = (tuple(base_scan.paths), tuple(sorted(base_scan.schema.keys()))) + if dedup_key in seen: + continue + seen.add(dedup_key) + + source = stats.scan_stats.get(base_scan) + if source is None: + continue + + plan = scan_partition_plan(base_scan, stats, config) + projected_bytes = sum( + sz + for col in base_scan.schema + if (sz := source.column_storage_size(col)) is not None + ) + partitions = partition_info[node].count + factor = plan.factor + flavor = plan.flavor + + match flavor: + case IOPartitionFlavor.SPLIT_FILES: + files = partitions // factor if factor > 0 else partitions + task_bytes = ( + projected_bytes // factor if factor > 0 else projected_bytes + ) + case IOPartitionFlavor.FUSED_FILES: + files = partitions * factor + task_bytes = projected_bytes * factor + case _: + files = partitions + task_bytes = projected_bytes + + p = Path(base_scan.paths[0]) + stem = p.stem + parent = p.parent.name + # Prefer the stem unless it looks like a partition filename (purely + # numeric like "1" or prefixed like "part-0"), in which case the + # parent directory holds the table name. + table = parent if (stem.isdigit() or stem.lower().startswith("part")) else stem + + rows.append( + PartitionPlanRow( + query=q_id, + table=table, + flavor=flavor, + factor=factor, + files=files, + projected_bytes=projected_bytes, + task_bytes=task_bytes, + partitions=partitions, + ) + ) + + return rows + + +def _fmt_partition_bytes(b: int) -> str: + if b < 1_000: + return f"{b} B" + elif b < 1_000_000: + return f"{round(b / 1_000, 2):g} KB" + elif b < 1_000_000_000: + return f"{round(b / 1_000_000, 2):g} MB" + else: + return f"{round(b / 1_000_000_000, 2):g} GB" + + +def factor_str(row: PartitionPlanRow) -> str: + """Format the factor field with units appropriate to the scan flavor.""" + match row.flavor: + case IOPartitionFlavor.SPLIT_FILES: + return f"{row.factor} tasks/file" + case IOPartitionFlavor.FUSED_FILES: + unit = "file" if row.factor == 1 else "files" + return f"{row.factor} {unit}/task" + case _: + return str(row.factor) + + +def format_partition_plan_table(rows: list[PartitionPlanRow]) -> str: + """Format a list of PartitionPlanRows as a fixed-width ASCII table.""" + if not rows: + return "" + + headers = [ + "Q", + "Table", + "Flavor", + "Factor", + "Files", + "Projected (bytes/file)", + "Size/task", + "Partitions", + ] + + formatted: list[list[str]] = [] + prev_q: int | None = None + for row in rows: + q_str = str(row.query) if row.query != prev_q else "" + prev_q = row.query + formatted.append( + [ + q_str, + row.table, + row.flavor.name, + factor_str(row), + str(row.files), + _fmt_partition_bytes(row.projected_bytes), + _fmt_partition_bytes(row.task_bytes), + str(row.partitions), + ] + ) + + col_widths = [len(h) for h in headers] + for cells in formatted: + for i, cell in enumerate(cells): + col_widths[i] = max(col_widths[i], len(cell)) + + sep = "+-" + "-+-".join("-" * w for w in col_widths) + "-+" + header_row = ( + "| " + " | ".join(h.ljust(col_widths[i]) for i, h in enumerate(headers)) + " |" + ) + lines = ["", "Partition Plan Summary", sep, header_row, sep] + lines.extend( + "| " + " | ".join(c.ljust(col_widths[i]) for i, c in enumerate(cells)) + " |" + for cells in formatted + ) + lines.append(sep) + return "\n".join(lines) + + def serialize_query( q: pl.LazyFrame, engine: pl.GPUEngine, @@ -207,6 +380,7 @@ def _repr_ir_tree( *, offset: str = "", stats: StatsCollector | None = None, + config: ConfigOptions | None = None, ) -> str: header = _repr_ir(ir, offset=offset) count = partition_info[ir].count if partition_info else None @@ -215,11 +389,31 @@ def _repr_ir_tree( row_count_estimate = _fmt_row_count(source.row_count) row_count = f"~{row_count_estimate}" if row_count_estimate else "unknown" header = header.rstrip("\n") + f" {row_count=}\n" + if ( + os.environ.get("CUDF_POLARS__EXPLAIN__PARTITION_PLAN", "0") == "1" + and config is not None + and stats is not None + and isinstance(ir, StreamingScan) + and (source := stats.scan_stats.get(ir.base_scan)) is not None + ): + plan = scan_partition_plan(ir.base_scan, stats, config) + projected_size = sum( + sz + for col in ir.base_scan.schema + if (sz := source.column_storage_size(col)) is not None + ) + plan_info = ( + f"flavor={plan.flavor.name} factor={plan.factor}" + f" projected={_fmt_partition_bytes(projected_size)}" + ) + header = header.rstrip("\n") + f" [{plan_info}]\n" if count is not None: header = header.rstrip("\n") + f" [{count}]\n" children_strs = [ - _repr_ir_tree(child, partition_info, offset=offset + " ", stats=stats) + _repr_ir_tree( + child, partition_info, offset=offset + " ", stats=stats, config=config + ) for child in ir.children ] diff --git a/python/cudf_polars/cudf_polars/streaming/io.py b/python/cudf_polars/cudf_polars/streaming/io.py index 075d6c840d94..315e294a21b7 100644 --- a/python/cudf_polars/cudf_polars/streaming/io.py +++ b/python/cudf_polars/cudf_polars/streaming/io.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 """Multi-partition IO Logic.""" @@ -98,7 +98,7 @@ def scan_partition_plan( ) else: # Fuse small files - factor = max(blocksize // int(file_size), 1) + factor = min(max(blocksize // int(file_size), 1), len(ir.paths)) return IOPartitionPlan( factor, IOPartitionFlavor.FUSED_FILES, diff --git a/python/cudf_polars/tests/streaming/test_explain.py b/python/cudf_polars/tests/streaming/test_explain.py index 41e0177dad10..5ea6be578ae4 100644 --- a/python/cudf_polars/tests/streaming/test_explain.py +++ b/python/cudf_polars/tests/streaming/test_explain.py @@ -19,10 +19,16 @@ from cudf_polars.dsl.expressions.base import Col from cudf_polars.dsl.expressions.binaryop import BinOp from cudf_polars.engine.options import StreamingOptions +from cudf_polars.streaming.base import IOPartitionFlavor from cudf_polars.streaming.explain import ( + PartitionPlanRow, + _fmt_partition_bytes, _fmt_row_count, _predicate_to_str, + collect_partition_plan, explain_query, + factor_str, + format_partition_plan_table, serialize_query, ) from cudf_polars.testing.asserts import assert_gpu_result_equal @@ -750,3 +756,167 @@ def test_dynamic_planning_adds_repartition(df, op): assert "REPARTITION" not in plan else: assert "REPARTITION" in plan + + +def test_collect_partition_plan_fused(tmp_path, df): + """Small files with a large target_partition_size → FUSED_FILES.""" + make_partitioned_source(df, tmp_path, fmt="parquet", n_files=4) + q = pl.scan_parquet(tmp_path) + engine = pl.GPUEngine( + executor="streaming", + raise_on_fail=True, + executor_options={"target_partition_size": 100_000_000}, + ) + rows = collect_partition_plan(q, engine, q_id=1) + assert len(rows) == 1 + row = rows[0] + assert row.query == 1 + assert row.flavor == IOPartitionFlavor.FUSED_FILES + assert row.factor >= 1 + assert row.files == row.partitions * row.factor + assert row.partitions > 0 + assert row.projected_bytes > 0 + assert row.task_bytes == row.projected_bytes * row.factor + + +def test_collect_partition_plan_split(tmp_path, df): + """Very small target_partition_size → SPLIT_FILES.""" + make_partitioned_source(df, tmp_path, fmt="parquet", n_files=1) + q = pl.scan_parquet(tmp_path) + engine = pl.GPUEngine( + executor="streaming", + raise_on_fail=True, + executor_options={"target_partition_size": 1}, + ) + rows = collect_partition_plan(q, engine, q_id=7) + assert len(rows) == 1 + row = rows[0] + assert row.query == 7 + assert row.flavor == IOPartitionFlavor.SPLIT_FILES + assert row.factor > 1 + assert row.files == row.partitions // row.factor + assert row.task_bytes == row.projected_bytes // row.factor + + +def test_collect_partition_plan_table_name_stem(tmp_path, df): + """Single named file: table name is taken from the file stem.""" + single_file = tmp_path / "lineitem.parquet" + df.write_parquet(single_file) + q = pl.scan_parquet(single_file) + engine = pl.GPUEngine(executor="streaming", raise_on_fail=True) + rows = collect_partition_plan(q, engine, q_id=1) + assert len(rows) == 1 + assert rows[0].table == "lineitem" + + +def test_collect_partition_plan_table_name_parent(tmp_path, df): + """Partitioned directory: table name is taken from the parent directory.""" + (tmp_path / "orders").mkdir() + make_partitioned_source(df, tmp_path / "orders", fmt="parquet", n_files=3) + q = pl.scan_parquet(tmp_path / "orders") + engine = pl.GPUEngine(executor="streaming", raise_on_fail=True) + rows = collect_partition_plan(q, engine, q_id=1) + assert len(rows) == 1 + assert rows[0].table == "orders" + + +def test_collect_partition_plan_deduplicates(tmp_path, df): + """A scan used twice in a join should produce only one PartitionPlanRow.""" + make_partitioned_source(df, tmp_path, fmt="parquet", n_files=2) + q = pl.scan_parquet(tmp_path) + q = q.join(q, on="x", how="inner") + engine = pl.GPUEngine(executor="streaming", raise_on_fail=True) + rows = collect_partition_plan(q, engine, q_id=1) + assert len(rows) == 1 + + +_SAMPLE_ROWS = [ + PartitionPlanRow( + query=1, + table="lineitem", + flavor=IOPartitionFlavor.SPLIT_FILES, + factor=2, + files=120, + projected_bytes=2_000_000_000, + task_bytes=1_000_000_000, + partitions=240, + ), + PartitionPlanRow( + query=1, + table="orders", + flavor=IOPartitionFlavor.FUSED_FILES, + factor=1, + files=60, + projected_bytes=500_000_000, + task_bytes=500_000_000, + partitions=60, + ), + PartitionPlanRow( + query=2, + table="lineitem", + flavor=IOPartitionFlavor.FUSED_FILES, + factor=1, + files=60, + projected_bytes=1_400_000_000, + task_bytes=1_400_000_000, + partitions=60, + ), +] + + +def test_format_partition_plan_table_empty(): + assert format_partition_plan_table([]) == "" + + +def test_format_partition_plan_table_content(): + table = format_partition_plan_table(_SAMPLE_ROWS) + assert "Partition Plan Summary" in table + assert "lineitem" in table + assert "orders" in table + assert "SPLIT" in table + assert "FUSED" in table + + +def test_format_partition_plan_table_query_shown_once(): + """Each query number should appear on exactly one data row (Q column only).""" + table = format_partition_plan_table(_SAMPLE_ROWS) + # The Q column is the first column; its cell is "| 1 |" for query=1 and "| |" for + # repeated rows. Count occurrences of the literal "| 1 |" at line start. + q1_lines = [line for line in table.splitlines() if line.startswith("| 1 |")] + assert len(q1_lines) == 1 + + +@pytest.mark.parametrize( + "b,expected", + [ + (500, "500 B"), + (1_500, "1.5 KB"), + (2_500_000, "2.5 MB"), + (1_500_000_000, "1.5 GB"), + ], +) +def test_fmt_partition_bytes(b, expected): + assert _fmt_partition_bytes(b) == expected + + +@pytest.mark.parametrize( + "flavor,factor,expected", + [ + (IOPartitionFlavor.SPLIT_FILES, 3, "3 tasks/file"), + (IOPartitionFlavor.FUSED_FILES, 1, "1 file/task"), + (IOPartitionFlavor.FUSED_FILES, 4, "4 files/task"), + (IOPartitionFlavor.SINGLE_FILE, 1, "1"), + ], +) +def test_factor_str(flavor, factor, expected): + row = PartitionPlanRow( + query=1, + table="t", + flavor=flavor, + factor=factor, + files=1, + projected_bytes=1, + task_bytes=1, + partitions=1, + ) + assert factor_str(row) == expected From 7f6473e93a7b2f6e232ea5411c66671ce2ef4cfe Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Fri, 26 Jun 2026 13:29:47 +0000 Subject: [PATCH 09/22] Add additional regex gtests for contains, count, findall, and replace (#22874) Adds some additional gtests for regex patterns to help validate optimizations in follow on PRs. This is part of splitting out some of the work for #21936 Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) - Basit Ayantunde (https://github.com/lamarrr) URL: https://github.com/rapidsai/cudf/pull/22874 --- cpp/tests/strings/contains_tests.cpp | 126 ++++++++++++++++++++++ cpp/tests/strings/findall_tests.cpp | 16 +++ cpp/tests/strings/replace_regex_tests.cpp | 27 +++++ 3 files changed, 169 insertions(+) diff --git a/cpp/tests/strings/contains_tests.cpp b/cpp/tests/strings/contains_tests.cpp index bb4111adb244..89ed9756d00a 100644 --- a/cpp/tests/strings/contains_tests.cpp +++ b/cpp/tests/strings/contains_tests.cpp @@ -1141,3 +1141,129 @@ TEST_F(StringsContainsTests, CrlfDefaultLfOnlyNoExtNewline) cudf::test::fixed_width_column_wrapper({0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*cudf::strings::contains_re(view, *prog), expected); } + +TEST_F(StringsContainsTests, AlternationNullableBranch) +{ + // "a(bc|de|fg|)h" has an explicit empty branch that makes 'h' directly reachable after 'a'. + auto input = cudf::test::strings_column_wrapper( + {"ah", "abch", "adeh", "afghh", "abcde", "a", "h", "", "abcdefgh", "xabchx"}); + auto sv = cudf::strings_column_view(input); + + auto prog = cudf::strings::regex_program::create("a(bc|de|fg|)h"); + { + auto results = cudf::strings::contains_re(sv, *prog); + cudf::test::fixed_width_column_wrapper expected({1, 1, 1, 1, 0, 0, 0, 0, 1, 1}); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected); + } + { + auto results = cudf::strings::count_re(sv, *prog); + cudf::test::fixed_width_column_wrapper expected({1, 1, 1, 1, 0, 0, 0, 0, 1, 1}); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected); + } +} + +TEST_F(StringsContainsTests, BoundedRepetitionGap) +{ + // "ab{0,4}cv" — 'b' may repeat 0–4 times; five or more b's yield no match. + auto input = cudf::test::strings_column_wrapper( + {"acv", "abcv", "abbcv", "abbbcv", "abbbbcv", "abbbbbcv", "av", "acvx", "xacvx", ""}); + auto sv = cudf::strings_column_view(input); + + auto prog = cudf::strings::regex_program::create("ab{0,4}cv"); + auto results = cudf::strings::contains_re(sv, *prog); + cudf::test::fixed_width_column_wrapper expected({1, 1, 1, 1, 1, 0, 0, 1, 1, 0}); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected); +} + +TEST_F(StringsContainsTests, ExtNewlineDotAny) +{ + // DEFAULT mode excludes only \n from '.'. + // EXT_NEWLINE additionally excludes \r, U+0085 (NEL), U+2028 (LS), and U+2029 (PS). + auto input = cudf::test::strings_column_wrapper({"axb", + "a\nb", + "a\rb", + "a\xc2\x85" + "b", // U+0085 NEL between 'a' and 'b' + "a\xe2\x80\xa8" + "b", // U+2028 LINE SEPARATOR + "a\xe2\x80\xa9" + "b", // U+2029 PARAGRAPH SEPARATOR + "abc", + ""}); + auto sv = cudf::strings_column_view(input); + + // DEFAULT: only \n excluded — \r and extended newlines are matched by '.' + { + auto prog = cudf::strings::regex_program::create("a.b"); + auto results = cudf::strings::contains_re(sv, *prog); + cudf::test::fixed_width_column_wrapper expected({1, 0, 1, 1, 1, 1, 0, 0}); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected); + } + // EXT_NEWLINE: \r and all extended newlines also excluded + { + auto prog = + cudf::strings::regex_program::create("a.b", cudf::strings::regex_flags::EXT_NEWLINE); + auto results = cudf::strings::contains_re(sv, *prog); + cudf::test::fixed_width_column_wrapper expected({1, 0, 0, 0, 0, 0, 0, 0}); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected); + } + // A string composed entirely of extended newlines yields no '.+' match under EXT_NEWLINE + { + auto input2 = cudf::test::strings_column_wrapper( + {"hello", + "\xc2\x85\xe2\x80\xa8\xe2\x80\xa9", // only extended newlines + "a\xc2\x85" + "b", + ""}); + auto sv2 = cudf::strings_column_view(input2); + auto prog = cudf::strings::regex_program::create(".+", cudf::strings::regex_flags::EXT_NEWLINE); + auto results = cudf::strings::contains_re(sv2, *prog); + cudf::test::fixed_width_column_wrapper expected2({1, 0, 1, 0}); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected2); + } +} + +TEST_F(StringsContainsTests, AlternationPriorityCount) +{ + // Leftmost-first (first-alternative-wins): the shorter first branch is consumed, leaving + // subsequent characters available for the next match. + { + // "a|aa": "a" wins, so "aaaa" counts as 4 individual matches, not 2 "aa" matches. + auto input = cudf::test::strings_column_wrapper({"aaaa", "aaaaaa", "aaab", "a", "b", ""}); + auto sv = cudf::strings_column_view(input); + auto prog = cudf::strings::regex_program::create("a|aa"); + auto results = cudf::strings::count_re(sv, *prog); + cudf::test::fixed_width_column_wrapper expected({4, 6, 3, 1, 0, 0}); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected); + } + { + // "foo|foobar": "foo" wins when both alternatives start at the same position. + auto input = cudf::test::strings_column_wrapper({"foo", "foobar", "foofoo", "bar", ""}); + auto sv = cudf::strings_column_view(input); + auto prog = cudf::strings::regex_program::create("foo|foobar"); + auto results = cudf::strings::count_re(sv, *prog); + cudf::test::fixed_width_column_wrapper expected({1, 1, 2, 0, 0}); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected); + } +} + +TEST_F(StringsContainsTests, LazyQuantifiers) +{ + // Lazy star/plus in non-DOTALL mode: prefer the shortest match. + auto input = cudf::test::strings_column_wrapper( + {"ab", "abc", "xdefx", "xghix", "jkl", "abc xdefx xghix jkl"}); + auto sv = cudf::strings_column_view(input); + + { + auto prog = cudf::strings::regex_program::create("x.*?x"); + auto results = cudf::strings::contains_re(sv, *prog); + cudf::test::fixed_width_column_wrapper expected({0, 0, 1, 1, 0, 1}); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected); + } + { + auto prog = cudf::strings::regex_program::create("x.+?x"); + auto results = cudf::strings::contains_re(sv, *prog); + cudf::test::fixed_width_column_wrapper expected({0, 0, 1, 1, 0, 1}); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected); + } +} diff --git a/cpp/tests/strings/findall_tests.cpp b/cpp/tests/strings/findall_tests.cpp index 22097ae79a6d..e4999a41bcc7 100644 --- a/cpp/tests/strings/findall_tests.cpp +++ b/cpp/tests/strings/findall_tests.cpp @@ -219,6 +219,22 @@ TEST_F(StringsFindallTests, OneCaptureGroup) CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(results->view(), expected); } +TEST_F(StringsFindallTests, AlternationPriorityFirstWins) +{ + // Leftmost-first (first-alternative-wins): "foo" is found instead of "foobar" when both + // alternatives start at the same position. + auto input = + cudf::test::strings_column_wrapper({"foo", "foobar", "foobarbaz", "bar", "xfoobar", ""}); + auto sv = cudf::strings_column_view(input); + auto prog = cudf::strings::regex_program::create( + "foo|foobar", cudf::strings::regex_flags::DEFAULT, cudf::strings::capture_groups::NON_CAPTURE); + auto results = cudf::strings::findall(sv, *prog); + + using LCW = cudf::test::lists_column_wrapper; + LCW expected({LCW{"foo"}, LCW{"foo"}, LCW{"foo"}, LCW{}, LCW{"foo"}, LCW{}}); + CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(results->view(), expected); +} + TEST_F(StringsFindallTests, EmptyMatch) { auto input = cudf::test::strings_column_wrapper({" ", "hello world", "é\r\ny"}); diff --git a/cpp/tests/strings/replace_regex_tests.cpp b/cpp/tests/strings/replace_regex_tests.cpp index 7fd0b4a00a08..8422264d4c36 100644 --- a/cpp/tests/strings/replace_regex_tests.cpp +++ b/cpp/tests/strings/replace_regex_tests.cpp @@ -535,3 +535,30 @@ TEST_F(StringsReplaceRegexTest, CrlfEdgeCasesExtNewline) str_col(&edge_case::exp_abc_backref)); } } + +TEST_F(StringsReplaceRegexTest, AlternationPriorityFirstWins) +{ + // Leftmost-first (first-alternative-wins): when a shorter first alternative is a prefix of a + // longer second, the shorter match is consumed and the remainder is left for the next search. + auto repl = cudf::string_scalar("X"); + + { + // "foo" wins over "foobar": "foobar" becomes "Xbar". + auto input = + cudf::test::strings_column_wrapper({"foo", "foobar", "foobarbaz", "bar", "xfoobar", ""}); + auto sv = cudf::strings_column_view(input); + auto prog = cudf::strings::regex_program::create("foo|foobar"); + auto results = cudf::strings::replace_re(sv, *prog, repl); + cudf::test::strings_column_wrapper expected({"X", "Xbar", "Xbarbaz", "bar", "xXbar", ""}); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected); + } + { + // "cat" wins over "catch": "catch" becomes "Xch". + auto input = cudf::test::strings_column_wrapper({"cat", "catch", "catfish", "dog", ""}); + auto sv = cudf::strings_column_view(input); + auto prog = cudf::strings::regex_program::create("cat|catch"); + auto results = cudf::strings::replace_re(sv, *prog, repl); + cudf::test::strings_column_wrapper expected({"X", "Xch", "Xfish", "dog", ""}); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected); + } +} From 82d06ad06699cf1375300fc7b590ac5adf1426a6 Mon Sep 17 00:00:00 2001 From: "Mads R. B. Kristensen" Date: Fri, 26 Jun 2026 17:18:03 +0200 Subject: [PATCH 10/22] Refactor packed metadata to use an explicit table header (#22951) ## Summary Refactors the packed metadata format to use an explicit `serialized_table_header` instead of encoding table-level metadata in a fake `serialized_column` entry. Previously, the metadata buffer began with a stub `serialized_column` whose `size` field was repurposed to store the number of top-level columns. This PR replaces that implicit convention with a dedicated table header. ## Motivation Using a column-level field (`serialized_column::size`) to store a table-level value is an awkward encoding that makes the format harder to understand and extend. An explicit table header keeps table metadata separate from column metadata and provides a natural place for future table-level fields, such as `num_rows` for zero-column table support (#22765). ## Breaking change This changes the packed metadata wire format: the buffer now begins with a `serialized_table_header` rather than a stub `serialized_column`, so the byte layout produced by `cudf::pack` / `cudf::contiguous_split` is different. Metadata serialized by an older version cannot be unpacked by this version, and vice versa. `pack` and `unpack` are updated together, so round-tripping within a single build is unaffected, but any consumer that reads the raw metadata bytes directly, or that exchanges packed metadata across cudf versions (e.g. persisted buffers or mixed-version workers), must be rebuilt against this format. This also breaks `CudfTable` files written before this PR using the experimental CudfTable API: the embedded metadata changed, so the file format version is bumped. Authors: - Mads R. B. Kristensen (https://github.com/madsbk) Approvers: - David Wendt (https://github.com/davidwendt) URL: https://github.com/rapidsai/cudf/pull/22951 --- cpp/include/cudf/detail/contiguous_split.hpp | 9 +- cpp/src/copying/pack.cpp | 124 +++++++++++++------ cpp/src/io/cudftable.cpp | 28 ++++- cpp/tests/copying/pack_tests.cpp | 84 ++++++++++--- cpp/tests/io/cudftable_test.cpp | 2 +- 5 files changed, 187 insertions(+), 60 deletions(-) diff --git a/cpp/include/cudf/detail/contiguous_split.hpp b/cpp/include/cudf/detail/contiguous_split.hpp index 4ba29a700835..ca94623a04cc 100644 --- a/cpp/include/cudf/detail/contiguous_split.hpp +++ b/cpp/include/cudf/detail/contiguous_split.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -12,6 +12,8 @@ #include +#include + namespace cudf { namespace detail { @@ -109,5 +111,10 @@ std::vector pack_metadata(table_view const& table, size_t buffer_size, metadata_builder& builder); +/** + * @brief Version of the packed metadata layout produced by `pack`/`pack_metadata`. + */ +constexpr std::int32_t packed_metadata_version = 1; + } // namespace detail } // namespace cudf diff --git a/cpp/src/copying/pack.cpp b/cpp/src/copying/pack.cpp index 8b2f8d8913f8..ebbda6089833 100644 --- a/cpp/src/copying/pack.cpp +++ b/cpp/src/copying/pack.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -15,6 +15,8 @@ #include #include #include +#include +#include #include #include @@ -27,9 +29,9 @@ namespace { * @brief The data that is stored as anonymous bytes in the `packed_columns` metadata * field. * - * The metadata field of the `packed_columns` struct is simply an array of these. - * This struct is exposed here because it is needed by both contiguous_split, pack - * and unpack. + * The metadata field of the `packed_columns` struct stores a `serialized_table_header` + * followed by an array of these entries. This struct is exposed here because it is needed + * by both contiguous_split, pack and unpack. */ struct serialized_column { serialized_column() = default; @@ -61,7 +63,43 @@ struct serialized_column { int pad{}; }; -constexpr auto serialized_column_size = sizeof(serialized_column); +/** + * @brief Table-level metadata stored before the serialized column entries. + */ +struct alignas(8) serialized_table_header { + serialized_table_header() = default; + explicit serialized_table_header(size_type _num_columns) : num_columns(_num_columns) {} + + int32_t version{packed_metadata_version}; + size_type num_columns{}; +}; + +// The header is serialized with memcpy, so it must not contain padding bytes +// (which the value constructor would leave uninitialized in the output). +static_assert(std::has_unique_object_representations_v); + +/** + * @brief Read the table header at `ptr`. + * + * @param ptr Pointer to the start of the header in the metadata buffer. + * @param buffer_end One past the end of the metadata buffer. When non-null, the + * read is bounds-checked against it; when null the check is skipped. + * @return The deserialized table header + */ +serialized_table_header read_header(std::uint8_t const* ptr, + std::uint8_t const* buffer_end = nullptr) +{ + if (buffer_end) { + CUDF_EXPECTS(std::cmp_greater_equal(buffer_end - ptr, sizeof(serialized_table_header)), + "packed metadata access is out of bounds"); + } + serialized_table_header header; + std::memcpy(&header, ptr, sizeof(serialized_table_header)); + CUDF_EXPECTS(header.version == packed_metadata_version, + "packed metadata has an unsupported format version"); + CUDF_EXPECTS(header.num_columns >= 0, "packed metadata header has negative column count"); + return header; +} // Read a serialized_column entry at `ptr`, optionally checking that the read // stays within [ptr, buffer_end). When buffer_end is nullptr the check is @@ -69,11 +107,12 @@ constexpr auto serialized_column_size = sizeof(serialized_column); serialized_column read_entry(std::uint8_t const* ptr, std::uint8_t const* buffer_end = nullptr) { if (buffer_end) { - CUDF_EXPECTS(std::cmp_greater_equal(buffer_end - ptr, serialized_column_size), + CUDF_EXPECTS(std::cmp_greater_equal(buffer_end - ptr, sizeof(serialized_column)), "packed metadata access is out of bounds"); } serialized_column entry; - std::memcpy(&entry, ptr, serialized_column_size); + std::memcpy(&entry, ptr, sizeof(serialized_column)); + CUDF_EXPECTS(entry.num_children >= 0, "packed metadata column has negative child count"); return entry; } @@ -85,7 +124,7 @@ size_type subtree_size(std::uint8_t const* ptr, std::uint8_t const* buffer_end = size_type count = 1; size_type remaining = entry.num_children; while (remaining > 0) { - ptr += serialized_column_size; + ptr += sizeof(serialized_column); entry = read_entry(ptr, buffer_end); ++count; remaining += entry.num_children - 1; @@ -100,7 +139,7 @@ uint8_t const* skip_subtrees(std::uint8_t const* ptr, std::uint8_t const* buffer_end = nullptr) { for (size_type i = 0; i < n; ++i) { - ptr += subtree_size(ptr, buffer_end) * serialized_column_size; + ptr += subtree_size(ptr, buffer_end) * sizeof(serialized_column); } return ptr; } @@ -183,17 +222,17 @@ table_view unpack(uint8_t const* metadata, uint8_t const* gpu_data) // gpu data can be null if everything is empty but the metadata must always be valid CUDF_EXPECTS(metadata != nullptr, "Encountered invalid packed column input"); uint8_t const* base_ptr = gpu_data; - // first entry is a stub where size == the total # of top level columns (see pack_metadata above) - auto const num_columns = read_entry(metadata).size; + auto const header = read_header(metadata); + auto const num_columns = header.num_columns; // current_ptr tracks position in the metadata byte buffer - auto const* current_ptr = metadata + serialized_column_size; + auto const* current_ptr = metadata + sizeof(serialized_table_header); std::function(size_type)> get_columns; get_columns = [¤t_ptr, base_ptr, &get_columns](size_t num_columns) { std::vector cols; for (size_t i = 0; i < num_columns; i++) { auto serial_column = read_entry(current_ptr); - current_ptr += serialized_column_size; + current_ptr += sizeof(serialized_column); std::vector const children = get_columns(serial_column.num_children); @@ -236,7 +275,11 @@ std::vector pack_metadata(table_view const& table, class metadata_builder_impl { public: - metadata_builder_impl(size_type const num_root_columns) { metadata.reserve(num_root_columns); } + metadata_builder_impl(size_type const num_root_columns) : _num_root_columns(num_root_columns) + { + // Lower bound: exact for flat tables but nested children add more entries and grow the vector. + _columns.reserve(num_root_columns); + } void add_column_info_to_meta(data_type const col_type, size_type const col_size, @@ -245,35 +288,36 @@ class metadata_builder_impl { int64_t const null_mask_offset, size_type const num_children) { - metadata.emplace_back( + _columns.emplace_back( col_type, col_size, col_null_count, data_offset, null_mask_offset, num_children); } [[nodiscard]] std::vector build() const { - auto output = std::vector(metadata.size() * sizeof(serialized_column)); - std::memcpy(output.data(), metadata.data(), output.size()); + auto const header = serialized_table_header{_num_root_columns}; + auto output = std::vector(sizeof(serialized_table_header) + + _columns.size() * sizeof(serialized_column)); + std::memcpy(output.data(), &header, sizeof(serialized_table_header)); + if (!_columns.empty()) { + std::memcpy(output.data() + sizeof(serialized_table_header), + _columns.data(), + _columns.size() * sizeof(serialized_column)); + } return output; } - void clear() - { - // Clear all, except the first metadata entry storing the number of top level columns that - // was added upon object construction. - metadata.resize(1); - } + void clear() { _columns.clear(); } private: - std::vector metadata; + // Number of top-level columns (excludes nested children) stored in the header. + size_type const _num_root_columns; + // Serialized column entries, depth-first with each column written before its children. + std::vector _columns; }; metadata_builder::metadata_builder(size_type const num_root_columns) - : impl(std::make_unique(num_root_columns + - 1 /*one more extra metadata entry as below*/)) + : impl(std::make_unique(num_root_columns)) { - // first metadata entry is a stub indicating how many total (top level) columns - // there are - impl->add_column_info_to_meta(data_type{type_id::EMPTY}, num_root_columns, 0, -1, -1, 0); } metadata_builder::~metadata_builder() = default; @@ -291,7 +335,7 @@ void metadata_builder::add_column_info_to_meta(data_type const col_type, std::vector metadata_builder::build() const { return impl->build(); } -void metadata_builder::clear() { return impl->clear(); } +void metadata_builder::clear() { impl->clear(); } } // namespace detail @@ -318,20 +362,23 @@ packed_metadata_view::column_view packed_metadata_view::column_view::child(size_ auto const* end = _buffer.data() + _buffer.size(); // Children start immediately after this entry in pre-order layout. auto const* child_ptr = - detail::skip_subtrees(_buffer.data() + detail::serialized_column_size, i, end); + detail::skip_subtrees(_buffer.data() + sizeof(detail::serialized_column), i, end); return packed_metadata_view::column_view{{child_ptr, end}}; } packed_metadata_view::packed_metadata_view(std::span buffer) { CUDF_EXPECTS(!buffer.empty(), "metadata buffer must not be empty"); - CUDF_EXPECTS(buffer.size() >= detail::serialized_column_size, "metadata buffer too small"); - CUDF_EXPECTS(buffer.size() % detail::serialized_column_size == 0, - "metadata buffer size is not a multiple of the entry size"); + CUDF_EXPECTS(buffer.size() >= sizeof(detail::serialized_table_header), + "metadata buffer too small"); + CUDF_EXPECTS( + (buffer.size() - sizeof(detail::serialized_table_header)) % sizeof(detail::serialized_column) == + 0, + "metadata buffer size is not a valid header plus column entry size"); auto const* end = buffer.data() + buffer.size(); - auto const* entries = buffer.data() + detail::serialized_column_size; - // The first entry is a stub whose `size` field holds the number of top-level columns. - _num_columns = detail::read_entry(buffer.data(), end).size; + auto const* entries = buffer.data() + sizeof(detail::serialized_table_header); + auto const header = detail::read_header(buffer.data(), end); + _num_columns = header.num_columns; // Validate that the column tree exactly fills the buffer. auto const* past_last = detail::skip_subtrees(entries, _num_columns, end); CUDF_EXPECTS(past_last == end, @@ -344,7 +391,8 @@ size_type packed_metadata_view::num_columns() const { return _num_columns; } size_type packed_metadata_view::num_rows() const { if (_num_columns == 0) { return 0; } - return detail::read_entry(_entries.data(), _entries.data() + detail::serialized_column_size).size; + return detail::read_entry(_entries.data(), _entries.data() + sizeof(detail::serialized_column)) + .size; } packed_metadata_view::column_view packed_metadata_view::column(size_type i) const diff --git a/cpp/src/io/cudftable.cpp b/cpp/src/io/cudftable.cpp index f0fde63d4391..01cdeff5da07 100644 --- a/cpp/src/io/cudftable.cpp +++ b/cpp/src/io/cudftable.cpp @@ -1,9 +1,10 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ #include +#include #include #include #include @@ -25,7 +26,9 @@ namespace { * * The CudfTable format stores a table in a simple binary layout: * - Magic number (4 bytes): "CTBL" - * - Version (4 bytes): uint32_t format version (currently 1) + * - Version (4 bytes): uint32_t format version (currently 2) + * - Metadata version (4 bytes): int32_t version of the embedded pack() metadata + * - Reserved (4 bytes): padding, kept zero * - Metadata length (8 bytes): uint64_t size of the metadata buffer in bytes * - Data length (8 bytes): uint64_t size of the data buffer in bytes * - Metadata (variable): serialized column metadata from pack() @@ -33,30 +36,41 @@ namespace { */ struct cudftable_header { static constexpr uint32_t magic_number = 0x4C425443; ///< "CTBL" in little-endian - static constexpr uint32_t version = 1; ///< Format version + // Bumped to 2 when the embedded pack() metadata layout changed to an explicit + // serialized_table_header, so older readers reject new files instead of + // misparsing the metadata. See cpp/src/copying/pack.cpp. + static constexpr uint32_t version = 2; ///< File format version uint32_t magic{}; ///< Magic number for format validation - uint32_t format_version{}; ///< Format version number + uint32_t format_version{}; ///< File format version number + int32_t metadata_version{}; ///< Version of the embedded pack() metadata layout + int32_t reserved{}; ///< Padding; kept zero for deterministic output uint64_t metadata_length{}; ///< Length of metadata buffer in bytes uint64_t data_length{}; ///< Length of data buffer in bytes cudftable_header() = default; - cudftable_header(uint64_t metadata_size, uint64_t data_size) + cudftable_header(uint64_t metadata_size, uint64_t data_size, int32_t meta_version) : magic{magic_number}, format_version{version}, + metadata_version{meta_version}, metadata_length{metadata_size}, data_length{data_size} { } }; +// Header is written/read via memcpy, so guard against accidental padding that +// would leave uninitialized bytes in the file. +static_assert(sizeof(cudftable_header) == 32); + } // anonymous namespace void write_cudftable(data_sink* sink, table_view const& input, rmm::cuda_stream_view stream) { auto const packed = cudf::pack(input, stream, cudf::get_current_device_resource_ref()); - auto const header = cudftable_header{packed.metadata->size(), packed.gpu_data->size()}; + auto const header = cudftable_header{ + packed.metadata->size(), packed.gpu_data->size(), cudf::detail::packed_metadata_version}; sink->host_write(&header, sizeof(cudftable_header)); sink->host_write(packed.metadata->data(), header.metadata_length); @@ -87,6 +101,8 @@ packed_table read_cudftable(datasource* source, "Invalid magic number in cudftable header"); CUDF_EXPECTS(header.format_version == cudftable_header::version, "Unsupported cudftable format version"); + CUDF_EXPECTS(header.metadata_version == cudf::detail::packed_metadata_version, + "Unsupported cudftable packed metadata version"); auto const metadata_offset = header_size; auto const data_offset = metadata_offset + header.metadata_length; diff --git a/cpp/tests/copying/pack_tests.cpp b/cpp/tests/copying/pack_tests.cpp index de1f2466508b..49ee4183f243 100644 --- a/cpp/tests/copying/pack_tests.cpp +++ b/cpp/tests/copying/pack_tests.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2021-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -630,16 +630,17 @@ TEST_F(PackUnpackTest, MetadataViewRejectsNonMultipleSize) TEST_F(PackUnpackTest, MetadataViewRejectsTruncatedBuffer) { // Pack a multi-column table, then lop off one serialized entry so - // the stub claims more columns than the buffer actually contains. + // the header claims more columns than the buffer actually contains. cudf::test::fixed_width_column_wrapper col1{1, 2, 3}; cudf::test::fixed_width_column_wrapper col2{4.0f, 5.0f, 6.0f}; cudf::test::fixed_width_column_wrapper col3{7.0, 8.0, 9.0}; auto packed = cudf::pack(cudf::table_view({col1, col2, col3})); - // Metadata has 4 entries (1 stub + 3 columns). Remove the last entry so - // the stub still says "3 columns" but only 2 column entries remain. - auto const entry_size = packed.metadata->size() / 4; - auto const truncated_size = packed.metadata->size() - entry_size; + // Metadata has a table header plus 3 column entries. Remove the last entry so + // the header still says "3 columns" but only 2 column entries remain. + auto constexpr header_size = 2 * sizeof(cudf::size_type); + auto const entry_size = (packed.metadata->size() - header_size) / 3; + auto const truncated_size = packed.metadata->size() - entry_size; auto truncated = std::span(packed.metadata->data(), truncated_size); EXPECT_THROW(cudf::packed_metadata_view{truncated}, cudf::logic_error); @@ -652,8 +653,9 @@ TEST_F(PackUnpackTest, MetadataViewRejectsTooLongBuffer) cudf::test::fixed_width_column_wrapper col{1, 2, 3}; auto packed = cudf::pack(cudf::table_view({col})); - auto const entry_size = packed.metadata->size() / 2; // 2 entries: stub + 1 column - auto extended = *packed.metadata; + auto constexpr header_size = 2 * sizeof(cudf::size_type); + auto const entry_size = packed.metadata->size() - header_size; // 1 column entry + auto extended = *packed.metadata; extended.resize(packed.metadata->size() + entry_size, 0); EXPECT_THROW(cudf::packed_metadata_view{extended}, cudf::logic_error); @@ -668,22 +670,76 @@ TEST_F(PackUnpackTest, MetadataViewRejectsCorruptedChildCount) auto struct_col = cudf::test::structs_column_wrapper({ints, floats}); auto packed = cudf::pack(cudf::table_view({struct_col})); - // The metadata layout is: [stub, struct, ints_child, floats_child] - // The struct entry is at index 1. We corrupt its num_children from 2 to + // The metadata layout is: [header, struct, ints_child, floats_child]. + // The struct entry is the first column entry. We corrupt its num_children from 2 to // something larger so the tree claims more entries than exist. auto corrupted = *packed.metadata; - // The struct entry is at index 1. The num_children field is the + // The num_children field is the // second-to-last 4-byte value in each entry (before the trailing pad). - auto const entry_size = corrupted.size() / 4; // 4 entries total - auto const num_children_offset = entry_size // skip stub entry - + entry_size - 2 * sizeof(int32_t); // num_children in struct + auto constexpr header_size = 2 * sizeof(cudf::size_type); + auto const entry_size = (corrupted.size() - header_size) / 3; // 3 column entries + auto const num_children_offset = header_size // skip table header + + entry_size - 2 * sizeof(int32_t); // num_children in struct cudf::size_type bad_children = 10; std::memcpy(corrupted.data() + num_children_offset, &bad_children, sizeof(bad_children)); EXPECT_THROW(cudf::packed_metadata_view{corrupted}, cudf::logic_error); } +TEST_F(PackUnpackTest, MetadataRejectsNegativeColumnCount) +{ + cudf::test::fixed_width_column_wrapper col{1, 2, 3}; + auto packed = cudf::pack(cudf::table_view({col})); + + auto corrupted = *packed.metadata; + // num_columns follows the leading version field in the header. + auto constexpr num_columns_offset = sizeof(std::int32_t); + cudf::size_type const negative = -1; + std::memcpy(corrupted.data() + num_columns_offset, &negative, sizeof(negative)); + + EXPECT_THROW(cudf::packed_metadata_view{corrupted}, cudf::logic_error); + EXPECT_THROW( + cudf::unpack(corrupted.data(), reinterpret_cast(packed.gpu_data->data())), + cudf::logic_error); +} + +TEST_F(PackUnpackTest, MetadataRejectsUnsupportedVersion) +{ + cudf::test::fixed_width_column_wrapper col{1, 2, 3}; + auto packed = cudf::pack(cudf::table_view({col})); + + auto corrupted = *packed.metadata; + // The version is the leading value of the header. + std::int32_t const unknown_version = 999; + std::memcpy(corrupted.data(), &unknown_version, sizeof(unknown_version)); + + EXPECT_THROW(cudf::packed_metadata_view{corrupted}, cudf::logic_error); + EXPECT_THROW( + cudf::unpack(corrupted.data(), reinterpret_cast(packed.gpu_data->data())), + cudf::logic_error); +} + +TEST_F(PackUnpackTest, MetadataRejectsNegativeChildCount) +{ + cudf::test::fixed_width_column_wrapper ints{1, 2, 3}; + cudf::test::fixed_width_column_wrapper floats{4.0f, 5.0f, 6.0f}; + auto struct_col = cudf::test::structs_column_wrapper({ints, floats}); + auto packed = cudf::pack(cudf::table_view({struct_col})); + + auto corrupted = *packed.metadata; + auto constexpr header_size = 2 * sizeof(cudf::size_type); + auto const entry_size = (corrupted.size() - header_size) / 3; // 3 column entries + auto const num_children_offset = header_size + entry_size - 2 * sizeof(int32_t); + cudf::size_type const negative = -1; + std::memcpy(corrupted.data() + num_children_offset, &negative, sizeof(negative)); + + EXPECT_THROW(cudf::packed_metadata_view{corrupted}, cudf::logic_error); + EXPECT_THROW( + cudf::unpack(corrupted.data(), reinterpret_cast(packed.gpu_data->data())), + cudf::logic_error); +} + TEST_F(PackUnpackTest, MetadataViewColumnIndexOutOfRange) { cudf::test::fixed_width_column_wrapper col{1, 2, 3}; diff --git a/cpp/tests/io/cudftable_test.cpp b/cpp/tests/io/cudftable_test.cpp index 7641e59b5766..63859cb99d30 100644 --- a/cpp/tests/io/cudftable_test.cpp +++ b/cpp/tests/io/cudftable_test.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ From 59f3b708120f6950968abb8fa882b7e39ac2a283 Mon Sep 17 00:00:00 2001 From: Igor Peshansky <7594381+igorpeshansky@users.noreply.github.com> Date: Fri, 26 Jun 2026 12:32:45 -0400 Subject: [PATCH 11/22] Support building and testing cudf-java on JDK 17/21 (#23006) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cudf Java bindings could not be built or tested on recent JDKs. This PR fixes that while preserving JDK 8 compatibility. It is split into two commits, addressing two independent problems. ### 1. Migrate `gmaven-plugin` → `gmavenplus-plugin` (build-time, all JDKs) `gmaven-plugin:1.5` bundles an old Groovy that fails under the strong encapsulation enforced from JDK 16+, breaking the `execute` step that derives `native.cudf.path` and `cuda.classifier`. - Replace it with `gmavenplus-plugin:3.0.0` (Groovy 4.0.21). - Port the `setproperty` execution to the `gmavenplus` config: wrap the script in ``/` + From bcbbd38ea2a208aea61e12bba2e5776a31897f80 Mon Sep 17 00:00:00 2001 From: Lawrence Mitchell Date: Fri, 26 Jun 2026 17:52:39 +0100 Subject: [PATCH 12/22] Work around pola-rs/polars#23214 in streaming dataframe scan (#23007) If we have a dataframe with struct columns then if those columns have nulls somewhere a slice of that dataframe is not exported to arrow correctly by Polars. To, partially, workaround this bug apply the big hammer of just serialising and deserialising the dataframe. This ensures that a sliced frame doesn't export in an invalid way to arrow. Authors: - Lawrence Mitchell (https://github.com/wence-) Approvers: - Tom Augspurger (https://github.com/TomAugspurger) - James Lamb (https://github.com/jameslamb) URL: https://github.com/rapidsai/cudf/pull/23007 --- ci/run_cudf_polars_polars_tests.sh | 1 + .../cudf_polars/streaming/actor_graph/io.py | 25 ++++++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/ci/run_cudf_polars_polars_tests.sh b/ci/run_cudf_polars_polars_tests.sh index faccfadd48c6..5302942db173 100755 --- a/ci/run_cudf_polars_polars_tests.sh +++ b/ci/run_cudf_polars_polars_tests.sh @@ -31,6 +31,7 @@ DESELECTED_TESTS=( "tests/unit/io/test_write.py::test_write_async[read_parquet-]" # kvikio file creation error in CI "tests/unit/io/test_write.py::test_write_async[-0]" # kvikio file creation error in CI "tests/unit/io/test_write.py::test_write_async[-2]" # kvikio file creation error in CI + "tests/unit/operations/test_random.py::test_shuffle_group_by_reseed" # https://github.com/rapidsai/cudf/issues/22964 ) if [[ $(arch) == "aarch64" ]]; then diff --git a/python/cudf_polars/cudf_polars/streaming/actor_graph/io.py b/python/cudf_polars/cudf_polars/streaming/actor_graph/io.py index c6e3110eec8b..e537d56ed408 100644 --- a/python/cudf_polars/cudf_polars/streaming/actor_graph/io.py +++ b/python/cudf_polars/cudf_polars/streaming/actor_graph/io.py @@ -5,9 +5,12 @@ from __future__ import annotations import asyncio +import io import math from typing import TYPE_CHECKING, Any +import polars as pl + import pylibcudf as plc from cudf_streaming.channel_metadata import ChannelMetadata from cudf_streaming.table_chunk import TableChunk @@ -193,14 +196,34 @@ async def dataframescan_node( # Build list of IR slices to read ir_slices = [] + # Partial workaround for + # https://github.com/pola-rs/polars/issues/23214 If a struct column + # has nulls and is sliced then polars exports invalid validity + # buffers. We can't detect this exact state because we can't know + # when the column is sliced. + copy_slice = any( + isinstance(dt, pl.Struct) + for dt in pl.datatypes.unpack_dtypes(ir.df.dtypes(), include_compound=True) + ) + for seq_num in range(local_count): offset = local_offset * rows_per_partition + seq_num * rows_per_partition if offset >= nrows: break + sliced = ir.df.slice(offset, rows_per_partition) + if copy_slice: + # OK, we have structs that might have nulls, and we're + # slicing. So let's copy to contiguous storage. This is + # hacky and doesn't handle the case where we didn't slice + # but the user sliced the input. + f = io.BytesIO() + sliced.serialize_binary(f) + f.seek(0) + sliced = pl._plr.PyDataFrame.deserialize_binary(f) ir_slices.append( DataFrameScan( ir.schema, - ir.df.slice(offset, rows_per_partition), + sliced, ir.projection, ) ) From 37f55e18ad256601650abbbe7c45ff27571afb98 Mon Sep 17 00:00:00 2001 From: Rutuja Pathade <73137503+rpathade@users.noreply.github.com> Date: Fri, 26 Jun 2026 11:50:56 -0700 Subject: [PATCH 13/22] Add pandas-compatible args and caching to RangeIndex.to_numpy (#21896) Closes #21347 `RangeIndex.to_numpy()` now accepts dtype, copy, and na_value to match the pandas API. A cached backing array avoids redundant host allocations, and the deprecated values_host shares the same cache. Authors: - Rutuja Pathade (https://github.com/rpathade) - Matthew Murray (https://github.com/Matt711) Approvers: - Matthew Murray (https://github.com/Matt711) - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/21896 --- python/cudf/cudf/core/index.py | 37 +++++++++++++++++-- .../indexes/rangeindex/methods/test_numpy.py | 36 ++++++++++++++++++ 2 files changed, 69 insertions(+), 4 deletions(-) create mode 100644 python/cudf/cudf/tests/indexes/rangeindex/methods/test_numpy.py diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index c883205f9cbe..8590d52f8690 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -2716,11 +2716,40 @@ def values(self) -> cupy.ndarray: return cupy.arange(self.start, self.stop, self.step) @_performance_tracking - def to_numpy(self) -> np.ndarray: - """ - Return a numpy array representation of the RangeIndex. + def to_numpy( + self, + dtype: Dtype | None = None, + copy: bool = False, + na_value=None, + ) -> np.ndarray: + """Convert the RangeIndex to a NumPy array. + + Parameters + ---------- + dtype : str or :class:`numpy.dtype`, optional + The dtype to cast the result to. Defaults to the RangeIndex dtype. + copy : bool, default False + Whether to ensure that the returned value is not a view on + another array. Note that ``copy=False`` does not ensure that + ``to_numpy()`` is no-copy. Rather, ``copy=True`` ensures that + a copy is made, even if not strictly necessary. + na_value : Any, default None + Value to use for missing values. Since ``RangeIndex`` cannot + contain missing values, this parameter has no effect. + + Returns + ------- + numpy.ndarray """ - return np.arange(self.start, self.stop, self.step) + return ( + self._to_numpy(dtype, na_value).copy() + if copy + else self._to_numpy(dtype, na_value) + ) + + @cache + def _to_numpy(self, dtype=None, na_value=None) -> np.ndarray: + return self.to_pandas().to_numpy(dtype=dtype, na_value=na_value) @_performance_tracking def to_cupy(self) -> cupy.ndarray: diff --git a/python/cudf/cudf/tests/indexes/rangeindex/methods/test_numpy.py b/python/cudf/cudf/tests/indexes/rangeindex/methods/test_numpy.py new file mode 100644 index 000000000000..f6451c40b4c7 --- /dev/null +++ b/python/cudf/cudf/tests/indexes/rangeindex/methods/test_numpy.py @@ -0,0 +1,36 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import numpy as np + +import cudf +from cudf.testing import assert_eq + + +def test_rangeindex_to_numpy_args(): + gidx = cudf.RangeIndex(start=1, stop=10, step=2) + pidx = gidx.to_pandas() + + assert_eq( + gidx.to_numpy(dtype=np.float64, copy=False, na_value=None), + pidx.to_numpy(dtype=np.float64, copy=False, na_value=None), + ) + + +def test_rangeindex_to_numpy_caches_host_array(): + gidx = cudf.RangeIndex(start=0, stop=10, step=1) + + first = gidx.to_numpy(copy=False) + second = gidx.to_numpy(copy=False) + + assert first is second + + +def test_rangeindex_to_numpy_copy_true_returns_new_array(): + gidx = cudf.RangeIndex(start=0, stop=10, step=1) + + base = gidx.to_numpy(copy=False) + copied = gidx.to_numpy(copy=True) + + assert copied is not base + np.testing.assert_array_equal(copied, base) From 98d16239c0c4fbe16e39ef0684c2be420caf54c2 Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Fri, 26 Jun 2026 19:14:28 +0000 Subject: [PATCH 14/22] Add regex-flags member variable to internal libcudf reprog class (#22994) Adds new member variable to hold the regex-flags specified by the regex-program interface so that it may be referenced more easily later for fast-path checking. This is some common changes needed in #22178 and #21936 and will help reduce the size of the changes for both and hopefully reduce confusion on the changes needed there. Cleanup of related files is included as well. Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Muhammad Haseeb (https://github.com/mhaseeb123) - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/22994 --- cpp/include/cudf/strings/regex/regex_program.hpp | 5 ++--- cpp/src/strings/regex/regcomp.cpp | 12 +++++++----- cpp/src/strings/regex/regcomp.h | 7 ++++--- cpp/src/strings/search/count.cu | 4 ++-- 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/cpp/include/cudf/strings/regex/regex_program.hpp b/cpp/include/cudf/strings/regex/regex_program.hpp index a5c92b23bfaf..78b75e51afdd 100644 --- a/cpp/include/cudf/strings/regex/regex_program.hpp +++ b/cpp/include/cudf/strings/regex/regex_program.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ #pragma once @@ -46,6 +46,7 @@ struct regex_program { regex_program() = delete; regex_program(regex_program const&) = delete; regex_program& operator=(regex_program const&) = delete; + ~regex_program(); /** * @brief Move constructor @@ -105,8 +106,6 @@ struct regex_program { */ [[nodiscard]] std::size_t compute_working_memory_size(int32_t num_strings) const; - ~regex_program(); - private: std::string _pattern; regex_flags _flags; diff --git a/cpp/src/strings/regex/regcomp.cpp b/cpp/src/strings/regex/regcomp.cpp index 1985de557ddf..af70f54eb737 100644 --- a/cpp/src/strings/regex/regcomp.cpp +++ b/cpp/src/strings/regex/regcomp.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -1071,14 +1071,16 @@ reprog reprog::create_from(std::string_view pattern, regex_flags const flags, capture_groups const capture) { - reprog rtn; + reprog rtn(flags); auto pattern32 = string_to_char32_vector(pattern); regex_compiler const compiler(pattern32.data(), flags, capture, rtn); - // for debugging, it can be helpful to call rtn.print(flags) here to dump + // for debugging, it can be helpful to call rtn.print() here to dump // out the instructions that have been created from the given pattern return rtn; } +reprog::reprog(regex_flags flags) : _flags{flags} {} + void reprog::optimize() { collapse_nops(); } void reprog::finalize() { build_start_ids(); } @@ -1247,9 +1249,9 @@ match_flags reprog::compute_match_flags() const } #ifndef NDEBUG -void reprog::print(regex_flags const flags) +void reprog::print() const { - printf("Flags = 0x%08x\n", static_cast(flags)); + printf("Flags = 0x%08x\n", static_cast(_flags)); printf("Instructions:\n"); for (std::size_t i = 0; i < _insts.size(); i++) { reinst const& inst = _insts[i]; diff --git a/cpp/src/strings/regex/regcomp.h b/cpp/src/strings/regex/regcomp.h index 5cb222427f97..0ce4df885d39 100644 --- a/cpp/src/strings/regex/regcomp.h +++ b/cpp/src/strings/regex/regcomp.h @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ #pragma once @@ -138,7 +138,7 @@ class reprog { [[nodiscard]] match_flags compute_match_flags() const; #ifndef NDEBUG - void print(regex_flags const flags); + void print() const; #endif private: @@ -147,8 +147,9 @@ class reprog { int32_t _startinst_id{}; // id of first instruction std::vector _startinst_ids; // short-cut to speed-up ORs int32_t _num_capturing_groups{}; + [[maybe_unused]] regex_flags _flags{}; - reprog() = default; + reprog(regex_flags); void collapse_nops(); void build_start_ids(); void check_for_errors(int32_t id, int32_t next_id); diff --git a/cpp/src/strings/search/count.cu b/cpp/src/strings/search/count.cu index ce03e3bea5a8..d0d994e6ce8c 100644 --- a/cpp/src/strings/search/count.cu +++ b/cpp/src/strings/search/count.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -49,6 +49,7 @@ struct counter_fn { return count; } }; +} // namespace std::unique_ptr count(strings_column_view const& input, string_scalar const& target, @@ -78,7 +79,6 @@ std::unique_ptr count(strings_column_view const& input, return results; } -} // namespace } // namespace detail From 4985b6ab5d17e4f6d22b635f555e6327120ab62f Mon Sep 17 00:00:00 2001 From: Yunsong Wang <12716979+PointKernel@users.noreply.github.com> Date: Fri, 26 Jun 2026 13:30:41 -0700 Subject: [PATCH 15/22] Add SUM_OVERFLOW in sort groupby (#22832) Closes #22576 This PR adds support for `SUM_OVERFLOW` (previously `SUM_WITH_OVERFLOW`) in the sort-based groupby execution path. Authors: - Yunsong Wang (https://github.com/PointKernel) - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Nghia Truong (https://github.com/ttnghia) - Muhammad Haseeb (https://github.com/mhaseeb123) URL: https://github.com/rapidsai/cudf/pull/22832 --- cpp/CMakeLists.txt | 1 + cpp/include/cudf/aggregation.hpp | 94 +++++++------ .../cudf/detail/aggregation/aggregation.hpp | 18 ++- .../cudf/reduction/detail/sum_overflow.cuh | 66 +++++++++ cpp/src/aggregation/aggregation.cpp | 17 ++- cpp/src/groupby/sort/aggregate.cpp | 19 ++- cpp/src/groupby/sort/group_reductions.hpp | 22 ++- cpp/src/groupby/sort/group_sum_overflow.cu | 127 ++++++++++++++++++ cpp/src/reductions/reductions.cpp | 6 +- cpp/src/reductions/sum_with_overflow.cu | 54 +------- cpp/tests/groupby/sum_with_overflow_tests.cpp | 118 ++++++++++++---- .../test/java/ai/rapids/cudf/TableTest.java | 82 ++++++----- 12 files changed, 454 insertions(+), 170 deletions(-) create mode 100644 cpp/include/cudf/reduction/detail/sum_overflow.cuh create mode 100644 cpp/src/groupby/sort/group_sum_overflow.cu diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 92e18a162805..eefb265464fe 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -620,6 +620,7 @@ add_library( src/groupby/sort/group_replace_nulls.cu src/groupby/sort/group_std.cu src/groupby/sort/group_sum.cu + src/groupby/sort/group_sum_overflow.cu src/groupby/sort/group_sum_scan.cu src/groupby/sort/group_topk.cu src/groupby/sort/host_udf_aggregation.cpp diff --git a/cpp/include/cudf/aggregation.hpp b/cpp/include/cudf/aggregation.hpp index 77c8836aabcf..5cbd81dc1ac8 100644 --- a/cpp/include/cudf/aggregation.hpp +++ b/cpp/include/cudf/aggregation.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -77,48 +77,50 @@ class aggregation { * @brief Possible aggregation operations. */ enum Kind : int32_t { - SUM = 0, ///< sum reduction - SUM_WITH_OVERFLOW, ///< sum reduction with overflow detection - PRODUCT, ///< product reduction - MIN, ///< min reduction - MAX, ///< max reduction - COUNT_VALID, ///< count number of valid elements - COUNT_ALL, ///< count number of elements - ANY, ///< any reduction - ALL, ///< all reduction - SUM_OF_SQUARES, ///< sum of squares reduction - MEAN, ///< arithmetic mean reduction - M2, ///< sum of squares of differences from the mean - VARIANCE, ///< variance - STD, ///< standard deviation - MEDIAN, ///< median reduction - QUANTILE, ///< compute specified quantile(s) - ARGMAX, ///< Index of max element - ARGMIN, ///< Index of min element - NUNIQUE, ///< count number of unique elements - NTH_ELEMENT, ///< get the nth element - ROW_NUMBER, ///< get row-number of current index (relative to rolling window) - EWMA, ///< get exponential weighted moving average at current index - RANK, ///< get rank of current index - COLLECT_LIST, ///< collect values into a list - COLLECT_SET, ///< collect values into a list without duplicate entries - LEAD, ///< window function, accesses row at specified offset following current row - LAG, ///< window function, accesses row at specified offset preceding current row - PTX, ///< PTX based UDF aggregation - CUDA, ///< CUDA based UDF aggregation - HOST_UDF, ///< host based UDF aggregation - MERGE_LISTS, ///< merge multiple lists values into one list - MERGE_SETS, ///< merge multiple lists values into one list then drop duplicate entries - MERGE_M2, ///< merge partial values of M2 aggregation, - COVARIANCE, ///< covariance between two sets of elements - CORRELATION, ///< correlation between two sets of elements - TDIGEST, ///< create a tdigest from a set of input values - MERGE_TDIGEST, ///< create a tdigest by merging multiple tdigests together - HISTOGRAM, ///< compute frequency of each element - MERGE_HISTOGRAM, ///< merge partial values of HISTOGRAM aggregation - BITWISE_AGG, ///< bitwise aggregation on numeric columns - TOP_K, ///< top k elements in a group - INVALID ///< invalid aggregation, used as a placeholder when default-constructed + SUM = 0, ///< sum reduction + SUM_OVERFLOW, ///< sum reduction with overflow detection + /// @deprecated Use SUM_OVERFLOW instead. + SUM_WITH_OVERFLOW = SUM_OVERFLOW, + PRODUCT, ///< product reduction + MIN, ///< min reduction + MAX, ///< max reduction + COUNT_VALID, ///< count number of valid elements + COUNT_ALL, ///< count number of elements + ANY, ///< any reduction + ALL, ///< all reduction + SUM_OF_SQUARES, ///< sum of squares reduction + MEAN, ///< arithmetic mean reduction + M2, ///< sum of squares of differences from the mean + VARIANCE, ///< variance + STD, ///< standard deviation + MEDIAN, ///< median reduction + QUANTILE, ///< compute specified quantile(s) + ARGMAX, ///< Index of max element + ARGMIN, ///< Index of min element + NUNIQUE, ///< count number of unique elements + NTH_ELEMENT, ///< get the nth element + ROW_NUMBER, ///< get row-number of current index (relative to rolling window) + EWMA, ///< get exponential weighted moving average at current index + RANK, ///< get rank of current index + COLLECT_LIST, ///< collect values into a list + COLLECT_SET, ///< collect values into a list without duplicate entries + LEAD, ///< window function, accesses row at specified offset following current row + LAG, ///< window function, accesses row at specified offset preceding current row + PTX, ///< PTX based UDF aggregation + CUDA, ///< CUDA based UDF aggregation + HOST_UDF, ///< host based UDF aggregation + MERGE_LISTS, ///< merge multiple lists values into one list + MERGE_SETS, ///< merge multiple lists values into one list then drop duplicate entries + MERGE_M2, ///< merge partial values of M2 aggregation, + COVARIANCE, ///< covariance between two sets of elements + CORRELATION, ///< correlation between two sets of elements + TDIGEST, ///< create a tdigest from a set of input values + MERGE_TDIGEST, ///< create a tdigest by merging multiple tdigests together + HISTOGRAM, ///< compute frequency of each element + MERGE_HISTOGRAM, ///< merge partial values of HISTOGRAM aggregation + BITWISE_AGG, ///< bitwise aggregation on numeric columns + TOP_K, ///< top k elements in a group + INVALID ///< invalid aggregation, used as a placeholder when default-constructed }; /** @@ -212,8 +214,14 @@ enum class ewm_history : int32_t { INFINITE, FINITE }; template std::unique_ptr make_sum_aggregation(); +/// Factory to create a SUM_OVERFLOW aggregation +/// @return A SUM_OVERFLOW aggregation object +template +std::unique_ptr make_sum_overflow_aggregation(); + /// Factory to create a SUM_WITH_OVERFLOW aggregation /// @return A SUM_WITH_OVERFLOW aggregation object +/// @deprecated Use make_sum_overflow_aggregation() instead. template std::unique_ptr make_sum_with_overflow_aggregation(); diff --git a/cpp/include/cudf/detail/aggregation/aggregation.hpp b/cpp/include/cudf/detail/aggregation/aggregation.hpp index b848f4417b57..1dbb6594ba50 100644 --- a/cpp/include/cudf/detail/aggregation/aggregation.hpp +++ b/cpp/include/cudf/detail/aggregation/aggregation.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -977,11 +977,19 @@ struct target_type_impl - requires((cudf::is_integral_not_bool() && cudf::is_signed()) || - cudf::is_fixed_point()) +concept sum_overflow_supported = + (cudf::is_integral_not_bool() && cudf::is_signed()) || + cudf::is_fixed_point(); + +// SUM_WITH_OVERFLOW outputs a struct {sum: Source, overflow: bool} where the sum matches the input +// type +template struct target_type_impl { using type = struct_view; // SUM_WITH_OVERFLOW outputs a struct with sum and overflow fields }; diff --git a/cpp/include/cudf/reduction/detail/sum_overflow.cuh b/cpp/include/cudf/reduction/detail/sum_overflow.cuh new file mode 100644 index 000000000000..87d4f656c73d --- /dev/null +++ b/cpp/include/cudf/reduction/detail/sum_overflow.cuh @@ -0,0 +1,66 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include +#include + +#include + +namespace cudf { +namespace reduction::detail { + +/** + * @brief Running accumulator for a sum that detects signed-integer overflow. + * + * `wraps` is the net number of times the running sum has stepped outside [MIN, MAX]. + * A final `wraps == 0` means the true sum fits in `DeviceType`, i.e. no overflow. + */ +template +struct sum_overflow_result { + DeviceType sum; + cudf::size_type wraps; + + CUDF_HOST_DEVICE sum_overflow_result() : sum{0}, wraps{0} {} + CUDF_HOST_DEVICE sum_overflow_result(DeviceType s, cudf::size_type w) : sum{s}, wraps{w} {} +}; + +/// @brief Associative combine: wrap the sums and track the net carry direction. +template +struct overflow_sum_op { + __device__ sum_overflow_result operator()( + sum_overflow_result const& lhs, sum_overflow_result const& rhs) const + { + auto const r = cuda::add_overflow(lhs.sum, rhs.sum); + auto const carry = r.overflow ? (rhs.sum > DeviceType{0} ? 1 : -1) : 0; + return sum_overflow_result{r.value, lhs.wraps + rhs.wraps + carry}; + } +}; + +/// @brief Maps a value to a zero-wrap accumulator. +template +struct to_sum_overflow { + __device__ sum_overflow_result operator()(DeviceType value) const + { + return sum_overflow_result{value, 0}; + } +}; + +/// @brief Maps a row index to an accumulator, treating nulls as a zero contribution. +template +struct null_aware_to_sum_overflow { + cudf::column_device_view dcol; + + CUDF_HOST_DEVICE null_aware_to_sum_overflow(cudf::column_device_view const& d) : dcol{d} {} + + __device__ sum_overflow_result operator()(cudf::size_type idx) const + { + return dcol.is_valid(idx) ? sum_overflow_result{dcol.element(idx), 0} + : sum_overflow_result{DeviceType{0}, 0}; + } +}; + +} // namespace reduction::detail +} // namespace cudf diff --git a/cpp/src/aggregation/aggregation.cpp b/cpp/src/aggregation/aggregation.cpp index ab871abf94ed..e82d602e3bb5 100644 --- a/cpp/src/aggregation/aggregation.cpp +++ b/cpp/src/aggregation/aggregation.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -35,6 +35,21 @@ template CUDF_EXPORT std::unique_ptr make_sum_aggregation(); /// Factory to create a SUM_WITH_OVERFLOW aggregation +template +std::unique_ptr make_sum_overflow_aggregation() +{ + return std::make_unique(); +} +template CUDF_EXPORT std::unique_ptr make_sum_overflow_aggregation(); +template CUDF_EXPORT std::unique_ptr +make_sum_overflow_aggregation(); +template CUDF_EXPORT std::unique_ptr +make_sum_overflow_aggregation(); +template CUDF_EXPORT std::unique_ptr +make_sum_overflow_aggregation(); +template CUDF_EXPORT std::unique_ptr +make_sum_overflow_aggregation(); + template std::unique_ptr make_sum_with_overflow_aggregation() { diff --git a/cpp/src/groupby/sort/aggregate.cpp b/cpp/src/groupby/sort/aggregate.cpp index 2e9521f538f8..ae56aa2327f2 100644 --- a/cpp/src/groupby/sort/aggregate.cpp +++ b/cpp/src/groupby/sort/aggregate.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -140,6 +140,18 @@ void aggregate_result_functor::operator()(aggregation const& a get_grouped_values(), helper.num_groups(stream), helper.group_labels(stream), stream, mr)); } +template <> +void aggregate_result_functor::operator()(aggregation const& agg) +{ + if (cache.has_result(values, agg)) return; + + cache.add_result( + values, + agg, + detail::group_sum_overflow( + get_grouped_values(), helper.num_groups(stream), helper.group_labels(stream), stream, mr)); +} + template <> void aggregate_result_functor::operator()(aggregation const& agg) { @@ -878,11 +890,6 @@ std::pair, std::vector> groupby::sort auto store_functor = detail::aggregate_result_functor(request.values, helper(), cache, stream, mr); for (auto const& agg : request.aggregations) { - // SUM_WITH_OVERFLOW is only supported with hash-based groupby, not sort-based - CUDF_EXPECTS(agg->kind != aggregation::SUM_WITH_OVERFLOW, - "SUM_WITH_OVERFLOW aggregation is only supported with hash-based groupby, not " - "sort-based groupby"); - // TODO (dm): single pass compute all supported reductions cudf::detail::aggregation_dispatcher(agg->kind, store_functor, *agg); } diff --git a/cpp/src/groupby/sort/group_reductions.hpp b/cpp/src/groupby/sort/group_reductions.hpp index db764cb02f25..389a5d7fb535 100644 --- a/cpp/src/groupby/sort/group_reductions.hpp +++ b/cpp/src/groupby/sort/group_reductions.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -44,6 +44,26 @@ std::unique_ptr group_sum(column_view const& values, rmm::cuda_stream_view stream, rmm::device_async_resource_ref mr); +/** + * @brief Internal API to calculate groupwise sum with overflow detection. + * + * Returns a STRUCT column with two children: the (wrapping) sum and a BOOL8 overflow flag that is + * true when the true sum does not fit in the value type. On overflow the sum value is unspecified; + * the flag is the meaningful output. A group is null only when all of its values are null. + * + * @param values Grouped values to sum + * @param num_groups Number of groups + * @param group_labels ID of group that the corresponding value belongs to + * @param stream CUDA stream used for device memory operations and kernel launches. + * @param mr Device memory resource used to allocate the returned column's device memory + */ +[[nodiscard]] std::unique_ptr group_sum_overflow( + column_view const& values, + size_type num_groups, + cudf::device_span group_labels, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr); + /** * @brief Internal API to calculate groupwise product * diff --git a/cpp/src/groupby/sort/group_sum_overflow.cu b/cpp/src/groupby/sort/group_sum_overflow.cu new file mode 100644 index 000000000000..a58aa3e2b37f --- /dev/null +++ b/cpp/src/groupby/sort/group_sum_overflow.cu @@ -0,0 +1,127 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "groupby/sort/group_reductions.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include + +namespace cudf::groupby::detail { +namespace { + +// Splits a reduced {sum, wraps} accumulator into the (sum, overflow-flag) pair of the output +// struct. +template +struct split_accumulator { + __device__ cuda::std::tuple operator()( + cudf::reduction::detail::sum_overflow_result const& acc) const + { + return {acc.sum, acc.wraps != 0}; + } +}; + +struct group_sum_overflow_fn { + template + std::unique_ptr operator()(column_view const& values, + size_type num_groups, + cudf::device_span group_labels, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) const + { + using DeviceType = cudf::device_storage_type_t; + + auto const dcol = cudf::column_device_view::create(values, stream); + + auto sum_child = + cudf::make_fixed_width_column(values.type(), num_groups, mask_state::UNALLOCATED, stream, mr); + auto overflow_child = cudf::make_fixed_width_column( + cudf::data_type{type_id::BOOL8}, num_groups, mask_state::UNALLOCATED, stream, mr); + + // Segmented reduction per group, written straight into the two struct children. + auto const values_in = cudf::detail::make_counting_transform_iterator( + 0, cudf::reduction::detail::null_aware_to_sum_overflow{*dcol}); + auto const children_out = cuda::transform_output_iterator{ + cuda::make_zip_iterator(sum_child->mutable_view().begin(), + overflow_child->mutable_view().begin()), + split_accumulator{}}; + + thrust::reduce_by_key(rmm::exec_policy_nosync(stream, cudf::get_current_device_resource_ref()), + group_labels.begin(), + group_labels.end(), + values_in, + cuda::make_discard_iterator(), + children_out, + cuda::std::equal_to{}, + cudf::reduction::detail::overflow_sum_op{}); + + // A group's struct entry is null only when every row in the group is null (mirrors group_sum): + // reduce per-row validity with logical-or, then build the mask from the per-group result. + auto [null_mask, null_count] = [&]() -> std::pair { + if (!values.has_nulls()) { return {rmm::device_buffer{}, size_type{0}}; } + rmm::device_uvector group_valid( + num_groups, stream, cudf::get_current_device_resource_ref()); + thrust::reduce_by_key( + rmm::exec_policy_nosync(stream, cudf::get_current_device_resource_ref()), + group_labels.begin(), + group_labels.end(), + cudf::detail::make_validity_iterator(*dcol), + cuda::make_discard_iterator(), + group_valid.begin(), + cuda::std::equal_to{}, + cuda::std::logical_or{}); + return cudf::detail::valid_if( + group_valid.begin(), group_valid.end(), cuda::std::identity{}, stream, mr); + }(); + + std::vector> children; + children.push_back(std::move(sum_child)); + children.push_back(std::move(overflow_child)); + return cudf::create_structs_hierarchy( + num_groups, std::move(children), null_count, std::move(null_mask), stream, mr); + } + + template + requires(!cudf::detail::sum_overflow_supported) + std::unique_ptr operator()(Args&&...) const + { + CUDF_FAIL("SUM_OVERFLOW is only supported for signed integral and fixed-point types"); + } +}; + +} // namespace + +std::unique_ptr group_sum_overflow(column_view const& values, + size_type num_groups, + cudf::device_span group_labels, + rmm::cuda_stream_view stream, + rmm::device_async_resource_ref mr) +{ + return cudf::type_dispatcher( + values.type(), group_sum_overflow_fn{}, values, num_groups, group_labels, stream, mr); +} + +} // namespace cudf::groupby::detail diff --git a/cpp/src/reductions/reductions.cpp b/cpp/src/reductions/reductions.cpp index d58f145d8a7e..4d4f31e17273 100644 --- a/cpp/src/reductions/reductions.cpp +++ b/cpp/src/reductions/reductions.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -95,9 +95,7 @@ struct reduction_function : public base_reductio } }; -template - requires((cudf::is_integral_not_bool() && cudf::is_signed()) || - cudf::is_fixed_point()) +template struct reduction_function : public base_reduction_function { [[nodiscard]] std::unique_ptr reduce(reduction_parameters const& params) const diff --git a/cpp/src/reductions/sum_with_overflow.cu b/cpp/src/reductions/sum_with_overflow.cu index 34fff81317c1..31ad3e0a28ad 100644 --- a/cpp/src/reductions/sum_with_overflow.cu +++ b/cpp/src/reductions/sum_with_overflow.cu @@ -1,15 +1,17 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ #include #include #include +#include #include #include #include #include +#include #include #include #include @@ -27,49 +29,6 @@ namespace cudf::reduction::detail { namespace { -// `wraps` is the net number of times the running sum has stepped outside [MIN, MAX]. -// A final `wraps == 0` means the true sum fits in DeviceType, i.e. no overflow. -template -struct sum_overflow_result { - DeviceType sum; - cudf::size_type wraps; - - CUDF_HOST_DEVICE sum_overflow_result() : sum{0}, wraps{0} {} - CUDF_HOST_DEVICE sum_overflow_result(DeviceType s, cudf::size_type w) : sum{s}, wraps{w} {} -}; - -template -struct overflow_sum_op { - __device__ sum_overflow_result operator()( - sum_overflow_result const& lhs, sum_overflow_result const& rhs) const - { - auto const r = cuda::add_overflow(lhs.sum, rhs.sum); - auto const carry = r.overflow ? (rhs.sum > DeviceType{0} ? 1 : -1) : 0; - return sum_overflow_result{r.value, lhs.wraps + rhs.wraps + carry}; - } -}; - -template -struct to_sum_overflow { - __device__ sum_overflow_result operator()(DeviceType value) const - { - return sum_overflow_result{value, 0}; - } -}; - -template -struct null_aware_to_sum_overflow { - cudf::column_device_view dcol; - - CUDF_HOST_DEVICE null_aware_to_sum_overflow(cudf::column_device_view const& d) : dcol{d} {} - - __device__ sum_overflow_result operator()(cudf::size_type idx) const - { - return dcol.is_valid(idx) ? sum_overflow_result{dcol.element(idx), 0} - : sum_overflow_result{DeviceType{0}, 0}; - } -}; - template std::unique_ptr make_sum_overflow_struct_scalar( device_storage_type_t sum_value, @@ -156,9 +115,7 @@ std::unique_ptr sum_with_overflow_impl( } struct sum_with_overflow_dispatcher { - template - requires((cudf::is_integral_not_bool() && cudf::is_signed()) || - cudf::is_fixed_point()) + template std::unique_ptr operator()(column_view const& col, std::optional> init, rmm::cuda_stream_view stream, @@ -168,8 +125,7 @@ struct sum_with_overflow_dispatcher { } template - requires(!((cudf::is_integral_not_bool() && cudf::is_signed()) || - cudf::is_fixed_point())) + requires(!cudf::detail::sum_overflow_supported) std::unique_ptr operator()(column_view const&, std::optional>, rmm::cuda_stream_view, diff --git a/cpp/tests/groupby/sum_with_overflow_tests.cpp b/cpp/tests/groupby/sum_with_overflow_tests.cpp index 74d89b0945aa..0183b8864bfd 100644 --- a/cpp/tests/groupby/sum_with_overflow_tests.cpp +++ b/cpp/tests/groupby/sum_with_overflow_tests.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -58,12 +58,9 @@ TYPED_TEST(groupby_sum_with_overflow_test, basic) auto agg = cudf::make_sum_with_overflow_aggregation(); test_single_agg(keys, vals, expect_keys, *expect_vals, std::move(agg)); - // SUM_WITH_OVERFLOW should throw with sort-based groupby auto agg_sort = cudf::make_sum_with_overflow_aggregation(); - EXPECT_THROW( - test_single_agg( - keys, vals, expect_keys, *expect_vals, std::move(agg_sort), force_use_sort_impl::YES), - cudf::logic_error); + test_single_agg( + keys, vals, expect_keys, *expect_vals, std::move(agg_sort), force_use_sort_impl::YES); } else { // For integer types cudf::test::fixed_width_column_wrapper vals{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; @@ -80,15 +77,62 @@ TYPED_TEST(groupby_sum_with_overflow_test, basic) auto agg = cudf::make_sum_with_overflow_aggregation(); test_single_agg(keys, vals, expect_keys, *expect_vals, std::move(agg)); - // SUM_WITH_OVERFLOW should throw with sort-based groupby auto agg_sort = cudf::make_sum_with_overflow_aggregation(); - EXPECT_THROW( - test_single_agg( - keys, vals, expect_keys, *expect_vals, std::move(agg_sort), force_use_sort_impl::YES), - cudf::logic_error); + test_single_agg( + keys, vals, expect_keys, *expect_vals, std::move(agg_sort), force_use_sort_impl::YES); } +} + +TYPED_TEST(groupby_sum_with_overflow_test, sort_path_with_tdigest) +{ + using K = int32_t; + using V = TypeParam; + + cudf::test::fixed_width_column_wrapper keys{1, 2, 3, 1, 2, 2, 1, 3, 3, 2}; + cudf::test::fixed_width_column_wrapper expect_keys{1, 2, 3}; + + // Co-request TDIGEST (a sort-only aggregation) so the whole groupby takes the sort-based path, + // then verify the SUM_WITH_OVERFLOW struct matches the hash result and TDIGEST also runs. + auto run_and_check = [&](cudf::column_view const& vals, cudf::column_view const& expect_vals) { + std::vector requests; + requests.emplace_back(); + requests[0].values = vals; + requests[0].aggregations.push_back( + cudf::make_sum_with_overflow_aggregation()); + requests[0].aggregations.push_back( + cudf::make_tdigest_aggregation(1000)); - // Note: SUM_WITH_OVERFLOW only works with hash groupby, not sort groupby + auto result = cudf::groupby::groupby(cudf::table_view{{keys}}).aggregate(requests); + + // Sort-based groupby returns keys in sorted order, aligning with expect_keys/expect_vals. + CUDF_TEST_EXPECT_COLUMNS_EQUAL(result.first->get_column(0).view(), expect_keys); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(result.second[0].results[0]->view(), expect_vals); + // TDIGEST produces one tdigest per group. + EXPECT_EQ(result.second[0].results[1]->size(), 3); + }; + + if constexpr (cudf::is_fixed_point()) { + using RepType = cudf::device_storage_type_t; + auto const scale = scale_type{0}; + auto vals = + cudf::test::fixed_point_column_wrapper{{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, scale}; + auto sum_col = cudf::test::fixed_point_column_wrapper{{9, 19, 17}, scale}; + auto overflow_col = cudf::test::fixed_width_column_wrapper{false, false, false}; + std::vector> children; + children.push_back(sum_col.release()); + children.push_back(overflow_col.release()); + auto expect_vals = cudf::create_structs_hierarchy(3, std::move(children), 0, {}); + run_and_check(vals, *expect_vals); + } else { + cudf::test::fixed_width_column_wrapper vals{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + auto sum_col = cudf::test::fixed_width_column_wrapper{9, 19, 17}; + auto overflow_col = cudf::test::fixed_width_column_wrapper{false, false, false}; + std::vector> children; + children.push_back(sum_col.release()); + children.push_back(overflow_col.release()); + auto expect_vals = cudf::create_structs_hierarchy(3, std::move(children), 0, {}); + run_and_check(vals, *expect_vals); + } } TYPED_TEST(groupby_sum_with_overflow_test, empty_cols) @@ -112,8 +156,6 @@ TYPED_TEST(groupby_sum_with_overflow_test, empty_cols) auto agg = cudf::make_sum_with_overflow_aggregation(); test_single_agg(keys, vals, expect_keys, *expect_vals, std::move(agg)); - - // Note: SUM_WITH_OVERFLOW only works with hash groupby, not sort groupby } TYPED_TEST(groupby_sum_with_overflow_test, zero_valid_keys) @@ -137,8 +179,6 @@ TYPED_TEST(groupby_sum_with_overflow_test, zero_valid_keys) auto agg = cudf::make_sum_with_overflow_aggregation(); test_single_agg(keys, vals, expect_keys, *expect_vals, std::move(agg)); - - // Note: SUM_WITH_OVERFLOW only works with hash groupby, not sort groupby } TYPED_TEST(groupby_sum_with_overflow_test, zero_valid_values) @@ -167,7 +207,10 @@ TYPED_TEST(groupby_sum_with_overflow_test, zero_valid_values) auto agg = cudf::make_sum_with_overflow_aggregation(); test_single_agg(keys, vals, expect_keys, *expect_vals, std::move(agg)); - // Note: SUM_WITH_OVERFLOW only works with hash groupby, not sort groupby + // Exercise the sort-based path for an all-null group. + auto agg_sort = cudf::make_sum_with_overflow_aggregation(); + test_single_agg( + keys, vals, expect_keys, *expect_vals, std::move(agg_sort), force_use_sort_impl::YES); } TYPED_TEST(groupby_sum_with_overflow_test, null_keys_and_values) @@ -202,7 +245,10 @@ TYPED_TEST(groupby_sum_with_overflow_test, null_keys_and_values) auto agg = cudf::make_sum_with_overflow_aggregation(); test_single_agg(keys, vals, expect_keys, *expect_vals, std::move(agg)); - // Note: SUM_WITH_OVERFLOW only works with hash groupby, not sort groupby + // Exercise the sort-based path with null keys and null values. + auto agg_sort = cudf::make_sum_with_overflow_aggregation(); + test_single_agg( + keys, vals, expect_keys, *expect_vals, std::move(agg_sort), force_use_sort_impl::YES); } TYPED_TEST(groupby_sum_with_overflow_test, overflow_detection) @@ -230,6 +276,29 @@ TYPED_TEST(groupby_sum_with_overflow_test, overflow_detection) CUDF_TEST_EXPECT_COLUMNS_EQUAL(sorted->view().column(1), expect_overflow); }; + // Same check, but a co-requested sort-only aggregation (TDIGEST) forces the sort path. + auto check_overflow_flags_sort = [](cudf::column_view const& keys, + cudf::column_view const& vals, + cudf::column_view const& expect_keys, + cudf::column_view const& expect_overflow) { + std::vector requests; + requests.emplace_back(); + requests[0].values = vals; + requests[0].aggregations.push_back( + cudf::make_sum_with_overflow_aggregation()); + requests[0].aggregations.push_back( + cudf::make_tdigest_aggregation(1000)); + + auto result = cudf::groupby::groupby(cudf::table_view{{keys}}).aggregate(requests); + auto const overflow_child = + cudf::structs_column_view{result.second[0].results[0]->view()}.get_sliced_child(1); + + auto sorted = cudf::sort_by_key( + cudf::table_view{{result.first->get_column(0).view(), overflow_child}}, result.first->view()); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(sorted->view().column(0), expect_keys); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(sorted->view().column(1), expect_overflow); + }; + cudf::test::fixed_width_column_wrapper keys{1, 2, 3, 4, 1, 2, 2, 1, 3, 3, 2, 4, 4}; cudf::test::fixed_width_column_wrapper expect_keys{1, 2, 3, 4}; cudf::test::fixed_width_column_wrapper expect_overflow{true, false, true, true}; @@ -267,16 +336,8 @@ TYPED_TEST(groupby_sum_with_overflow_test, overflow_detection) check_overflow_flags(keys, vals, expect_keys, expect_overflow); - // Adding nth_element forces sort-based groupby, which must throw for SUM_WITH_OVERFLOW. - std::vector sort_requests; - sort_requests.emplace_back(); - sort_requests[0].values = vals; - sort_requests[0].aggregations.push_back( - cudf::make_sum_with_overflow_aggregation()); - sort_requests[0].aggregations.push_back( - cudf::make_nth_element_aggregation(0)); - EXPECT_THROW(cudf::groupby::groupby(cudf::table_view{{keys}}).aggregate(sort_requests), - cudf::logic_error); + // Adding TDIGEST forces sort-based groupby; the overflow flags must match the hash path. + check_overflow_flags_sort(keys, vals, expect_keys, expect_overflow); } else { using DeviceType = cudf::device_storage_type_t; @@ -306,6 +367,7 @@ TYPED_TEST(groupby_sum_with_overflow_test, overflow_detection) static_cast(large_negative)}; check_overflow_flags(keys, vals, expect_keys, expect_overflow); + check_overflow_flags_sort(keys, vals, expect_keys, expect_overflow); } } diff --git a/java/src/test/java/ai/rapids/cudf/TableTest.java b/java/src/test/java/ai/rapids/cudf/TableTest.java index be4ea13e0f14..01ec01f35157 100644 --- a/java/src/test/java/ai/rapids/cudf/TableTest.java +++ b/java/src/test/java/ai/rapids/cudf/TableTest.java @@ -1,6 +1,6 @@ /* * - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * */ @@ -7848,9 +7848,31 @@ void testGroupByM2() { } } + private static void assertSumWithOverflowResult(Table results, + int[] expectedKeys, + long[] expectedSums, + boolean[] expectedOvf) { + ColumnVector structCol = results.getColumn(1); + assertEquals(DType.STRUCT, structCol.getType()); + try (ColumnView ovfChild = structCol.getChildColumnView(1); + ColumnVector ovfCol = ovfChild.copyToColumnVector(); + ColumnVector expectedKeyCol = ColumnVector.fromInts(expectedKeys); + ColumnVector expectedOvfCol = ColumnVector.fromBooleans(expectedOvf)) { + assertColumnsAreEqual(expectedKeyCol, results.getColumn(0)); + assertColumnsAreEqual(expectedOvfCol, ovfCol); + if (expectedSums != null) { + try (ColumnView sumChild = structCol.getChildColumnView(0); + ColumnVector sumCol = sumChild.copyToColumnVector(); + ColumnVector expectedSumCol = ColumnVector.fromLongs(expectedSums)) { + assertEquals(DType.INT64, sumCol.getType()); + assertColumnsAreEqual(expectedSumCol, sumCol); + } + } + } + } + @Test void testGroupByHashSumWithOverflow() { - // int64 keys 1, 2, 3 with values that fit comfortably in int64. try (Table input = new Table.TestBuilder() .column(1, 2, 3, 1, 2, 2, 1, 3, 3, 2) .column(10L, 20L, 30L, 11L, 21L, 22L, 12L, 31L, 32L, 23L) @@ -7860,28 +7882,13 @@ void testGroupByHashSumWithOverflow() { Table sorted = results.orderBy(OrderByArg.asc(0))) { assertEquals(2, sorted.getNumberOfColumns()); assertEquals(3, sorted.getRowCount()); - - ColumnVector keyCol = sorted.getColumn(0); - ColumnVector structCol = sorted.getColumn(1); - assertEquals(DType.STRUCT, structCol.getType()); - - try (ColumnView sumChild = structCol.getChildColumnView(0); - ColumnView ovfChild = structCol.getChildColumnView(1); - ColumnVector sumCol = sumChild.copyToColumnVector(); - ColumnVector ovfCol = ovfChild.copyToColumnVector(); - ColumnVector expectedKeys = ColumnVector.fromInts(1, 2, 3); - ColumnVector expectedSum = ColumnVector.fromLongs(33L, 86L, 93L); - ColumnVector expectedOvf = ColumnVector.fromBooleans(false, false, false)) { - assertColumnsAreEqual(expectedKeys, keyCol); - assertColumnsAreEqual(expectedSum, sumCol); - assertColumnsAreEqual(expectedOvf, ovfCol); - } + assertSumWithOverflowResult(sorted, + new int[]{1, 2, 3}, new long[]{33L, 86L, 93L}, new boolean[]{false, false, false}); } } @Test void testGroupByHashSumWithOverflowDetectsOverflow() { - // Group 1 overflows (max + max), group 2 stays in range. try (Table input = new Table.TestBuilder() .column(1, 1, 2, 2) .column(Long.MAX_VALUE, Long.MAX_VALUE, 3L, 4L) @@ -7889,12 +7896,7 @@ void testGroupByHashSumWithOverflowDetectsOverflow() { Table results = input.groupBy(0).aggregate( GroupByAggregation.sumWithOverflow().onColumn(1)); Table sorted = results.orderBy(OrderByArg.asc(0))) { - ColumnVector structCol = sorted.getColumn(1); - try (ColumnView ovfChild = structCol.getChildColumnView(1); - ColumnVector ovfCol = ovfChild.copyToColumnVector(); - ColumnVector expectedOvf = ColumnVector.fromBooleans(true, false)) { - assertColumnsAreEqual(expectedOvf, ovfCol); - } + assertSumWithOverflowResult(sorted, new int[]{1, 2}, null, new boolean[]{true, false}); } } @@ -7924,17 +7926,31 @@ void testGroupByHashSumWithOverflowInt32() { } @Test - void testGroupBySortSumWithOverflowThrows() { - // Sort-based groupby (keysSorted=true forces the sort impl in cudf). - // SUM_WITH_OVERFLOW is hash-only, so cudf should throw. - GroupByOptions sortOpts = GroupByOptions.builder().withKeysSorted(true).build(); + void testGroupBySortSumWithOverflow() { try (Table input = new Table.TestBuilder() .column(1, 1, 2, 2) .column(1L, 2L, 3L, 4L) - .build()) { - assertThrows(CudfException.class, () -> - input.groupBy(sortOpts, 0).aggregate( - GroupByAggregation.sumWithOverflow().onColumn(1)).close()); + .build(); + // median() is sort-only, so it forces the sort-based groupby path + Table results = input.groupBy(0).aggregate( + GroupByAggregation.sumWithOverflow().onColumn(1), + GroupByAggregation.median().onColumn(1))) { + assertSumWithOverflowResult(results, + new int[]{1, 2}, new long[]{3L, 7L}, new boolean[]{false, false}); + } + } + + @Test + void testGroupBySortSumWithOverflowDetectsOverflow() { + try (Table input = new Table.TestBuilder() + .column(1, 1, 2, 2) + .column(Long.MAX_VALUE, Long.MAX_VALUE, 3L, 4L) + .build(); + // median() is sort-only, so it forces the sort-based groupby path + Table results = input.groupBy(0).aggregate( + GroupByAggregation.sumWithOverflow().onColumn(1), + GroupByAggregation.median().onColumn(1))) { + assertSumWithOverflowResult(results, new int[]{1, 2}, null, new boolean[]{true, false}); } } From 46f57fb50b165ab8790bad2f7ec990ada75cbaa1 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 26 Jun 2026 14:11:35 -0700 Subject: [PATCH 16/22] Remove nogil from pylibcudf view()/mutable_view() and hoist calls out of nogil blocks (#23002) Remove `nogil` from pylibcudf `view()`/`mutable_view()` method declarations and hoist all pylibcudf `.view()`/`.mutable_view()` calls out of `with nogil:` blocks into typed `cdef` variables. This eliminates the Cython "Exception check after calling 'view' will always require the GIL" warnings since the C++ `column_view` constructor can throw. Closes #19720 Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Bradley Dice (https://github.com/bdice) - Matthew Roeschke (https://github.com/mroeschke) URL: https://github.com/rapidsai/cudf/pull/23002 --- ci/build_python.sh | 20 ++- ci/build_wheel_cudf_streaming.sh | 11 +- ci/build_wheel_pylibcudf.sh | 11 +- python/pylibcudf/pylibcudf/binaryop.pyx | 18 ++- python/pylibcudf/pylibcudf/column.pxd | 8 +- python/pylibcudf/pylibcudf/column.pyx | 83 +++++------ .../pylibcudf/pylibcudf/contiguous_split.pyx | 6 +- python/pylibcudf/pylibcudf/copying.pyx | 119 ++++++++++----- python/pylibcudf/pylibcudf/datetime.pyx | 40 ++++-- python/pylibcudf/pylibcudf/filling.pyx | 23 ++- python/pylibcudf/pylibcudf/groupby.pyx | 10 +- python/pylibcudf/pylibcudf/hashing.pyx | 33 +++-- python/pylibcudf/pylibcudf/interop.pyx | 9 +- python/pylibcudf/pylibcudf/io/json.pyx | 6 +- python/pylibcudf/pylibcudf/io/orc.pyx | 6 +- python/pylibcudf/pylibcudf/io/parquet.pyx | 6 +- python/pylibcudf/pylibcudf/join.pyx | 136 ++++++++++++------ python/pylibcudf/pylibcudf/json.pyx | 8 +- python/pylibcudf/pylibcudf/labeling.pyx | 12 +- python/pylibcudf/pylibcudf/lists.pyx | 118 ++++++++++----- python/pylibcudf/pylibcudf/null_mask.pyx | 13 +- .../pylibcudf/nvtext/byte_pair_encode.pyx | 5 +- .../pylibcudf/nvtext/deduplicate.pyx | 22 ++- python/pylibcudf/pylibcudf/nvtext/minhash.pyx | 39 +++-- .../pylibcudf/nvtext/ngrams_tokenize.pyx | 6 +- .../pylibcudf/pylibcudf/nvtext/normalize.pyx | 8 +- python/pylibcudf/pylibcudf/nvtext/replace.pyx | 15 +- python/pylibcudf/pylibcudf/nvtext/stemmer.pyx | 14 +- .../pylibcudf/pylibcudf/nvtext/tokenize.pyx | 32 +++-- .../pylibcudf/nvtext/wordpiece_tokenize.pyx | 5 +- python/pylibcudf/pylibcudf/partitioning.pyx | 22 ++- python/pylibcudf/pylibcudf/quantiles.pyx | 9 +- python/pylibcudf/pylibcudf/reduce.pyx | 25 ++-- python/pylibcudf/pylibcudf/replace.pyx | 40 ++++-- python/pylibcudf/pylibcudf/reshape.pyx | 12 +- python/pylibcudf/pylibcudf/rolling.pyx | 29 ++-- python/pylibcudf/pylibcudf/round.pyx | 9 +- python/pylibcudf/pylibcudf/search.pyx | 22 ++- python/pylibcudf/pylibcudf/sorting.pyx | 58 +++++--- .../pylibcudf/pylibcudf/stream_compaction.pyx | 36 +++-- .../pylibcudf/strings/attributes.pyx | 15 +- .../pylibcudf/strings/capitalize.pyx | 13 +- python/pylibcudf/pylibcudf/strings/case.pyx | 12 +- .../pylibcudf/strings/char_types.pyx | 11 +- .../pylibcudf/pylibcudf/strings/combine.pyx | 30 ++-- .../pylibcudf/pylibcudf/strings/contains.pyx | 19 +-- .../strings/convert/convert_booleans.pyx | 10 +- .../strings/convert/convert_datetime.pyx | 16 ++- .../strings/convert/convert_durations.pyx | 10 +- .../strings/convert/convert_fixed_point.pyx | 13 +- .../strings/convert/convert_floats.pyx | 13 +- .../strings/convert/convert_integers.pyx | 24 ++-- .../strings/convert/convert_ipv4.pyx | 13 +- .../strings/convert/convert_lists.pyx | 10 +- .../strings/convert/convert_urls.pyx | 10 +- .../pylibcudf/pylibcudf/strings/extract.pyx | 15 +- python/pylibcudf/pylibcudf/strings/find.pyx | 50 +++++-- .../pylibcudf/strings/find_multiple.pyx | 17 ++- .../pylibcudf/pylibcudf/strings/findall.pyx | 11 +- .../pylibcudf/pylibcudf/strings/padding.pyx | 18 +-- python/pylibcudf/pylibcudf/strings/repeat.pyx | 14 +- .../pylibcudf/pylibcudf/strings/replace.pyx | 21 +-- .../pylibcudf/strings/replace_re.pyx | 10 +- .../pylibcudf/pylibcudf/strings/reverse.pyx | 6 +- python/pylibcudf/pylibcudf/strings/slice.pyx | 18 ++- .../pylibcudf/strings/split/partition.pyx | 9 +- .../pylibcudf/strings/split/split.pyx | 31 ++-- python/pylibcudf/pylibcudf/strings/strip.pyx | 7 +- .../pylibcudf/pylibcudf/strings/translate.pyx | 11 +- python/pylibcudf/pylibcudf/strings/wrap.pyx | 7 +- python/pylibcudf/pylibcudf/table.pxd | 4 +- python/pylibcudf/pylibcudf/table.pyx | 19 +-- python/pylibcudf/pylibcudf/table_equality.pyx | 7 +- python/pylibcudf/pylibcudf/transform.pyx | 26 ++-- python/pylibcudf/pylibcudf/transpose.pyx | 5 +- python/pylibcudf/pylibcudf/unary.pyx | 21 ++- 76 files changed, 1097 insertions(+), 553 deletions(-) diff --git a/ci/build_python.sh b/ci/build_python.sh index 33701bbc833e..3858ebc6fd7a 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail @@ -38,7 +38,14 @@ rapids-logger "Building pylibcudf" rapids-telemetry-record build-pylibcudf.log \ rattler-build build --recipe conda/recipes/pylibcudf \ "${RATTLER_ARGS[@]}" \ - "${RATTLER_CHANNELS[@]}" + "${RATTLER_CHANNELS[@]}" 2>&1 | tee pylibcudf-build-output.log + +rapids-logger "Checking for Cython performance warnings in pylibcudf" +if grep -Fq "performance hint:" pylibcudf-build-output.log; then + echo "Cython performance hints found in pylibcudf build:" + grep -F "performance hint:" pylibcudf-build-output.log + exit 1 +fi rapids-telemetry-record sccache-stats-pylibcudf.txt sccache --show-adv-stats sccache --stop-server >/dev/null 2>&1 || true @@ -68,7 +75,14 @@ rapids-logger "Building cudf_streaming" rapids-telemetry-record build-cudf_streaming.log \ rattler-build build --recipe conda/recipes/cudf_streaming \ "${RATTLER_ARGS[@]}" \ - "${RATTLER_CHANNELS[@]}" + "${RATTLER_CHANNELS[@]}" 2>&1 | tee cudf_streaming-build-output.log + +rapids-logger "Checking for Cython performance warnings in cudf_streaming" +if grep -Fq "performance hint:" cudf_streaming-build-output.log; then + echo "Cython performance hints found in cudf_streaming build:" + grep -F "performance hint:" cudf_streaming-build-output.log + exit 1 +fi rapids-telemetry-record sccache-stats-cudf_streaming.txt sccache --show-adv-stats sccache --stop-server >/dev/null 2>&1 || true diff --git a/ci/build_wheel_cudf_streaming.sh b/ci/build_wheel_cudf_streaming.sh index ad6bbdc4b28e..f265528848c4 100755 --- a/ci/build_wheel_cudf_streaming.sh +++ b/ci/build_wheel_cudf_streaming.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail @@ -43,7 +43,14 @@ export PIP_NO_BUILD_ISOLATION=0 RAPIDS_PY_API="cp${RAPIDS_PY_VERSION//./}" export RAPIDS_PY_API -./ci/build_wheel.sh "${package_name}" "${package_dir}" --stable +./ci/build_wheel.sh "${package_name}" "${package_dir}" --stable 2>&1 | tee cudf-streaming-wheel-build-output.log + +rapids-logger "Checking for Cython performance warnings" +if grep -Fq "performance hint:" cudf-streaming-wheel-build-output.log; then + echo "Cython performance hints found in ${package_name} build:" + grep -F "performance hint:" cudf-streaming-wheel-build-output.log + exit 1 +fi # repair wheels and write to the location that artifact-uploading code expects to find them python -m auditwheel repair \ diff --git a/ci/build_wheel_pylibcudf.sh b/ci/build_wheel_pylibcudf.sh index 462c01bb294b..b78c762acb7c 100755 --- a/ci/build_wheel_pylibcudf.sh +++ b/ci/build_wheel_pylibcudf.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail @@ -23,7 +23,14 @@ echo "libcudf-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo "${LIBCUDF_WHEELHOUSE}"/l RAPIDS_PY_API="cp${RAPIDS_PY_VERSION//./}" export RAPIDS_PY_API -./ci/build_wheel.sh pylibcudf ${package_dir} --stable +./ci/build_wheel.sh pylibcudf ${package_dir} --stable 2>&1 | tee pylibcudf-wheel-build-output.log + +rapids-logger "Checking for Cython performance warnings" +if grep -Fq "performance hint:" pylibcudf-wheel-build-output.log; then + echo "Cython performance hints found in pylibcudf build:" + grep -F "performance hint:" pylibcudf-wheel-build-output.log + exit 1 +fi # repair wheels and write to the location that artifact-uploading code expects to find them python -m auditwheel repair \ diff --git a/python/pylibcudf/pylibcudf/binaryop.pyx b/python/pylibcudf/pylibcudf/binaryop.pyx index 20a69d607276..ea72b2d3d4e8 100644 --- a/python/pylibcudf/pylibcudf/binaryop.pyx +++ b/python/pylibcudf/pylibcudf/binaryop.pyx @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from cython.operator import dereference @@ -9,6 +9,7 @@ from libcpp.utility cimport move from pylibcudf.libcudf cimport binaryop as cpp_binaryop from pylibcudf.libcudf.binaryop cimport binary_operator from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view from pylibcudf.libcudf.binaryop import \ binary_operator as BinaryOperator # no-cython-lint @@ -64,22 +65,28 @@ cpdef Column binary_operation( cdef unique_ptr[column] result cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() + cdef column_view c_lhs_column + cdef column_view c_rhs_column + mr = _get_memory_resource(mr) if LeftBinaryOperand is Column and RightBinaryOperand is Column: + c_lhs_column = lhs.view() + c_rhs_column = rhs.view() with nogil: result = cpp_binaryop.binary_operation( - lhs.view(), - rhs.view(), + c_lhs_column, + c_rhs_column, op, output_type.c_obj, _cs, mr.get_mr() ) elif LeftBinaryOperand is Column and RightBinaryOperand is Scalar: + c_lhs_column = lhs.view() with nogil: result = cpp_binaryop.binary_operation( - lhs.view(), + c_lhs_column, dereference(rhs.c_obj), op, output_type.c_obj, @@ -87,10 +94,11 @@ cpdef Column binary_operation( mr.get_mr() ) elif LeftBinaryOperand is Scalar and RightBinaryOperand is Column: + c_rhs_column = rhs.view() with nogil: result = cpp_binaryop.binary_operation( dereference(lhs.c_obj), - rhs.view(), + c_rhs_column, op, output_type.c_obj, _cs, diff --git a/python/pylibcudf/pylibcudf/column.pxd b/python/pylibcudf/pylibcudf/column.pxd index 08fa94b9a573..8cb3b7d4093f 100644 --- a/python/pylibcudf/pylibcudf/column.pxd +++ b/python/pylibcudf/pylibcudf/column.pxd @@ -54,8 +54,8 @@ cdef class Column: list _children size_type _num_children - cdef column_view view(self) nogil - cdef mutable_column_view mutable_view(self) nogil + cdef column_view view(self) + cdef mutable_column_view mutable_view(self) @staticmethod cdef Column from_libcudf( @@ -107,11 +107,11 @@ cdef class ListsColumnView: cdef Column _column cpdef child(self) cpdef offsets(self) - cdef lists_column_view view(self) nogil + cdef lists_column_view view(self) cpdef Column get_sliced_child(self, object stream=*) cdef class StructsColumnView: cdef Column _column - cdef structs_column_view view(self) nogil + cdef structs_column_view view(self) cpdef Column get_sliced_child(self, int index, object stream=*) diff --git a/python/pylibcudf/pylibcudf/column.pyx b/python/pylibcudf/pylibcudf/column.pyx index 1827340c0ce8..925a05273640 100644 --- a/python/pylibcudf/pylibcudf/column.pyx +++ b/python/pylibcudf/pylibcudf/column.pyx @@ -14,6 +14,7 @@ from libcpp.memory cimport make_unique, unique_ptr from libcpp.utility cimport move from pylibcudf.libcudf.column.column cimport column, column_contents +from pylibcudf.libcudf.column.column_view cimport column_view from pylibcudf.libcudf.column.column_factories cimport make_column_from_scalar from pylibcudf.libcudf.copying cimport get_element from pylibcudf.libcudf.interop cimport ( @@ -533,7 +534,7 @@ cdef class Column: else: raise ValueError("Invalid Arrow-like object") - cdef column_view view(self) nogil: + cdef column_view view(self): """Generate a libcudf column_view to pass to libcudf algorithms. This method is for pylibcudf's functions to use to generate inputs when @@ -545,37 +546,26 @@ cdef class Column: cdef size_t data_ptr cdef size_t mask_ptr - with gil: - if self._data is not None: - data_ptr = self._data.ptr - data = data_ptr - if self._mask is not None: - mask_ptr = self._mask.ptr - null_mask = mask_ptr + if self._data is not None: + data_ptr = self._data.ptr + data = data_ptr + if self._mask is not None: + mask_ptr = self._mask.ptr + null_mask = mask_ptr # TODO: Check if children can ever change. If not, this could be # computed once in the constructor and always be reused. cdef vector[column_view] c_children - with gil: - if self._children is not None: - for child in self._children: - # Need to cast to Column here so that Cython knows that - # `view` returns a typed object, not a Python object. We - # cannot use a typed variable for `child` because cdef - # declarations cannot be inside nested blocks (`if` or - # `with` blocks) so we cannot declare it inside the `with - # gil` block, but we also cannot declare it outside the - # `with gil` block because it is erroneous to declare a - # variable of a cdef class type in a `nogil` context (which - # this whole function is). - c_children.push_back(( child).view()) + if self._children is not None: + for child in self._children: + c_children.push_back(( child).view()) return column_view( self._data_type.c_obj, self._size, data, null_mask, self._null_count, self._offset, c_children ) - cdef mutable_column_view mutable_view(self) nogil: + cdef mutable_column_view mutable_view(self): """Generate a libcudf mutable_column_view to pass to libcudf algorithms. This method is for pylibcudf's functions to use to generate inputs when @@ -587,20 +577,18 @@ cdef class Column: cdef size_t data_ptr cdef size_t mask_ptr - with gil: - if self._data is not None: - data_ptr = self._data.ptr - data = data_ptr - if self._mask is not None: - mask_ptr = self._mask.ptr - null_mask = mask_ptr + if self._data is not None: + data_ptr = self._data.ptr + data = data_ptr + if self._mask is not None: + mask_ptr = self._mask.ptr + null_mask = mask_ptr cdef vector[mutable_column_view] c_children - with gil: - if self._children is not None: - for child in self._children: - # See the view method for why this needs to be cast. - c_children.push_back(( child).mutable_view()) + if self._children is not None: + for child in self._children: + # See the view method for why this needs to be cast. + c_children.push_back(( child).mutable_view()) return mutable_column_view( self._data_type.c_obj, self._size, data, null_mask, @@ -1271,6 +1259,8 @@ cdef class Column: NotImplementedError If the column type is not fixed-width or string. """ + cdef column_view c_self + cdef type_id dtype = self.type().id() cdef bint large_offsets = ( dtype == type_id.STRING @@ -1280,8 +1270,9 @@ cdef class Column: cdef Stream _stream = _get_stream(None) cdef cudaStream_t _cs = _stream.view().value() cdef ArrowArray* raw_host_array_ptr = NULL + c_self = self.view() with nogil: - raw_host_array_ptr = to_arrow_host_raw(self.view(), _cs) + raw_host_array_ptr = to_arrow_host_raw(c_self, _cs) try: return _arrow_to_pylist_impl(dtype, raw_host_array_ptr, large_offsets) finally: @@ -1408,9 +1399,11 @@ cdef class Column: cdef unique_ptr[column] c_result cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() + mr = _get_memory_resource(mr) + cdef column_view c_self = self.view() with nogil: - c_result = make_unique[column](self.view(), _cs, mr.get_mr()) + c_result = make_unique[column](c_self, _cs, mr.get_mr()) return Column.from_libcudf(move(c_result), _stream, mr) cpdef uint64_t device_buffer_size(self): @@ -1446,6 +1439,9 @@ cdef class Column: def _to_schema(self, metadata=None): """Create an Arrow schema from this Column.""" + + cdef column_view c_self + if metadata is None: metadata = self._create_nested_column_metadata() elif isinstance(metadata, str): @@ -1454,8 +1450,9 @@ cdef class Column: cdef column_metadata c_metadata = _metadata_to_libcudf(metadata) cdef ArrowSchema* raw_schema_ptr + c_self = self.view() with nogil: - raw_schema_ptr = to_arrow_schema_raw(self.view(), c_metadata) + raw_schema_ptr = to_arrow_schema_raw(c_self, c_metadata) return PyCapsule_New(raw_schema_ptr, 'arrow_schema', _release_schema) @@ -1463,15 +1460,19 @@ cdef class Column: cdef ArrowArray* raw_host_array_ptr cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() + + cdef column_view c_self = self.view() with nogil: - raw_host_array_ptr = to_arrow_host_raw(self.view(), _cs) + raw_host_array_ptr = to_arrow_host_raw(c_self, _cs) return PyCapsule_New(raw_host_array_ptr, "arrow_array", _release_array) def _to_device_array(self): cdef ArrowDeviceArray* raw_device_array_ptr + + cdef column_view c_self = self.view() with nogil: - raw_device_array_ptr = to_arrow_device_raw(self.view(), self) + raw_device_array_ptr = to_arrow_device_raw(c_self, self) return PyCapsule_New( raw_device_array_ptr, @@ -1517,7 +1518,7 @@ cdef class ListsColumnView: """The offsets column of the underlying list column.""" return self._column.child(0) - cdef lists_column_view view(self) nogil: + cdef lists_column_view view(self): """Generate a libcudf lists_column_view to pass to libcudf algorithms. This method is for pylibcudf's functions to use to generate inputs when @@ -1555,7 +1556,7 @@ cdef class StructsColumnView: __hash__ = None - cdef structs_column_view view(self) nogil: + cdef structs_column_view view(self): """Generate a libcudf structs_column_view to pass to libcudf algorithms. This method is for pylibcudf's functions to use to generate inputs when diff --git a/python/pylibcudf/pylibcudf/contiguous_split.pyx b/python/pylibcudf/pylibcudf/contiguous_split.pyx index 239d89d64705..1f5cb5c52f89 100644 --- a/python/pylibcudf/pylibcudf/contiguous_split.pyx +++ b/python/pylibcudf/pylibcudf/contiguous_split.pyx @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from cpython.buffer cimport PyBuffer_FillInfo @@ -351,10 +351,12 @@ cpdef PackedColumns pack(Table input, object stream=None, DeviceMemoryResource m cdef unique_ptr[packed_columns] pack cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() + mr = _get_memory_resource(mr) + cdef table_view c_input = input.view() with nogil: pack = move(make_unique[packed_columns]( - cpp_pack(input.view(), _cs, mr.get_mr()) + cpp_pack(c_input, _cs, mr.get_mr()) )) return PackedColumns.from_libcudf(move(pack), _stream, mr) diff --git a/python/pylibcudf/pylibcudf/copying.pyx b/python/pylibcudf/pylibcudf/copying.pyx index 30be1ea7d0a9..22289c325835 100644 --- a/python/pylibcudf/pylibcudf/copying.pyx +++ b/python/pylibcudf/pylibcudf/copying.pyx @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from cython.operator import dereference @@ -99,10 +99,12 @@ cpdef Table gather( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_source_table = source_table.view() + cdef column_view c_gather_map = gather_map.view() with nogil: c_result = cpp_copying.gather( - source_table.view(), - gather_map.view(), + c_source_table, + c_gather_map, bounds_policy, _cs, mr.get_mr() @@ -159,24 +161,33 @@ cpdef Table scatter( cdef vector[reference_wrapper[const scalar]] source_scalars cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() + cdef table_view c_source_table + cdef column_view c_scatter_map + cdef table_view c_target_table + mr = _get_memory_resource(mr) if TableOrListOfScalars is Table: + c_source_table = source.view() + c_scatter_map = scatter_map.view() + c_target_table = target_table.view() with nogil: c_result = cpp_copying.scatter( - source.view(), - scatter_map.view(), - target_table.view(), + c_source_table, + c_scatter_map, + c_target_table, _cs, mr.get_mr() ) else: source_scalars = _as_vector(source) + c_scatter_map = scatter_map.view() + c_target_table = target_table.view() with nogil: c_result = cpp_copying.scatter( source_scalars, - scatter_map.view(), - target_table.view(), + c_scatter_map, + c_target_table, _cs, mr.get_mr() ) @@ -205,14 +216,19 @@ cpdef ColumnOrTable empty_like( cdef unique_ptr[table] c_tbl_result cdef unique_ptr[column] c_col_result cdef Stream _stream = _get_stream(stream) + cdef column_view c_input_column + cdef table_view c_input_table + mr = _get_memory_resource(mr) if ColumnOrTable is Column: + c_input_column = input.view() with nogil: - c_col_result = cpp_copying.empty_like(input.view()) + c_col_result = cpp_copying.empty_like(c_input_column) return Column.from_libcudf(move(c_col_result), _stream, mr) else: + c_input_table = input.view() with nogil: - c_tbl_result = cpp_copying.empty_like(input.view()) + c_tbl_result = cpp_copying.empty_like(c_input_table) return Table.from_libcudf(move(c_tbl_result), _stream, mr) @@ -251,9 +267,10 @@ cpdef Column allocate_like( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input_column = input_column.view() with nogil: c_result = cpp_copying.allocate_like( - input_column.view(), + c_input_column, c_size, policy, _cs, @@ -308,9 +325,10 @@ cpdef Column copy_range_in_place( cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() + cdef column_view c_input_column = input_column.view() with nogil: cpp_copying.copy_range_in_place( - input_column.view(), + c_input_column, target_view, input_begin, input_end, @@ -366,10 +384,12 @@ cpdef Column copy_range( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input_column = input_column.view() + cdef column_view c_target_column = target_column.view() with nogil: c_result = cpp_copying.copy_range( - input_column.view(), - target_column.view(), + c_input_column, + c_target_column, input_begin, input_end, target_begin, @@ -419,9 +439,10 @@ cpdef Column shift( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: c_result = cpp_copying.shift( - input.view(), + c_input, offset, dereference(fill_value.c_obj), _cs, @@ -464,17 +485,22 @@ cpdef list slice(ColumnOrTable input, list indices, object stream=None): cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() + cdef column_view c_input_column + cdef table_view c_input_table + if ColumnOrTable is Column: + c_input_column = input.view() with nogil: - c_col_result = cpp_copying.slice(input.view(), c_indices, _cs) + c_col_result = cpp_copying.slice(c_input_column, c_indices, _cs) return [ Column.from_column_view(c_col_result[i], input) for i in range(c_col_result.size()) ] else: + c_input_table = input.view() with nogil: - c_tbl_result = cpp_copying.slice(input.view(), c_indices, _cs) + c_tbl_result = cpp_copying.slice(c_input_table, c_indices, _cs) return [ Table.from_table_view(c_tbl_result[i], input) @@ -508,17 +534,22 @@ cpdef list split(ColumnOrTable input, list splits, object stream=None): cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() + cdef column_view c_input_column + cdef table_view c_input_table + if ColumnOrTable is Column: + c_input_column = input.view() with nogil: - c_col_result = cpp_copying.split(input.view(), c_splits, _cs) + c_col_result = cpp_copying.split(c_input_column, c_splits, _cs) return [ Column.from_column_view(c_col_result[i], input) for i in range(c_col_result.size()) ] else: + c_input_table = input.view() with nogil: - c_tbl_result = cpp_copying.split(input.view(), c_splits, _cs) + c_tbl_result = cpp_copying.split(c_input_table, c_splits, _cs) return [ Table.from_table_view(c_tbl_result[i], input) @@ -567,41 +598,53 @@ cpdef Column copy_if_else( cdef unique_ptr[column] result cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() + cdef column_view c_lhs_column + cdef column_view c_rhs_column + cdef column_view c_boolean_mask + mr = _get_memory_resource(mr) if LeftCopyIfElseOperand is Column and RightCopyIfElseOperand is Column: + c_lhs_column = lhs.view() + c_rhs_column = rhs.view() + c_boolean_mask = boolean_mask.view() with nogil: result = cpp_copying.copy_if_else( - lhs.view(), - rhs.view(), - boolean_mask.view(), + c_lhs_column, + c_rhs_column, + c_boolean_mask, _cs, mr.get_mr() ) elif LeftCopyIfElseOperand is Column and RightCopyIfElseOperand is Scalar: + c_lhs_column = lhs.view() + c_boolean_mask = boolean_mask.view() with nogil: result = cpp_copying.copy_if_else( - lhs.view(), + c_lhs_column, dereference(rhs.c_obj), - boolean_mask.view(), + c_boolean_mask, _cs, mr.get_mr() ) elif LeftCopyIfElseOperand is Scalar and RightCopyIfElseOperand is Column: + c_rhs_column = rhs.view() + c_boolean_mask = boolean_mask.view() with nogil: result = cpp_copying.copy_if_else( dereference(lhs.c_obj), - rhs.view(), - boolean_mask.view(), + c_rhs_column, + c_boolean_mask, _cs, mr.get_mr() ) else: + c_boolean_mask = boolean_mask.view() with nogil: result = cpp_copying.copy_if_else( dereference(lhs.c_obj), dereference(rhs.c_obj), - boolean_mask.view(), + c_boolean_mask, _cs, mr.get_mr() ) @@ -653,24 +696,33 @@ cpdef Table boolean_mask_scatter( cdef vector[reference_wrapper[const scalar]] source_scalars cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() + cdef table_view c_input_table + cdef table_view c_target + cdef column_view c_boolean_mask + mr = _get_memory_resource(mr) if TableOrListOfScalars is Table: + c_input_table = input.view() + c_target = target.view() + c_boolean_mask = boolean_mask.view() with nogil: result = cpp_copying.boolean_mask_scatter( - input.view(), - target.view(), - boolean_mask.view(), + c_input_table, + c_target, + c_boolean_mask, _cs, mr.get_mr() ) else: source_scalars = _as_vector(input) + c_target = target.view() + c_boolean_mask = boolean_mask.view() with nogil: result = cpp_copying.boolean_mask_scatter( source_scalars, - target.view(), - boolean_mask.view(), + c_target, + c_boolean_mask, _cs, mr.get_mr() ) @@ -712,9 +764,10 @@ cpdef Scalar get_element( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input_column = input_column.view() with nogil: c_output = cpp_copying.get_element( - input_column.view(), index, _cs, mr.get_mr() + c_input_column, index, _cs, mr.get_mr() ) return Scalar.from_libcudf(move(c_output)) diff --git a/python/pylibcudf/pylibcudf/datetime.pyx b/python/pylibcudf/pylibcudf/datetime.pyx index 1e5270bad92e..91b6ef83037e 100644 --- a/python/pylibcudf/pylibcudf/datetime.pyx +++ b/python/pylibcudf/pylibcudf/datetime.pyx @@ -1,8 +1,9 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libcpp.memory cimport unique_ptr from libcpp.utility cimport move from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view from pylibcudf.libcudf.datetime cimport ( add_calendrical_months as cpp_add_calendrical_months, ceil_datetimes as cpp_ceil_datetimes, @@ -78,9 +79,10 @@ cpdef Column extract_datetime_component( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: result = cpp_extract_datetime_component( - input.view(), component, _cs, mr.get_mr() + c_input, component, _cs, mr.get_mr() ) return Column.from_libcudf(move(result), _stream, mr) @@ -115,8 +117,9 @@ cpdef Column ceil_datetimes( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: - result = cpp_ceil_datetimes(input.view(), freq, _cs, mr.get_mr()) + result = cpp_ceil_datetimes(c_input, freq, _cs, mr.get_mr()) return Column.from_libcudf(move(result), _stream, mr) cpdef Column floor_datetimes( @@ -150,8 +153,9 @@ cpdef Column floor_datetimes( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: - result = cpp_floor_datetimes(input.view(), freq, _cs, mr.get_mr()) + result = cpp_floor_datetimes(c_input, freq, _cs, mr.get_mr()) return Column.from_libcudf(move(result), _stream, mr) cpdef Column round_datetimes( @@ -185,8 +189,9 @@ cpdef Column round_datetimes( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: - result = cpp_round_datetimes(input.view(), freq, _cs, mr.get_mr()) + result = cpp_round_datetimes(c_input, freq, _cs, mr.get_mr()) return Column.from_libcudf(move(result), _stream, mr) cpdef Column add_calendrical_months( @@ -216,6 +221,9 @@ cpdef Column add_calendrical_months( Column Column of computed timestamps. """ + cdef column_view c_input + cdef column_view c_months_column + if not isinstance(months, (Column, Scalar)): raise TypeError("Must pass a Column or Scalar") @@ -225,10 +233,13 @@ cpdef Column add_calendrical_months( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + c_input = input.view() + if ColumnOrScalar is Column: + c_months_column = months.view() with nogil: result = cpp_add_calendrical_months( - input.view(), - months.view() if ColumnOrScalar is Column else + c_input, + c_months_column if ColumnOrScalar is Column else dereference(months.get()), _cs, mr.get_mr() @@ -263,8 +274,9 @@ cpdef Column day_of_year( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: - result = cpp_day_of_year(input.view(), _cs, mr.get_mr()) + result = cpp_day_of_year(c_input, _cs, mr.get_mr()) return Column.from_libcudf(move(result), _stream, mr) cpdef Column is_leap_year( @@ -294,8 +306,9 @@ cpdef Column is_leap_year( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: - result = cpp_is_leap_year(input.view(), _cs, mr.get_mr()) + result = cpp_is_leap_year(c_input, _cs, mr.get_mr()) return Column.from_libcudf(move(result), _stream, mr) cpdef Column last_day_of_month( @@ -325,8 +338,9 @@ cpdef Column last_day_of_month( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: - result = cpp_last_day_of_month(input.view(), _cs, mr.get_mr()) + result = cpp_last_day_of_month(c_input, _cs, mr.get_mr()) return Column.from_libcudf(move(result), _stream, mr) cpdef Column extract_quarter( @@ -356,8 +370,9 @@ cpdef Column extract_quarter( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: - result = cpp_extract_quarter(input.view(), _cs, mr.get_mr()) + result = cpp_extract_quarter(c_input, _cs, mr.get_mr()) return Column.from_libcudf(move(result), _stream, mr) cpdef Column days_in_month( @@ -386,8 +401,9 @@ cpdef Column days_in_month( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: - result = cpp_days_in_month(input.view(), _cs, mr.get_mr()) + result = cpp_days_in_month(c_input, _cs, mr.get_mr()) return Column.from_libcudf(move(result), _stream, mr) DatetimeComponent.__str__ = DatetimeComponent.__repr__ diff --git a/python/pylibcudf/pylibcudf/filling.pyx b/python/pylibcudf/pylibcudf/filling.pyx index ce6002eb24ef..4a11a33ea8a4 100644 --- a/python/pylibcudf/pylibcudf/filling.pyx +++ b/python/pylibcudf/pylibcudf/filling.pyx @@ -1,11 +1,14 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from cython.operator cimport dereference from libcpp.memory cimport unique_ptr from libcpp.utility cimport move from pylibcudf.libcudf.column.column cimport column -from pylibcudf.libcudf.column.column_view cimport mutable_column_view +from pylibcudf.libcudf.column.column_view cimport ( + column_view, + mutable_column_view, +) from pylibcudf.libcudf.filling cimport ( fill as cpp_fill, fill_in_place as cpp_fill_in_place, @@ -14,6 +17,7 @@ from pylibcudf.libcudf.filling cimport ( calendrical_month_sequence as cpp_calendrical_month_sequence ) from pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.table.table_view cimport table_view from pylibcudf.libcudf.types cimport size_type from rmm.pylibrmm.stream cimport Stream from rmm.pylibrmm.memory_resource cimport DeviceMemoryResource @@ -73,9 +77,10 @@ cpdef Column fill( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_destination = destination.view() with nogil: result = cpp_fill( - destination.view(), + c_destination, begin, end, dereference(( value).c_obj), @@ -209,20 +214,26 @@ cpdef Table repeat( cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() + cdef table_view c_input_table + cdef column_view c_count_column + mr = _get_memory_resource(mr) if ColumnOrSize is Column: + c_input_table = input_table.view() + c_count_column = count.view() with nogil: result = cpp_repeat( - input_table.view(), - count.view(), + c_input_table, + c_count_column, _cs, mr.get_mr() ) if ColumnOrSize is size_type: + c_input_table = input_table.view() with nogil: result = cpp_repeat( - input_table.view(), + c_input_table, count, _cs, mr.get_mr() diff --git a/python/pylibcudf/pylibcudf/groupby.pyx b/python/pylibcudf/pylibcudf/groupby.pyx index 4b2f842a3600..1646639bbb83 100644 --- a/python/pylibcudf/pylibcudf/groupby.pyx +++ b/python/pylibcudf/pylibcudf/groupby.pyx @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from cython.operator cimport dereference @@ -272,6 +272,8 @@ cdef class GroupBy: A tuple whose first element is the group's keys and whose second element is a table of shifted values. """ + cdef table_view c_values + cdef vector[reference_wrapper[const scalar]] c_fill_values = \ _as_vector(fill_values) @@ -280,9 +282,10 @@ cdef class GroupBy: cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + c_values = values.view() with nogil: c_res = dereference(self.c_obj).shift( - values.view(), + c_values, c_offset, c_fill_values, _cs, @@ -324,9 +327,10 @@ cdef class GroupBy: cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_value = value.view() with nogil: c_res = dereference(self.c_obj).replace_nulls( - value.view(), + c_value, c_replace_policies, _cs, mr.get_mr() diff --git a/python/pylibcudf/pylibcudf/hashing.pyx b/python/pylibcudf/pylibcudf/hashing.pyx index 941393cf9498..fb2fcad3c82a 100644 --- a/python/pylibcudf/pylibcudf/hashing.pyx +++ b/python/pylibcudf/pylibcudf/hashing.pyx @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libc.stdint cimport uint32_t, uint64_t from libcpp.memory cimport unique_ptr @@ -18,6 +18,7 @@ from pylibcudf.libcudf.hash cimport ( xxhash_64 as cpp_xxhash_64, ) from pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.table.table_view cimport table_view from rmm.pylibrmm.memory_resource cimport DeviceMemoryResource from rmm.pylibrmm.stream cimport Stream @@ -70,9 +71,10 @@ cpdef Column murmurhash3_x86_32( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_input = input.view() with nogil: c_result = cpp_murmurhash3_x86_32( - input.view(), + c_input, seed, _cs, mr.get_mr() @@ -109,9 +111,10 @@ cpdef Table murmurhash3_x64_128( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_input = input.view() with nogil: c_result = cpp_murmurhash3_x64_128( - input.view(), + c_input, seed, _cs, mr.get_mr() @@ -149,9 +152,10 @@ cpdef Column xxhash_32( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_input = input.view() with nogil: c_result = cpp_xxhash_32( - input.view(), + c_input, seed, _cs, mr.get_mr() @@ -189,9 +193,10 @@ cpdef Column xxhash_64( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_input = input.view() with nogil: c_result = cpp_xxhash_64( - input.view(), + c_input, seed, _cs, mr.get_mr() @@ -229,8 +234,9 @@ cpdef Column md5( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_input = input.view() with nogil: - c_result = cpp_md5(input.view(), _cs, mr.get_mr()) + c_result = cpp_md5(c_input, _cs, mr.get_mr()) return Column.from_libcudf(move(c_result), _stream, mr) cpdef Column sha1( @@ -260,8 +266,9 @@ cpdef Column sha1( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_input = input.view() with nogil: - c_result = cpp_sha1(input.view(), _cs, mr.get_mr()) + c_result = cpp_sha1(c_input, _cs, mr.get_mr()) return Column.from_libcudf(move(c_result), _stream, mr) @@ -292,8 +299,9 @@ cpdef Column sha224( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_input = input.view() with nogil: - c_result = cpp_sha224(input.view(), _cs, mr.get_mr()) + c_result = cpp_sha224(c_input, _cs, mr.get_mr()) return Column.from_libcudf(move(c_result), _stream, mr) @@ -324,8 +332,9 @@ cpdef Column sha256( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_input = input.view() with nogil: - c_result = cpp_sha256(input.view(), _cs, mr.get_mr()) + c_result = cpp_sha256(c_input, _cs, mr.get_mr()) return Column.from_libcudf(move(c_result), _stream, mr) @@ -356,8 +365,9 @@ cpdef Column sha384( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_input = input.view() with nogil: - c_result = cpp_sha384(input.view(), _cs, mr.get_mr()) + c_result = cpp_sha384(c_input, _cs, mr.get_mr()) return Column.from_libcudf(move(c_result), _stream, mr) @@ -388,6 +398,7 @@ cpdef Column sha512( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_input = input.view() with nogil: - c_result = cpp_sha512(input.view(), _cs, mr.get_mr()) + c_result = cpp_sha512(c_input, _cs, mr.get_mr()) return Column.from_libcudf(move(c_result), _stream, mr) diff --git a/python/pylibcudf/pylibcudf/interop.pyx b/python/pylibcudf/pylibcudf/interop.pyx index b43233ef5492..b31792f13b2f 100644 --- a/python/pylibcudf/pylibcudf/interop.pyx +++ b/python/pylibcudf/pylibcudf/interop.pyx @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from cpython.pycapsule cimport ( @@ -16,6 +16,7 @@ from pylibcudf.libcudf.interop cimport ( to_dlpack as cpp_to_dlpack, ) from pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.table.table_view cimport table_view from rmm.pylibrmm.stream cimport Stream from rmm.pylibrmm.memory_resource cimport DeviceMemoryResource @@ -102,6 +103,9 @@ cpdef object to_dlpack(Table input, object stream=None, DeviceMemoryResource mr= PyCapsule 1D or 2D DLPack tensor with a copy of the table data, or nullptr. """ + + cdef table_view c_input + for col in input._columns: if col.null_count(): raise ValueError( @@ -113,8 +117,9 @@ cpdef object to_dlpack(Table input, object stream=None, DeviceMemoryResource mr= cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + c_input = input.view() with nogil: - dlpack_tensor = cpp_to_dlpack(input.view(), _cs, mr.get_mr()) + dlpack_tensor = cpp_to_dlpack(c_input, _cs, mr.get_mr()) return PyCapsule_New( dlpack_tensor, diff --git a/python/pylibcudf/pylibcudf/io/json.pyx b/python/pylibcudf/pylibcudf/io/json.pyx index 1bce364fdd8c..a1c716f18a66 100644 --- a/python/pylibcudf/pylibcudf/io/json.pyx +++ b/python/pylibcudf/pylibcudf/io/json.pyx @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libcpp cimport bool from libcpp.map cimport map @@ -41,6 +41,7 @@ from pylibcudf.libcudf.io.json import json_recovery_mode_t as JsonRecoveryModeTy from pylibcudf.libcudf.types cimport data_type, size_type from pylibcudf.libcudf.column.column cimport column, column_contents +from pylibcudf.libcudf.column.column_view cimport column_view from pylibcudf.types cimport DataType @@ -860,10 +861,11 @@ cpdef TableWithMetadata read_json_from_string_column( mr = _get_memory_resource(mr) # Join the string column into a single string + cdef column_view c_input = input.view() with nogil: c_join_string_column = move( cpp_combine.join_strings( - input.view(), + c_input, dereference(c_separator), dereference(c_narep), _cs, diff --git a/python/pylibcudf/pylibcudf/io/orc.pyx b/python/pylibcudf/pylibcudf/io/orc.pyx index 3a2fabc5683b..a25151bd0365 100644 --- a/python/pylibcudf/pylibcudf/io/orc.pyx +++ b/python/pylibcudf/pylibcudf/io/orc.pyx @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libcpp cimport bool from libcpp.string cimport string @@ -49,6 +49,7 @@ from pylibcudf.libcudf.io.orc cimport ( ) from pylibcudf.libcudf.types cimport size_type +from pylibcudf.libcudf.table.table_view cimport table_view from pylibcudf.types cimport DataType @@ -720,8 +721,9 @@ cdef class OrcChunkedWriter: ------- None """ + cdef table_view c_table = table.view() with nogil: - self.c_obj.get()[0].write(table.view()) + self.c_obj.get()[0].write(c_table) @staticmethod def from_options(ChunkedOrcWriterOptions options, object stream = None): diff --git a/python/pylibcudf/pylibcudf/io/parquet.pyx b/python/pylibcudf/pylibcudf/io/parquet.pyx index d43d956960f1..fdffb02b0f32 100644 --- a/python/pylibcudf/pylibcudf/io/parquet.pyx +++ b/python/pylibcudf/pylibcudf/io/parquet.pyx @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from cython.operator cimport dereference import warnings @@ -46,6 +46,7 @@ from pylibcudf.libcudf.io.types cimport ( statistics_freq, table_with_metadata, ) +from pylibcudf.libcudf.table.table_view cimport table_view from pylibcudf.libcudf.types cimport size_type, type_id from pylibcudf.table cimport Table from pylibcudf.utils cimport _get_stream, _get_memory_resource @@ -741,8 +742,9 @@ cdef class ChunkedParquetWriter: partitions.push_back( partition_info(part[0], part[1]) ) + cdef table_view c_table = table.view() with nogil: - self.c_obj.get()[0].write(table.view(), partitions) + self.c_obj.get()[0].write(c_table, partitions) @staticmethod def from_options(ChunkedParquetWriterOptions options, object stream = None): diff --git a/python/pylibcudf/pylibcudf/join.pyx b/python/pylibcudf/pylibcudf/join.pyx index 9a2f84448b09..d0f90f777ea2 100644 --- a/python/pylibcudf/pylibcudf/join.pyx +++ b/python/pylibcudf/pylibcudf/join.pyx @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from cython.operator import dereference @@ -10,6 +10,7 @@ from libcpp.utility cimport move from pylibcudf.libcudf cimport join as cpp_join from pylibcudf.libcudf.column.column cimport column from pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.table.table_view cimport table_view from pylibcudf.libcudf.types cimport null_equality from rmm.librmm.device_buffer cimport device_buffer @@ -91,10 +92,12 @@ cpdef tuple inner_join( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_left_keys = left_keys.view() + cdef table_view c_right_keys = right_keys.view() with nogil: c_result = cpp_join.inner_join( - left_keys.view(), - right_keys.view(), + c_left_keys, + c_right_keys, nulls_equal, _cs, mr.get_mr() @@ -137,10 +140,12 @@ cpdef tuple left_join( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_left_keys = left_keys.view() + cdef table_view c_right_keys = right_keys.view() with nogil: c_result = cpp_join.left_join( - left_keys.view(), - right_keys.view(), + c_left_keys, + c_right_keys, nulls_equal, _cs, mr.get_mr() @@ -183,10 +188,12 @@ cpdef tuple full_join( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_left_keys = left_keys.view() + cdef table_view c_right_keys = right_keys.view() with nogil: c_result = cpp_join.full_join( - left_keys.view(), - right_keys.view(), + c_left_keys, + c_right_keys, nulls_equal, _cs, mr.get_mr() @@ -230,16 +237,18 @@ cpdef Column left_semi_join( cdef unique_ptr[cpp_join.filtered_join] join_obj + cdef table_view c_right_keys = right_keys.view() + cdef table_view c_left_keys = left_keys.view() with nogil: join_obj.reset( new cpp_join.filtered_join( - right_keys.view(), + c_right_keys, nulls_equal, _cs ) ) c_result = join_obj.get()[0].semi_join( - left_keys.view(), + c_left_keys, _cs, mr.get_mr() ) @@ -279,16 +288,18 @@ cpdef Column left_anti_join( cdef unique_ptr[cpp_join.filtered_join] join_obj + cdef table_view c_right_keys = right_keys.view() + cdef table_view c_left_keys = left_keys.view() with nogil: join_obj.reset( new cpp_join.filtered_join( - right_keys.view(), + c_right_keys, nulls_equal, _cs ) ) c_result = join_obj.get()[0].anti_join( - left_keys.view(), + c_left_keys, _cs, mr.get_mr() ) @@ -324,9 +335,11 @@ cpdef Table cross_join( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_left = left.view() + cdef table_view c_right = right.view() with nogil: result = cpp_join.cross_join( - left.view(), right.view(), _cs, mr.get_mr() + c_left, c_right, _cs, mr.get_mr() ) return Table.from_libcudf(move(result), _stream, mr) @@ -364,10 +377,12 @@ cpdef tuple conditional_inner_join( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_left = left.view() + cdef table_view c_right = right.view() with nogil: c_result = cpp_join.conditional_inner_join( - left.view(), - right.view(), + c_left, + c_right, dereference(binary_predicate.c_obj.get()), output_size, _cs, @@ -412,10 +427,12 @@ cpdef tuple conditional_left_join( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_left = left.view() + cdef table_view c_right = right.view() with nogil: c_result = cpp_join.conditional_left_join( - left.view(), - right.view(), + c_left, + c_right, dereference(binary_predicate.c_obj.get()), output_size, _cs, @@ -459,10 +476,12 @@ cpdef tuple conditional_full_join( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_left = left.view() + cdef table_view c_right = right.view() with nogil: c_result = cpp_join.conditional_full_join( - left.view(), - right.view(), + c_left, + c_right, dereference(binary_predicate.c_obj.get()), _cs, mr.get_mr() @@ -505,10 +524,12 @@ cpdef Column conditional_left_semi_join( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_left = left.view() + cdef table_view c_right = right.view() with nogil: c_result = cpp_join.conditional_left_semi_join( - left.view(), - right.view(), + c_left, + c_right, dereference(binary_predicate.c_obj.get()), output_size, _cs, @@ -549,10 +570,12 @@ cpdef Column conditional_left_anti_join( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_left = left.view() + cdef table_view c_right = right.view() with nogil: c_result = cpp_join.conditional_left_anti_join( - left.view(), - right.view(), + c_left, + c_right, dereference(binary_predicate.c_obj.get()), output_size, _cs, @@ -603,12 +626,16 @@ cpdef tuple mixed_inner_join( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_left_keys = left_keys.view() + cdef table_view c_right_keys = right_keys.view() + cdef table_view c_left_conditional = left_conditional.view() + cdef table_view c_right_conditional = right_conditional.view() with nogil: c_result = cpp_join.mixed_inner_join( - left_keys.view(), - right_keys.view(), - left_conditional.view(), - right_conditional.view(), + c_left_keys, + c_right_keys, + c_left_conditional, + c_right_conditional, dereference(binary_predicate.c_obj.get()), nulls_equal, empty_optional, @@ -663,12 +690,16 @@ cpdef tuple mixed_left_join( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_left_keys = left_keys.view() + cdef table_view c_right_keys = right_keys.view() + cdef table_view c_left_conditional = left_conditional.view() + cdef table_view c_right_conditional = right_conditional.view() with nogil: c_result = cpp_join.mixed_left_join( - left_keys.view(), - right_keys.view(), - left_conditional.view(), - right_conditional.view(), + c_left_keys, + c_right_keys, + c_left_conditional, + c_right_conditional, dereference(binary_predicate.c_obj.get()), nulls_equal, empty_optional, @@ -723,12 +754,16 @@ cpdef tuple mixed_full_join( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_left_keys = left_keys.view() + cdef table_view c_right_keys = right_keys.view() + cdef table_view c_left_conditional = left_conditional.view() + cdef table_view c_right_conditional = right_conditional.view() with nogil: c_result = cpp_join.mixed_full_join( - left_keys.view(), - right_keys.view(), - left_conditional.view(), - right_conditional.view(), + c_left_keys, + c_right_keys, + c_left_conditional, + c_right_conditional, dereference(binary_predicate.c_obj.get()), nulls_equal, empty_optional, @@ -781,12 +816,16 @@ cpdef Column mixed_left_semi_join( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_left_keys = left_keys.view() + cdef table_view c_right_keys = right_keys.view() + cdef table_view c_left_conditional = left_conditional.view() + cdef table_view c_right_conditional = right_conditional.view() with nogil: c_result = cpp_join.mixed_left_semi_join( - left_keys.view(), - right_keys.view(), - left_conditional.view(), - right_conditional.view(), + c_left_keys, + c_right_keys, + c_left_conditional, + c_right_conditional, dereference(binary_predicate.c_obj.get()), nulls_equal, _cs, @@ -835,12 +874,16 @@ cpdef Column mixed_left_anti_join( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_left_keys = left_keys.view() + cdef table_view c_right_keys = right_keys.view() + cdef table_view c_left_conditional = left_conditional.view() + cdef table_view c_right_conditional = right_conditional.view() with nogil: c_result = cpp_join.mixed_left_anti_join( - left_keys.view(), - right_keys.view(), - left_conditional.view(), - right_conditional.view(), + c_left_keys, + c_right_keys, + c_left_conditional, + c_right_conditional, dereference(binary_predicate.c_obj.get()), nulls_equal, _cs, @@ -887,10 +930,11 @@ cdef class FilteredJoin: cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() + cdef table_view c_right = right.view() with nogil: self.c_obj.reset( new cpp_join.filtered_join( - right.view(), + c_right, compare_nulls, load_factor, _cs @@ -929,9 +973,10 @@ cdef class FilteredJoin: cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_left = left.view() with nogil: c_result = self.c_obj.get()[0].semi_join( - left.view(), + c_left, _cs, mr.get_mr() ) @@ -969,9 +1014,10 @@ cdef class FilteredJoin: cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_left = left.view() with nogil: c_result = self.c_obj.get()[0].anti_join( - left.view(), + c_left, _cs, mr.get_mr() ) diff --git a/python/pylibcudf/pylibcudf/json.pyx b/python/pylibcudf/pylibcudf/json.pyx index a470f6a1cb3a..7d71f2a9dca4 100644 --- a/python/pylibcudf/pylibcudf/json.pyx +++ b/python/pylibcudf/pylibcudf/json.pyx @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from cython.operator cimport dereference @@ -8,6 +8,7 @@ from libcpp.utility cimport move from pylibcudf.column cimport Column from pylibcudf.libcudf cimport json as cpp_json from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view from pylibcudf.libcudf.scalar.scalar cimport string_scalar from pylibcudf.scalar cimport Scalar @@ -149,6 +150,8 @@ cpdef Column get_json_object( New strings column containing the retrieved json object strings. """ cdef unique_ptr[column] c_result + cdef column_view c_col + cdef string_scalar* c_json_path = ( json_path.c_obj.get() ) @@ -160,9 +163,10 @@ cpdef Column get_json_object( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + c_col = col.view() with nogil: c_result = cpp_json.get_json_object( - col.view(), + c_col, dereference(c_json_path), c_options, _cs, diff --git a/python/pylibcudf/pylibcudf/labeling.pyx b/python/pylibcudf/pylibcudf/labeling.pyx index e3a052f7cb89..9d52e308f7c4 100644 --- a/python/pylibcudf/pylibcudf/labeling.pyx +++ b/python/pylibcudf/pylibcudf/labeling.pyx @@ -1,10 +1,11 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libcpp.memory cimport unique_ptr from libcpp.utility cimport move from pylibcudf.libcudf cimport labeling as cpp_labeling from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view from pylibcudf.libcudf.labeling cimport inclusive from pylibcudf.libcudf.labeling import inclusive as Inclusive # no-cython-lint @@ -59,12 +60,15 @@ cpdef Column label_bins( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() + cdef column_view c_left_edges = left_edges.view() + cdef column_view c_right_edges = right_edges.view() with nogil: c_result = cpp_labeling.label_bins( - input.view(), - left_edges.view(), + c_input, + c_left_edges, left_inclusive, - right_edges.view(), + c_right_edges, right_inclusive, _cs, mr.get_mr() diff --git a/python/pylibcudf/pylibcudf/lists.pyx b/python/pylibcudf/pylibcudf/lists.pyx index fbc07eebb8a2..71996340e5db 100644 --- a/python/pylibcudf/pylibcudf/lists.pyx +++ b/python/pylibcudf/pylibcudf/lists.pyx @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from cython.operator cimport dereference @@ -6,7 +6,9 @@ from libcpp cimport bool from libcpp.memory cimport unique_ptr from libcpp.utility cimport move from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view from pylibcudf.libcudf.copying cimport out_of_bounds_policy +from pylibcudf.libcudf.lists.lists_column_view cimport lists_column_view from pylibcudf.libcudf.lists cimport ( contains as cpp_contains, explode as cpp_explode, @@ -37,6 +39,7 @@ from pylibcudf.libcudf.lists.stream_compaction cimport ( ) from pylibcudf.libcudf.stream_compaction cimport duplicate_keep_option from pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.table.table_view cimport table_view from pylibcudf.libcudf.types cimport ( nan_equality, null_equality, @@ -112,9 +115,10 @@ cpdef Table explode_outer( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_input = input.view() with nogil: c_result = cpp_explode.explode_outer( - input.view(), explode_column_idx, _cs, mr.get_mr() + c_input, explode_column_idx, _cs, mr.get_mr() ) return Table.from_libcudf(move(c_result), _stream, mr) @@ -147,9 +151,10 @@ cpdef Column concatenate_rows( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_input = input.view() with nogil: c_result = cpp_concatenate_rows( - input.view(), concatenate_null_policy.IGNORE, _cs, mr.get_mr() + c_input, concatenate_null_policy.IGNORE, _cs, mr.get_mr() ) return Column.from_libcudf(move(c_result), _stream, mr) @@ -183,9 +188,10 @@ cpdef Column concatenate_list_elements( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: c_result = cpp_concatenate_list_elements( - input.view(), null_policy, _cs, mr.get_mr() + c_input, null_policy, _cs, mr.get_mr() ) return Column.from_libcudf(move(c_result), _stream, mr) @@ -226,15 +232,21 @@ cpdef Column contains( cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() + cdef lists_column_view c_list_view + cdef column_view c_search_key_column + mr = _get_memory_resource(mr) if not isinstance(search_key, (Column, Scalar)): raise TypeError("Must pass a Column or Scalar") + c_list_view = list_view.view() + if ColumnOrScalar is Column: + c_search_key_column = search_key.view() with nogil: c_result = cpp_contains.contains( - list_view.view(), - search_key.view() if ColumnOrScalar is Column else dereference( + c_list_view, + c_search_key_column if ColumnOrScalar is Column else dereference( search_key.get() ), _cs, @@ -273,9 +285,10 @@ cpdef Column contains_nulls( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef lists_column_view c_list_view = list_view.view() with nogil: c_result = cpp_contains.contains_nulls( - list_view.view(), _cs, mr.get_mr() + c_list_view, _cs, mr.get_mr() ) return Column.from_libcudf(move(c_result), _stream, mr) @@ -317,12 +330,17 @@ cpdef Column index_of( cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() + cdef column_view c_search_key_column + mr = _get_memory_resource(mr) + cdef lists_column_view c_list_view = list_view.view() + if ColumnOrScalar is Column: + c_search_key_column = search_key.view() with nogil: c_result = cpp_contains.index_of( - list_view.view(), - search_key.view() if ColumnOrScalar is Column else dereference( + c_list_view, + c_search_key_column if ColumnOrScalar is Column else dereference( search_key.get() ), find_option, @@ -360,8 +378,9 @@ cpdef Column reverse( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef lists_column_view c_list_view = list_view.view() with nogil: - c_result = cpp_reverse.reverse(list_view.view(), _cs, mr.get_mr()) + c_result = cpp_reverse.reverse(c_list_view, _cs, mr.get_mr()) return Column.from_libcudf(move(c_result), _stream, mr) @@ -408,10 +427,12 @@ cpdef Column segmented_gather( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef lists_column_view c_list_view1 = list_view1.view() + cdef lists_column_view c_list_view2 = list_view2.view() with nogil: c_result = cpp_gather.segmented_gather( - list_view1.view(), - list_view2.view(), + c_list_view1, + c_list_view2, bounds_policy, _cs, mr.get_mr(), @@ -446,12 +467,17 @@ cpdef Column extract_list_element( cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() + cdef column_view c_index_column + mr = _get_memory_resource(mr) + cdef lists_column_view c_list_view = list_view.view() + if ColumnOrSizeType is Column: + c_index_column = index.view() with nogil: c_result = cpp_extract_list_element( - list_view.view(), - index.view() if ColumnOrSizeType is Column else index, + c_list_view, + c_index_column if ColumnOrSizeType is Column else index, _cs, mr.get_mr(), ) @@ -488,8 +514,9 @@ cpdef Column count_elements( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef lists_column_view c_list_view = list_view.view() with nogil: - c_result = cpp_count_elements(list_view.view(), _cs, mr.get_mr()) + c_result = cpp_count_elements(c_list_view, _cs, mr.get_mr()) return Column.from_libcudf(move(c_result), _stream, mr) @@ -524,21 +551,30 @@ cpdef Column sequences( cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() + cdef column_view c_starts + cdef column_view c_steps + cdef column_view c_sizes + mr = _get_memory_resource(mr) if steps is not None: + c_starts = starts.view() + c_steps = steps.view() + c_sizes = sizes.view() with nogil: c_result = cpp_filling.sequences( - starts.view(), - steps.view(), - sizes.view(), + c_starts, + c_steps, + c_sizes, _cs, mr.get_mr(), ) else: + c_starts = starts.view() + c_sizes = sizes.view() with nogil: c_result = cpp_filling.sequences( - starts.view(), sizes.view(), _cs, mr.get_mr() + c_starts, c_sizes, _cs, mr.get_mr() ) return Column.from_libcudf(move(c_result), _stream, mr) @@ -579,10 +615,11 @@ cpdef Column sort_lists( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef lists_column_view c_list_view = list_view.view() with nogil: if stable: c_result = cpp_stable_sort_lists( - list_view.view(), + c_list_view, sort_order, na_position, _cs, @@ -590,7 +627,7 @@ cpdef Column sort_lists( ) else: c_result = cpp_sort_lists( - list_view.view(), + c_list_view, sort_order, na_position, _cs, @@ -636,10 +673,12 @@ cpdef Column difference_distinct( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef lists_column_view c_lhs_view = lhs_view.view() + cdef lists_column_view c_rhs_view = rhs_view.view() with nogil: c_result = cpp_set_operations.difference_distinct( - lhs_view.view(), - rhs_view.view(), + c_lhs_view, + c_rhs_view, nulls_equal, nans_equal, _cs, @@ -684,10 +723,12 @@ cpdef Column have_overlap( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef lists_column_view c_lhs_view = lhs_view.view() + cdef lists_column_view c_rhs_view = rhs_view.view() with nogil: c_result = cpp_set_operations.have_overlap( - lhs_view.view(), - rhs_view.view(), + c_lhs_view, + c_rhs_view, nulls_equal, nans_equal, _cs, @@ -732,10 +773,12 @@ cpdef Column intersect_distinct( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef lists_column_view c_lhs_view = lhs_view.view() + cdef lists_column_view c_rhs_view = rhs_view.view() with nogil: c_result = cpp_set_operations.intersect_distinct( - lhs_view.view(), - rhs_view.view(), + c_lhs_view, + c_rhs_view, nulls_equal, nans_equal, _cs, @@ -781,10 +824,12 @@ cpdef Column union_distinct( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef lists_column_view c_lhs_view = lhs_view.view() + cdef lists_column_view c_rhs_view = rhs_view.view() with nogil: c_result = cpp_set_operations.union_distinct( - lhs_view.view(), - rhs_view.view(), + c_lhs_view, + c_rhs_view, nulls_equal, nans_equal, _cs, @@ -825,10 +870,12 @@ cpdef Column apply_boolean_mask( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef lists_column_view c_list_view = list_view.view() + cdef lists_column_view c_mask_view = mask_view.view() with nogil: c_result = cpp_apply_boolean_mask( - list_view.view(), - mask_view.view(), + c_list_view, + c_mask_view, _cs, mr.get_mr(), ) @@ -865,10 +912,12 @@ cpdef Column apply_deletion_mask( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef lists_column_view c_list_view = list_view.view() + cdef lists_column_view c_mask_view = mask_view.view() with nogil: c_result = cpp_apply_deletion_mask( - list_view.view(), - mask_view.view(), + c_list_view, + c_mask_view, _cs, mr.get_mr(), ) @@ -907,9 +956,10 @@ cpdef Column distinct( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef lists_column_view c_list_view = list_view.view() with nogil: c_result = cpp_distinct( - list_view.view(), + c_list_view, nulls_equal, nans_equal, duplicate_keep_option.KEEP_ANY, diff --git a/python/pylibcudf/pylibcudf/null_mask.pyx b/python/pylibcudf/pylibcudf/null_mask.pyx index 164c51aca9f7..5178f979d248 100644 --- a/python/pylibcudf/pylibcudf/null_mask.pyx +++ b/python/pylibcudf/pylibcudf/null_mask.pyx @@ -1,10 +1,12 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libc.stdint cimport uintptr_t from libcpp.memory cimport make_unique from libcpp.pair cimport pair from libcpp.utility cimport move from pylibcudf.libcudf cimport null_mask as cpp_null_mask +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.table.table_view cimport table_view from pylibcudf.libcudf.types cimport mask_state, size_type, bitmask_type from rmm.librmm.device_buffer cimport device_buffer @@ -68,8 +70,9 @@ cpdef DeviceBuffer copy_bitmask( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_col = col.view() with nogil: - db = cpp_null_mask.copy_bitmask(col.view(), _cs, mr.get_mr()) + db = cpp_null_mask.copy_bitmask(c_col, _cs, mr.get_mr()) return buffer_to_python(move(db), _stream, mr) @@ -214,9 +217,10 @@ cpdef tuple bitmask_and(list columns, object stream=None, DeviceMemoryResource m cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_input = c_table.view() with nogil: c_result = cpp_null_mask.bitmask_and( - c_table.view(), _cs, mr.get_mr() + c_input, _cs, mr.get_mr() ) return buffer_to_python(move(c_result.first), _stream, mr), c_result.second @@ -247,8 +251,9 @@ cpdef tuple bitmask_or(list columns, object stream=None, DeviceMemoryResource mr cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_input = c_table.view() with nogil: - c_result = cpp_null_mask.bitmask_or(c_table.view(), _cs, mr.get_mr()) + c_result = cpp_null_mask.bitmask_or(c_input, _cs, mr.get_mr()) return buffer_to_python(move(c_result.first), _stream, mr), c_result.second diff --git a/python/pylibcudf/pylibcudf/nvtext/byte_pair_encode.pyx b/python/pylibcudf/pylibcudf/nvtext/byte_pair_encode.pyx index 023e00a1169e..0db5a345f43c 100644 --- a/python/pylibcudf/pylibcudf/nvtext/byte_pair_encode.pyx +++ b/python/pylibcudf/pylibcudf/nvtext/byte_pair_encode.pyx @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from cython.operator cimport dereference @@ -83,10 +83,11 @@ cpdef Column byte_pair_encoding( cpp_make_string_scalar(" ".encode(), _stream.view().value(), mr.get_mr()) ) + cdef column_view c_input = input.view() with nogil: c_result = move( cpp_byte_pair_encoding( - input.view(), + c_input, dereference(merge_pairs.c_obj.get()), dereference(separator.c_obj.get()), _cs, diff --git a/python/pylibcudf/pylibcudf/nvtext/deduplicate.pyx b/python/pylibcudf/pylibcudf/nvtext/deduplicate.pyx index e679841a7928..1dbbe3e066e6 100644 --- a/python/pylibcudf/pylibcudf/nvtext/deduplicate.pyx +++ b/python/pylibcudf/pylibcudf/nvtext/deduplicate.pyx @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from cython.operator import dereference @@ -7,6 +7,7 @@ from libcpp.memory cimport unique_ptr, make_unique from libcpp.utility cimport move from pylibcudf.column cimport Column from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view from pylibcudf.libcudf.nvtext.deduplicate cimport ( build_suffix_array as cpp_build_suffix_array, suffix_array_type as cpp_suffix_array_type, @@ -71,9 +72,10 @@ cpdef Column build_suffix_array( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: c_result = cpp_build_suffix_array( - input.view(), min_width, _cs, mr.get_mr() + c_input, min_width, _cs, mr.get_mr() ) return _column_from_suffix_array(move(c_result), _stream, mr) @@ -115,9 +117,11 @@ cpdef Column resolve_duplicates( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() + cdef column_view c_indices = indices.view() with nogil: c_result = cpp_resolve_duplicates( - input.view(), indices.view(), min_width, _cs, mr.get_mr() + c_input, c_indices, min_width, _cs, mr.get_mr() ) return Column.from_libcudf(move(c_result), _stream, mr) @@ -166,12 +170,16 @@ cpdef Column resolve_duplicates_pair( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input1 = input1.view() + cdef column_view c_indices1 = indices1.view() + cdef column_view c_input2 = input2.view() + cdef column_view c_indices2 = indices2.view() with nogil: c_result = cpp_resolve_duplicates_pair( - input1.view(), - indices1.view(), - input2.view(), - indices2.view(), + c_input1, + c_indices1, + c_input2, + c_indices2, min_width, _cs, mr.get_mr(), diff --git a/python/pylibcudf/pylibcudf/nvtext/minhash.pyx b/python/pylibcudf/pylibcudf/nvtext/minhash.pyx index 3029ed54c503..035c4370acd2 100644 --- a/python/pylibcudf/pylibcudf/nvtext/minhash.pyx +++ b/python/pylibcudf/pylibcudf/nvtext/minhash.pyx @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libc.stdint cimport uint32_t, uint64_t @@ -6,6 +6,7 @@ from libcpp.memory cimport unique_ptr from libcpp.utility cimport move from pylibcudf.column cimport Column from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view from pylibcudf.libcudf.nvtext.minhash cimport ( minhash as cpp_minhash, minhash64 as cpp_minhash64, @@ -63,12 +64,15 @@ cpdef Column minhash( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() + cdef column_view c_a = a.view() + cdef column_view c_b = b.view() with nogil: c_result = cpp_minhash( - input.view(), + c_input, seed, - a.view(), - b.view(), + c_a, + c_b, width, _cs, mr.get_mr() @@ -116,12 +120,15 @@ cpdef Column minhash64( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() + cdef column_view c_a = a.view() + cdef column_view c_b = b.view() with nogil: c_result = cpp_minhash64( - input.view(), + c_input, seed, - a.view(), - b.view(), + c_a, + c_b, width, _cs, mr.get_mr() @@ -170,13 +177,16 @@ cpdef Column minhash_ngrams( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() + cdef column_view c_a = a.view() + cdef column_view c_b = b.view() with nogil: c_result = cpp_minhash_ngrams( - input.view(), + c_input, ngrams, seed, - a.view(), - b.view(), + c_a, + c_b, _cs, mr.get_mr() ) @@ -224,13 +234,16 @@ cpdef Column minhash64_ngrams( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() + cdef column_view c_a = a.view() + cdef column_view c_b = b.view() with nogil: c_result = cpp_minhash64_ngrams( - input.view(), + c_input, ngrams, seed, - a.view(), - b.view(), + c_a, + c_b, _cs, mr.get_mr() ) diff --git a/python/pylibcudf/pylibcudf/nvtext/ngrams_tokenize.pyx b/python/pylibcudf/pylibcudf/nvtext/ngrams_tokenize.pyx index 959c47d595d7..7dfba5c6b989 100644 --- a/python/pylibcudf/pylibcudf/nvtext/ngrams_tokenize.pyx +++ b/python/pylibcudf/pylibcudf/nvtext/ngrams_tokenize.pyx @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from cython.operator cimport dereference @@ -6,6 +6,7 @@ from libcpp.memory cimport unique_ptr from libcpp.utility cimport move from pylibcudf.column cimport Column from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view from pylibcudf.libcudf.nvtext.ngrams_tokenize cimport ( ngrams_tokenize as cpp_ngrams_tokenize, ) @@ -57,9 +58,10 @@ cpdef Column ngrams_tokenize( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: c_result = cpp_ngrams_tokenize( - input.view(), + c_input, ngrams, dereference(delimiter.get()), dereference(separator.get()), diff --git a/python/pylibcudf/pylibcudf/nvtext/normalize.pyx b/python/pylibcudf/pylibcudf/nvtext/normalize.pyx index 8e29aad9121e..889792d67383 100644 --- a/python/pylibcudf/pylibcudf/nvtext/normalize.pyx +++ b/python/pylibcudf/pylibcudf/nvtext/normalize.pyx @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from cython.operator cimport dereference @@ -74,9 +74,10 @@ cpdef Column normalize_spaces( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: c_result = cpp_normalize.normalize_spaces( - input.view(), _cs, mr.get_mr() + c_input, _cs, mr.get_mr() ) return Column.from_libcudf(move(c_result), _stream, mr) @@ -112,9 +113,10 @@ cpdef Column normalize_characters( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: c_result = cpp_normalize.normalize_characters( - input.view(), + c_input, dereference(normalizer.c_obj.get()), _cs, mr.get_mr() diff --git a/python/pylibcudf/pylibcudf/nvtext/replace.pyx b/python/pylibcudf/pylibcudf/nvtext/replace.pyx index 4b00d76bd640..70fec3d96640 100644 --- a/python/pylibcudf/pylibcudf/nvtext/replace.pyx +++ b/python/pylibcudf/pylibcudf/nvtext/replace.pyx @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from cython.operator cimport dereference @@ -6,6 +6,7 @@ from libcpp.memory cimport unique_ptr from libcpp.utility cimport move from pylibcudf.column cimport Column from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view from pylibcudf.libcudf.nvtext.replace cimport ( filter_tokens as cpp_filter_tokens, replace_tokens as cpp_replace_tokens, @@ -63,11 +64,14 @@ cpdef Column replace_tokens( delimiter = Scalar.from_libcudf( cpp_make_string_scalar("".encode(), _stream.view().value(), mr.get_mr()) ) + cdef column_view c_input = input.view() + cdef column_view c_targets = targets.view() + cdef column_view c_replacements = replacements.view() with nogil: c_result = cpp_replace_tokens( - input.view(), - targets.view(), - replacements.view(), + c_input, + c_targets, + c_replacements, dereference(delimiter.get()), _cs, mr.get_mr() @@ -121,9 +125,10 @@ cpdef Column filter_tokens( cpp_make_string_scalar("".encode(), _stream.view().value(), mr.get_mr()) ) + cdef column_view c_input = input.view() with nogil: c_result = cpp_filter_tokens( - input.view(), + c_input, min_token_length, dereference(replacement.get()), dereference(delimiter.get()), diff --git a/python/pylibcudf/pylibcudf/nvtext/stemmer.pyx b/python/pylibcudf/pylibcudf/nvtext/stemmer.pyx index e038cd03fb2a..cb5c5f69cc23 100644 --- a/python/pylibcudf/pylibcudf/nvtext/stemmer.pyx +++ b/python/pylibcudf/pylibcudf/nvtext/stemmer.pyx @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libcpp cimport bool @@ -6,6 +6,7 @@ from libcpp.memory cimport unique_ptr from libcpp.utility cimport move from pylibcudf.column cimport Column from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view from pylibcudf.libcudf.nvtext.stemmer cimport ( is_letter as cpp_is_letter, letter_type, @@ -60,11 +61,15 @@ cpdef Column is_letter( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() + cdef column_view c_indices + if ColumnOrSize is Column: + c_indices = indices.view() with nogil: c_result = cpp_is_letter( - input.view(), + c_input, letter_type.VOWEL if check_vowels else letter_type.CONSONANT, - indices if ColumnOrSize is size_type else indices.view(), + indices if ColumnOrSize is size_type else c_indices, _cs ) @@ -98,8 +103,9 @@ cpdef Column porter_stemmer_measure( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: - c_result = cpp_porter_stemmer_measure(input.view(), _cs, mr.get_mr()) + c_result = cpp_porter_stemmer_measure(c_input, _cs, mr.get_mr()) return Column.from_libcudf(move(c_result), _stream, mr) diff --git a/python/pylibcudf/pylibcudf/nvtext/tokenize.pyx b/python/pylibcudf/pylibcudf/nvtext/tokenize.pyx index 4e44d781cc46..2459760a9d4c 100644 --- a/python/pylibcudf/pylibcudf/nvtext/tokenize.pyx +++ b/python/pylibcudf/pylibcudf/nvtext/tokenize.pyx @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from cython.operator cimport dereference @@ -88,9 +88,10 @@ cpdef Column tokenize_scalar( cpp_make_string_scalar("".encode(), _stream.view().value(), mr.get_mr()) ) + cdef column_view c_input = input.view() with nogil: c_result = cpp_tokenize( - input.view(), + c_input, dereference(delimiter.c_obj.get()), _cs, mr.get_mr() @@ -126,10 +127,12 @@ cpdef Column tokenize_column( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() + cdef column_view c_delimiters = delimiters.view() with nogil: c_result = cpp_tokenize( - input.view(), - delimiters.view(), + c_input, + c_delimiters, _cs, mr.get_mr() ) @@ -172,9 +175,10 @@ cpdef Column count_tokens_scalar( cpp_make_string_scalar("".encode(), _stream.view().value(), mr.get_mr()) ) + cdef column_view c_input = input.view() with nogil: c_result = cpp_count_tokens( - input.view(), + c_input, dereference(delimiter.c_obj.get()), _cs, mr.get_mr() @@ -210,10 +214,12 @@ cpdef Column count_tokens_column( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() + cdef column_view c_delimiters = delimiters.view() with nogil: c_result = cpp_count_tokens( - input.view(), - delimiters.view(), + c_input, + c_delimiters, _cs, mr.get_mr() ) @@ -245,8 +251,9 @@ cpdef Column character_tokenize( cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: - c_result = cpp_character_tokenize(input.view(), _cs, mr.get_mr()) + c_result = cpp_character_tokenize(c_input, _cs, mr.get_mr()) return Column.from_libcudf(move(c_result), _stream, mr) @@ -289,10 +296,12 @@ cpdef Column detokenize( cpp_make_string_scalar(" ".encode(), _stream.view().value(), mr.get_mr()) ) + cdef column_view c_input = input.view() + cdef column_view c_row_indices = row_indices.view() with nogil: c_result = cpp_detokenize( - input.view(), - row_indices.view(), + c_input, + c_row_indices, dereference(separator.c_obj.get()), _cs, mr.get_mr() @@ -337,9 +346,10 @@ cpdef Column tokenize_with_vocabulary( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: c_result = cpp_tokenize_with_vocabulary( - input.view(), + c_input, dereference(vocabulary.c_obj.get()), dereference(delimiter.c_obj.get()), default_id, diff --git a/python/pylibcudf/pylibcudf/nvtext/wordpiece_tokenize.pyx b/python/pylibcudf/pylibcudf/nvtext/wordpiece_tokenize.pyx index dfdb563087d2..815139152b59 100644 --- a/python/pylibcudf/pylibcudf/nvtext/wordpiece_tokenize.pyx +++ b/python/pylibcudf/pylibcudf/nvtext/wordpiece_tokenize.pyx @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from cython.operator cimport dereference @@ -79,9 +79,10 @@ cpdef Column wordpiece_tokenize( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: c_result = cpp_wordpiece_tokenize( - input.view(), + c_input, dereference(vocabulary.c_obj.get()), max_words_per_row, _cs, diff --git a/python/pylibcudf/pylibcudf/partitioning.pyx b/python/pylibcudf/pylibcudf/partitioning.pyx index 62e35ab9cca5..88f7f9de086e 100644 --- a/python/pylibcudf/pylibcudf/partitioning.pyx +++ b/python/pylibcudf/pylibcudf/partitioning.pyx @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 cimport pylibcudf.libcudf.types as libcudf_types @@ -8,7 +8,9 @@ from libcpp.utility cimport move from libcpp.vector cimport vector from pylibcudf.libcudf cimport partitioning as cpp_partitioning from pylibcudf.libcudf.partitioning import hash_id as HashId # no-cython-lint +from pylibcudf.libcudf.column.column_view cimport column_view from pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.table.table_view cimport table_view from rmm.pylibrmm.stream cimport Stream from rmm.pylibrmm.memory_resource cimport DeviceMemoryResource @@ -67,11 +69,14 @@ cpdef tuple[Table, list] hash_partition( cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_input = input.view() + cdef table_view c_keys if TableOrList is Table: + c_keys = keys.view() with nogil: c_result = cpp_partitioning.hash_partition( - input.view(), - keys.view(), + c_input, + c_keys, c_num_partitions, hash_function, seed, @@ -82,7 +87,7 @@ cpdef tuple[Table, list] hash_partition( columns_to_hash = keys with nogil: c_result = cpp_partitioning.hash_partition( - input.view(), + c_input, columns_to_hash, c_num_partitions, hash_function, @@ -132,10 +137,12 @@ cpdef tuple[Table, list] partition( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_input = t.view() + cdef column_view c_partition_map = partition_map.view() with nogil: c_result = cpp_partitioning.partition( - t.view(), - partition_map.view(), + c_input, + c_partition_map, c_num_partitions, _cs, mr.get_mr() @@ -183,9 +190,10 @@ cpdef tuple[Table, list] round_robin_partition( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_input = input.view() with nogil: c_result = cpp_partitioning.round_robin_partition( - input.view(), + c_input, c_num_partitions, c_start_partition, _cs, diff --git a/python/pylibcudf/pylibcudf/quantiles.pyx b/python/pylibcudf/pylibcudf/quantiles.pyx index f02643754cbd..96d2a0d8bc99 100644 --- a/python/pylibcudf/pylibcudf/quantiles.pyx +++ b/python/pylibcudf/pylibcudf/quantiles.pyx @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libcpp cimport bool @@ -12,6 +12,7 @@ from pylibcudf.libcudf.quantiles cimport ( quantiles as cpp_quantiles, ) from pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.table.table_view cimport table_view from pylibcudf.libcudf.types cimport null_order, order, sorted from rmm.pylibrmm.memory_resource cimport DeviceMemoryResource from rmm.pylibrmm.stream cimport Stream @@ -79,9 +80,10 @@ cpdef Column quantile( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: c_result = cpp_quantile( - input.view(), + c_input, q, interp, ordered_indices_view, @@ -162,9 +164,10 @@ cpdef Table quantiles( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_input = input.view() with nogil: c_result = cpp_quantiles( - input.view(), + c_input, q, interp, is_input_sorted, diff --git a/python/pylibcudf/pylibcudf/reduce.pyx b/python/pylibcudf/pylibcudf/reduce.pyx index c4b5731f0665..b2bdfa720872 100644 --- a/python/pylibcudf/pylibcudf/reduce.pyx +++ b/python/pylibcudf/pylibcudf/reduce.pyx @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from cython.operator cimport dereference @@ -11,6 +11,8 @@ from pylibcudf.libcudf cimport distinct_count as cpp_distinct_count from pylibcudf.libcudf cimport unique_count as cpp_unique_count from pylibcudf.libcudf.aggregation cimport reduce_aggregation, scan_aggregation from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.table.table_view cimport table_view from pylibcudf.libcudf.reduce cimport ( reduce as cpp_reduce, scan as cpp_scan, @@ -92,9 +94,10 @@ cpdef Scalar reduce( else: c_init = nullopt + cdef column_view c_col = col.view() with nogil: result = cpp_reduce( - col.view(), + c_col, dereference(c_agg), data_type.c_obj, c_init, @@ -140,9 +143,10 @@ cpdef Column scan( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_col = col.view() with nogil: result = cpp_scan( - col.view(), + c_col, dereference(c_agg), inclusive, null_policy.EXCLUDE, @@ -180,8 +184,9 @@ cpdef tuple minmax(Column col, object stream=None, DeviceMemoryResource mr=None) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_col = col.view() with nogil: - result = cpp_minmax(col.view(), _cs, mr.get_mr()) + result = cpp_minmax(c_col, _cs, mr.get_mr()) min_scalar = Scalar.from_libcudf(move(result.first)) max_scalar = Scalar.from_libcudf(move(result.second)) @@ -238,10 +243,11 @@ cpdef size_type unique_count( same result as distinct_count, but faster. """ cdef Stream _stream = _get_stream(stream) + cdef column_view c_source = source.view() with nogil: return cpp_unique_count.unique_count( - source.view(), null_handling, nan_handling, _stream.view().value() + c_source, null_handling, nan_handling, _stream.view().value() ) @@ -272,10 +278,11 @@ cpdef size_type distinct_count( The number of distinct elements in the input column. """ cdef Stream _stream = _get_stream(stream) + cdef column_view c_source = source.view() with nogil: return cpp_distinct_count.distinct_count( - source.view(), null_handling, nan_handling, _stream.view().value() + c_source, null_handling, nan_handling, _stream.view().value() ) @@ -305,10 +312,11 @@ cpdef size_type unique_count_table( NaNs compare equal in this comparison. """ cdef Stream _stream = _get_stream(stream) + cdef table_view c_source = source.view() with nogil: return cpp_unique_count.unique_count( - source.view(), nulls_equal, _stream.view().value() + c_source, nulls_equal, _stream.view().value() ) @@ -338,10 +346,11 @@ cpdef size_type distinct_count_table( NaNs compare equal in this comparison. """ cdef Stream _stream = _get_stream(stream) + cdef table_view c_source = source.view() with nogil: return cpp_distinct_count.distinct_count( - source.view(), nulls_equal, _stream.view().value() + c_source, nulls_equal, _stream.view().value() ) diff --git a/python/pylibcudf/pylibcudf/replace.pyx b/python/pylibcudf/pylibcudf/replace.pyx index 4a5cc1625513..e411ae645062 100644 --- a/python/pylibcudf/pylibcudf/replace.pyx +++ b/python/pylibcudf/pylibcudf/replace.pyx @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 @@ -9,6 +9,7 @@ from libcpp.memory cimport unique_ptr from libcpp.utility cimport move from pylibcudf.libcudf cimport replace as cpp_replace from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view, mutable_column_view from rmm.pylibrmm.stream cimport Stream from rmm.pylibrmm.memory_resource cimport DeviceMemoryResource @@ -70,11 +71,13 @@ cpdef Column replace_nulls( """ cdef unique_ptr[column] c_result cdef replace_policy policy + cdef column_view c_replacement cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_source_column = source_column.view() # Due to https://github.com/cython/cython/issues/5984, if this function is # called as a Python function (i.e. without typed inputs, which is always # true in pure Python files), the type of `replacement` will be `object` @@ -84,7 +87,7 @@ cpdef Column replace_nulls( policy = replacement with nogil: c_result = cpp_replace.replace_nulls( - source_column.view(), + c_source_column, policy, _cs, mr.get_mr() @@ -93,24 +96,27 @@ cpdef Column replace_nulls( else: raise TypeError("replacement must be a Column, Scalar, or replace_policy") + if ReplacementType is Column: + c_replacement = replacement.view() + with nogil: if ReplacementType is Column: c_result = cpp_replace.replace_nulls( - source_column.view(), - replacement.view(), + c_source_column, + c_replacement, _cs, mr.get_mr() ) elif ReplacementType is Scalar: c_result = cpp_replace.replace_nulls( - source_column.view(), + c_source_column, dereference(replacement.c_obj), _cs, mr.get_mr() ) elif ReplacementType is replace_policy: c_result = cpp_replace.replace_nulls( - source_column.view(), + c_source_column, replacement, _cs, mr.get_mr() @@ -156,11 +162,14 @@ cpdef Column find_and_replace_all( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_source_column = source_column.view() + cdef column_view c_values_to_replace = values_to_replace.view() + cdef column_view c_replacement_values = replacement_values.view() with nogil: c_result = cpp_replace.find_and_replace_all( - source_column.view(), - values_to_replace.view(), - replacement_values.view(), + c_source_column, + c_values_to_replace, + c_replacement_values, _cs, mr.get_mr() ) @@ -213,10 +222,11 @@ cpdef Column clamp( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_source_column = source_column.view() with nogil: if lo_replace is None: c_result = cpp_replace.clamp( - source_column.view(), + c_source_column, dereference(lo.c_obj), dereference(hi.c_obj), _cs, @@ -224,7 +234,7 @@ cpdef Column clamp( ) else: c_result = cpp_replace.clamp( - source_column.view(), + c_source_column, dereference(lo.c_obj), dereference(lo_replace.c_obj), dereference(hi.c_obj), @@ -268,16 +278,20 @@ cpdef Column normalize_nans_and_zeros( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_source_column = source_column.view() + cdef mutable_column_view c_mutable_source_column + if inplace: + c_mutable_source_column = source_column.mutable_view() with nogil: if inplace: cpp_replace.normalize_nans_and_zeros( - source_column.mutable_view(), + c_mutable_source_column, _cs, mr.get_mr() ) else: c_result = cpp_replace.normalize_nans_and_zeros( - source_column.view(), + c_source_column, _cs, mr.get_mr() ) diff --git a/python/pylibcudf/pylibcudf/reshape.pyx b/python/pylibcudf/pylibcudf/reshape.pyx index a81dadf62cec..b1b60f4444dd 100644 --- a/python/pylibcudf/pylibcudf/reshape.pyx +++ b/python/pylibcudf/pylibcudf/reshape.pyx @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libc.stddef cimport size_t @@ -14,6 +14,7 @@ from pylibcudf.libcudf.reshape cimport ( byte, ) from pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.table.table_view cimport table_view from pylibcudf.libcudf.types cimport size_type from pylibcudf.libcudf.utilities.span cimport device_span @@ -60,9 +61,10 @@ cpdef Column interleave_columns( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_source_table = source_table.view() with nogil: c_result = cpp_interleave_columns( - source_table.view(), _cs, mr.get_mr() + c_source_table, _cs, mr.get_mr() ) return Column.from_libcudf(move(c_result), _stream, mr) @@ -99,9 +101,10 @@ cpdef Table tile( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_source_table = source_table.view() with nogil: c_result = cpp_tile( - source_table.view(), count, _cs, mr.get_mr() + c_source_table, count, _cs, mr.get_mr() ) return Table.from_libcudf(move(c_result), _stream, mr) @@ -138,10 +141,11 @@ cpdef void table_to_array( cdef device_span[byte] span = device_span[byte]( ptr, size ) + cdef table_view c_input_table = input_table.view() with nogil: cpp_table_to_array( - input_table.view(), + c_input_table, span, _cs ) diff --git a/python/pylibcudf/pylibcudf/rolling.pyx b/python/pylibcudf/pylibcudf/rolling.pyx index ae9d7665d695..d12a40592983 100644 --- a/python/pylibcudf/pylibcudf/rolling.pyx +++ b/python/pylibcudf/pylibcudf/rolling.pyx @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from cython.operator cimport dereference @@ -10,7 +10,9 @@ from libcpp.vector cimport vector from pylibcudf.libcudf cimport rolling as cpp_rolling from pylibcudf.libcudf.aggregation cimport rolling_aggregation from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view from pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.table.table_view cimport table_view from pylibcudf.libcudf.types cimport size_type from rmm.pylibrmm.stream cimport Stream from rmm.pylibrmm.memory_resource cimport DeviceMemoryResource @@ -169,10 +171,12 @@ cpdef Table grouped_range_rolling_window( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_group_keys = group_keys.view() + cdef column_view c_orderby = orderby.view() with nogil: result = cpp_rolling.grouped_range_rolling_window( - group_keys.view(), - orderby.view(), + c_group_keys, + c_orderby, order, null_order, dereference(preceding.c_obj.get()), @@ -230,12 +234,17 @@ cpdef Column rolling_window( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_source = source.view() + cdef column_view c_preceding_window + cdef column_view c_following_window if WindowType is Column: + c_preceding_window = preceding_window.view() + c_following_window = following_window.view() with nogil: result = cpp_rolling.rolling_window( - source.view(), - preceding_window.view(), - following_window.view(), + c_source, + c_preceding_window, + c_following_window, min_periods, dereference(c_agg), _cs, @@ -244,7 +253,7 @@ cpdef Column rolling_window( else: with nogil: result = cpp_rolling.rolling_window( - source.view(), + c_source, preceding_window, following_window, min_periods, @@ -315,10 +324,12 @@ cpdef tuple make_range_windows( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_group_keys = group_keys.view() + cdef column_view c_orderby = orderby.view() with nogil: result = cpp_rolling.make_range_windows( - group_keys.view(), - orderby.view(), + c_group_keys, + c_orderby, order, null_order, dereference(preceding.c_obj.get()), diff --git a/python/pylibcudf/pylibcudf/round.pyx b/python/pylibcudf/pylibcudf/round.pyx index f5baa6bbd230..a3de7add7533 100644 --- a/python/pylibcudf/pylibcudf/round.pyx +++ b/python/pylibcudf/pylibcudf/round.pyx @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libc.stdint cimport int32_t from libcpp.memory cimport unique_ptr @@ -13,6 +13,7 @@ from pylibcudf.libcudf.round import \ rounding_method as RoundingMethod # no-cython-lint from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view from rmm.pylibrmm.stream cimport Stream from rmm.pylibrmm.memory_resource cimport DeviceMemoryResource @@ -63,9 +64,10 @@ cpdef Column round( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_source = source.view() with nogil: c_result = cpp_round( - source.view(), + c_source, decimal_places, round_method, _cs, @@ -112,9 +114,10 @@ cpdef Column round_decimal( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_source = source.view() with nogil: c_result = cpp_round_decimal( - source.view(), + c_source, decimal_places, round_method, _cs, diff --git a/python/pylibcudf/pylibcudf/search.pyx b/python/pylibcudf/pylibcudf/search.pyx index 885d25f2d490..d67229729d7b 100644 --- a/python/pylibcudf/pylibcudf/search.pyx +++ b/python/pylibcudf/pylibcudf/search.pyx @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libcpp.memory cimport unique_ptr @@ -6,6 +6,8 @@ from libcpp.utility cimport move from libcpp.vector cimport vector from pylibcudf.libcudf cimport search as cpp_search from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.table.table_view cimport table_view from pylibcudf.libcudf.types cimport null_order, order from rmm.pylibrmm.stream cimport Stream from rmm.pylibrmm.memory_resource cimport DeviceMemoryResource @@ -57,10 +59,12 @@ cpdef Column lower_bound( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_haystack = haystack.view() + cdef table_view c_needles = needles.view() with nogil: c_result = cpp_search.lower_bound( - haystack.view(), - needles.view(), + c_haystack, + c_needles, c_orders, c_null_precedence, _cs, @@ -109,10 +113,12 @@ cpdef Column upper_bound( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_haystack = haystack.view() + cdef table_view c_needles = needles.view() with nogil: c_result = cpp_search.upper_bound( - haystack.view(), - needles.view(), + c_haystack, + c_needles, c_orders, c_null_precedence, _cs, @@ -150,10 +156,12 @@ cpdef Column contains( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_haystack = haystack.view() + cdef column_view c_needles = needles.view() with nogil: c_result = cpp_search.contains( - haystack.view(), - needles.view(), + c_haystack, + c_needles, _cs, mr.get_mr() ) diff --git a/python/pylibcudf/pylibcudf/sorting.pyx b/python/pylibcudf/pylibcudf/sorting.pyx index fa0ed78b709c..cb8d9909eb60 100644 --- a/python/pylibcudf/pylibcudf/sorting.pyx +++ b/python/pylibcudf/pylibcudf/sorting.pyx @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libcpp.memory cimport unique_ptr @@ -7,7 +7,9 @@ from libcpp.vector cimport vector from pylibcudf.libcudf cimport sorting as cpp_sorting from pylibcudf.libcudf.aggregation cimport rank_method from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view from pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.table.table_view cimport table_view from pylibcudf.libcudf.types cimport null_order, null_policy, order, size_type from rmm.pylibrmm.memory_resource cimport DeviceMemoryResource from rmm.pylibrmm.stream cimport Stream @@ -63,9 +65,10 @@ cpdef Column sorted_order( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_source_table = source_table.view() with nogil: c_result = cpp_sorting.sorted_order( - source_table.view(), + c_source_table, c_orders, c_null_precedence, _cs, @@ -108,9 +111,10 @@ cpdef Column stable_sorted_order( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_source_table = source_table.view() with nogil: c_result = cpp_sorting.stable_sorted_order( - source_table.view(), + c_source_table, c_orders, c_null_precedence, _cs, @@ -159,9 +163,10 @@ cpdef Column rank( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input_view = input_view.view() with nogil: c_result = cpp_sorting.rank( - input_view.view(), + c_input_view, method, column_order, null_handling, @@ -200,10 +205,11 @@ cpdef bool is_sorted( cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() + cdef table_view c_tbl = tbl.view() with nogil: c_result = cpp_sorting.is_sorted( - tbl.view(), + c_tbl, c_orders, c_null_precedence, _cs @@ -250,11 +256,14 @@ cpdef Table segmented_sort_by_key( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_values = values.view() + cdef table_view c_keys = keys.view() + cdef column_view c_segment_offsets = segment_offsets.view() with nogil: c_result = cpp_sorting.segmented_sort_by_key( - values.view(), - keys.view(), - segment_offsets.view(), + c_values, + c_keys, + c_segment_offsets, c_orders, c_null_precedence, _cs, @@ -303,11 +312,14 @@ cpdef Table stable_segmented_sort_by_key( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_values = values.view() + cdef table_view c_keys = keys.view() + cdef column_view c_segment_offsets = segment_offsets.view() with nogil: c_result = cpp_sorting.stable_segmented_sort_by_key( - values.view(), - keys.view(), - segment_offsets.view(), + c_values, + c_keys, + c_segment_offsets, c_orders, c_null_precedence, _cs, @@ -352,10 +364,12 @@ cpdef Table sort_by_key( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_values = values.view() + cdef table_view c_keys = keys.view() with nogil: c_result = cpp_sorting.sort_by_key( - values.view(), - keys.view(), + c_values, + c_keys, c_orders, c_null_precedence, _cs, @@ -400,10 +414,12 @@ cpdef Table stable_sort_by_key( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_values = values.view() + cdef table_view c_keys = keys.view() with nogil: c_result = cpp_sorting.stable_sort_by_key( - values.view(), - keys.view(), + c_values, + c_keys, c_orders, c_null_precedence, _cs, @@ -445,9 +461,10 @@ cpdef Table sort( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_source_table = source_table.view() with nogil: c_result = cpp_sorting.sort( - source_table.view(), + c_source_table, c_orders, c_null_precedence, _cs, @@ -489,9 +506,10 @@ cpdef Table stable_sort( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_source_table = source_table.view() with nogil: c_result = cpp_sorting.stable_sort( - source_table.view(), + c_source_table, c_orders, c_null_precedence, _cs, @@ -533,9 +551,10 @@ cpdef Column top_k( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_col = col.view() with nogil: c_result = cpp_sorting.top_k( - col.view(), + c_col, k, sort_order, _cs, @@ -580,9 +599,10 @@ cpdef Column top_k_order( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_col = col.view() with nogil: c_result = cpp_sorting.top_k_order( - col.view(), + c_col, k, sort_order, _cs, diff --git a/python/pylibcudf/pylibcudf/stream_compaction.pyx b/python/pylibcudf/pylibcudf/stream_compaction.pyx index 2fe8705ea527..457c82f63a7e 100644 --- a/python/pylibcudf/pylibcudf/stream_compaction.pyx +++ b/python/pylibcudf/pylibcudf/stream_compaction.pyx @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from cython.operator cimport dereference @@ -7,6 +7,8 @@ from libcpp.utility cimport move from libcpp.vector cimport vector from pylibcudf.libcudf cimport stream_compaction as cpp_stream_compaction from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view +from pylibcudf.libcudf.table.table_view cimport table_view from pylibcudf.libcudf.stream_compaction cimport duplicate_keep_option from pylibcudf.libcudf.table.table cimport table from pylibcudf.libcudf.types cimport ( @@ -71,9 +73,10 @@ cpdef Table drop_nulls( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_source_table = source_table.view() with nogil: c_result = cpp_stream_compaction.drop_nulls( - source_table.view(), c_keys, keep_threshold, _cs, mr.get_mr() + c_source_table, c_keys, keep_threshold, _cs, mr.get_mr() ) return Table.from_libcudf(move(c_result), _stream, mr) @@ -110,9 +113,10 @@ cpdef Table drop_nans( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_source_table = source_table.view() with nogil: c_result = cpp_stream_compaction.drop_nans( - source_table.view(), c_keys, keep_threshold, _cs, mr.get_mr() + c_source_table, c_keys, keep_threshold, _cs, mr.get_mr() ) return Table.from_libcudf(move(c_result), _stream, mr) @@ -145,9 +149,11 @@ cpdef Table apply_boolean_mask( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_source_table = source_table.view() + cdef column_view c_boolean_mask = boolean_mask.view() with nogil: c_result = cpp_stream_compaction.apply_boolean_mask( - source_table.view(), boolean_mask.view(), _cs, mr.get_mr() + c_source_table, c_boolean_mask, _cs, mr.get_mr() ) return Table.from_libcudf(move(c_result), _stream, mr) @@ -180,9 +186,11 @@ cpdef Table apply_deletion_mask( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_source_table = source_table.view() + cdef column_view c_deletion_mask = deletion_mask.view() with nogil: c_result = cpp_stream_compaction.apply_deletion_mask( - source_table.view(), deletion_mask.view(), _cs, mr.get_mr() + c_source_table, c_deletion_mask, _cs, mr.get_mr() ) return Table.from_libcudf(move(c_result), _stream, mr) @@ -228,9 +236,10 @@ cpdef Table unique( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_input = input.view() with nogil: c_result = cpp_stream_compaction.unique( - input.view(), c_keys, keep, nulls_equal, _cs, mr.get_mr() + c_input, c_keys, keep, nulls_equal, _cs, mr.get_mr() ) return Table.from_libcudf(move(c_result), _stream, mr) @@ -274,9 +283,10 @@ cpdef Table distinct( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_input = input.view() with nogil: c_result = cpp_stream_compaction.distinct( - input.view(), c_keys, keep, nulls_equal, nans_equal, _cs, + c_input, c_keys, keep, nulls_equal, nans_equal, _cs, mr.get_mr() ) return Table.from_libcudf(move(c_result), _stream, mr) @@ -316,9 +326,10 @@ cpdef Column distinct_indices( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_input = input.view() with nogil: c_result = cpp_stream_compaction.distinct_indices( - input.view(), keep, nulls_equal, nans_equal, _cs, mr.get_mr() + c_input, keep, nulls_equal, nans_equal, _cs, mr.get_mr() ) return Column.from_libcudf(move(c_result), _stream, mr) @@ -362,9 +373,10 @@ cpdef Table stable_distinct( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_input = input.view() with nogil: c_result = cpp_stream_compaction.stable_distinct( - input.view(), c_keys, keep, nulls_equal, nans_equal, _cs, + c_input, c_keys, keep, nulls_equal, nans_equal, _cs, mr.get_mr() ) return Table.from_libcudf(move(c_result), _stream, mr) @@ -401,11 +413,13 @@ cpdef Table filter( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_predicate_table = predicate_table.view() + cdef table_view c_filter_table = filter_table.view() with nogil: c_result = cpp_stream_compaction.filter( - predicate_table.view(), + c_predicate_table, dereference(predicate_expr.c_obj.get()), - filter_table.view(), + c_filter_table, _cs, mr.get_mr() ) diff --git a/python/pylibcudf/pylibcudf/strings/attributes.pyx b/python/pylibcudf/pylibcudf/strings/attributes.pyx index 334270ea8347..ead494d047f3 100644 --- a/python/pylibcudf/pylibcudf/strings/attributes.pyx +++ b/python/pylibcudf/pylibcudf/strings/attributes.pyx @@ -1,10 +1,11 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libcpp.memory cimport unique_ptr from libcpp.utility cimport move from pylibcudf.column cimport Column from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view from pylibcudf.libcudf.strings cimport attributes as cpp_attributes from pylibcudf.utils cimport _get_stream, _get_memory_resource from rmm.pylibrmm.memory_resource cimport DeviceMemoryResource @@ -36,10 +37,10 @@ cpdef Column count_characters( cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) - + cdef column_view c_source_strings = source_strings.view() with nogil: c_result = cpp_attributes.count_characters( - source_strings.view(), _cs, mr.get_mr() + c_source_strings, _cs, mr.get_mr() ) return Column.from_libcudf(move(c_result), _stream, mr) @@ -68,10 +69,10 @@ cpdef Column count_bytes( cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) - + cdef column_view c_source_strings = source_strings.view() with nogil: c_result = cpp_attributes.count_bytes( - source_strings.view(), _cs, mr.get_mr() + c_source_strings, _cs, mr.get_mr() ) return Column.from_libcudf(move(c_result), _stream, mr) @@ -100,10 +101,10 @@ cpdef Column code_points( cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) - + cdef column_view c_source_strings = source_strings.view() with nogil: c_result = cpp_attributes.code_points( - source_strings.view(), _cs, mr.get_mr() + c_source_strings, _cs, mr.get_mr() ) return Column.from_libcudf(move(c_result), _stream, mr) diff --git a/python/pylibcudf/pylibcudf/strings/capitalize.pyx b/python/pylibcudf/pylibcudf/strings/capitalize.pyx index be8c52a59b54..2c5683980191 100644 --- a/python/pylibcudf/pylibcudf/strings/capitalize.pyx +++ b/python/pylibcudf/pylibcudf/strings/capitalize.pyx @@ -1,10 +1,11 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libcpp.memory cimport unique_ptr from libcpp.utility cimport move from pylibcudf.column cimport Column from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view from pylibcudf.libcudf.scalar.scalar cimport string_scalar from pylibcudf.libcudf.scalar.scalar_factories cimport ( make_string_scalar as cpp_make_string_scalar, @@ -49,6 +50,7 @@ cpdef Column capitalize( cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input if delimiters is None: delimiters = Scalar.from_libcudf( @@ -59,9 +61,10 @@ cpdef Column capitalize( delimiters.c_obj.get() ) + c_input = input.view() with nogil: c_result = cpp_capitalize.capitalize( - input.view(), + c_input, dereference(cpp_delimiters), _cs, mr.get_mr() @@ -97,9 +100,10 @@ cpdef Column title( cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: c_result = cpp_capitalize.title( - input.view(), sequence_type, _cs, mr.get_mr() + c_input, sequence_type, _cs, mr.get_mr() ) return Column.from_libcudf(move(c_result), _stream, mr) @@ -124,7 +128,8 @@ cpdef Column is_title(Column input, object stream=None, DeviceMemoryResource mr= cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: - c_result = cpp_capitalize.is_title(input.view(), _cs, mr.get_mr()) + c_result = cpp_capitalize.is_title(c_input, _cs, mr.get_mr()) return Column.from_libcudf(move(c_result), _stream, mr) diff --git a/python/pylibcudf/pylibcudf/strings/case.pyx b/python/pylibcudf/pylibcudf/strings/case.pyx index ec6539f42e18..5a122faacb16 100644 --- a/python/pylibcudf/pylibcudf/strings/case.pyx +++ b/python/pylibcudf/pylibcudf/strings/case.pyx @@ -1,10 +1,11 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libcpp.memory cimport unique_ptr from libcpp.utility cimport move from pylibcudf.column cimport Column from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view from pylibcudf.libcudf.strings cimport case as cpp_case from pylibcudf.utils cimport _get_stream, _get_memory_resource from rmm.pylibrmm.memory_resource cimport DeviceMemoryResource @@ -36,8 +37,9 @@ cpdef Column to_lower(Column input, object stream=None, DeviceMemoryResource mr= cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: - c_result = cpp_case.to_lower(input.view(), _cs, mr.get_mr()) + c_result = cpp_case.to_lower(c_input, _cs, mr.get_mr()) return Column.from_libcudf(move(c_result), _stream, mr) @@ -64,8 +66,9 @@ cpdef Column to_upper(Column input, object stream=None, DeviceMemoryResource mr= cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: - c_result = cpp_case.to_upper(input.view(), _cs, mr.get_mr()) + c_result = cpp_case.to_upper(c_input, _cs, mr.get_mr()) return Column.from_libcudf(move(c_result), _stream, mr) @@ -94,7 +97,8 @@ cpdef Column swapcase(Column input, object stream=None, DeviceMemoryResource mr= cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: - c_result = cpp_case.swapcase(input.view(), _cs, mr.get_mr()) + c_result = cpp_case.swapcase(c_input, _cs, mr.get_mr()) return Column.from_libcudf(move(c_result), _stream, mr) diff --git a/python/pylibcudf/pylibcudf/strings/char_types.pyx b/python/pylibcudf/pylibcudf/strings/char_types.pyx index 2567ab8ee4b6..d7e155f548a5 100644 --- a/python/pylibcudf/pylibcudf/strings/char_types.pyx +++ b/python/pylibcudf/pylibcudf/strings/char_types.pyx @@ -1,10 +1,11 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libcpp.memory cimport unique_ptr from libcpp.utility cimport move from pylibcudf.column cimport Column from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view from pylibcudf.libcudf.scalar.scalar cimport string_scalar from pylibcudf.libcudf.strings cimport char_types as cpp_char_types from pylibcudf.libcudf.strings.char_types cimport string_character_types @@ -54,10 +55,10 @@ cpdef Column all_characters_of_type( cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) - + cdef column_view c_source_strings = source_strings.view() with nogil: c_result = cpp_char_types.all_characters_of_type( - source_strings.view(), + c_source_strings, types, verify_types, _cs, @@ -104,10 +105,10 @@ cpdef Column filter_characters_of_type( cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) - + cdef column_view c_source_strings = source_strings.view() with nogil: c_result = cpp_char_types.filter_characters_of_type( - source_strings.view(), + c_source_strings, types_to_remove, dereference(c_replacement), types_to_keep, diff --git a/python/pylibcudf/pylibcudf/strings/combine.pyx b/python/pylibcudf/pylibcudf/strings/combine.pyx index 829030029074..f147374df11d 100644 --- a/python/pylibcudf/pylibcudf/strings/combine.pyx +++ b/python/pylibcudf/pylibcudf/strings/combine.pyx @@ -1,9 +1,10 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libcpp.memory cimport unique_ptr from libcpp.utility cimport move from pylibcudf.column cimport Column from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view from pylibcudf.libcudf.scalar.scalar cimport string_scalar from pylibcudf.libcudf.scalar.scalar_factories cimport ( make_string_scalar as cpp_make_string_scalar, @@ -11,6 +12,7 @@ from pylibcudf.libcudf.scalar.scalar_factories cimport ( from pylibcudf.libcudf.strings cimport combine as cpp_combine from pylibcudf.scalar cimport Scalar from pylibcudf.table cimport Table +from pylibcudf.libcudf.table.table_view cimport table_view from pylibcudf.utils cimport _get_stream, _get_memory_resource from rmm.pylibrmm.memory_resource cimport DeviceMemoryResource from rmm.pylibrmm.stream cimport Stream @@ -72,6 +74,8 @@ cpdef Column concatenate( cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_strings_columns + cdef column_view c_separator_view if narep is None: narep = Scalar.from_libcudf( @@ -89,11 +93,13 @@ cpdef Column concatenate( c_col_narep = ( col_narep.c_obj.get() ) + c_strings_columns = strings_columns.view() + c_separator_view = separator.view() with nogil: c_result = move( cpp_combine.concatenate( - strings_columns.view(), - separator.view(), + c_strings_columns, + c_separator_view, dereference(c_narep), dereference(c_col_narep), separate_nulls, @@ -107,10 +113,11 @@ cpdef Column concatenate( "col_narep cannot be specified when separator is a Scalar" ) c_separator = (separator.c_obj.get()) + c_strings_columns = strings_columns.view() with nogil: c_result = move( cpp_combine.concatenate( - strings_columns.view(), + c_strings_columns, dereference(c_separator), dereference(c_narep), separate_nulls, @@ -154,16 +161,18 @@ cpdef Column join_strings( cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input cdef const string_scalar* c_separator = ( separator.c_obj.get() ) cdef const string_scalar* c_narep = ( narep.c_obj.get() ) + c_input = input.view() with nogil: c_result = move( cpp_combine.join_strings( - input.view(), + c_input, dereference(c_separator), dereference(c_narep), _cs, @@ -223,6 +232,8 @@ cpdef Column join_list_elements( cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_lists_strings_column + cdef column_view c_separator_view cdef const string_scalar* c_separator_narep = ( separator_narep.c_obj.get() ) @@ -232,11 +243,13 @@ cpdef Column join_list_elements( cdef const string_scalar* c_separator if ColumnOrScalar is Column: + c_lists_strings_column = lists_strings_column.view() + c_separator_view = separator.view() with nogil: c_result = move( cpp_combine.join_list_elements( - lists_strings_column.view(), - separator.view(), + c_lists_strings_column, + c_separator_view, dereference(c_separator_narep), dereference(c_string_narep), separate_nulls, @@ -247,10 +260,11 @@ cpdef Column join_list_elements( ) elif ColumnOrScalar is Scalar: c_separator = (separator.c_obj.get()) + c_lists_strings_column = lists_strings_column.view() with nogil: c_result = move( cpp_combine.join_list_elements( - lists_strings_column.view(), + c_lists_strings_column, dereference(c_separator), dereference(c_separator_narep), separate_nulls, diff --git a/python/pylibcudf/pylibcudf/strings/contains.pyx b/python/pylibcudf/pylibcudf/strings/contains.pyx index 495d1637d8a8..2e2e89de2795 100644 --- a/python/pylibcudf/pylibcudf/strings/contains.pyx +++ b/python/pylibcudf/pylibcudf/strings/contains.pyx @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libcpp.memory cimport unique_ptr from libcpp.utility cimport move @@ -6,6 +6,7 @@ from libcpp.string cimport string from pylibcudf.column cimport Column from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view from pylibcudf.libcudf.strings cimport contains as cpp_contains from pylibcudf.strings.regex_program cimport RegexProgram from pylibcudf.utils cimport _get_stream, _get_memory_resource @@ -45,10 +46,10 @@ cpdef Column contains_re( if _stream is None: _stream = _get_stream(None) mr = _get_memory_resource(mr) - + cdef column_view c_input = input.view() with nogil: result = cpp_contains.contains_re( - input.view(), + c_input, prog.c_obj.get()[0], _cs, mr.get_mr() @@ -85,10 +86,10 @@ cpdef Column count_re( cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) - + cdef column_view c_input = input.view() with nogil: result = cpp_contains.count_re( - input.view(), + c_input, prog.c_obj.get()[0], _cs, mr.get_mr() @@ -126,10 +127,10 @@ cpdef Column matches_re( cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) - + cdef column_view c_input = input.view() with nogil: result = cpp_contains.matches_re( - input.view(), + c_input, prog.c_obj.get()[0], _cs, mr.get_mr() @@ -170,6 +171,7 @@ cpdef Column like( cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input if escape_character is None: escape_character = "" @@ -177,9 +179,10 @@ cpdef Column like( cdef string c_escape_character = escape_character.encode() cdef string c_pattern = pattern.encode() + c_input = input.view() with nogil: result = cpp_contains.like( - input.view(), + c_input, c_pattern, c_escape_character, _cs, diff --git a/python/pylibcudf/pylibcudf/strings/convert/convert_booleans.pyx b/python/pylibcudf/pylibcudf/strings/convert/convert_booleans.pyx index e8f963cf0f3b..2a4f77f4c0e8 100644 --- a/python/pylibcudf/pylibcudf/strings/convert/convert_booleans.pyx +++ b/python/pylibcudf/pylibcudf/strings/convert/convert_booleans.pyx @@ -1,10 +1,10 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libcpp.memory cimport unique_ptr from libcpp.utility cimport move from pylibcudf.column cimport Column -from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column cimport column, column_view from pylibcudf.libcudf.scalar.scalar cimport string_scalar from pylibcudf.libcudf.strings.convert cimport ( convert_booleans as cpp_convert_booleans, @@ -52,9 +52,10 @@ cpdef Column to_booleans( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: c_result = cpp_convert_booleans.to_booleans( - input.view(), + c_input, dereference(c_true_string), _cs, mr.get_mr() @@ -105,9 +106,10 @@ cpdef Column from_booleans( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_booleans = booleans.view() with nogil: c_result = cpp_convert_booleans.from_booleans( - booleans.view(), + c_booleans, dereference(c_true_string), dereference(c_false_string), _cs, diff --git a/python/pylibcudf/pylibcudf/strings/convert/convert_datetime.pyx b/python/pylibcudf/pylibcudf/strings/convert/convert_datetime.pyx index 633445a7383f..9aab9ae06dce 100644 --- a/python/pylibcudf/pylibcudf/strings/convert/convert_datetime.pyx +++ b/python/pylibcudf/pylibcudf/strings/convert/convert_datetime.pyx @@ -1,11 +1,11 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libcpp.memory cimport unique_ptr from libcpp.string cimport string from libcpp.utility cimport move from pylibcudf.column cimport Column -from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column cimport column, column_view from pylibcudf.libcudf.strings.convert cimport ( convert_datetime as cpp_convert_datetime, ) @@ -55,9 +55,10 @@ cpdef Column to_timestamps( cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: c_result = cpp_convert_datetime.to_timestamps( - input.view(), + c_input, timestamp_type.c_obj, c_format, _cs, @@ -103,11 +104,13 @@ cpdef Column from_timestamps( cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_timestamps = timestamps.view() + cdef column_view c_input_strings_names = input_strings_names.view() with nogil: c_result = cpp_convert_datetime.from_timestamps( - timestamps.view(), + c_timestamps, c_format, - input_strings_names.view(), + c_input_strings_names, _cs, mr.get_mr() ) @@ -147,9 +150,10 @@ cpdef Column is_timestamp( cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: c_result = cpp_convert_datetime.is_timestamp( - input.view(), + c_input, c_format, _cs, mr.get_mr() diff --git a/python/pylibcudf/pylibcudf/strings/convert/convert_durations.pyx b/python/pylibcudf/pylibcudf/strings/convert/convert_durations.pyx index 548df7398b4b..8068d80a8e92 100644 --- a/python/pylibcudf/pylibcudf/strings/convert/convert_durations.pyx +++ b/python/pylibcudf/pylibcudf/strings/convert/convert_durations.pyx @@ -1,11 +1,11 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libcpp.memory cimport unique_ptr from libcpp.string cimport string from libcpp.utility cimport move from pylibcudf.column cimport Column -from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column cimport column, column_view from pylibcudf.libcudf.strings.convert cimport ( convert_durations as cpp_convert_durations, ) @@ -56,9 +56,10 @@ cpdef Column to_durations( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: c_result = cpp_convert_durations.to_durations( - input.view(), + c_input, duration_type.c_obj, c_format, _cs, @@ -105,9 +106,10 @@ cpdef Column from_durations( format = "%D days %H:%M:%S" cdef string c_format = format.encode() + cdef column_view c_durations = durations.view() with nogil: c_result = cpp_convert_durations.from_durations( - durations.view(), + c_durations, c_format, _cs, mr.get_mr() diff --git a/python/pylibcudf/pylibcudf/strings/convert/convert_fixed_point.pyx b/python/pylibcudf/pylibcudf/strings/convert/convert_fixed_point.pyx index 059373790c59..abc7d7a7848a 100644 --- a/python/pylibcudf/pylibcudf/strings/convert/convert_fixed_point.pyx +++ b/python/pylibcudf/pylibcudf/strings/convert/convert_fixed_point.pyx @@ -1,10 +1,10 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libcpp.memory cimport unique_ptr from libcpp.utility cimport move from pylibcudf.column cimport Column -from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column cimport column, column_view from pylibcudf.libcudf.strings.convert cimport ( convert_fixed_point as cpp_fixed_point, ) @@ -47,9 +47,10 @@ cpdef Column to_fixed_point( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: c_result = cpp_fixed_point.to_fixed_point( - input.view(), + c_input, output_type.c_obj, _cs, mr.get_mr() @@ -84,9 +85,10 @@ cpdef Column from_fixed_point( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: c_result = cpp_fixed_point.from_fixed_point( - input.view(), _cs, mr.get_mr() + c_input, _cs, mr.get_mr() ) return Column.from_libcudf(move(c_result), _stream, mr) @@ -128,9 +130,10 @@ cpdef Column is_fixed_point( if decimal_type is None: decimal_type = DataType(type_id.DECIMAL64) + cdef column_view c_input = input.view() with nogil: c_result = cpp_fixed_point.is_fixed_point( - input.view(), + c_input, decimal_type.c_obj, _cs, mr.get_mr() diff --git a/python/pylibcudf/pylibcudf/strings/convert/convert_floats.pyx b/python/pylibcudf/pylibcudf/strings/convert/convert_floats.pyx index d4901ce7be67..cd48c98e9d92 100644 --- a/python/pylibcudf/pylibcudf/strings/convert/convert_floats.pyx +++ b/python/pylibcudf/pylibcudf/strings/convert/convert_floats.pyx @@ -1,10 +1,10 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libcpp.memory cimport unique_ptr from libcpp.utility cimport move from pylibcudf.column cimport Column -from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column cimport column, column_view from pylibcudf.libcudf.strings.convert cimport ( convert_floats as cpp_convert_floats, ) @@ -49,9 +49,10 @@ cpdef Column to_floats( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_strings = strings.view() with nogil: c_result = cpp_convert_floats.to_floats( - strings.view(), + c_strings, output_type.c_obj, _cs, mr.get_mr() @@ -87,9 +88,10 @@ cpdef Column from_floats( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_floats = floats.view() with nogil: c_result = cpp_convert_floats.from_floats( - floats.view(), _cs, mr.get_mr() + c_floats, _cs, mr.get_mr() ) return Column.from_libcudf(move(c_result), _stream, mr) @@ -120,9 +122,10 @@ cpdef Column is_float(Column input, object stream=None, DeviceMemoryResource mr= cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: c_result = cpp_convert_floats.is_float( - input.view(), _cs, mr.get_mr() + c_input, _cs, mr.get_mr() ) return Column.from_libcudf(move(c_result), _stream, mr) diff --git a/python/pylibcudf/pylibcudf/strings/convert/convert_integers.pyx b/python/pylibcudf/pylibcudf/strings/convert/convert_integers.pyx index b717ddbbcdaf..95db3b8829b5 100644 --- a/python/pylibcudf/pylibcudf/strings/convert/convert_integers.pyx +++ b/python/pylibcudf/pylibcudf/strings/convert/convert_integers.pyx @@ -1,10 +1,10 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libcpp.memory cimport unique_ptr from libcpp.utility cimport move from pylibcudf.column cimport Column -from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column cimport column, column_view from pylibcudf.libcudf.strings.convert cimport ( convert_integers as cpp_convert_integers, ) @@ -53,10 +53,11 @@ cpdef Column to_integers( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: c_result = move( cpp_convert_integers.to_integers( - input.view(), + c_input, output_type.c_obj, _cs, mr.get_mr() @@ -93,10 +94,11 @@ cpdef Column from_integers( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_integers = integers.view() with nogil: c_result = move( cpp_convert_integers.from_integers( - integers.view(), + c_integers, _cs, mr.get_mr() ) @@ -140,11 +142,12 @@ cpdef Column is_integer( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() if int_type is None: with nogil: c_result = move( cpp_convert_integers.is_integer( - input.view(), + c_input, _cs, mr.get_mr() ) @@ -153,7 +156,7 @@ cpdef Column is_integer( with nogil: c_result = move( cpp_convert_integers.is_integer( - input.view(), + c_input, int_type.c_obj, _cs, mr.get_mr() @@ -193,10 +196,11 @@ cpdef Column hex_to_integers( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: c_result = move( cpp_convert_integers.hex_to_integers( - input.view(), + c_input, output_type.c_obj, _cs, mr.get_mr() @@ -231,10 +235,11 @@ cpdef Column is_hex(Column input, object stream=None, DeviceMemoryResource mr=No cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: c_result = move( cpp_convert_integers.is_hex( - input.view(), + c_input, _cs, mr.get_mr() ) @@ -270,10 +275,11 @@ cpdef Column integers_to_hex( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: c_result = move( cpp_convert_integers.integers_to_hex( - input.view(), + c_input, _cs, mr.get_mr() ) diff --git a/python/pylibcudf/pylibcudf/strings/convert/convert_ipv4.pyx b/python/pylibcudf/pylibcudf/strings/convert/convert_ipv4.pyx index 45b98190aa77..f0a262192b8b 100644 --- a/python/pylibcudf/pylibcudf/strings/convert/convert_ipv4.pyx +++ b/python/pylibcudf/pylibcudf/strings/convert/convert_ipv4.pyx @@ -1,10 +1,10 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libcpp.memory cimport unique_ptr from libcpp.utility cimport move from pylibcudf.column cimport Column -from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column cimport column, column_view from pylibcudf.libcudf.strings.convert cimport convert_ipv4 as cpp_convert_ipv4 from pylibcudf.utils cimport _get_stream, _get_memory_resource from rmm.pylibrmm.memory_resource cimport DeviceMemoryResource @@ -39,9 +39,10 @@ cpdef Column ipv4_to_integers( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: c_result = cpp_convert_ipv4.ipv4_to_integers( - input.view(), _cs, mr.get_mr() + c_input, _cs, mr.get_mr() ) return Column.from_libcudf(move(c_result), _stream, mr) @@ -73,9 +74,10 @@ cpdef Column integers_to_ipv4( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_integers = integers.view() with nogil: c_result = cpp_convert_ipv4.integers_to_ipv4( - integers.view(), _cs, mr.get_mr() + c_integers, _cs, mr.get_mr() ) return Column.from_libcudf(move(c_result), _stream, mr) @@ -106,7 +108,8 @@ cpdef Column is_ipv4(Column input, object stream=None, DeviceMemoryResource mr=N cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: - c_result = cpp_convert_ipv4.is_ipv4(input.view(), _cs, mr.get_mr()) + c_result = cpp_convert_ipv4.is_ipv4(c_input, _cs, mr.get_mr()) return Column.from_libcudf(move(c_result), _stream, mr) diff --git a/python/pylibcudf/pylibcudf/strings/convert/convert_lists.pyx b/python/pylibcudf/pylibcudf/strings/convert/convert_lists.pyx index 9c8f9d7b02eb..903b83a9ea92 100644 --- a/python/pylibcudf/pylibcudf/strings/convert/convert_lists.pyx +++ b/python/pylibcudf/pylibcudf/strings/convert/convert_lists.pyx @@ -1,11 +1,11 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libcpp.memory cimport unique_ptr from libcpp.utility cimport move from pylibcudf.column cimport Column from pylibcudf.column_factories cimport make_empty_column -from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column cimport column, column_view from pylibcudf.libcudf.scalar.scalar cimport string_scalar from pylibcudf.libcudf.scalar.scalar_factories cimport ( make_string_scalar as cpp_make_string_scalar, @@ -75,11 +75,13 @@ cpdef Column format_list_column( if separators is None: separators = make_empty_column(type_id.STRING) + cdef column_view c_input = input.view() + cdef column_view c_separators = separators.view() with nogil: c_result = cpp_convert_lists.format_list_column( - input.view(), + c_input, dereference(c_na_rep), - separators.view(), + c_separators, _cs, mr.get_mr() ) diff --git a/python/pylibcudf/pylibcudf/strings/convert/convert_urls.pyx b/python/pylibcudf/pylibcudf/strings/convert/convert_urls.pyx index efe009e6c02a..5c9d4bf17a79 100644 --- a/python/pylibcudf/pylibcudf/strings/convert/convert_urls.pyx +++ b/python/pylibcudf/pylibcudf/strings/convert/convert_urls.pyx @@ -1,10 +1,10 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libcpp.memory cimport unique_ptr from libcpp.utility cimport move from pylibcudf.column cimport Column -from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column cimport column, column_view from pylibcudf.libcudf.strings.convert cimport convert_urls as cpp_convert_urls from pylibcudf.utils cimport _get_stream, _get_memory_resource @@ -38,9 +38,10 @@ cpdef Column url_encode(Column input, object stream=None, DeviceMemoryResource m cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: c_result = cpp_convert_urls.url_encode( - input.view(), _cs, mr.get_mr() + c_input, _cs, mr.get_mr() ) return Column.from_libcudf(move(c_result), _stream, mr) @@ -70,9 +71,10 @@ cpdef Column url_decode(Column input, object stream=None, DeviceMemoryResource m cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: c_result = cpp_convert_urls.url_decode( - input.view(), _cs, mr.get_mr() + c_input, _cs, mr.get_mr() ) return Column.from_libcudf(move(c_result), _stream, mr) diff --git a/python/pylibcudf/pylibcudf/strings/extract.pyx b/python/pylibcudf/pylibcudf/strings/extract.pyx index c670b226e842..3dee27693a37 100644 --- a/python/pylibcudf/pylibcudf/strings/extract.pyx +++ b/python/pylibcudf/pylibcudf/strings/extract.pyx @@ -1,10 +1,11 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libcpp.memory cimport unique_ptr from libcpp.utility cimport move from pylibcudf.column cimport Column from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view from pylibcudf.libcudf.strings cimport extract as cpp_extract from pylibcudf.libcudf.table.table cimport table from pylibcudf.strings.regex_program cimport RegexProgram @@ -45,10 +46,10 @@ cpdef Table extract( cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) - + cdef column_view c_input = input.view() with nogil: c_result = cpp_extract.extract( - input.view(), + c_input, prog.c_obj.get()[0], _cs, mr.get_mr() @@ -85,10 +86,10 @@ cpdef Column extract_all_record( cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) - + cdef column_view c_input = input.view() with nogil: c_result = cpp_extract.extract_all_record( - input.view(), + c_input, prog.c_obj.get()[0], _cs, mr.get_mr() @@ -130,10 +131,10 @@ cpdef Column extract_single( cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) - + cdef column_view c_input = input.view() with nogil: c_result = cpp_extract.extract_single( - input.view(), + c_input, prog.c_obj.get()[0], group, _cs, diff --git a/python/pylibcudf/pylibcudf/strings/find.pyx b/python/pylibcudf/pylibcudf/strings/find.pyx index 102a8787651f..d9c9e81dea7d 100644 --- a/python/pylibcudf/pylibcudf/strings/find.pyx +++ b/python/pylibcudf/pylibcudf/strings/find.pyx @@ -1,9 +1,10 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libcpp.memory cimport unique_ptr from libcpp.utility cimport move from pylibcudf.column cimport Column from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view from pylibcudf.libcudf.strings cimport find as cpp_find from pylibcudf.libcudf.types cimport size_type from pylibcudf.scalar cimport Scalar @@ -62,19 +63,24 @@ cpdef Column find( cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input + cdef column_view c_target if ColumnOrScalar is Column: + c_input = input.view() + c_target = target.view() with nogil: result = cpp_find.find( - input.view(), - target.view(), + c_input, + c_target, start, _cs, mr.get_mr() ) elif ColumnOrScalar is Scalar: + c_input = input.view() with nogil: result = cpp_find.find( - input.view(), + c_input, dereference((target.c_obj.get())), start, stop, @@ -124,9 +130,10 @@ cpdef Column rfind( cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: result = cpp_find.rfind( - input.view(), + c_input, dereference((target.c_obj.get())), start, stop, @@ -175,18 +182,23 @@ cpdef Column contains( cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input + cdef column_view c_target if ColumnOrScalar is Column: + c_input = input.view() + c_target = target.view() with nogil: result = cpp_find.contains( - input.view(), - target.view(), + c_input, + c_target, _cs, mr.get_mr() ) elif ColumnOrScalar is Scalar: + c_input = input.view() with nogil: result = cpp_find.contains( - input.view(), + c_input, dereference((target.c_obj.get())), _cs, mr.get_mr() @@ -236,19 +248,24 @@ cpdef Column starts_with( cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input + cdef column_view c_target if ColumnOrScalar is Column: + c_input = input.view() + c_target = target.view() with nogil: result = cpp_find.starts_with( - input.view(), - target.view(), + c_input, + c_target, _cs, mr.get_mr() ) elif ColumnOrScalar is Scalar: + c_input = input.view() with nogil: result = cpp_find.starts_with( - input.view(), + c_input, dereference((target.c_obj.get())), _cs, mr.get_mr() @@ -296,18 +313,23 @@ cpdef Column ends_with( cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input + cdef column_view c_target if ColumnOrScalar is Column: + c_input = input.view() + c_target = target.view() with nogil: result = cpp_find.ends_with( - input.view(), - target.view(), + c_input, + c_target, _cs, mr.get_mr() ) elif ColumnOrScalar is Scalar: + c_input = input.view() with nogil: result = cpp_find.ends_with( - input.view(), + c_input, dereference((target.c_obj.get())), _cs, mr.get_mr() diff --git a/python/pylibcudf/pylibcudf/strings/find_multiple.pyx b/python/pylibcudf/pylibcudf/strings/find_multiple.pyx index ed5f0d785062..3ac87788cd6f 100644 --- a/python/pylibcudf/pylibcudf/strings/find_multiple.pyx +++ b/python/pylibcudf/pylibcudf/strings/find_multiple.pyx @@ -1,10 +1,11 @@ -# SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libcpp.memory cimport unique_ptr from libcpp.utility cimport move from pylibcudf.column cimport Column from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view from pylibcudf.libcudf.strings cimport find_multiple as cpp_find_multiple from pylibcudf.libcudf.table.table cimport table from pylibcudf.table cimport Table @@ -45,11 +46,12 @@ cpdef Column find_multiple( cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) - + cdef column_view c_input = input.view() + cdef column_view c_targets = targets.view() with nogil: c_result = cpp_find_multiple.find_multiple( - input.view(), - targets.view(), + c_input, + c_targets, _cs, mr.get_mr() ) @@ -87,11 +89,12 @@ cpdef Table contains_multiple( cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) - + cdef column_view c_input = input.view() + cdef column_view c_targets = targets.view() with nogil: c_result = cpp_find_multiple.contains_multiple( - input.view(), - targets.view(), + c_input, + c_targets, _cs, mr.get_mr() ) diff --git a/python/pylibcudf/pylibcudf/strings/findall.pyx b/python/pylibcudf/pylibcudf/strings/findall.pyx index 5647a791ef12..17099870f3aa 100644 --- a/python/pylibcudf/pylibcudf/strings/findall.pyx +++ b/python/pylibcudf/pylibcudf/strings/findall.pyx @@ -1,10 +1,11 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libcpp.memory cimport unique_ptr from libcpp.utility cimport move from pylibcudf.column cimport Column from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view from pylibcudf.libcudf.strings cimport findall as cpp_findall from pylibcudf.strings.regex_program cimport RegexProgram from pylibcudf.utils cimport _get_stream, _get_memory_resource @@ -41,10 +42,10 @@ cpdef Column findall( cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) - + cdef column_view c_input = input.view() with nogil: c_result = cpp_findall.findall( - input.view(), + c_input, pattern.c_obj.get()[0], _cs, mr.get_mr() @@ -80,10 +81,10 @@ cpdef Column find_re( cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) - + cdef column_view c_input = input.view() with nogil: c_result = cpp_findall.find_re( - input.view(), + c_input, pattern.c_obj.get()[0], _cs, mr.get_mr() diff --git a/python/pylibcudf/pylibcudf/strings/padding.pyx b/python/pylibcudf/pylibcudf/strings/padding.pyx index d8eb4f1da4a0..56949d84eb5f 100644 --- a/python/pylibcudf/pylibcudf/strings/padding.pyx +++ b/python/pylibcudf/pylibcudf/strings/padding.pyx @@ -1,9 +1,10 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libcpp.memory cimport unique_ptr from libcpp.utility cimport move from pylibcudf.column cimport Column from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view from pylibcudf.libcudf.strings cimport padding as cpp_padding from pylibcudf.libcudf.strings.side_type cimport side_type from pylibcudf.libcudf.types cimport size_type @@ -50,10 +51,10 @@ cpdef Column pad( cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) - + cdef column_view c_input = input.view() with nogil: c_result = cpp_padding.pad( - input.view(), + c_input, width, side, c_fill_char, @@ -89,10 +90,10 @@ cpdef Column zfill( cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) - + cdef column_view c_input = input.view() with nogil: c_result = cpp_padding.zfill( - input.view(), + c_input, width, _cs, mr.get_mr() @@ -126,11 +127,12 @@ cpdef Column zfill_by_widths( cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) - + cdef column_view c_input = input.view() + cdef column_view c_widths = widths.view() with nogil: c_result = cpp_padding.zfill_by_widths( - input.view(), - widths.view(), + c_input, + c_widths, _cs, mr.get_mr() ) diff --git a/python/pylibcudf/pylibcudf/strings/repeat.pyx b/python/pylibcudf/pylibcudf/strings/repeat.pyx index 7a9c5285d02b..3f3fd1f0310a 100644 --- a/python/pylibcudf/pylibcudf/strings/repeat.pyx +++ b/python/pylibcudf/pylibcudf/strings/repeat.pyx @@ -1,9 +1,10 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libcpp.memory cimport unique_ptr from libcpp.utility cimport move from pylibcudf.column cimport Column from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view from pylibcudf.libcudf.strings cimport repeat as cpp_repeat from pylibcudf.libcudf.types cimport size_type @@ -48,19 +49,24 @@ cpdef Column repeat_strings( cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input + cdef column_view c_repeat_times if ColumnorSizeType is Column: + c_input = input.view() + c_repeat_times = repeat_times.view() with nogil: c_result = cpp_repeat.repeat_strings( - input.view(), - repeat_times.view(), + c_input, + c_repeat_times, _cs, mr.get_mr() ) elif ColumnorSizeType is size_type: + c_input = input.view() with nogil: c_result = cpp_repeat.repeat_strings( - input.view(), + c_input, repeat_times, _cs, mr.get_mr() diff --git a/python/pylibcudf/pylibcudf/strings/replace.pyx b/python/pylibcudf/pylibcudf/strings/replace.pyx index ccd6c9244417..5603ac849f0c 100644 --- a/python/pylibcudf/pylibcudf/strings/replace.pyx +++ b/python/pylibcudf/pylibcudf/strings/replace.pyx @@ -1,10 +1,11 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libcpp.memory cimport unique_ptr from libcpp.utility cimport move from pylibcudf.column cimport Column from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view from pylibcudf.libcudf.scalar.scalar cimport string_scalar from pylibcudf.libcudf.scalar.scalar_factories cimport ( make_string_scalar as cpp_make_string_scalar, @@ -64,10 +65,10 @@ cpdef Column replace( cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) - + cdef column_view c_input = input.view() with nogil: c_result = cpp_replace( - input.view(), + c_input, target_str[0], repl_str[0], maxrepl, @@ -114,12 +115,14 @@ cpdef Column replace_multiple( cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) - + cdef column_view c_input = input.view() + cdef column_view c_target = target.view() + cdef column_view c_repl = repl.view() with nogil: c_result = cpp_replace_multiple( - input.view(), - target.view(), - repl.view(), + c_input, + c_target, + c_repl, _cs, mr.get_mr() ) @@ -168,6 +171,7 @@ cpdef Column replace_slice( cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input if repl is None: repl = Scalar.from_libcudf( @@ -176,9 +180,10 @@ cpdef Column replace_slice( cdef const string_scalar* scalar_str = (repl.c_obj.get()) + c_input = input.view() with nogil: c_result = cpp_replace_slice( - input.view(), + c_input, scalar_str[0], start, stop, diff --git a/python/pylibcudf/pylibcudf/strings/replace_re.pyx b/python/pylibcudf/pylibcudf/strings/replace_re.pyx index 2b266eb3eac9..7112379911d5 100644 --- a/python/pylibcudf/pylibcudf/strings/replace_re.pyx +++ b/python/pylibcudf/pylibcudf/strings/replace_re.pyx @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from cython.operator cimport dereference from libcpp.memory cimport unique_ptr @@ -6,6 +6,7 @@ from libcpp.string cimport string from libcpp.utility cimport move from pylibcudf.column cimport Column from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view from pylibcudf.libcudf.scalar.scalar cimport string_scalar from pylibcudf.libcudf.scalar.scalar_factories cimport ( make_string_scalar as cpp_make_string_scalar, @@ -57,15 +58,17 @@ cpdef Column replace_re( cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input if replacement is None: replacement = Scalar.from_libcudf( cpp_make_string_scalar("".encode(), _stream.view().value(), mr.get_mr()) ) + c_input = input.view() with nogil: c_result = move( cpp_replace_re.replace_re( - input.view(), + c_input, pattern.c_obj.get()[0], dereference((replacement.get())), max_replace_count, @@ -111,9 +114,10 @@ cpdef Column replace_with_backrefs( mr = _get_memory_resource(mr) cdef string c_replacement = replacement.encode() + cdef column_view c_input = input.view() with nogil: c_result = cpp_replace_re.replace_with_backrefs( - input.view(), + c_input, prog.c_obj.get()[0], c_replacement, _cs, diff --git a/python/pylibcudf/pylibcudf/strings/reverse.pyx b/python/pylibcudf/pylibcudf/strings/reverse.pyx index f1d06248523e..ba31d98f20b4 100644 --- a/python/pylibcudf/pylibcudf/strings/reverse.pyx +++ b/python/pylibcudf/pylibcudf/strings/reverse.pyx @@ -1,10 +1,11 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libcpp.memory cimport unique_ptr from libcpp.utility cimport move from pylibcudf.column cimport Column from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view from pylibcudf.libcudf.strings cimport reverse as cpp_reverse from pylibcudf.utils cimport _get_stream, _get_memory_resource from rmm.pylibrmm.memory_resource cimport DeviceMemoryResource @@ -36,7 +37,8 @@ cpdef Column reverse(Column input, object stream=None, DeviceMemoryResource mr=N cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: - c_result = cpp_reverse.reverse(input.view(), _cs, mr.get_mr()) + c_result = cpp_reverse.reverse(c_input, _cs, mr.get_mr()) return Column.from_libcudf(move(c_result), _stream, mr) diff --git a/python/pylibcudf/pylibcudf/strings/slice.pyx b/python/pylibcudf/pylibcudf/strings/slice.pyx index b3ac2cd8bfe0..dc86db07ce93 100644 --- a/python/pylibcudf/pylibcudf/strings/slice.pyx +++ b/python/pylibcudf/pylibcudf/strings/slice.pyx @@ -1,10 +1,11 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libcpp.memory cimport unique_ptr from libcpp.utility cimport move from pylibcudf.column cimport Column from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view from pylibcudf.libcudf.scalar.scalar cimport numeric_scalar from pylibcudf.libcudf.scalar.scalar_factories cimport ( make_fixed_width_scalar as cpp_make_fixed_width_scalar, @@ -64,6 +65,9 @@ cpdef Column slice_strings( cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input + cdef column_view c_start + cdef column_view c_stop if input is None: raise ValueError("input cannot be None") @@ -77,11 +81,14 @@ cpdef Column slice_strings( "start and stop must be provided for Column-wise slice" ) + c_input = input.view() + c_start = start.view() + c_stop = stop.view() with nogil: c_result = cpp_slice.slice_strings( - input.view(), - start.view(), - stop.view(), + c_input, + c_start, + c_stop, _cs, mr.get_mr() ) @@ -104,9 +111,10 @@ cpdef Column slice_strings( cpp_stop = stop.c_obj.get() cpp_step = step.c_obj.get() + c_input = input.view() with nogil: c_result = cpp_slice.slice_strings( - input.view(), + c_input, dereference(cpp_start), dereference(cpp_stop), dereference(cpp_step), diff --git a/python/pylibcudf/pylibcudf/strings/split/partition.pyx b/python/pylibcudf/pylibcudf/strings/split/partition.pyx index ce813c10bbad..ea8735da729e 100644 --- a/python/pylibcudf/pylibcudf/strings/split/partition.pyx +++ b/python/pylibcudf/pylibcudf/strings/split/partition.pyx @@ -1,8 +1,9 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libcpp.memory cimport unique_ptr from libcpp.utility cimport move from pylibcudf.column cimport Column +from pylibcudf.libcudf.column.column cimport column_view from pylibcudf.libcudf.scalar.scalar cimport string_scalar from pylibcudf.libcudf.scalar.scalar_factories cimport ( make_string_scalar as cpp_make_string_scalar, @@ -60,9 +61,10 @@ cpdef Table partition( delimiter.c_obj.get() ) + cdef column_view c_input = input.view() with nogil: c_result = cpp_partition.partition( - input.view(), + c_input, dereference(c_delimiter), _cs, mr.get_mr() @@ -110,9 +112,10 @@ cpdef Table rpartition( delimiter.c_obj.get() ) + cdef column_view c_input = input.view() with nogil: c_result = cpp_partition.rpartition( - input.view(), + c_input, dereference(c_delimiter), _cs, mr.get_mr() diff --git a/python/pylibcudf/pylibcudf/strings/split/split.pyx b/python/pylibcudf/pylibcudf/strings/split/split.pyx index 52803b08eb00..0a6c71ee77c6 100644 --- a/python/pylibcudf/pylibcudf/strings/split/split.pyx +++ b/python/pylibcudf/pylibcudf/strings/split/split.pyx @@ -1,9 +1,9 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libcpp.memory cimport unique_ptr from libcpp.utility cimport move from pylibcudf.column cimport Column -from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column cimport column, column_view from pylibcudf.libcudf.scalar.scalar cimport string_scalar from pylibcudf.libcudf.strings.split cimport split as cpp_split from pylibcudf.libcudf.table.table cimport table @@ -70,9 +70,10 @@ cpdef Table split( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_strings_column = strings_column.view() with nogil: c_result = cpp_split.split( - strings_column.view(), + c_strings_column, dereference(c_delimiter), maxsplit, _cs, @@ -123,9 +124,10 @@ cpdef Table rsplit( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_strings_column = strings_column.view() with nogil: c_result = cpp_split.rsplit( - strings_column.view(), + c_strings_column, dereference(c_delimiter), maxsplit, _cs, @@ -171,9 +173,10 @@ cpdef Column split_record( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_strings = strings.view() with nogil: c_result = cpp_split.split_record( - strings.view(), + c_strings, dereference(c_delimiter), maxsplit, _cs, @@ -221,9 +224,10 @@ cpdef Column rsplit_record( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_strings = strings.view() with nogil: c_result = cpp_split.rsplit_record( - strings.view(), + c_strings, dereference(c_delimiter), maxsplit, _cs, @@ -268,9 +272,10 @@ cpdef Table split_re( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: c_result = cpp_split.split_re( - input.view(), + c_input, prog.c_obj.get()[0], maxsplit, _cs, @@ -315,9 +320,10 @@ cpdef Table rsplit_re( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: c_result = cpp_split.rsplit_re( - input.view(), + c_input, prog.c_obj.get()[0], maxsplit, _cs, @@ -361,9 +367,10 @@ cpdef Column split_record_re( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: c_result = cpp_split.split_record_re( - input.view(), + c_input, prog.c_obj.get()[0], maxsplit, _cs, @@ -404,9 +411,10 @@ cpdef Column rsplit_record_re( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: c_result = cpp_split.rsplit_record_re( - input.view(), + c_input, prog.c_obj.get()[0], maxsplit, _cs, @@ -428,9 +436,10 @@ cpdef Column split_part( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: c_result = cpp_split.split_part( - input.view(), + c_input, dereference(c_delimiter), index, _cs, diff --git a/python/pylibcudf/pylibcudf/strings/strip.pyx b/python/pylibcudf/pylibcudf/strings/strip.pyx index 607428b6f69f..0e4dd400d53d 100644 --- a/python/pylibcudf/pylibcudf/strings/strip.pyx +++ b/python/pylibcudf/pylibcudf/strings/strip.pyx @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from cython.operator cimport dereference @@ -6,6 +6,7 @@ from libcpp.memory cimport unique_ptr from libcpp.utility cimport move from pylibcudf.column cimport Column from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view from pylibcudf.libcudf.scalar.scalar cimport string_scalar from pylibcudf.libcudf.scalar.scalar_factories cimport ( make_string_scalar as cpp_make_string_scalar, @@ -51,6 +52,7 @@ cpdef Column strip( cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input if to_strip is None: to_strip = Scalar.from_libcudf( @@ -61,9 +63,10 @@ cpdef Column strip( cdef string_scalar* cpp_to_strip cpp_to_strip = (to_strip.c_obj.get()) + c_input = input.view() with nogil: c_result = cpp_strip.strip( - input.view(), + c_input, side, dereference(cpp_to_strip), _cs, diff --git a/python/pylibcudf/pylibcudf/strings/translate.pyx b/python/pylibcudf/pylibcudf/strings/translate.pyx index 2a60ff881d42..4c48fdd72ca3 100644 --- a/python/pylibcudf/pylibcudf/strings/translate.pyx +++ b/python/pylibcudf/pylibcudf/strings/translate.pyx @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libcpp.memory cimport unique_ptr from libcpp.pair cimport pair @@ -6,6 +6,7 @@ from libcpp.utility cimport move from libcpp.vector cimport vector from pylibcudf.column cimport Column from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view from pylibcudf.libcudf.scalar.scalar cimport string_scalar from pylibcudf.libcudf.strings cimport translate as cpp_translate from pylibcudf.libcudf.types cimport char_utf8 @@ -73,10 +74,10 @@ cpdef Column translate( cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) - + cdef column_view c_input = input.view() with nogil: c_result = cpp_translate.translate( - input.view(), + c_input, c_chars_table, _cs, mr.get_mr() @@ -129,10 +130,10 @@ cpdef Column filter_characters( cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) - + cdef column_view c_input = input.view() with nogil: c_result = cpp_translate.filter_characters( - input.view(), + c_input, c_characters_to_filter, keep_characters, dereference(c_replacement), diff --git a/python/pylibcudf/pylibcudf/strings/wrap.pyx b/python/pylibcudf/pylibcudf/strings/wrap.pyx index 28bc310b5a44..f6180c7f56dd 100644 --- a/python/pylibcudf/pylibcudf/strings/wrap.pyx +++ b/python/pylibcudf/pylibcudf/strings/wrap.pyx @@ -1,10 +1,11 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libcpp.memory cimport unique_ptr from libcpp.utility cimport move from pylibcudf.column cimport Column from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view from pylibcudf.libcudf.strings cimport wrap as cpp_wrap from pylibcudf.libcudf.types cimport size_type from pylibcudf.utils cimport _get_stream, _get_memory_resource @@ -45,10 +46,10 @@ cpdef Column wrap( cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) - + cdef column_view c_input = input.view() with nogil: c_result = cpp_wrap.wrap( - input.view(), + c_input, width, _cs, mr.get_mr() diff --git a/python/pylibcudf/pylibcudf/table.pxd b/python/pylibcudf/pylibcudf/table.pxd index 76c38dacf3f6..5e9b9834f623 100644 --- a/python/pylibcudf/pylibcudf/table.pxd +++ b/python/pylibcudf/pylibcudf/table.pxd @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libcpp.memory cimport unique_ptr @@ -10,7 +10,7 @@ cdef class Table: # List[pylibcudf.Column] cdef public list _columns - cdef table_view view(self) nogil + cdef table_view view(self) cpdef int num_columns(self) cpdef int num_rows(self) diff --git a/python/pylibcudf/pylibcudf/table.pyx b/python/pylibcudf/pylibcudf/table.pyx index 6b62a5428f9e..d6b3de8dfeeb 100644 --- a/python/pylibcudf/pylibcudf/table.pyx +++ b/python/pylibcudf/pylibcudf/table.pyx @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from cython.operator cimport dereference @@ -28,6 +28,7 @@ from pylibcudf.libcudf.interop cimport ( to_arrow_schema_raw, ) from pylibcudf.libcudf.table.table cimport table +from pylibcudf.libcudf.table.table_view cimport table_view from .column cimport Column from .types cimport DataType @@ -215,7 +216,7 @@ cdef class Table: else: raise ValueError("Invalid Arrow-like object") - cdef table_view view(self) nogil: + cdef table_view view(self): """Generate a libcudf table_view to pass to libcudf algorithms. This method is for pylibcudf's functions to use to generate inputs when @@ -226,9 +227,8 @@ cdef class Table: # self._columns whenever new columns are added or columns are removed. cdef vector[column_view] c_columns - with gil: - for col in self._columns: - c_columns.push_back(( col).view()) + for col in self._columns: + c_columns.push_back(( col).view()) return table_view(c_columns) @@ -354,8 +354,9 @@ cdef class Table: c_metadata.push_back(_metadata_to_libcudf(meta)) cdef ArrowSchema* raw_schema_ptr + cdef table_view c_self = self.view() with nogil: - raw_schema_ptr = to_arrow_schema_raw(self.view(), c_metadata) + raw_schema_ptr = to_arrow_schema_raw(c_self, c_metadata) return PyCapsule_New(raw_schema_ptr, "arrow_schema", _release_schema) @@ -363,16 +364,18 @@ cdef class Table: cdef ArrowArray* raw_host_array_ptr cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() + cdef table_view c_self = self.view() with nogil: - raw_host_array_ptr = to_arrow_host_raw(self.view(), _cs) + raw_host_array_ptr = to_arrow_host_raw(c_self, _cs) return PyCapsule_New(raw_host_array_ptr, "arrow_array", _release_array) def _to_device_array(self): cdef ArrowDeviceArray* raw_device_array_ptr + cdef table_view c_self = self.view() with nogil: - raw_device_array_ptr = to_arrow_device_raw(self.view(), self) + raw_device_array_ptr = to_arrow_device_raw(c_self, self) return PyCapsule_New( raw_device_array_ptr, diff --git a/python/pylibcudf/pylibcudf/table_equality.pyx b/python/pylibcudf/pylibcudf/table_equality.pyx index b6f78ddce600..e57ae491b76c 100644 --- a/python/pylibcudf/pylibcudf/table_equality.pyx +++ b/python/pylibcudf/pylibcudf/table_equality.pyx @@ -1,8 +1,9 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libcpp cimport bool from pylibcudf.libcudf.table cimport equality as cpp_table_equality +from pylibcudf.libcudf.table.table_view cimport table_view from pylibcudf.libcudf.types cimport null_equality from rmm.pylibrmm.stream cimport Stream @@ -48,9 +49,11 @@ cpdef bool tables_equal( cdef bool c_result cdef Stream _stream = _get_stream(stream) cdef cudaStream_t _cs = _stream.view().value() + cdef table_view c_left = left.view() + cdef table_view c_right = right.view() with nogil: c_result = cpp_table_equality.tables_equal( - left.view(), right.view(), nulls_equal, _cs + c_left, c_right, nulls_equal, _cs ) return c_result diff --git a/python/pylibcudf/pylibcudf/transform.pyx b/python/pylibcudf/pylibcudf/transform.pyx index 0025ed7d5668..bc92959c8d7c 100644 --- a/python/pylibcudf/pylibcudf/transform.pyx +++ b/python/pylibcudf/pylibcudf/transform.pyx @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from cython.operator cimport dereference @@ -68,9 +68,10 @@ cpdef tuple[gpumemoryview, int] nans_to_nulls( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: c_result = cpp_transform.nans_to_nulls( - input.view(), _cs, mr.get_mr() + c_input, _cs, mr.get_mr() ) return ( @@ -110,9 +111,10 @@ cpdef Column column_nans_to_nulls( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: c_result = cpp_transform.column_nans_to_nulls( - input.view(), _cs, mr.get_mr() + c_input, _cs, mr.get_mr() ) return Column.from_libcudf(move(c_result), _stream, mr) @@ -146,9 +148,10 @@ cpdef Column compute_column( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_input = input.view() with nogil: c_result = cpp_transform.compute_column( - input.view(), dereference(expr.c_obj.get()), _cs, mr.get_mr() + c_input, dereference(expr.c_obj.get()), _cs, mr.get_mr() ) return Column.from_libcudf(move(c_result), _stream, mr) @@ -184,9 +187,10 @@ cpdef Column compute_column_jit( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_input = input.view() with nogil: c_result = cpp_transform.compute_column_jit( - input.view(), dereference(expr.c_obj.get()), _cs, mr.get_mr() + c_input, dereference(expr.c_obj.get()), _cs, mr.get_mr() ) return Column.from_libcudf(move(c_result), _stream, mr) @@ -219,9 +223,10 @@ cpdef tuple[gpumemoryview, int] bools_to_mask( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: c_result = cpp_transform.bools_to_mask( - input.view(), _cs, mr.get_mr() + c_input, _cs, mr.get_mr() ) return ( @@ -375,8 +380,9 @@ cpdef tuple[Table, Column] encode( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_input = input.view() with nogil: - c_result = cpp_transform.encode(input.view(), _cs, mr.get_mr()) + c_result = cpp_transform.encode(c_input, _cs, mr.get_mr()) return ( Table.from_libcudf(move(c_result.first), _stream, mr), @@ -416,10 +422,12 @@ cpdef Table one_hot_encode( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() + cdef column_view c_categories = categories.view() with nogil: c_result = cpp_transform.one_hot_encode( - input.view(), - categories.view(), + c_input, + c_categories, _cs, mr.get_mr() ) diff --git a/python/pylibcudf/pylibcudf/transpose.pyx b/python/pylibcudf/pylibcudf/transpose.pyx index e15aa45ce775..38ad6e68e17f 100644 --- a/python/pylibcudf/pylibcudf/transpose.pyx +++ b/python/pylibcudf/pylibcudf/transpose.pyx @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libcpp.memory cimport unique_ptr from libcpp.pair cimport pair @@ -44,9 +44,10 @@ cpdef Table transpose( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef table_view c_input_table = input_table.view() with nogil: c_result = cpp_transpose.transpose( - input_table.view(), _cs, mr.get_mr() + c_input_table, _cs, mr.get_mr() ) owner_table = Table( diff --git a/python/pylibcudf/pylibcudf/unary.pyx b/python/pylibcudf/pylibcudf/unary.pyx index e06140370124..2c0a01334b01 100644 --- a/python/pylibcudf/pylibcudf/unary.pyx +++ b/python/pylibcudf/pylibcudf/unary.pyx @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libcpp cimport bool @@ -6,6 +6,7 @@ from libcpp.memory cimport unique_ptr from libcpp.utility cimport move from pylibcudf.libcudf cimport unary as cpp_unary from pylibcudf.libcudf.column.column cimport column +from pylibcudf.libcudf.column.column_view cimport column_view from pylibcudf.libcudf.unary cimport unary_operator from rmm.pylibrmm.stream cimport Stream from rmm.pylibrmm.memory_resource cimport DeviceMemoryResource @@ -58,9 +59,10 @@ cpdef Column unary_operation( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: result = cpp_unary.unary_operation( - input.view(), op, _cs, mr.get_mr() + c_input, op, _cs, mr.get_mr() ) return Column.from_libcudf(move(result), _stream, mr) @@ -91,8 +93,9 @@ cpdef Column is_null(Column input, object stream=None, DeviceMemoryResource mr=N cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: - result = cpp_unary.is_null(input.view(), _cs, mr.get_mr()) + result = cpp_unary.is_null(c_input, _cs, mr.get_mr()) return Column.from_libcudf(move(result), _stream, mr) @@ -122,8 +125,9 @@ cpdef Column is_valid(Column input, object stream=None, DeviceMemoryResource mr= cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: - result = cpp_unary.is_valid(input.view(), _cs, mr.get_mr()) + result = cpp_unary.is_valid(c_input, _cs, mr.get_mr()) return Column.from_libcudf(move(result), _stream, mr) @@ -157,9 +161,10 @@ cpdef Column cast( cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: result = cpp_unary.cast( - input.view(), data_type.c_obj, _cs, mr.get_mr() + c_input, data_type.c_obj, _cs, mr.get_mr() ) return Column.from_libcudf(move(result), _stream, mr) @@ -190,8 +195,9 @@ cpdef Column is_nan(Column input, object stream=None, DeviceMemoryResource mr=No cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: - result = cpp_unary.is_nan(input.view(), _cs, mr.get_mr()) + result = cpp_unary.is_nan(c_input, _cs, mr.get_mr()) return Column.from_libcudf(move(result), _stream, mr) @@ -221,8 +227,9 @@ cpdef Column is_not_nan(Column input, object stream=None, DeviceMemoryResource m cdef cudaStream_t _cs = _stream.view().value() mr = _get_memory_resource(mr) + cdef column_view c_input = input.view() with nogil: - result = cpp_unary.is_not_nan(input.view(), _cs, mr.get_mr()) + result = cpp_unary.is_not_nan(c_input, _cs, mr.get_mr()) return Column.from_libcudf(move(result), _stream, mr) From 308f7304e34d99cb06c2e28ae38ef51f7a3f7152 Mon Sep 17 00:00:00 2001 From: Muhammad Haseeb <14217455+mhaseeb123@users.noreply.github.com> Date: Fri, 26 Jun 2026 14:16:11 -0700 Subject: [PATCH 17/22] Fuse multi-column range window offset generation (#22863) Closes #22830 This PR fuses the multi-column range window offset generation Authors: - Muhammad Haseeb (https://github.com/mhaseeb123) - Nghia Truong (https://github.com/ttnghia) Approvers: - Nghia Truong (https://github.com/ttnghia) - Bradley Dice (https://github.com/bdice) - Yunsong Wang (https://github.com/PointKernel) URL: https://github.com/rapidsai/cudf/pull/22863 --- cpp/src/rolling/range_rolling.cu | 143 +++++++++++++++++++++++++------ 1 file changed, 119 insertions(+), 24 deletions(-) diff --git a/cpp/src/rolling/range_rolling.cu b/cpp/src/rolling/range_rolling.cu index 9de9ce4d9fcf..ac8cfe412679 100644 --- a/cpp/src/rolling/range_rolling.cu +++ b/cpp/src/rolling/range_rolling.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -27,10 +27,13 @@ #include #include +#include +#include #include #include #include +#include #include namespace CUDF_EXPORT cudf { @@ -154,6 +157,106 @@ std::pair, std::unique_ptr> make_range_windows( } } +namespace { + +/** + * @brief Computes preceding and following offsets from one group lookup + */ +template +struct unbounded_distance_fn { + Grouping const groups; + + [[nodiscard]] __device__ cuda::std::tuple operator()( + size_type i) const noexcept + { + auto const row_info = groups.row_info(i); + return cuda::std::tuple{i - row_info.group_start() + 1, + row_info.group_end() - i - 1}; + } +}; + +/** + * @brief Computes preceding and following offsets from different groupings. + */ +template +struct mixed_unbounded_distance_fn { + rolling::unbounded_distance_functor const preceding_fn; + rolling::unbounded_distance_functor const following_fn; + + [[nodiscard]] __device__ cuda::std::tuple operator()( + size_type i) const noexcept + { + return cuda::std::tuple{preceding_fn(i), following_fn(i)}; + } +}; + +/** + * @brief Selects grouping and writes preceding and following offsets. + */ +template +void select_and_write_offsets(range_window_type const& preceding, + range_window_type const& following, + rolling::grouped peers, + std::optional& group_helper, + mutable_column_view preceding_view, + mutable_column_view following_view, + size_type num_rows, + rmm::cuda_stream_view stream) +{ + // Write the offsets to the output columns + auto const write_offsets = [&](auto offset_fn) { + auto const src_iter = cudf::detail::make_counting_transform_iterator( + size_type{0}, cuda::proclaim_return_type>(offset_fn)); + thrust::copy_n( + rmm::exec_policy_nosync(stream, cudf::get_current_device_resource_ref()), + src_iter, + num_rows, + cuda::zip_iterator(preceding_view.begin(), following_view.begin())); + }; + + // Write offsets for the current row case + if (std::holds_alternative(preceding) && + std::holds_alternative(following)) { + write_offsets(unbounded_distance_fn{peers}); + // Write offsets for the unbounded case + } else if (std::holds_alternative(preceding) && + std::holds_alternative(following)) { + if (group_helper.has_value()) { + write_offsets(unbounded_distance_fn{ + {group_helper->group_labels(stream).data(), group_helper->group_offsets(stream).data()}}); + } else { + write_offsets(unbounded_distance_fn{{num_rows}}); + } + } else { + // Select groupings for preceding and following windows + auto const select_grouping = + [&](range_window_type const& window) -> std::variant { + if (std::holds_alternative(window)) { + return peers; + } else if (group_helper.has_value()) { + return rolling::grouped{group_helper->group_labels(stream).data(), + group_helper->group_offsets(stream).data()}; + } else { + return rolling::ungrouped{num_rows}; + } + }; + + auto preceding_grouping = select_grouping(preceding); + auto following_grouping = select_grouping(following); + std::visit( + [&](auto preceding_grouping, auto following_grouping) { + write_offsets( + mixed_unbounded_distance_fn{ + {preceding_grouping, rolling::direction::PRECEDING}, + {following_grouping, rolling::direction::FOLLOWING}}); + }, + preceding_grouping, + following_grouping); + } +} + +} // namespace + std::pair, std::unique_ptr> make_range_windows( table_view const& group_keys, table_view const& orderby, @@ -204,30 +307,22 @@ std::pair, std::unique_ptr> make_range_windows( group_helper.emplace(group_keys, null_policy::INCLUDE, sorted::YES, std::vector{}); } - auto const num_rows = orderby.num_rows(); - auto make_offsets = [&](range_window_type const& window, rolling::direction direction) { - auto result = make_numeric_column( - data_type{type_to_id()}, num_rows, mask_state::UNALLOCATED, stream, mr); - auto write_offsets = [&](auto grouping) { - thrust::copy_n(rmm::exec_policy_nosync(stream, cudf::get_current_device_resource_ref()), - cudf::detail::make_counting_transform_iterator( - size_type{0}, rolling::unbounded_distance_functor{grouping, direction}), - num_rows, - result->mutable_view().begin()); - }; - if (std::holds_alternative(window)) { - write_offsets(peers); - } else if (group_helper.has_value()) { - write_offsets(rolling::grouped{group_helper->group_labels(stream).data(), - group_helper->group_offsets(stream).data()}); - } else { - write_offsets(rolling::ungrouped{num_rows}); - } - return result; - }; + auto const num_rows = orderby.num_rows(); + auto preceding_result = make_numeric_column( + data_type{type_to_id()}, num_rows, mask_state::UNALLOCATED, stream, mr); + auto following_result = make_numeric_column( + data_type{type_to_id()}, num_rows, mask_state::UNALLOCATED, stream, mr); + + select_and_write_offsets(preceding, + following, + peers, + group_helper, + preceding_result->mutable_view(), + following_result->mutable_view(), + num_rows, + stream); - return {make_offsets(preceding, rolling::direction::PRECEDING), - make_offsets(following, rolling::direction::FOLLOWING)}; + return std::pair{std::move(preceding_result), std::move(following_result)}; } } // namespace detail From f6b9113ade3588111a6794672aca17fcb2f54b03 Mon Sep 17 00:00:00 2001 From: Muhammad Haseeb <14217455+mhaseeb123@users.noreply.github.com> Date: Fri, 26 Jun 2026 15:51:58 -0700 Subject: [PATCH 18/22] Skill to compare performance of a branch or PR with main (#22725) This PR adds a new AI-agent skill to automatically compare the performance of branch or a PR against the `rapidsai/cudf/main` branch and produce a report Authors: - Muhammad Haseeb (https://github.com/mhaseeb123) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) - Bradley Dice (https://github.com/bdice) - Yunsong Wang (https://github.com/PointKernel) URL: https://github.com/rapidsai/cudf/pull/22725 --- .agents/skills/perf-compare-cudf/SKILL.md | 141 ++++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 .agents/skills/perf-compare-cudf/SKILL.md diff --git a/.agents/skills/perf-compare-cudf/SKILL.md b/.agents/skills/perf-compare-cudf/SKILL.md new file mode 100644 index 000000000000..30bdd2423486 --- /dev/null +++ b/.agents/skills/perf-compare-cudf/SKILL.md @@ -0,0 +1,141 @@ +--- +name: perf-compare-cudf +description: Benchmark a cuDF branch, WIP changes, or a PR against the `main` branch +--- + +Use this skill when the user asks to compare libcudf benchmark performance for: +- **the current branch or WIP changes** against `rapidsai/cudf` `main`. +- **a cudf PR link or number** against `rapidsai/cudf` `main`. + +# Goal + +Run the same selected libcudf NVBench benchmarks on the target (current WIP or cudf PR) and then on `rapidsai/cudf` `main`, then report meaningful differences. + +`` is the git remote for `https://github.com/rapidsai/cudf` (often `upstream`). Detect it with `git remote -v`. + +## Prerequisites + +- For PR targets, **`gh` CLI** authenticated — run `gh auth status`. If not authenticated, guide the user to run: + ```bash + gh auth login + ``` + The token needs `repo` scope. Do **not** run `gh auth token` from within the agent. +- Ensure we are in the cudf devcontainer (username `coder`). If not, stop and ask the user for instructions. + +## 1. Prepare + +- Record the starting branch, `git status --short`, and the exact target (current WIP or cudf PR). +- Run order: Target side first, then `main`. +- Record current timestamp as `ts = ` +- Create result directories: + ```bash + mkdir -p benchmark_compare//{target,main} + ``` + +## 2. Build Target + +- For current-branch or WIP targets: keep target changes applied for the target run. +- For PR targets: Stash any unrelated local changes, record the stash name, and check out the PR: + ```bash + gh pr checkout --repo rapidsai/cudf + ``` +- For PR targets: After switching, check if the PR branch is behind `/main` and add a merge commit. DO **NOT** push anything. If there are merge conflicts, stop and guide the user to fix them. + +- On the first build for a checkout, force CMake reconfiguration to enable benchmarks: + +```bash +configure-cudf-cpp -DBUILD_BENCHMARKS=ON +build-cudf-cpp +``` +- Re-run `configure-cudf-cpp -DBUILD_BENCHMARKS=ON` if the build directory is cleaned or CMake options may have changed. +- If needed, refer to the `build-test-cudf` skill for instructions and troubleshooting. + +## 3. Choose Benchmarks + +- Fetch current main with `git fetch main`. +- Infer candidate benchmark suites from: + ```bash + git diff --name-only /main...HEAD + ``` +- Benchmark binaries live under `cpp/build/latest/benchmarks/*_NVBENCH`. +- Inspect candidate binaries from the target build: + ```bash + cpp/build/latest/benchmarks/ --list + cpp/build/latest/benchmarks/ --help-axes + ``` +- Confirm benchmark binaries and axis coverage with the user. Use a small, representative axis subset by default; use full coverage only when requested or necessary. +- Record exact `-b` and `-a` options. Reuse them unchanged on both branches. + +## 4. Run Target + +- Pick an idle GPU with `nvidia-smi`. Do this every time before running anything (target or main run); if the same GPU is no longer idle, pick another one, wait, or ask before continuing. +- Run on one masked device only: `CUDA_VISIBLE_DEVICES=` and `-d 0`. +- Write target JSON and log files under `benchmark_compare//target/`, for example: + ```bash + CUDA_VISIBLE_DEVICES= cpp/build/latest/benchmarks/ -d 0 \ + -b -a ... \ + --json benchmark_compare//target/.json 2>&1 | tee benchmark_compare//target/.log + ``` +- If nvbench emits an end-of-suite segfault after writing results, note it and continue. If a config throws, verify that both branches (main and target) behave the same. + +## 5. Switch over to main + +- To switch to main, stash any target WIP if needed, record the stash name, and use a clean branch: + ```bash + git fetch main + git checkout -B _bench_main /main + ``` +- Do not apply any WIP or target changes on `_bench_main`. + +## 6. Build and run main + +Follow configure, build and benchmark run steps as for the target. Run the same set of benchmarks chosen above, but write JSON and log files to `benchmark_compare//main/` instead. + +## 7. Compare + +Use NVBench's comparison script from the build tree: + +```bash +NVBENCH_SCRIPTS=cpp/build/latest/_deps/nvbench-src/python/scripts +test -f "$NVBENCH_SCRIPTS/nvbench_compare.py" || \ + NVBENCH_SCRIPTS=cpp/build/latest/_deps/nvbench-src/scripts +PYTHONPATH="$NVBENCH_SCRIPTS" python "$NVBENCH_SCRIPTS/nvbench_compare.py" \ + --threshold-diff 0.05 --no-color benchmark_compare//main benchmark_compare//target \ + | tee benchmark_compare//COMPARISON.md +``` + +- The first path is the reference (`main`), the second is the comparison (`target`). Re-run surprising failures once, especially small or noisy configs. + +## 8. Restore and report + +- Return to the starting branch/state, pop any stash you created, delete temporary branches, and confirm `git status` matches the starting state. +- Remember to note if there were any end-of-suite segfaults or config throws and if the behavior was the same on both branches. +- Use the below template for `COMPARISON.md`, adapting the metric columns to the benchmark. GPU time is always useful, but other metrics such as output file size, throughput, compression ratio, or memory usage are also of interest when they change significantly in target vs main. +- Summarize chat with the headline result (regression, improvement, or within noise), relevant metrics, hardware used, branch SHAs, axis coverage, the summary table from the template, and generated files. + + ```markdown + # Benchmark Comparison: /main vs target (`WIP` or `PR`) + + - Primary metric(s): + - Δ = (target - main) / main. Interpret direction per metric. + - Significant timing deltas: |Δ| >= 5% AND larger than max(noise) of either side. + - Hardware: , driver/CUDA if available. + , model, architecture, if available. + - Branches: target `` vs main ``. Axis coverage: (list values used). + + ## Summary + | Benchmark Suite | Primary metric | # Configs | # Meaningful Changes | + | ... | + + ## Top N Changes + | Suite / bench | axes | metric | main | target | Δ | noise, if timing | + | ... | + + ## Per-suite tables + (one table per benchmark, axes as columns, include all relevant metrics) + + ## Notes + - Exceptions excluded (same on both branches): ... + - End-of-suite segfaults ignored. + - Files generated: list of JSON/log paths + this report + ``` From b5d2addd675d4828f6e82f194d254bc16994428d Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Sat, 27 Jun 2026 00:38:41 -0500 Subject: [PATCH 19/22] Fix `cudf.pandas --line-profile` clobbering `__file__` (#23017) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit closes #23010 `python -m cudf.pandas --line-profile