Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 14 additions & 2 deletions python/cudf/cudf/core/groupby/groupby.py
Original file line number Diff line number Diff line change
Expand Up @@ -2308,8 +2308,14 @@ def _post_process_chunk_results(
)
else:
index_data = group_keys._data.copy(deep=True)
inner_name = grouped_values.index.name
index_data[None] = grouped_values.index._column
result.index = MultiIndex._from_data(index_data)
mi = MultiIndex._from_data(index_data)
# ColumnAccessor keys must be unique, so the inner
# level's name (which may duplicate a key name) is
# restored after construction.
mi.names = [*mi.names[:-1], inner_name]
result.index = mi
elif len(chunk_results) == len(group_names):
result = concat(chunk_results, axis=1).T
result.index = group_names
Expand All @@ -2329,8 +2335,14 @@ def _post_process_chunk_results(
# row positions of the grouped values. This matches pandas,
# e.g. a UDF returning ``DataFrame({"values": range(len(grp))})``
# contributes a fresh 0..len(grp)-1 range per group.
inner_name = result.index.name
index_data[None] = result.index._column
result.index = MultiIndex._from_data(index_data)
mi = MultiIndex._from_data(index_data)
# ColumnAccessor keys must be unique, so the inner level's
# name (which may duplicate a key name) is restored after
# construction.
mi.names = [*mi.names[:-1], inner_name]
result.index = mi
return result

@_performance_tracking
Expand Down
60 changes: 57 additions & 3 deletions python/cudf/cudf/core/window/rolling.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION
# SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
from __future__ import annotations

Expand All @@ -7,6 +7,7 @@
import warnings
from typing import TYPE_CHECKING, Any, TypeVar

import cupy
import numpy as np
import pandas as pd
from pandas.api.indexers import BaseIndexer
Expand Down Expand Up @@ -672,8 +673,38 @@ class RollingGroupby(Rolling):
"""

def __init__(self, groupby, window, min_periods=None, center=False):
if isinstance(window, BaseIndexer):
raise NotImplementedError(
"BaseIndexer subclasses are not yet supported with "
"groupby.rolling: the window bounds would not be computed "
"per group"
)
self._as_index = groupby._as_index
sort_inds = groupby.grouping.keys._get_sorted_inds()
if not groupby._sort:
# With sort=False pandas keeps groups in order of first
# appearance; reorder the key-sorted blocks accordingly while
# keeping the original row order within each block.
offsets, _, (positions,) = groupby._groups(
[groupby._range_column_from_obj]
)
pos = cupy.asarray(positions.values)
off = cupy.asarray(offsets)
# broadcast each group's first-appearance position to its rows
# (searchsorted maps each row to its group block; older cupy
# does not support an ndarray ``repeats`` in ``cupy.repeat``)
row_group = (
cupy.searchsorted(off, cupy.arange(len(pos)), side="right") - 1
)
row_first_pos = pos[off[:-1]][row_group]
# lexsort: primary key is each row's group-first-appearance
# position, ties broken by the current (key-sorted) order
order = cupy.lexsort(
cupy.stack([cupy.arange(len(pos)), row_first_pos])
)
sort_inds = as_column(pos[order])
sort_order = GatherMap.from_column_unchecked(
groupby.grouping.keys._get_sorted_inds(),
sort_inds,
len(groupby.obj),
nullify=False,
)
Expand Down Expand Up @@ -704,9 +735,33 @@ def __getitem__(self, arg) -> Self:
center=self.center,
)
new._group_keys = self._group_keys
new._as_index = self._as_index
return new

def _apply_agg(self, agg_name: str, **agg_kwargs) -> DataFrame | Series:
from cudf.core.dataframe import DataFrame

result = super()._apply_agg(agg_name, **agg_kwargs)

if self._as_index is False and isinstance(result, DataFrame):
# pandas returns the group keys as leading columns with the
# original (group-ordered) index when as_index=False.
data = dict(
zip(
self._group_keys._column_names, # type: ignore[union-attr]
self._group_keys._columns, # type: ignore[union-attr]
strict=True,
)
)
for name, col in result._data.items():
if name in data:
raise NotImplementedError(
"as_index=False with a group key sharing a column "
"name with the result is not supported"
)
data[name] = col
return DataFrame._from_data(data, index=self.obj.index)

index = MultiIndex._from_data(
dict(
enumerate(
Expand All @@ -723,6 +778,5 @@ def _apply_agg(self, agg_name: str, **agg_kwargs) -> DataFrame | Series:
self.obj.index._column_names,
)
)
result = super()._apply_agg(agg_name, **agg_kwargs)
result.index = index
return result
17 changes: 2 additions & 15 deletions python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py
Original file line number Diff line number Diff line change
Expand Up @@ -2080,7 +2080,6 @@ def pytest_unconfigure(config):
"tests/groupby/test_api.py::test_all_methods_categorized": "TODO: Add a reason for failure",
"tests/groupby/test_api.py::test_tab_completion": "TODO: Add a reason for failure",
"tests/groupby/test_apply.py::test_apply_as_index_constant_lambda[False-expected0]": "AssertionError: DataFrame.columns are different",
"tests/groupby/test_apply.py::test_apply_concat_preserve_names": "TODO: Add a reason for failure",
"tests/groupby/test_apply.py::test_apply_datetime_issue[group_column_dtlike0]": "TODO: Add a reason for failure",
"tests/groupby/test_apply.py::test_apply_datetime_issue[group_column_dtlike1]": "TODO: Add a reason for failure",
"tests/groupby/test_apply.py::test_apply_frame_concat_series": "TODO: Add a reason for failure",
Expand Down Expand Up @@ -4600,20 +4599,8 @@ def pytest_unconfigure(config):
"tests/window/test_dtypes.py::test_series_dtypes[category-None-std-data16-expected_data16-True-None]": "TODO: Add a reason for failure",
"tests/window/test_dtypes.py::test_series_dtypes[category-None-sum-data10-expected_data10-True-None]": "TODO: Add a reason for failure",
"tests/window/test_dtypes.py::test_series_dtypes[category-None-var-data19-expected_data19-True-None]": "TODO: Add a reason for failure",
"tests/window/test_groupby.py::TestRolling::test_as_index_false[ms-by0-expected_data0]": "AssertionError: left and right shape mismatch",
"tests/window/test_groupby.py::TestRolling::test_as_index_false[ms-by1-expected_data1]": "AssertionError: left and right shape mismatch",
"tests/window/test_groupby.py::TestRolling::test_as_index_false[ns-by0-expected_data0]": "AssertionError: left and right shape mismatch",
"tests/window/test_groupby.py::TestRolling::test_as_index_false[ns-by1-expected_data1]": "AssertionError: left and right shape mismatch",
"tests/window/test_groupby.py::TestRolling::test_as_index_false[s-by0-expected_data0]": "AssertionError: left and right shape mismatch",
"tests/window/test_groupby.py::TestRolling::test_as_index_false[s-by1-expected_data1]": "AssertionError: left and right shape mismatch",
"tests/window/test_groupby.py::TestRolling::test_as_index_false[us-by0-expected_data0]": "AssertionError: left and right shape mismatch",
"tests/window/test_groupby.py::TestRolling::test_as_index_false[us-by1-expected_data1]": "AssertionError: left and right shape mismatch",
"tests/window/test_groupby.py::TestRolling::test_datelike_on_monotonic_within_each_group": "AssertionError: MultiIndex level [1] are different",
"tests/window/test_groupby.py::TestRolling::test_groupby_monotonic": "AssertionError: MultiIndex level [1] are different",
"tests/window/test_groupby.py::TestRolling::test_groupby_rolling_custom_indexer": 'AssertionError: Column name="a" are different',
"tests/window/test_groupby.py::TestRolling::test_groupby_rolling_no_sort": "AssertionError: MultiIndex level [0] are different",
"tests/window/test_groupby.py::test_rolling_corr_with_single_integer_in_index": "NotImplementedError: Unsupported column type passed to create an Index: <class 'cudf.core.column.lists.ListColumn'>",
"tests/window/test_groupby.py::test_rolling_corr_with_tuples_in_index": "NotImplementedError: Unsupported column type passed to create an Index: <class 'cudf.core.column.lists.ListColumn'>",
"tests/window/test_groupby.py::test_rolling_corr_with_single_integer_in_index": "cudf stores tuple values as list rows, which do not round-trip (the pandas-fallback frame holds unhashable lists)",
"tests/window/test_groupby.py::test_rolling_corr_with_tuples_in_index": "cudf stores tuple values as list rows, which do not round-trip (the pandas-fallback frame holds unhashable lists)",
"tests/window/test_pairwise.py::TestPairwise::test_no_flex[pairwise_frames1-<lambda>1]": "TODO: Add a reason for failure",
"tests/window/test_pairwise.py::TestPairwise::test_no_flex[pairwise_frames2-<lambda>1]": "TODO: Add a reason for failure",
"tests/window/test_pairwise.py::TestPairwise::test_no_flex[pairwise_frames5-<lambda>1]": "TODO: Add a reason for failure",
Expand Down
21 changes: 21 additions & 0 deletions python/cudf/cudf/tests/groupby/test_apply.py
Original file line number Diff line number Diff line change
Expand Up @@ -957,3 +957,24 @@ def test_group_by_empty_apply(request, dtype, apply_op):
check_dtype=True,
check_index_type=True,
)


def test_groupby_apply_preserves_inner_index_name():
# The concatenated apply result keeps the UDF result's index name as
# the inner MultiIndex level name, matching pandas.
pdf = pd.DataFrame(
{
"name": ["a", "a", "b"],
"amount": [100.0, 200.0, 300.0],
},
index=pd.Index([1, 2, 3], name="stamp"),
)
gdf = cudf.from_pandas(pdf)
expect = pdf.groupby("name").apply(
lambda x: x["amount"].cumsum(), include_groups=False
)
got = gdf.groupby("name").apply(
lambda x: x["amount"].cumsum(), include_groups=False
)
assert expect.index.names == got.index.names
assert_eq(expect, got)
46 changes: 45 additions & 1 deletion python/cudf/cudf/tests/window/test_rolling.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION.
# SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
import math
import pickle
Expand Down Expand Up @@ -545,3 +545,47 @@ def test_rolling_min_periods_zero():
result = s.rolling(2, min_periods=0).sum()
expected = ps.rolling(2, min_periods=0).sum()
assert_eq(result, expected)


def test_groupby_rolling_as_index_false():
# pandas returns the group keys as leading columns with the original
# (group-ordered) index when as_index=False.
pdf = pd.DataFrame(
{"id": ["A", "A", "B", "B"], "num": [100.0, 200.0, 150.0, 250.0]},
index=pd.Index([10, 11, 12, 13], name="idx"),
)
gdf = cudf.from_pandas(pdf)
assert_eq(
pdf.groupby("id", as_index=False).rolling(2, min_periods=1).mean(),
gdf.groupby("id", as_index=False).rolling(2, min_periods=1).mean(),
)


def test_groupby_rolling_no_sort_first_appearance_order():
# With sort=False pandas keeps groups in order of first appearance.
pdf = pd.DataFrame({"foo": [2, 1, 2], "bar": [2.0, 1.0, 3.0]})
gdf = cudf.from_pandas(pdf)
assert_eq(
pdf.groupby("foo", sort=False).rolling(1).min(),
gdf.groupby("foo", sort=False).rolling(1).min(),
)


def test_groupby_rolling_base_indexer_raises():
gdf = cudf.DataFrame({"a": [1.0, 2.0, 3.0]}, index=[0, 0, 1])

class SimpleIndexer(BaseIndexer):
def get_window_bounds(
self,
num_values=0,
min_periods=None,
center=None,
closed=None,
step=None,
):
end = np.arange(num_values, dtype=np.int64) + 1
start = np.maximum(end - self.window_size, 0)
return start, end

with pytest.raises(NotImplementedError):
gdf.groupby(gdf.index).rolling(SimpleIndexer(window_size=2)).sum()
Loading