diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index b025ddec782b..62c164873ff4 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -5634,7 +5634,7 @@ def describe( for col in data_to_describe._column_names ] if len(describe_series_list) == 1: - return describe_series_list[0].to_frame() + result = describe_series_list[0].to_frame() else: ldesc_indexes = sorted( (x.index for x in describe_series_list), key=len @@ -5648,7 +5648,7 @@ def describe( None, ) - return cudf.concat( + result = cudf.concat( [ series.reindex(names, copy=False) for series in describe_series_list @@ -5656,6 +5656,11 @@ def describe( axis=1, sort=False, ) + # ``describe`` produces one column per described column in the same + # order, so preserve the source column index (incl. its dtype/name, + # e.g. a ``CategoricalIndex``), which ``concat``/``to_frame`` drop. + result.columns = data_to_describe._data.to_pandas_index + return result @_performance_tracking def to_pandas( diff --git a/python/cudf/cudf/pandas/fast_slow_proxy.py b/python/cudf/cudf/pandas/fast_slow_proxy.py index fd35e4c87d80..148bf1041ddc 100644 --- a/python/cudf/cudf/pandas/fast_slow_proxy.py +++ b/python/cudf/cudf/pandas/fast_slow_proxy.py @@ -1421,11 +1421,26 @@ def _transform_arg( ) result[...] = transformed return result.reshape(arg.shape) + elif isinstance(arg, (types.GeneratorType, map, filter, zip, enumerate)): + # "Pure" (resource-free) iterators. On the fast path we force a + # fallback because they are consumable (see below). On the slow path we + # transform the elements lazily into a new generator -- a bare iterator + # would otherwise hand wrapped proxies straight to the slow library, + # which can lose type information (e.g. ``MultiIndex.from_product`` of a + # ``map`` over categoricals dropping the ``category`` dtype). We yield a + # generator (rather than materializing a ``list``) so the slow library + # still sees a bare iterator: pandas treats iterators differently from + # lists in places (e.g. ``is_nested_list_like`` returns ``False`` for an + # iterator, and ``Index.get_loc`` reports the iterator in its error + # message), and materializing would silently change that behavior. + if attribute_name == "_fsproxy_fast": + raise Exception() + return (_transform_arg(a, attribute_name, seen) for a in arg) elif isinstance(arg, Iterator) and attribute_name == "_fsproxy_fast": - # this may include consumable objects like generators or - # IOBase objects, which we don't want unavailable to the slow - # path in case of fallback. So, we raise here and ensure the - # slow path is taken: + # Other consumable objects such as generators or IOBase objects, which + # we don't want unavailable to the slow path in case of fallback. So, + # we raise here and ensure the slow path is taken (and the object is + # left intact for it): raise Exception() elif isinstance(arg, types.FunctionType): if id(arg) in seen: diff --git a/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py b/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py index 9d7fa510d695..047360ebe946 100644 --- a/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py +++ b/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py @@ -2319,19 +2319,7 @@ def pytest_unconfigure(config): "tests/groupby/test_apply.py::test_include_groups": "Failed: DID NOT RAISE ", "tests/groupby/test_apply.py::test_positional_slice_groups_datetimelike": "AssertionError: DataFrame are different", "tests/groupby/test_apply.py::test_time_field_bug": "TODO: Add a reason for failure", - "tests/groupby/test_categorical.py::test_category_order_reducer[False-False-True-all-False-range]": "AttributeError: Can only use .cat accessor with a 'category' dtype. Did you mean: 'at'?", - "tests/groupby/test_categorical.py::test_category_order_reducer[False-False-True-all-True-range]": "AttributeError: Can only use .cat accessor with a 'category' dtype. Did you mean: 'at'?", - "tests/groupby/test_categorical.py::test_category_order_reducer[False-False-True-any-False-range]": "AttributeError: Can only use .cat accessor with a 'category' dtype. Did you mean: 'at'?", - "tests/groupby/test_categorical.py::test_category_order_reducer[False-False-True-any-True-range]": "AttributeError: Can only use .cat accessor with a 'category' dtype. Did you mean: 'at'?", - "tests/groupby/test_categorical.py::test_category_order_reducer[False-True-True-all-False-range]": "AttributeError: Can only use .cat accessor with a 'category' dtype. Did you mean: 'at'?", - "tests/groupby/test_categorical.py::test_category_order_reducer[False-True-True-all-True-range]": "AttributeError: Can only use .cat accessor with a 'category' dtype. Did you mean: 'at'?", - "tests/groupby/test_categorical.py::test_category_order_reducer[False-True-True-any-False-range]": "AttributeError: Can only use .cat accessor with a 'category' dtype. Did you mean: 'at'?", - "tests/groupby/test_categorical.py::test_category_order_reducer[False-True-True-any-True-range]": "AttributeError: Can only use .cat accessor with a 'category' dtype. Did you mean: 'at'?", - "tests/groupby/test_categorical.py::test_describe_categorical_columns": "TODO: Add a reason for failure", - "tests/groupby/test_categorical.py::test_observed[False]": "TODO: Add a reason for failure", - "tests/groupby/test_categorical.py::test_observed_codes_remap[False]": "TODO: Add a reason for failure", - "tests/groupby/test_categorical.py::test_observed_two_columns[False]": "AssertionError: DataFrame.index level [0] are different", - "tests/groupby/test_categorical.py::test_unstack_categorical": "TODO: Add a reason for failure", + "tests/groupby/test_categorical.py::test_describe_categorical_columns": "cudf's multi-level groupby aggregation and stack() drop the categorical column-index dtype", "tests/groupby/test_counting.py::TestCounting::test_ngroup_distinct": "TODO: Add a reason for failure", "tests/groupby/test_cumulative.py::test_cython_api2[False]": "AssertionError: DataFrame are different", "tests/groupby/test_cumulative.py::test_groupby_cumprod_nan_influences_other_columns": "TODO: Add a reason for failure", diff --git a/python/cudf/cudf/tests/groupby/test_reductions.py b/python/cudf/cudf/tests/groupby/test_reductions.py index 03bd566cab6c..d0f606bbbd87 100644 --- a/python/cudf/cudf/tests/groupby/test_reductions.py +++ b/python/cudf/cudf/tests/groupby/test_reductions.py @@ -1242,6 +1242,22 @@ def test_groupby_all_any_empty(op): assert_eq(expect, got, check_index_type=False) +@pytest.mark.parametrize("op", ["all", "any"]) +@pytest.mark.parametrize("key_is_categorical", [False, True]) +def test_groupby_all_any_as_index_false(op, key_is_categorical): + # With as_index=False the grouping key is reset as a column; it must not + # be coerced to bool along with the (reduced) value columns. + key = [2, 1, 2, 3] + if key_is_categorical: + key = pd.Categorical(key, categories=[1, 4, 3, 2], ordered=True) + pdf = pd.DataFrame({"a": key, "b": range(4)}) + gdf = cudf.from_pandas(pdf) + with cudf.option_context("mode.pandas_compatible", True): + got = getattr(gdf.groupby("a", as_index=False), op)() + expect = getattr(pdf.groupby("a", as_index=False), op)() + assert_eq(expect, got) + + @pytest.mark.parametrize( "string_dtype", [ diff --git a/python/cudf/cudf_pandas_tests/test_fast_slow_proxy.py b/python/cudf/cudf_pandas_tests/test_fast_slow_proxy.py index 406feaab8242..8ae72d4810b6 100644 --- a/python/cudf/cudf_pandas_tests/test_fast_slow_proxy.py +++ b/python/cudf/cudf_pandas_tests/test_fast_slow_proxy.py @@ -233,6 +233,66 @@ def fast(s): assert pxy(StringIO("hello")) == "hello" +# "Pure" (resource-free) iterators that ``_transform_arg`` handles specially: +# they force a fallback on the fast path and are transformed lazily into a new +# generator on the slow path. +_PURE_ITERATOR_FACTORIES = [ + lambda items: (item for item in items), + lambda items: map(lambda item: item, items), + lambda items: filter(lambda item: True, items), + lambda items: zip(items, strict=True), + lambda items: enumerate(items), +] +_PURE_ITERATOR_IDS = ["generator", "map", "filter", "zip", "enumerate"] + + +@pytest.mark.parametrize( + "make_iterator", _PURE_ITERATOR_FACTORIES, ids=_PURE_ITERATOR_IDS +) +def test_pure_iterator_forces_fallback_and_stays_iterator( + make_iterator, final_proxy +): + # A pure iterator argument forces a fallback to the slow path (the fast + # path raises before consuming it), and the slow callable must receive a + # *bare* iterator rather than a materialized list (pandas treats iterators + # and lists differently, e.g. ``is_nested_list_like``). + _, _, x = final_proxy + received = {} + + def fast(it): + raise AssertionError("fast path must not run for pure iterators") + + def slow(it): + received["bare_iterator"] = iter(it) is it and not isinstance( + it, (list, tuple) + ) + return list(it) + + pxy = _FunctionProxy(fast=fast, slow=slow) + result = pxy(make_iterator([x, x])) + assert received["bare_iterator"] + assert len(result) == 2 + + +@pytest.mark.parametrize( + "make_iterator", + _PURE_ITERATOR_FACTORIES[:3], + ids=_PURE_ITERATOR_IDS[:3], +) +def test_pure_iterator_slow_path_unwraps_elements(make_iterator, final_proxy): + # The lazily produced generator still unwraps proxy elements to their slow + # counterparts (the motivation for transforming rather than passing the + # iterator through untouched, which dropped type info such as the + # ``category`` dtype in ``MultiIndex.from_product(map(...))``). + _, slow_x, x = final_proxy + transformed = _slow_arg(make_iterator([x, x])) + assert iter(transformed) is transformed + items = list(transformed) + assert items == [slow_x, slow_x] + # Each element is the unwrapped *slow* object, not the proxy. + assert all(type(item) is type(slow_x) for item in items) + + def test_access_class(): def func(): pass