Preserve categorical column index in DataFrame.describe and handle pure iterators in cudf.pandas fallback

python/cudf/cudf/core/dataframe.py

@@ -5634,7 +5634,7 @@ def describe(
             for col in data_to_describe._column_names
         ]
         if len(describe_series_list) == 1:
-            return describe_series_list[0].to_frame()
+            result = describe_series_list[0].to_frame()
         else:
             ldesc_indexes = sorted(
                 (x.index for x in describe_series_list), key=len
@@ -5648,14 +5648,19 @@ def describe(
                 None,
             )
 
-            return cudf.concat(
+            result = cudf.concat(
                 [
                     series.reindex(names, copy=False)
                     for series in describe_series_list
                 ],
                 axis=1,
                 sort=False,
             )
+        # ``describe`` produces one column per described column in the same
+        # order, so preserve the source column index (incl. its dtype/name,
+        # e.g. a ``CategoricalIndex``), which ``concat``/``to_frame`` drop.
+        result.columns = data_to_describe._data.to_pandas_index
+        return result
 
     @_performance_tracking
     def to_pandas(

python/cudf/cudf/pandas/fast_slow_proxy.py

@@ -1421,11 +1421,26 @@ def _transform_arg(
         )
         result[...] = transformed
         return result.reshape(arg.shape)
+    elif isinstance(arg, (types.GeneratorType, map, filter, zip, enumerate)):
+        # "Pure" (resource-free) iterators. On the fast path we force a
+        # fallback because they are consumable (see below). On the slow path we
+        # transform the elements lazily into a new generator -- a bare iterator
+        # would otherwise hand wrapped proxies straight to the slow library,
+        # which can lose type information (e.g. ``MultiIndex.from_product`` of a
+        # ``map`` over categoricals dropping the ``category`` dtype). We yield a
+        # generator (rather than materializing a ``list``) so the slow library
+        # still sees a bare iterator: pandas treats iterators differently from
+        # lists in places (e.g. ``is_nested_list_like`` returns ``False`` for an
+        # iterator, and ``Index.get_loc`` reports the iterator in its error
+        # message), and materializing would silently change that behavior.
+        if attribute_name == "_fsproxy_fast":
+            raise Exception()
+        return (_transform_arg(a, attribute_name, seen) for a in arg)
     elif isinstance(arg, Iterator) and attribute_name == "_fsproxy_fast":
-        # this may include consumable objects like generators or
-        # IOBase objects, which we don't want unavailable to the slow
-        # path in case of fallback. So, we raise here and ensure the
-        # slow path is taken:
+        # Other consumable objects such as generators or IOBase objects, which
+        # we don't want unavailable to the slow path in case of fallback. So,
+        # we raise here and ensure the slow path is taken (and the object is
+        # left intact for it):
         raise Exception()
     elif isinstance(arg, types.FunctionType):
         if id(arg) in seen:

python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py

@@ -2319,19 +2319,7 @@ def pytest_unconfigure(config):
     "tests/groupby/test_apply.py::test_include_groups": "Failed: DID NOT RAISE <class 'ValueError'>",
     "tests/groupby/test_apply.py::test_positional_slice_groups_datetimelike": "AssertionError: DataFrame are different",
     "tests/groupby/test_apply.py::test_time_field_bug": "TODO: Add a reason for failure",
-    "tests/groupby/test_categorical.py::test_category_order_reducer[False-False-True-all-False-range]": "AttributeError: Can only use .cat accessor with a 'category' dtype. Did you mean: 'at'?",
-    "tests/groupby/test_categorical.py::test_category_order_reducer[False-False-True-all-True-range]": "AttributeError: Can only use .cat accessor with a 'category' dtype. Did you mean: 'at'?",
-    "tests/groupby/test_categorical.py::test_category_order_reducer[False-False-True-any-False-range]": "AttributeError: Can only use .cat accessor with a 'category' dtype. Did you mean: 'at'?",
-    "tests/groupby/test_categorical.py::test_category_order_reducer[False-False-True-any-True-range]": "AttributeError: Can only use .cat accessor with a 'category' dtype. Did you mean: 'at'?",
-    "tests/groupby/test_categorical.py::test_category_order_reducer[False-True-True-all-False-range]": "AttributeError: Can only use .cat accessor with a 'category' dtype. Did you mean: 'at'?",
-    "tests/groupby/test_categorical.py::test_category_order_reducer[False-True-True-all-True-range]": "AttributeError: Can only use .cat accessor with a 'category' dtype. Did you mean: 'at'?",
-    "tests/groupby/test_categorical.py::test_category_order_reducer[False-True-True-any-False-range]": "AttributeError: Can only use .cat accessor with a 'category' dtype. Did you mean: 'at'?",
-    "tests/groupby/test_categorical.py::test_category_order_reducer[False-True-True-any-True-range]": "AttributeError: Can only use .cat accessor with a 'category' dtype. Did you mean: 'at'?",
-    "tests/groupby/test_categorical.py::test_describe_categorical_columns": "TODO: Add a reason for failure",
-    "tests/groupby/test_categorical.py::test_observed[False]": "TODO: Add a reason for failure",
-    "tests/groupby/test_categorical.py::test_observed_codes_remap[False]": "TODO: Add a reason for failure",
-    "tests/groupby/test_categorical.py::test_observed_two_columns[False]": "AssertionError: DataFrame.index level [0] are different",
-    "tests/groupby/test_categorical.py::test_unstack_categorical": "TODO: Add a reason for failure",
+    "tests/groupby/test_categorical.py::test_describe_categorical_columns": "cudf's multi-level groupby aggregation and stack() drop the categorical column-index dtype",
     "tests/groupby/test_counting.py::TestCounting::test_ngroup_distinct": "TODO: Add a reason for failure",
     "tests/groupby/test_cumulative.py::test_cython_api2[False]": "AssertionError: DataFrame are different",
     "tests/groupby/test_cumulative.py::test_groupby_cumprod_nan_influences_other_columns": "TODO: Add a reason for failure",

python/cudf/cudf/tests/groupby/test_reductions.py

@@ -1242,6 +1242,22 @@ def test_groupby_all_any_empty(op):
     assert_eq(expect, got, check_index_type=False)
 
 
+@pytest.mark.parametrize("op", ["all", "any"])
+@pytest.mark.parametrize("key_is_categorical", [False, True])
+def test_groupby_all_any_as_index_false(op, key_is_categorical):
+    # With as_index=False the grouping key is reset as a column; it must not
+    # be coerced to bool along with the (reduced) value columns.
+    key = [2, 1, 2, 3]
+    if key_is_categorical:
+        key = pd.Categorical(key, categories=[1, 4, 3, 2], ordered=True)
+    pdf = pd.DataFrame({"a": key, "b": range(4)})
+    gdf = cudf.from_pandas(pdf)
+    with cudf.option_context("mode.pandas_compatible", True):
+        got = getattr(gdf.groupby("a", as_index=False), op)()
+    expect = getattr(pdf.groupby("a", as_index=False), op)()
+    assert_eq(expect, got)
+
+
 @pytest.mark.parametrize(
     "string_dtype",
     [

python/cudf/cudf_pandas_tests/test_fast_slow_proxy.py

@@ -233,6 +233,66 @@ def fast(s):
     assert pxy(StringIO("hello")) == "hello"
 
 
+# "Pure" (resource-free) iterators that ``_transform_arg`` handles specially:
+# they force a fallback on the fast path and are transformed lazily into a new
+# generator on the slow path.
+_PURE_ITERATOR_FACTORIES = [
+    lambda items: (item for item in items),
+    lambda items: map(lambda item: item, items),
+    lambda items: filter(lambda item: True, items),
+    lambda items: zip(items, strict=True),
+    lambda items: enumerate(items),
+]
+_PURE_ITERATOR_IDS = ["generator", "map", "filter", "zip", "enumerate"]
+
+
+@pytest.mark.parametrize(
+    "make_iterator", _PURE_ITERATOR_FACTORIES, ids=_PURE_ITERATOR_IDS
+)
+def test_pure_iterator_forces_fallback_and_stays_iterator(
+    make_iterator, final_proxy
+):
+    # A pure iterator argument forces a fallback to the slow path (the fast
+    # path raises before consuming it), and the slow callable must receive a
+    # *bare* iterator rather than a materialized list (pandas treats iterators
+    # and lists differently, e.g. ``is_nested_list_like``).
+    _, _, x = final_proxy
+    received = {}
+
+    def fast(it):
+        raise AssertionError("fast path must not run for pure iterators")
+
+    def slow(it):
+        received["bare_iterator"] = iter(it) is it and not isinstance(
+            it, (list, tuple)
+        )
+        return list(it)
+
+    pxy = _FunctionProxy(fast=fast, slow=slow)
+    result = pxy(make_iterator([x, x]))
+    assert received["bare_iterator"]
+    assert len(result) == 2
+
+
+@pytest.mark.parametrize(
+    "make_iterator",
+    _PURE_ITERATOR_FACTORIES[:3],
+    ids=_PURE_ITERATOR_IDS[:3],
+)
+def test_pure_iterator_slow_path_unwraps_elements(make_iterator, final_proxy):
+    # The lazily produced generator still unwraps proxy elements to their slow
+    # counterparts (the motivation for transforming rather than passing the
+    # iterator through untouched, which dropped type info such as the
+    # ``category`` dtype in ``MultiIndex.from_product(map(...))``).
+    _, slow_x, x = final_proxy
+    transformed = _slow_arg(make_iterator([x, x]))
+    assert iter(transformed) is transformed
+    items = list(transformed)
+    assert items == [slow_x, slow_x]
+    # Each element is the unwrapped *slow* object, not the proxy.
+    assert all(type(item) is type(slow_x) for item in items)
+
+
 def test_access_class():
     def func():
         pass