Preserve categorical column index in DataFrame.describe and handle pure iterators in cudf.pandas fallback

python/cudf/cudf/core/dataframe.py

@@ -5613,7 +5613,7 @@ def describe(
             for col in data_to_describe._column_names
         ]
         if len(describe_series_list) == 1:
-            return describe_series_list[0].to_frame()
+            result = describe_series_list[0].to_frame()
         else:
             ldesc_indexes = sorted(
                 (x.index for x in describe_series_list), key=len
@@ -5627,14 +5627,19 @@ def describe(
                 None,
             )
 
-            return cudf.concat(
+            result = cudf.concat(
                 [
                     series.reindex(names, copy=False)
                     for series in describe_series_list
                 ],
                 axis=1,
                 sort=False,
             )
+        # ``describe`` produces one column per described column in the same
+        # order, so preserve the source column index (incl. its dtype/name,
+        # e.g. a ``CategoricalIndex``), which ``concat``/``to_frame`` drop.
+        result.columns = data_to_describe._data.to_pandas_index
+        return result
 
     @_performance_tracking
     def to_pandas(

python/cudf/cudf/core/groupby/groupby.py

@@ -3344,7 +3344,11 @@ def _to_bool_col(col):
         if isinstance(result, Series):
             result = result.fillna(fill_value).astype(bool_np)
         else:
-            for col_name in result._column_names:
+            # With ``as_index=False`` the key columns are reset onto ``result``
+            # as ordinary columns; only the value columns were reduced, so
+            # restrict the fill/cast to them (otherwise the key columns -- e.g.
+            # a categorical grouper -- would be clobbered to ``bool``).
+            for col_name in self.grouping._values_column_names:
                 result[col_name] = (
                     result[col_name].fillna(fill_value).astype(bool_np)
                 )

python/cudf/cudf/pandas/fast_slow_proxy.py

@@ -1421,11 +1421,26 @@ def _transform_arg(
         )
         result[...] = transformed
         return result.reshape(arg.shape)
+    elif isinstance(arg, (types.GeneratorType, map, filter, zip, enumerate)):
+        # "Pure" (resource-free) iterators. On the fast path we force a
+        # fallback because they are consumable (see below). On the slow path we
+        # transform the elements lazily into a new generator -- a bare iterator
+        # would otherwise hand wrapped proxies straight to the slow library,
+        # which can lose type information (e.g. ``MultiIndex.from_product`` of a
+        # ``map`` over categoricals dropping the ``category`` dtype). We yield a
+        # generator (rather than materializing a ``list``) so the slow library
+        # still sees a bare iterator: pandas treats iterators differently from
+        # lists in places (e.g. ``is_nested_list_like`` returns ``False`` for an
+        # iterator, and ``Index.get_loc`` reports the iterator in its error
+        # message), and materializing would silently change that behavior.
+        if attribute_name == "_fsproxy_fast":
+            raise Exception()
+        return (_transform_arg(a, attribute_name, seen) for a in arg)
     elif isinstance(arg, Iterator) and attribute_name == "_fsproxy_fast":
-        # this may include consumable objects like generators or
-        # IOBase objects, which we don't want unavailable to the slow
-        # path in case of fallback. So, we raise here and ensure the
-        # slow path is taken:
+        # Other consumable objects such as generators or IOBase objects, which
+        # we don't want unavailable to the slow path in case of fallback. So,
+        # we raise here and ensure the slow path is taken (and the object is
+        # left intact for it):
         raise Exception()
     elif isinstance(arg, types.FunctionType):
         if id(arg) in seen:

python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py

@@ -2472,19 +2472,7 @@ def pytest_unconfigure(config):
     "tests/groupby/test_categorical.py::test_agg_list[False-True-std-keys0-True]": "TODO: Add a reason for failure",
     "tests/groupby/test_categorical.py::test_agg_list[False-True-sum-keys0-True]": "TODO: Add a reason for failure",
     "tests/groupby/test_categorical.py::test_agg_list[False-True-var-keys0-True]": "TODO: Add a reason for failure",
-    "tests/groupby/test_categorical.py::test_category_order_reducer[False-False-True-all-False-range]": "AttributeError: Can only use .cat accessor with a 'category' dtype. Did you mean: 'at'?",
-    "tests/groupby/test_categorical.py::test_category_order_reducer[False-False-True-all-True-range]": "AttributeError: Can only use .cat accessor with a 'category' dtype. Did you mean: 'at'?",
-    "tests/groupby/test_categorical.py::test_category_order_reducer[False-False-True-any-False-range]": "AttributeError: Can only use .cat accessor with a 'category' dtype. Did you mean: 'at'?",
-    "tests/groupby/test_categorical.py::test_category_order_reducer[False-False-True-any-True-range]": "AttributeError: Can only use .cat accessor with a 'category' dtype. Did you mean: 'at'?",
-    "tests/groupby/test_categorical.py::test_category_order_reducer[False-True-True-all-False-range]": "AttributeError: Can only use .cat accessor with a 'category' dtype. Did you mean: 'at'?",
-    "tests/groupby/test_categorical.py::test_category_order_reducer[False-True-True-all-True-range]": "AttributeError: Can only use .cat accessor with a 'category' dtype. Did you mean: 'at'?",
-    "tests/groupby/test_categorical.py::test_category_order_reducer[False-True-True-any-False-range]": "AttributeError: Can only use .cat accessor with a 'category' dtype. Did you mean: 'at'?",
-    "tests/groupby/test_categorical.py::test_category_order_reducer[False-True-True-any-True-range]": "AttributeError: Can only use .cat accessor with a 'category' dtype. Did you mean: 'at'?",
-    "tests/groupby/test_categorical.py::test_describe_categorical_columns": "TODO: Add a reason for failure",
-    "tests/groupby/test_categorical.py::test_observed[False]": "TODO: Add a reason for failure",
-    "tests/groupby/test_categorical.py::test_observed_codes_remap[False]": "TODO: Add a reason for failure",
-    "tests/groupby/test_categorical.py::test_observed_two_columns[False]": "AssertionError: DataFrame.index level [0] are different",
-    "tests/groupby/test_categorical.py::test_unstack_categorical": "TODO: Add a reason for failure",
+    "tests/groupby/test_categorical.py::test_describe_categorical_columns": "cudf's multi-level groupby aggregation and stack() drop the categorical column-index dtype",
     "tests/groupby/test_counting.py::TestCounting::test_ngroup_distinct": "TODO: Add a reason for failure",
     "tests/groupby/test_cumulative.py::test_cython_api2[False]": "AssertionError: DataFrame are different",
     "tests/groupby/test_cumulative.py::test_groupby_cumprod_nan_influences_other_columns": "TODO: Add a reason for failure",

python/cudf/cudf/tests/groupby/test_reductions.py

@@ -1242,6 +1242,22 @@ def test_groupby_all_any_empty(op):
     assert_eq(expect, got, check_index_type=False)
 
 
+@pytest.mark.parametrize("op", ["all", "any"])
+@pytest.mark.parametrize("key_is_categorical", [False, True])
+def test_groupby_all_any_as_index_false(op, key_is_categorical):
+    # With as_index=False the grouping key is reset as a column; it must not
+    # be coerced to bool along with the (reduced) value columns.
+    key = [2, 1, 2, 3]
+    if key_is_categorical:
+        key = pd.Categorical(key, categories=[1, 4, 3, 2], ordered=True)
+    pdf = pd.DataFrame({"a": key, "b": range(4)})
+    gdf = cudf.from_pandas(pdf)
+    with cudf.option_context("mode.pandas_compatible", True):
+        got = getattr(gdf.groupby("a", as_index=False), op)()
+    expect = getattr(pdf.groupby("a", as_index=False), op)()
+    assert_eq(expect, got)
+
+
 @pytest.mark.parametrize(
     "string_dtype",
     [