Fix groupby size dtype for masked series and apply group order for sort=False

python/cudf/cudf/core/groupby/groupby.py

@@ -791,8 +791,14 @@ def size(self) -> Series:
             isinstance(obj_dtype, pd.StringDtype)
             and obj_dtype.storage == "pyarrow"
             and obj_dtype.na_value is pd.NA
+        ) or (
+            self.obj.ndim == 1
+            and not isinstance(obj_dtype, pd.StringDtype)
+            and is_pandas_nullable_extension_dtype(obj_dtype)
         ):
-            # Series.groupby.size() on ``string[pyarrow]`` returns Int64.
+            # Series.groupby.size() returns Int64 for ``string[pyarrow]``
+            # and for masked (Int*/UInt*/Float*/boolean) dtypes
+            # (pandas GH#54132).
             int64_dtype = pd.Int64Dtype()
             if isinstance(result, Series):
                 result = Series._from_column(
@@ -2485,6 +2491,34 @@ def mult(df):
             include_groups=include_groups
         )
 
+        if not self._sort and len(offsets) > 2:
+            # libcudf returns groups sorted by key, but with ``sort=False``
+            # pandas processes groups in order of first appearance. Permute
+            # the grouped layout accordingly so both engines and the result
+            # assembly see pandas' iteration order.
+            pos_offsets, _, (positions,) = self._groups(
+                [self._range_column_from_obj]
+            )
+            first_pos = positions.take(as_column(pos_offsets[:-1]))
+            group_order = first_pos.argsort().to_numpy()
+            sizes = np.diff(np.asarray(offsets, dtype=SIZE_TYPE_DTYPE))
+            row_order = as_column(
+                np.concatenate(
+                    [
+                        np.arange(
+                            offsets[i], offsets[i + 1], dtype=SIZE_TYPE_DTYPE
+                        )
+                        for i in group_order
+                    ]
+                )
+            )
+            group_names = group_names.take(group_order)
+            group_keys = group_keys.take(row_order)
+            grouped_values = grouped_values.take(row_order)
+            new_offsets = np.zeros(len(sizes) + 1, dtype=SIZE_TYPE_DTYPE)
+            np.cumsum(sizes[group_order], out=new_offsets[1:])
+            offsets = new_offsets.tolist()
+
         if engine == "auto":
             if _can_be_jitted(grouped_values, func, args):
                 engine = "jit"

python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py

@@ -1912,12 +1912,6 @@ def pytest_unconfigure(config):
     "tests/groupby/methods/test_rank.py::test_rank_avg_even_vals[True-int64]": "TODO: Add a reason for failure",
     "tests/groupby/methods/test_rank.py::test_rank_avg_even_vals[True-uint32]": "TODO: Add a reason for failure",
     "tests/groupby/methods/test_rank.py::test_rank_avg_even_vals[True-uint64]": "TODO: Add a reason for failure",
-    "tests/groupby/methods/test_size.py::test_size_series_masked_type_returns_Int64[Float64]": "TODO: Add a reason for failure",
-    "tests/groupby/methods/test_size.py::test_size_series_masked_type_returns_Int64[Int64]": "TODO: Add a reason for failure",
-    "tests/groupby/methods/test_size.py::test_size_series_masked_type_returns_Int64[boolean]": "TODO: Add a reason for failure",
-    "tests/groupby/methods/test_size.py::test_size_sort[False-A]": "TODO: Add a reason for failure",
-    "tests/groupby/methods/test_size.py::test_size_sort[False-B]": "TODO: Add a reason for failure",
-    "tests/groupby/methods/test_size.py::test_size_sort[False-by2]": "TODO: Add a reason for failure",
     "tests/groupby/test_all_methods.py::test_not_c_contiguous_mask[all]": "assert not True",
     "tests/groupby/test_all_methods.py::test_not_c_contiguous_mask[any]": "assert not True",
     "tests/groupby/test_all_methods.py::test_not_c_contiguous_mask[bfill]": "assert not True",

python/cudf/cudf/tests/groupby/test_size.py

@@ -1,6 +1,7 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION.
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 import pandas as pd
+import pytest
 
 import cudf
 from cudf.testing import assert_groupby_results_equal
@@ -18,3 +19,29 @@ def test_size_series_with_name():
     expected = ser.groupby(ser).size()
     result = cudf.from_pandas(ser).groupby(ser).size()
     assert_groupby_results_equal(result, expected)
+
+
+@pytest.mark.parametrize("dtype", ["Int64", "Float64", "boolean"])
+def test_size_series_masked_dtype(dtype):
+    # pandas GH#54132: SeriesGroupBy.size on masked dtypes returns Int64
+    psr = pd.Series([1, 1, 1], index=["a", "a", "b"], dtype=dtype)
+    gsr = cudf.from_pandas(psr)
+
+    expect = psr.groupby(level=0).size()
+    got = gsr.groupby(level=0).size()
+
+    assert str(got.dtype) == "Int64"
+    assert_groupby_results_equal(expect, got)
+
+
+def test_apply_sort_false_first_appearance_order():
+    # groups are processed in order of first appearance with sort=False,
+    # like pandas
+    pdf = pd.DataFrame({"k": [3, 1, 3, 2, 1], "v": [1, 2, 3, 4, 5]})
+    gdf = cudf.from_pandas(pdf)
+
+    expect = pdf.groupby("k", sort=False)["v"].apply(lambda s: s.sum())
+    got = gdf.groupby("k", sort=False)["v"].apply(lambda s: s.sum())
+
+    assert list(got.index.to_pandas()) == [3, 1, 2]
+    assert_groupby_results_equal(expect, got)