Fix various pandas issues

ci/test_narwhals.sh

@@ -189,6 +189,13 @@ test_pandas_object_series \
 # test_contains_case_insensitive[pandas], test_contains_series_case_insensitive[pandas]: String contains with case_insensitive returns False instead of None for nulls
 # test_fill_null_pandas_downcast: asserts dtype is 'object' after fill_null on [True, None]; cudf.pandas reports 'bool'
 #   because cudf represents nullable bool natively (not as object) — fundamental design difference, not fixable in cudf
+# test_self_equal[pandas]: Under cudf.pandas, narwhals identifies proxy DataFrames as Implementation.PANDAS
+#   but it detects cudf list/struct/decimal dtypes via str(series.dtype).startswith(("list","struct","decimal"))
+#   in _pandas_like/utils.py (CUDF_BASE_DTYPE_PREFIX), and cudf.pandas' _Series_dtype now correctly returns
+#   pandas-compatible 'object' for these types, so narwhals maps the column to its generic Object dtype
+#   instead of List, skipping the specialized _check_list_like path in assert_series_equal. The fallback
+#   `left != right` comparison then fails because element-wise != on nested lists containing NaN is undefined.
+#   narwhals needs to a different mechanism for detecting which path to follow.
 TESTS_THAT_NEED_NARWHALS_FIX_FOR_CUDF_PANDAS=" \
 test_dtypes or \
 test_explode_multiple_cols or \
@@ -209,7 +216,8 @@ test_check_row_order_nested_only[pandas] or \
 test_cast_string or \
 (test_contains_case_insensitive and pandas) or \
 (test_contains_series_case_insensitive and pandas) or \
-test_fill_null_pandas_downcast \
+test_fill_null_pandas_downcast or \
+test_self_equal[pandas] \
 "
 
 PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 \

conda/environments/all_cuda-129_arch-aarch64.yaml

@@ -59,6 +59,7 @@ dependencies:
 - notebook
 - numba-cuda>=0.22.2,<0.29.0
 - numba>=0.60.0,<0.65.0
+- numexpr
 - numpy>=1.26,<3.0
 - numpydoc
 - nvidia-ml-py>=12

conda/environments/all_cuda-129_arch-x86_64.yaml

@@ -59,6 +59,7 @@ dependencies:
 - notebook
 - numba-cuda>=0.22.2,<0.29.0
 - numba>=0.60.0,<0.65.0
+- numexpr
 - numpy>=1.26,<3.0
 - numpydoc
 - nvidia-ml-py>=12

conda/environments/all_cuda-132_arch-aarch64.yaml

@@ -59,6 +59,7 @@ dependencies:
 - notebook
 - numba-cuda>=0.22.2,<0.29.0
 - numba>=0.60.0,<0.65.0
+- numexpr
 - numpy>=1.26,<3.0
 - numpydoc
 - nvidia-ml-py>=12

conda/environments/all_cuda-132_arch-x86_64.yaml

@@ -59,6 +59,7 @@ dependencies:
 - notebook
 - numba-cuda>=0.22.2,<0.29.0
 - numba>=0.60.0,<0.65.0
+- numexpr
 - numpy>=1.26,<3.0
 - numpydoc
 - nvidia-ml-py>=12

dependencies.yaml

@@ -49,6 +49,7 @@ files:
       - test_python_cudf_common
       - test_python_pylibcudf
       - test_python_cudf_pandas
+      - test_cudf_pandas_pandas_tests
       - test_python_cudf_polars
       - test_python_s3
   test_static_build:
@@ -1316,6 +1317,14 @@ dependencies:
           - openpyxl
           # https://github.com/pytest-dev/pytest-rerunfailures/issues/302
           - pytest-rerunfailures!=16.0.0
+  # Additional dependencies for running the pandas test suite under cudf.pandas.
+  # Unlike test_python_pandas_cudf (which uses pip extras like pandas[performance]),
+  # conda environments need these listed explicitly.
+  test_cudf_pandas_pandas_tests:
+    common:
+      - output_types: [conda]
+        packages:
+          - numexpr
   depends_on_dask_cuda:
     common:
       - output_types: conda

python/cudf/cudf/core/column/interval.py

@@ -27,6 +27,11 @@
 
 
 class IntervalColumn(ColumnBase):
+    _VALID_BINARY_OPERATIONS = {
+        "__eq__",
+        "__ne__",
+    }
+
     @functools.cached_property
     def subtype(self) -> DtypeObj:
         return subtype_from_interval_dtype(self.dtype)
@@ -161,15 +166,18 @@ def _binaryop(self, other: ColumnBinaryOperand, op: str) -> ColumnBase:
         reflect, op = self._check_reflected_op(op)
         if not isinstance(other, type(self)):
             return NotImplemented
-        if op == "NULL_EQUALS":
+        if op in {"__eq__", "__ne__", "NULL_EQUALS", "NULL_NOT_EQUALS"}:
             lefts_equal = self.left._binaryop(other.left, "NULL_EQUALS")
             rights_equal = self.right._binaryop(other.right, "NULL_EQUALS")
-            return binaryop.binaryop(
+            result = binaryop.binaryop(
                 lefts_equal,
                 rights_equal,
                 "__and__",
                 get_dtype_of_same_kind(self.dtype, lefts_equal.dtype),
             )
+            if op in {"__ne__", "NULL_NOT_EQUALS"}:
+                result = ~result
+            return result
         else:
             raise TypeError(f"{op} not supported with {type(other).__name__}")
 

python/cudf/cudf/core/groupby/groupby.py

@@ -1937,7 +1937,7 @@ def _grouped(self, *, include_groups: bool = True):
         grouped_values = self.obj._from_columns_like_self(
             grouped_value_cols,
             column_names=self.obj._column_names,
-            index_names=self.obj._index_names,  # type: ignore[arg-type]
+            index_names=self.obj.index.names,
         )
         if not include_groups and isinstance(grouped_values, DataFrame):
             selection = getattr(self, "_selection", None)

python/cudf/cudf/pandas/_wrappers/pandas.py

@@ -335,9 +335,19 @@ def custom_repr_html(obj):
 
 
 def _Series_dtype(self):
-    # Fast-path to extract dtype from the current
-    # object without round-tripping through the slow<->fast
-    return _maybe_wrap_result(self._fsproxy_wrapped.dtype, None)
+    dtype = self._fsproxy_wrapped.dtype
+    if isinstance(
+        dtype,
+        (
+            cudf.ListDtype,
+            cudf.StructDtype,
+            cudf.Decimal32Dtype,
+            cudf.Decimal64Dtype,
+            cudf.Decimal128Dtype,
+        ),
+    ):
+        dtype = self._fsproxy_slow.dtype
+    return _maybe_wrap_result(dtype, None)
 
 
 _SeriesAtIndexer = make_intermediate_proxy_type(
@@ -1233,6 +1243,12 @@ def Index__setattr__(self, name, value):
     pd.core.window.ewm.ExponentialMovingWindowGroupby,
 )
 
+OnlineExponentialMovingWindow = make_intermediate_proxy_type(
+    "OnlineExponentialMovingWindow",
+    _Unusable,
+    pd.core.window.ewm.OnlineExponentialMovingWindow,
+)
+
 EWMMeanState = make_intermediate_proxy_type(
     "EWMMeanState",
     _Unusable,

python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py

@@ -124,7 +124,7 @@ def pytest_unconfigure(config):
     "tests/arithmetic/test_numeric.py::TestAdditionSubtraction::test_series_operators_arithmetic[python-pow-slice]": "AssertionError: Series are different",
     "tests/arithmetic/test_numeric.py::TestDivisionByZero::test_df_mod_zero_df[numexpr]": "TODO: Add a reason for failure",
     "tests/arithmetic/test_numeric.py::TestDivisionByZero::test_df_mod_zero_df[python]": "TODO: Add a reason for failure",
-    "tests/arithmetic/test_numeric.py::TestDivisionByZero::test_ser_div_ser[numexpr-float32-int64]": "TODO: Add a reason for failure",
+    "tests/arithmetic/test_numeric.py::TestDivisionByZero::test_ser_div_ser[numexpr-float32-int64]": "cudf promotes float32/int64 division to float64",
     "tests/arithmetic/test_numeric.py::TestNumericArithmeticUnsorted::test_numeric_compat2[numexpr]": "TODO: Add a reason for failure",
     "tests/arithmetic/test_numeric.py::TestNumericArithmeticUnsorted::test_numeric_compat2[python]": "TODO: Add a reason for failure",
     "tests/arithmetic/test_numeric.py::TestNumericArithmeticUnsorted::test_numeric_compat2_floordiv[numexpr-idx0-2-expected0]": "TODO: Add a reason for failure",
@@ -1153,8 +1153,6 @@ def pytest_unconfigure(config):
     "tests/extension/test_interval.py::TestIntervalArray::test_EA_types[c]": "TODO: Add a reason for failure",
     "tests/extension/test_interval.py::TestIntervalArray::test_EA_types[python]": "TODO: Add a reason for failure",
     "tests/extension/test_interval.py::TestIntervalArray::test_astype_own_type[False]": "TODO: Add a reason for failure",
-    "tests/extension/test_interval.py::TestIntervalArray::test_compare_array[eq]": "TODO: Add a reason for failure",
-    "tests/extension/test_interval.py::TestIntervalArray::test_compare_array[ne]": "TODO: Add a reason for failure",
     "tests/extension/test_interval.py::TestIntervalArray::test_grouping_grouper": "AssertionError: ndarray Expected type <class 'numpy.ndarray'>, found <class 'pandas.arrays.ArrowStringArray'> instead",
     "tests/extension/test_interval.py::TestIntervalArray::test_in_numeric_groupby": "TODO: Add a reason for failure",
     "tests/extension/test_interval.py::TestIntervalArray::test_is_extension_array_dtype": "TODO: Add a reason for failure",
@@ -1383,9 +1381,6 @@ def pytest_unconfigure(config):
     "tests/extension/test_numpy.py::TestNumpyExtensionArray::test_set_frame_overwrite_object[float]": "TODO: Add a reason for failure",
     "tests/extension/test_numpy.py::TestNumpyExtensionArray::test_set_frame_overwrite_object[object]": "TODO: Add a reason for failure",
     "tests/extension/test_numpy.py::TestNumpyExtensionArray::test_setitem_2d_values[object]": "TODO: Add a reason for failure",
-    "tests/extension/test_numpy.py::TestNumpyExtensionArray::test_stack[object-False-columns0]": "AssertionError",
-    "tests/extension/test_numpy.py::TestNumpyExtensionArray::test_stack[object-True-columns0]": "AssertionError",
-    "tests/extension/test_numpy.py::TestNumpyExtensionArray::test_take_series[object]": "TODO: Add a reason for failure",
     "tests/extension/test_numpy.py::TestNumpyExtensionArray::test_to_numpy[object]": "TODO: Add a reason for failure",
     "tests/extension/test_numpy.py::TestNumpyExtensionArray::test_unary_ufunc_dunder_equivalence[float-absolute]": "TODO: Add a reason for failure",
     "tests/extension/test_numpy.py::TestNumpyExtensionArray::test_unary_ufunc_dunder_equivalence[float-negative]": "TODO: Add a reason for failure",
@@ -3775,7 +3770,7 @@ def pytest_unconfigure(config):
     "tests/indexes/test_numpy_compat.py::test_numpy_ufuncs_basic[complex64-exp]": "TODO: Add a reason for failure",
     "tests/indexes/test_numpy_compat.py::test_numpy_ufuncs_basic[complex64-expm1]": "TODO: Add a reason for failure",
     "tests/indexes/test_numpy_compat.py::test_numpy_ufuncs_basic[complex64-log10]": "TODO: Add a reason for failure",
-    "tests/indexes/test_numpy_compat.py::test_numpy_ufuncs_basic[complex64-log1p]": "TODO: Add a reason for failure",
+    "tests/indexes/test_numpy_compat.py::test_numpy_ufuncs_basic[complex64-log1p]": "GPU complex64 precision differs from CPU",
     "tests/indexes/test_numpy_compat.py::test_numpy_ufuncs_basic[complex64-log2]": "TODO: Add a reason for failure",
     "tests/indexes/test_numpy_compat.py::test_numpy_ufuncs_basic[complex64-log]": "TODO: Add a reason for failure",
     "tests/indexes/test_numpy_compat.py::test_numpy_ufuncs_basic[complex64-sin]": "TODO: Add a reason for failure",
@@ -6674,6 +6669,17 @@ def pytest_unconfigure(config):
     "tests/groupby/test_groupby_dropna.py::test_null_is_null_for_dtype[False-float0-NoneType-None-False]": "Flaky/version-sensitive cudf.pandas dispatch",
     "tests/groupby/test_groupby_dropna.py::test_null_is_null_for_dtype[False-float1-NoneType-None-False]": "Flaky/version-sensitive cudf.pandas dispatch",
     "tests/groupby/test_grouping.py::TestGrouping::test_groupby_level_index_value_all_na": "Flaky/version-sensitive cudf.pandas dispatch",
+    "tests/groupby/test_numba.py::TestEngine::test_as_index_false_unsupported[max]": "cuDF computes reductions on GPU without numba; pandas-specific limitation does not apply",
+    "tests/groupby/test_numba.py::TestEngine::test_as_index_false_unsupported[max-min_count]": "cuDF computes reductions on GPU without numba; pandas-specific limitation does not apply",
+    "tests/groupby/test_numba.py::TestEngine::test_as_index_false_unsupported[mean]": "cuDF computes reductions on GPU without numba; pandas-specific limitation does not apply",
+    "tests/groupby/test_numba.py::TestEngine::test_as_index_false_unsupported[min]": "cuDF computes reductions on GPU without numba; pandas-specific limitation does not apply",
+    "tests/groupby/test_numba.py::TestEngine::test_as_index_false_unsupported[min-min_count]": "cuDF computes reductions on GPU without numba; pandas-specific limitation does not apply",
+    "tests/groupby/test_numba.py::TestEngine::test_as_index_false_unsupported[std_0]": "cuDF computes reductions on GPU without numba; pandas-specific limitation does not apply",
+    "tests/groupby/test_numba.py::TestEngine::test_as_index_false_unsupported[std_1]": "cuDF computes reductions on GPU without numba; pandas-specific limitation does not apply",
+    "tests/groupby/test_numba.py::TestEngine::test_as_index_false_unsupported[sum]": "cuDF computes reductions on GPU without numba; pandas-specific limitation does not apply",
+    "tests/groupby/test_numba.py::TestEngine::test_as_index_false_unsupported[sum-min_count]": "cuDF computes reductions on GPU without numba; pandas-specific limitation does not apply",
+    "tests/groupby/test_numba.py::TestEngine::test_as_index_false_unsupported[var_0]": "cuDF computes reductions on GPU without numba; pandas-specific limitation does not apply",
+    "tests/groupby/test_numba.py::TestEngine::test_as_index_false_unsupported[var_1]": "cuDF computes reductions on GPU without numba; pandas-specific limitation does not apply",
     "tests/indexes/datetimes/test_partial_slicing.py::TestSlicing::test_slice_month": "Flaky xfails (TODO: Validate with pandas 3)",
     "tests/indexes/interval/test_interval.py::TestIntervalIndex::test_maybe_convert_i8_numeric_identical[float-IntervalIndex]": "Asserts private APIs",
     "tests/indexes/interval/test_interval.py::TestIntervalIndex::test_maybe_convert_i8_numeric_identical[float-Interval]": "Asserts private APIs",

python/cudf/cudf/tests/groupby/test_apply.py

@@ -854,6 +854,24 @@ def test_groupby_apply_series_args(func, args):
     assert_groupby_results_equal(expect, got)
 
 
+def test_groupby_apply_series_preserves_multiindex_names():
+    pdf = pd.DataFrame(
+        {"value": [1.0, 2.0, 3.0, 4.0]},
+        index=pd.MultiIndex.from_product([range(2), range(2)]),
+    )
+    gdf = cudf.from_pandas(pdf)
+    by = np.array([0, 0, 1, 1])
+
+    expect = pdf.groupby(by=by, group_keys=False)["value"].apply(
+        lambda x: x.cumprod()
+    )
+    got = gdf.groupby(by=by, group_keys=False)["value"].apply(
+        lambda x: x.cumprod()
+    )
+
+    assert_eq(expect, got)
+
+
 @pytest.mark.parametrize("group_keys", [None, True, False])
 @pytest.mark.parametrize("by", ["A", ["A", "B"]])
 def test_groupby_group_keys(group_keys, by):

python/cudf/cudf/tests/indexes/test_interval.py

@@ -393,3 +393,47 @@ def test_from_interval_range_indexing():
     result = cudf.interval_range(start=0, end=1, name="a").repeat(2)
     expected = pd.interval_range(start=0, end=1, name="a").repeat(2)
     assert_eq(result, expected)
+
+
+def test_interval_equality_series_eq():
+    s = cudf.Series(pd.arrays.IntervalArray.from_breaks([0, 1, 2, 3]))
+    result = s == s
+    expected = cudf.Series([True, True, True])
+    assert_eq(result, expected)
+
+
+def test_interval_equality_series_ne():
+    s = cudf.Series(pd.arrays.IntervalArray.from_breaks([0, 1, 2, 3]))
+    other = cudf.Series(pd.arrays.IntervalArray.from_breaks([0, 1, 2, 4]))
+    result = s != other
+    expected = cudf.Series([False, False, True])
+    assert_eq(result, expected)
+
+
+def test_interval_equality_series_eq_with_nulls():
+    pi = pd.arrays.IntervalArray.from_breaks([0.0, 1.0, 2.0, 3.0])
+    ps = pd.array(pi, dtype=pi.dtype)
+    ps[1] = pd.NA
+    s = cudf.Series(ps)
+    result = s == s
+    # cudf uses NULL_EQUALS semantics: null == null is True
+    expected = cudf.Series([True, True, True])
+    assert_eq(result, expected)
+
+
+def test_interval_equality_index_eq():
+    idx = cudf.IntervalIndex.from_breaks([0, 1, 2, 3])
+    other = cudf.IntervalIndex.from_breaks([0, 1, 5, 3])
+    result = idx == other
+    expected = np.array([True, False, False])
+    np.testing.assert_array_equal(result.get(), expected)
+
+
+@pytest.mark.parametrize("closed", ["left", "right", "both", "neither"])
+def test_interval_equality_eq_respects_closed(closed):
+    s = cudf.Series(
+        pd.arrays.IntervalArray.from_breaks([0, 1, 2], closed=closed)
+    )
+    result = s == s
+    expected = cudf.Series([True, True])
+    assert_eq(result, expected)