From 201eb380ef92715dc134fef8e84ec2d8daaf54b1 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Mon, 13 Jul 2026 23:18:10 +0000 Subject: [PATCH 1/6] Fix DataFrame.stack/unstack pandas incompatibilities Fixes 60 pandas unit-test failures in tests/frame/test_stack_unstack.py under cudf.pandas (64 -> 4; the remaining 4 assert BlockManager internals or monkeypatch pandas' private _Unstacker). DataFrame.stack: * Resolve levels positionally: integer column-level names no longer collide with level positions in get_level_values lookups. * Validate out-of-bounds integer levels (IndexError) and duplicated level names (ValueError) like pandas. * Build the stacked level keys from the column MultiIndex's own levels/codes so level dtypes survive (int64 levels with missing entries no longer upcast to float64; categorical levels stay categorical through the tile step). * Emit stacked keys in appearance order like pandas, replacing the argsort-based reordering that misaligned data for non-involution column permutations and NaN keys. * Attach pandas-faithful levels/codes to the result index eagerly (reusing the original MultiIndex's levels) so a later unstack restores the original row/column order; legacy dropna keeps them. unstack/_pivot: * Order result rows/columns by the removed level's codes (level order, missing keys first) instead of sorted values, matching pandas. * Propagate the source frame's column-axis level names instead of hardcoding None; fixes the 'Length of names must match number of levels' failure for frames with MultiIndex columns. * Promote integer columns to float64 when unstack introduces missing cells (pandas block semantics); pivot_table/crosstab opt out since they fill missing cells afterwards. * Preserve unused categories of the removed level in the result's column levels (pandas GH 17845). * Validate flat-index level (KeyError) and duplicated index names (ValueError) like pandas. Supporting fixes: * ColumnAccessor: NaN-containing labels now match under pandas' all-NaNs-equal semantics; to_pandas_index restores recorded per-level dtypes when the cast round-trips losslessly; the primed/cached pandas columns index survives accessor copies so explicit unsorted level layouts are not lost on fast-to-slow conversion. * MultiIndex: lazy codes/levels materialization sorts levels (pandas-canonical for per-row-value construction). * sort_index(axis=1) now honors level= and sort_remaining=. * GroupBy.agg keeps MultiIndex columns for MultiIndex-column sources. * NumericalColumn.as_numerical_column no longer mutates the column dtype in place on equal-pylibcudf-type casts. * Bool columns with nulls convert to pandas with np.nan (not None) in pandas-compatible mode. * cudf.pandas: do not bake one instance's transfer-blocking state into the class-level cached _MethodProxy (order-dependent test poisoning). Removes the 60 fixed xfail entries from the pandas-testing plugin and un-xfails now-passing cudf unstack tests with categorical indexes. --- python/cudf/cudf/core/column/column.py | 11 + python/cudf/cudf/core/column/numerical.py | 19 +- python/cudf/cudf/core/column_accessor.py | 72 ++++- python/cudf/cudf/core/dataframe.py | 293 ++++++++++++++---- python/cudf/cudf/core/groupby/groupby.py | 3 +- python/cudf/cudf/core/indexed_frame.py | 25 +- python/cudf/cudf/core/reshape.py | 194 +++++++++++- python/cudf/cudf/pandas/fast_slow_proxy.py | 17 +- .../pandas/scripts/pandas-testing-plugin.py | 63 +--- .../cudf/cudf/tests/reshape/test_unstack.py | 23 +- 10 files changed, 549 insertions(+), 171 deletions(-) diff --git a/python/cudf/cudf/core/column/column.py b/python/cudf/cudf/core/column/column.py index a69e6e1dfcae..d2505f416cf0 100644 --- a/python/cudf/cudf/core/column/column.py +++ b/python/cudf/cudf/core/column/column.py @@ -1175,6 +1175,17 @@ def to_pandas( # xref https://github.com/rapidsai/cudf/issues/21120 # TODO: Revisit using pa_array.to_pandas() once pandas 3.0 is supported np_array = pa_array.to_numpy(zero_copy_only=False, writable=True) + if ( + cudf.get_option("mode.pandas_compatible") + and isinstance(self.dtype, np.dtype) + and self.dtype.kind == "b" + and self.has_nulls() + ): + # pandas represents missing values in an (upcast-to-object) + # bool column as np.nan; pyarrow's to_numpy yields None. + np_array[pa_array.is_null().to_numpy(zero_copy_only=False)] = ( + np.nan + ) return pd.Index( np_array, dtype=np_array.dtype, diff --git a/python/cudf/cudf/core/column/numerical.py b/python/cudf/cudf/core/column/numerical.py index a1d14959e7ca..251514aa66ef 100644 --- a/python/cudf/cudf/core/column/numerical.py +++ b/python/cudf/cudf/core/column/numerical.py @@ -911,20 +911,23 @@ def as_numerical_column(self, dtype: DtypeObj) -> NumericalColumn: self.dtype ): # Short-circuit the cast if the dtypes are equivalent - # but not the same type object. + # but not the same type object. Do NOT mutate self._dtype: + # the column object may be shared with the caller's frame. if ( is_pandas_nullable_extension_dtype(dtype) and isinstance(self.dtype, np.dtype) and self.dtype.kind == "f" ): - # If the dtype is a pandas nullable extension type, we need to - # float column doesn't have any NaNs. + # NaNs must become nulls before viewing as a masked dtype. res = self.nans_to_nulls() - res._dtype = dtype - return res - else: - self._dtype = dtype - return self + return cast( + "NumericalColumn", + ColumnBase.create(res.plc_column, dtype), + ) + return cast( + "NumericalColumn", + ColumnBase.create(self.plc_column, dtype), + ) if self.dtype.kind == "f" and dtype.kind in "iu": if not is_pandas_nullable_extension_dtype(dtype) and ( self.nan_count > 0 diff --git a/python/cudf/cudf/core/column_accessor.py b/python/cudf/cudf/core/column_accessor.py index 5944e5486fe3..19da148086b1 100644 --- a/python/cudf/cudf/core/column_accessor.py +++ b/python/cudf/cudf/core/column_accessor.py @@ -35,6 +35,32 @@ def _is_bool(val: Any) -> bool: return isinstance(val, (bool, np.bool_)) +def _is_nan_scalar(val: Any) -> bool: + return isinstance(val, float) and val != val + + +def _label_contains_nan(label: Any) -> bool: + if isinstance(label, tuple): + return any(_is_nan_scalar(lv) for lv in label) + return _is_nan_scalar(label) + + +def _canonicalize_nan_label(label: Any) -> Any: + """Map float NaN elements of a label to the np.nan singleton. + + Distinct float('nan') objects hash and compare unequal, so NaN-containing + labels round-tripped through a pandas Index (which materializes fresh NaN + objects on iteration) never match dict keys. Canonicalizing to the np.nan + singleton restores pandas' all-NaNs-are-equal label semantics (dict/tuple + comparison uses the per-element identity shortcut). + """ + if isinstance(label, tuple): + return tuple(np.nan if _is_nan_scalar(lv) else lv for lv in label) + if _is_nan_scalar(label): + return np.nan + return label + + class _NestedGetItemDict(dict): """A dictionary whose __getitem__ method accesses nested dicts. @@ -125,6 +151,11 @@ def __init__( self.rangeindex: bool = data.rangeindex self.label_dtype: DtypeObj | None = data.label_dtype self._level_dtypes = data._level_dtypes + if "to_pandas_index" in data.__dict__: + # carry over the primed/cached pandas index: it holds + # fidelity (e.g. explicit unsorted level order) that a + # rebuild from tuples would lose + self.to_pandas_index = data.__dict__["to_pandas_index"] elif isinstance(data, MutableMapping): # This code path is performance-critical for copies and should be # modified with care. @@ -162,7 +193,19 @@ def __iter__(self) -> Iterator: return iter(self._data) def __getitem__(self, key: Hashable) -> ColumnBase: - return self._data[key] + try: + return self._data[key] + except KeyError: + if _label_contains_nan(key): + # NaN labels lose object identity when round-tripped through + # a pandas Index; retry with NaNs canonicalized so all NaNs + # compare equal, matching pandas label semantics. + canon = _canonicalize_nan_label(key) + for existing in self._data: + c = _canonicalize_nan_label(existing) + if c is canon or c == canon: + return self._data[existing] + raise def __setitem__(self, key: Hashable, value: ColumnBase) -> None: self.set_by_label(key, value) @@ -322,6 +365,33 @@ def to_pandas_index(self) -> pd.Index: self.names, names=self.level_names, ) + if ( + self._level_dtypes is not None + and len(self._level_dtypes) == result.nlevels + ): + # ``from_tuples`` re-infers every level dtype from the + # materialized labels, degrading e.g. categorical levels + # to str, object levels to str once mixed-type labels are + # selected away, and int64 levels with missing entries to + # float64. Restore each preserved level dtype when the + # cast is lossless (round-trips to the inferred values). + new_levels = [] + changed = False + for lvl, level_dtype in zip( + result.levels, self._level_dtypes, strict=True + ): + if lvl.dtype != level_dtype: + try: + cast_lvl = lvl.astype(level_dtype) + except (TypeError, ValueError): + pass + else: + if (cast_lvl.astype(lvl.dtype) == lvl).all(): + lvl = cast_lvl + changed = True + new_levels.append(lvl) + if changed: + result = result.set_levels(new_levels) else: # Determine if we can return a RangeIndex if self.rangeindex: diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index b03f3f0c7fd7..7cab9984fb32 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -47,10 +47,12 @@ is_decimal128_dtype, is_dict_like, is_dtype_equal, + is_integer, is_list_like, is_scalar, ) from cudf.core import indexing_utils, reshape +from cudf.core.algorithms import factorize from cudf.core.column import ( CategoricalColumn, ColumnBase, @@ -89,6 +91,7 @@ ) from cudf.core.indexed_frame import ( IndexedFrame, + _check_duplicate_level_names, _FrameIndexer, _indices_from_labels, doc_reset_index_template, @@ -615,7 +618,12 @@ def _pd_index_level_dtypes(idx) -> tuple | None: dtype cannot be inferred from zero entries). """ if isinstance(idx, pd.MultiIndex): - return tuple(idx.get_level_values(i).dtype for i in range(idx.nlevels)) + # use the levels, not get_level_values: materializing a level that + # has missing entries (-1 codes) upcasts e.g. int64 to float64 + return tuple(level.dtype for level in idx.levels) + if isinstance(idx, cudf.MultiIndex): + # the per-row columns share their dtype with the levels + return tuple(dtype for _, dtype in idx._dtypes) return None @@ -837,7 +845,7 @@ def _array_to_column_accessor( columns_labels = columns else: columns_labels = pd.RangeIndex(data.shape[1]) - return ColumnAccessor( + ca = ColumnAccessor( { column_label: as_column(data[:, i], nan_as_null=nan_as_null) for column_label, i in zip( @@ -851,6 +859,11 @@ def _array_to_column_accessor( level_names=tuple(columns_labels.names), level_dtypes=_pd_index_level_dtypes(columns_labels), ) + if isinstance(columns_labels, pd.MultiIndex): + # prime the cache with the exact source MultiIndex (rebuilding from + # tuples would re-sort the levels) + ca.to_pandas_index = columns_labels + return ca @_performance_tracking @@ -1279,6 +1292,10 @@ def __init__( label_dtype=columns.dtype, level_dtypes=_pd_index_level_dtypes(columns), ) + if isinstance(columns, pd.MultiIndex): + # prime the cache with the exact source MultiIndex + # (rebuilding from tuples would re-sort the levels) + col_accessor.to_pandas_index = columns elif isinstance(data, Mapping): # Note: We excluded ColumnAccessor already above result = _mapping_to_column_accessor( @@ -1350,6 +1367,20 @@ def __init__( if dtype: self._data = self.astype(dtype)._data + final_pd_columns = ( + second_columns if second_columns is not None else columns + ) + if ( + self._data.multiindex + and isinstance(final_pd_columns, pd.MultiIndex) + and len(self._data) == len(final_pd_columns) + ): + # prime the cache with the exact source MultiIndex: rebuilding + # from tuples would re-sort the levels, losing e.g. an explicit + # unsorted level layout (level order affects pandas operations + # that work on codes, like legacy stack's sort) + self._data.to_pandas_index = final_pd_columns + @classmethod def _from_data( # type: ignore[override] cls, @@ -3235,6 +3266,7 @@ def columns(self, columns): rangeindex = False label_dtype = None level_names = None + level_dtypes = None if isinstance(columns, (pd.MultiIndex, cudf.MultiIndex)): multiindex = True if isinstance(columns, cudf.MultiIndex): @@ -3244,6 +3276,7 @@ def columns(self, columns): if pd_columns.nunique(dropna=False) != len(pd_columns): raise ValueError("Duplicate column names are not allowed") level_names = list(pd_columns.names) + level_dtypes = _pd_index_level_dtypes(pd_columns) elif isinstance(columns, (Index, ColumnBase, Series)): level_names = (getattr(columns, "name", None),) rangeindex = isinstance(columns, cudf.RangeIndex) @@ -3280,8 +3313,15 @@ def columns(self, columns): level_names=level_names, label_dtype=label_dtype, rangeindex=rangeindex, + level_dtypes=level_dtypes, verify=False, ) + if multiindex: + # prime the cache with the exact source MultiIndex: rebuilding + # from tuples would re-sort the levels, losing e.g. an explicit + # unsorted level layout (levels order affects pandas operations + # that work on codes, like legacy stack's sort) + self._data.to_pandas_index = pd_columns def _set_columns_like(self, other: ColumnAccessor) -> None: """ @@ -8081,6 +8121,11 @@ def stack( "https://github.com/pandas-dev/pandas/issues/53515" ) + _check_duplicate_level_names( + [lv for lv in level if not is_integer(lv)], + self._data.level_names, + ) + # Compute the columns to stack based on specified levels level_indices: list[int] = [] @@ -8095,10 +8140,24 @@ def stack( "mixture of both." ) else: - # Must be a list of positions, normalize negative positions - level_indices = [ - lv + self._data.nlevels if lv < 0 else lv for lv in level - ] + # Must be a list of positions; normalize negative positions + # and validate bounds to match pandas MultiIndex._get_level_number + nlevels = self._data.nlevels + for lv in level: + if lv < 0: + if lv + nlevels < 0: + raise IndexError( + f"Too many levels: Index has only {nlevels} " + f"levels, {lv} is not a valid level number" + ) + level_indices.append(lv + nlevels) + else: + if lv >= nlevels: + raise IndexError( + f"Too many levels: Index has only {nlevels} " + f"levels, not {lv + 1}" + ) + level_indices.append(lv) unnamed_levels_indices = [ i for i in range(self._data.nlevels) if i not in level_indices @@ -8106,35 +8165,84 @@ def stack( has_unnamed_levels = len(unnamed_levels_indices) > 0 column_name_idx = self._data.to_pandas_index - # Construct new index from the levels specified by `level` - named_levels = pd.MultiIndex.from_arrays( - [column_name_idx.get_level_values(lv) for lv in level_indices] + # pandas' Index.get_level_values resolves an integer argument by + # name first: if a level is *named* that integer, that level is + # returned regardless of position. All lookups below use positional + # indices, so strip the names to force positional resolution and + # re-attach the real names afterwards. + nameless_column_name_idx = column_name_idx.set_names( + [None] * column_name_idx.nlevels ) + # Construct new index from the levels specified by `level` + if isinstance(column_name_idx, pd.MultiIndex): + # build from codes/levels to keep the level dtypes: materializing + # via get_level_values/from_arrays turns missing entries into NaN + # and upcasts e.g. int64 levels to float64 + named_levels = pd.MultiIndex( + levels=[column_name_idx.levels[i] for i in level_indices], + codes=[column_name_idx.codes[i] for i in level_indices], + names=[column_name_idx.names[i] for i in level_indices], + verify_integrity=False, + ) + else: + named_levels = pd.MultiIndex.from_arrays( + [ + nameless_column_name_idx.get_level_values(lv).rename( + column_name_idx.names[lv] + ) + for lv in level_indices + ] + ) # Since `level` may only specify a subset of all levels, `unique()` is - # required to remove duplicates. In pandas, the order of the keys in - # the specified levels are always sorted. + # required to remove duplicates. In pandas legacy stack, the keys of + # the specified levels are sorted by their level *codes* when the + # columns have multiple levels (flat column labels keep their + # original order): level order is preserved even for unsorted levels + # and missing labels (code -1) come first. unique_named_levels = named_levels.unique() - if not future_stack: - unique_named_levels = unique_named_levels.sort_values() + if not future_stack and self._data.nlevels > 1: + unique_named_levels = unique_named_levels.take( + np.lexsort(tuple(reversed(unique_named_levels.codes))) + ) # Each index from the original dataframe should repeat by the number # of unique values in the named_levels repeated_index = self.index.repeat(len(unique_named_levels)) # Each column name should tile itself by len(df) times - cols = [ - as_column(unique_named_levels.get_level_values(i)) - for i in range(unique_named_levels.nlevels) - ] + nameless_unique_named_levels = unique_named_levels.set_names( + [None] * unique_named_levels.nlevels + ) + cols = [] + for i in range(unique_named_levels.nlevels): + if future_stack: + # pandas future stack materializes the level values (a + # level with missing entries becomes e.g. float64 with NaN) + cols.append( + as_column(nameless_unique_named_levels.get_level_values(i)) + ) + else: + # pandas legacy stack keeps the original level dtype and + # represents missing entries as nulls (-1 codes) + level_col = as_column(unique_named_levels.levels[i]) + level_codes = np.asarray(unique_named_levels.codes[i]).astype( + "int64" + ) + level_codes[level_codes == -1] = np.iinfo(SIZE_TYPE_DTYPE).min + cols.append( + level_col.take(as_column(level_codes), nullify=True) + ) with access_columns(*cols, mode="read", scope="internal"): plc_table = plc.reshape.tile( plc.Table([col.plc_column for col in cols]), self.shape[0], ) tiled_index = [ - ColumnBase.create(plc, dtype=dtype_from_pylibcudf_column(plc)) - for plc in plc_table.columns() + ColumnBase.create(plc_col, dtype=src_col.dtype) + for src_col, plc_col in zip( + cols, plc_table.columns(), strict=True + ) ] # Assemble the final index @@ -8148,6 +8256,42 @@ def stack( # axis (e.g. ``"foo_10"`` before ``"foo_2"``). new_index._maybe_materialize_codes_and_levels(sort=False) new_index.names = index_names + # Attach codes/levels eagerly, matching how pandas' stack builds the + # result MultiIndex, so a later unstack can restore the original + # row/column order (lazy materialization would sort the levels): + # the original index contributes its own levels/codes (repeated); + # a flat original index and the tiled stacked level(s) get + # appearance-order factorization. + new_levels = [] + new_codes = [] + n_tile = len(unique_named_levels) + if isinstance(self.index, MultiIndex): + src = self.index._maybe_materialize_codes_and_levels() + for src_level, src_code in zip( + src._levels, + src._codes, + strict=True, + ): + new_levels.append(src_level) + new_codes.append( + Index._from_column(src_code.astype(np.dtype(np.int64))) + .repeat(n_tile) + ._column + ) + else: + code, cats = factorize(self.index) + new_levels.append(cats) + new_codes.append( + Index._from_column(as_column(code).astype(np.dtype(np.int64))) + .repeat(n_tile) + ._column + ) + for tiled_col in tiled_index: + code, cats = factorize(Index._from_column(tiled_col)) + new_codes.append(as_column(code).astype(np.dtype(np.int64))) + new_levels.append(cats) + new_index._levels = new_levels + new_index._codes = new_codes # Compute the column indices that serves as the input for # `interleave_columns` @@ -8156,41 +8300,49 @@ def stack( ) if has_unnamed_levels: - unnamed_level_values = pd.MultiIndex.from_arrays( - list( - map( - column_name_idx.get_level_values, - unnamed_levels_indices, - ) - ) + # the columns axis has multiple levels here, so column_name_idx + # is always a pd.MultiIndex; build from codes/levels to keep the + # level dtypes and to resolve the levels positionally + unnamed_level_values = pd.MultiIndex( + levels=[ + column_name_idx.levels[i] for i in unnamed_levels_indices + ], + codes=[ + column_name_idx.codes[i] for i in unnamed_levels_indices + ], + names=[ + column_name_idx.names[i] for i in unnamed_levels_indices + ], + verify_integrity=False, ) def unnamed_group_generator(): if has_unnamed_levels: - for _, grpdf in column_idx_df.groupby(by=unnamed_level_values): + # sort=False iterates groups in first-appearance order, i.e. + # exactly ``unnamed_level_values.unique()`` order (also for + # NaN-containing tuple keys, which sorted groupby would + # reorder via codes), so the stacked columns can be zipped + # 1:1 with those keys when assembling the result. + for _, grpdf in column_idx_df.groupby( + by=unnamed_level_values, sort=False + ): # When stacking part of the levels, some combinations # of keys may not be present in this group but can be # present in others. Reindexing with the globally computed # `unique_named_levels` assigns -1 to these key # combinations, representing an all-null column that # is used in the subsequent libcudf call. - if future_stack: - yield grpdf.reindex( - unique_named_levels, axis=0, fill_value=-1 - ).values - else: - yield ( - grpdf.reindex( - unique_named_levels, axis=0, fill_value=-1 - ) - .sort_index() - .values - ) + # ``reindex`` returns rows in target order, so the + # legacy path needs no further sorting (the target was + # already sorted above). + yield grpdf.reindex( + unique_named_levels, axis=0, fill_value=-1 + ).values else: - if future_stack: + if future_stack or self._data.nlevels == 1: yield column_idx_df.values else: - yield column_idx_df.sort_index().values + yield column_idx_df.reindex(unique_named_levels).values # For each of the group constructed from the unnamed levels, # invoke `interleave_columns` to stack the values. @@ -8246,23 +8398,35 @@ def unnamed_group_generator(): unnamed_level_values = unnamed_level_values.get_level_values(0) unnamed_level_values = unnamed_level_values.unique() - data = ColumnAccessor( - dict( - zip( - unnamed_level_values, - [ - stacked[i] - for i in unnamed_level_values.argsort().argsort() - ] - if not future_stack - else [ - stacked[i] for i in unnamed_level_values.argsort() - ], - strict=True, + if isinstance(unnamed_level_values, pd.MultiIndex): + # build the labels from levels/codes to preserve scalar + # types: iterating a MultiIndex materializes e.g. an int64 + # level containing a missing entry as float + keys: Any = [ + tuple( + unnamed_level_values.levels[j][c] + if c != -1 + else np.nan + for j, c in enumerate(row) ) - ), + for row in zip(*unnamed_level_values.codes, strict=True) + ] + else: + keys = unnamed_level_values + + # ``stacked`` is in group first-appearance order (groupby with + # sort=False above), which is exactly the order of + # ``unnamed_level_values.unique()``: zip 1:1. + data = ColumnAccessor( + dict(zip(keys, stacked, strict=True)), isinstance(unnamed_level_values, pd.MultiIndex), unnamed_level_values.names, + label_dtype=( + None + if isinstance(unnamed_level_values, pd.MultiIndex) + else unnamed_level_values.dtype + ), + level_dtypes=_pd_index_level_dtypes(unnamed_level_values), ) result = DataFrame._from_data( @@ -8270,7 +8434,24 @@ def unnamed_group_generator(): ) if not future_stack and dropna: - return result.dropna(how="all") + # dropna would gather the index and discard the eagerly + # attached appearance-order codes/levels; compute the row mask + # explicitly so the codes can be subset alongside (pandas keeps + # the full pre-drop levels through dropna). + if isinstance(result, Series): + keep = result.notna() + else: + keep = ~result.isna().all(axis=1) + keep_col = keep._column + dropped = result._apply_boolean_mask( + BooleanMask(keep_col, len(result)) + ) + if isinstance(dropped.index, MultiIndex): + dropped.index._levels = new_levels + dropped.index._codes = [ + code.apply_boolean_mask(keep_col) for code in new_codes + ] + return dropped else: return result diff --git a/python/cudf/cudf/core/groupby/groupby.py b/python/cudf/cudf/core/groupby/groupby.py index 73e5ac65b4e5..7c0ea6e247ea 100644 --- a/python/cudf/cudf/core/groupby/groupby.py +++ b/python/cudf/cudf/core/groupby/groupby.py @@ -1305,9 +1305,10 @@ def agg(self, func=None, *args, engine=None, engine_kwargs=None, **kwargs): ): data = ColumnAccessor( data, - multiindex=False, + multiindex=self.obj._data.multiindex, level_names=self.obj._data.level_names, label_dtype=self.obj._data.label_dtype, + level_dtypes=self.obj._data.level_dtypes, ) else: data = ColumnAccessor(data, multiindex=multilevel) diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index 8a1db59fa71a..b3f68d61d8bb 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -2921,7 +2921,30 @@ def sort_index( if ignore_index: out = out.reset_index(drop=True) else: - labels = sorted(self._column_names, reverse=not ascending) + if level is not None and self._data.multiindex: + if not is_list_like(level): + level = [level] + nlevels = self._data.nlevels + level_names = list(self._data.level_names) + + def _level_number(lvl): + if isinstance(lvl, int): + return lvl + nlevels if lvl < 0 else lvl + return level_names.index(lvl) + + key_order = [_level_number(lvl) for lvl in level] + if sort_remaining: + seen = set(key_order) + key_order.extend( + i for i in range(nlevels) if i not in seen + ) + labels = sorted( + self._column_names, + key=lambda label: tuple(label[i] for i in key_order), + reverse=not ascending, + ) + else: + labels = sorted(self._column_names, reverse=not ascending) result_columns = (self._data[label] for label in labels) if ignore_index: ca = ColumnAccessor( diff --git a/python/cudf/cudf/core/reshape.py b/python/cudf/cudf/core/reshape.py index e8736f789377..4abef6d26b74 100644 --- a/python/cudf/cudf/core/reshape.py +++ b/python/cudf/cudf/core/reshape.py @@ -11,7 +11,7 @@ import cudf from cudf.api.extensions import no_default -from cudf.api.types import is_list_like, is_scalar +from cudf.api.types import is_integer, is_list_like, is_scalar from cudf.core.column import ( ColumnBase, as_column, @@ -938,8 +938,11 @@ def get_dummies( def _pivot( col_accessor: ColumnAccessor, - index: Index | MultiIndex, - columns: Index | MultiIndex, + index_labels: Index | MultiIndex, + index_idx: ColumnBase, + columns_labels: Index | MultiIndex, + columns_idx: ColumnBase, + promote_ints_on_missing: bool = False, ) -> DataFrame: """ Reorganize the values of the DataFrame according to the given @@ -947,14 +950,22 @@ def _pivot( Parameters ---------- - col_accessor : DataFrame - index : Index - Index labels of the result - columns : Index - Column labels of the result + col_accessor : ColumnAccessor + Values to pivot into the result's columns. + index_labels : Index + Distinct index keys; row labels of the result. + index_idx : ColumnBase + Position of each source row's key within ``index_labels``. + columns_labels : Index + Distinct column keys; labels of the result's new column level(s). + columns_idx : ColumnBase + Position of each source row's key within ``columns_labels``. + promote_ints_on_missing : bool + Promote integer source columns to float64 when the reshape + introduces missing cells, as pandas' unstack does. Only the + unstack path wants this: pivot_table/crosstab fill missing + cells afterwards and keep the integer dtype. """ - columns_labels, columns_idx = columns._encode() - index_labels, index_idx = index._encode() column_labels = columns_labels.to_pandas().to_flat_index() result = {} @@ -964,12 +975,26 @@ def as_tuple(x): return x if isinstance(x, tuple) else (x,) nrows = len(index_labels) + promote_ints = promote_ints_on_missing and cudf.get_option( + "mode.pandas_compatible" + ) for col_label, col in col_accessor.items(): names = [ as_tuple(col_label) + as_tuple(name) for name in column_labels ] new_size = nrows * len(names) scatter_map = (columns_idx * np.int32(nrows)) + index_idx + if ( + promote_ints + and new_size > len(col) + and isinstance(col.dtype, np.dtype) + and col.dtype.kind in "iu" + ): + # pandas builds one 2-D values block per source column and + # promotes the whole block to float64 when the reshape + # introduces missing entries, so even gap-free result + # columns become float64 + col = col.astype(np.dtype(np.float64)) target_col = column_empty(row_count=new_size, dtype=col.dtype) target_col[scatter_map] = col result.update( @@ -984,16 +1009,82 @@ def as_tuple(x): ) ) - # the result of pivot always has a MultiIndex + # the result of pivot always has a MultiIndex; the leading level(s) + # come from the source frame's column labels, so preserve their names ca = ColumnAccessor( result, multiindex=True, - level_names=(None, *columns._column_names), + level_names=( + *col_accessor.level_names, + *columns_labels._column_names, + ), verify=False, ) return cudf.DataFrame._from_data(ca, index=index_labels) +def _unstack_encode_by_codes( + mi: MultiIndex, level +) -> tuple[Index | MultiIndex, ColumnBase, Index | MultiIndex, ColumnBase]: + """Encode unstack keys ordered by the MultiIndex level codes. + + libcudf's ``encode`` orders distinct keys by sorted value with nulls + last, but pandas' unstack orders keys by the index's level codes: the + level order is preserved and missing entries (code -1) come first. + Encoding the integer code columns instead of the level values yields + exactly that order. + """ + lvl_idx = mi._level_index_from_level(level) + mi._maybe_materialize_codes_and_levels() + names = mi.names + + def encode_side(sel: list[int]) -> tuple[Index | MultiIndex, ColumnBase]: + code_cols = [] + for i in sel: + code = mi._codes[i].astype(np.dtype(np.int64)).copy() # type: ignore[index] + # Normalize the NA sentinel (``MultiIndex.__init__`` stores + # ``iinfo(SIZE_TYPE_DTYPE).min``, lazy factorization stores -1) + # so the missing-key group encodes as one key that sorts first. + code[code < 0] = -1 + code_cols.append(code) + code_frame = cudf.DataFrame._from_data( + ColumnAccessor(dict(enumerate(code_cols)), verify=False) + ) + key_codes, idx = code_frame._encode() + labels_data = {} + out_levels = [] + out_codes = [] + for j, i in enumerate(sel): + kc = key_codes._columns[j].astype(np.dtype(np.int64)) + out_levels.append(mi._levels[i]) # type: ignore[index] + out_codes.append(kc) + gather_codes = kc.copy() + gather_codes[gather_codes == -1] = np.iinfo(SIZE_TYPE_DTYPE).min + # key by position: level names may be duplicated or None + labels_data[j] = mi._levels[i]._column.take( # type: ignore[index] + gather_codes, nullify=True + ) + if len(labels_data) == 1: + labels: Index | MultiIndex = cudf.Index._from_column( + next(iter(labels_data.values())), name=names[sel[0]] + ) + else: + mi_labels = cudf.MultiIndex._from_data(labels_data) + mi_labels.names = [names[i] for i in sel] + # carry the original level objects and the keys' codes so that + # a subsequent unstack/stack keeps ordering by the original + # levels, exactly like pandas (which reuses the level objects) + mi_labels._levels = out_levels + mi_labels._codes = out_codes + labels = mi_labels + return labels, idx + + remaining = [i for i in range(mi.nlevels) if i != lvl_idx] + index_labels, index_idx = encode_side(remaining) + columns_labels, columns_idx = encode_side([lvl_idx]) + return index_labels, index_idx, columns_labels, columns_idx + + def pivot( data: DataFrame, columns=None, index=no_default, values=no_default ) -> DataFrame: @@ -1125,8 +1216,15 @@ def pivot( if len(columns_index) != len(columns_index.drop_duplicates()): raise ValueError("Duplicate index-column pairs found. Cannot reshape.") + selection = data._data.select_by_label(cols_to_select) + if values is not no_default: + # pandas rebuilds the columns axis from ``values`` and drops the + # original columns-axis name(s) + selection._level_names = (None,) * selection.nlevels + columns_labels, columns_idx = column_data._encode() + index_labels, index_idx = index_data._encode() result = _pivot( - data._data.select_by_label(cols_to_select), index_data, column_data + selection, index_labels, index_idx, columns_labels, columns_idx ) result._attrs = data.attrs @@ -1229,6 +1327,23 @@ def unstack(df, level, fill_value=None, sort: bool = True): 2 7 dtype: int64 """ + return _unstack(df, level, fill_value=fill_value, sort=sort) + + +def _unstack( + df, + level, + fill_value=None, + sort: bool = True, + promote_ints_on_missing: bool = True, +): + """``unstack`` implementation. + + ``promote_ints_on_missing`` promotes integer source columns to float64 + when the reshape introduces missing cells, like pandas' unstack. + ``pivot_table`` (and thereby ``crosstab``) disables it because those fill + the missing cells afterwards and keep the integer dtype. + """ if not isinstance(df, cudf.DataFrame): raise ValueError("`df` should be a cudf Dataframe object.") @@ -1256,7 +1371,14 @@ def unstack(df, level, fill_value=None, sort: bool = True): if not is_scalar(level): if not level: return df + if len(level) == 1: + # pandas normalizes a length-1 list-like level to a scalar + level = level[0] if not isinstance(df.index, cudf.MultiIndex): + if not is_integer(level): + # pandas validates non-integer levels against the flat index + # name and raises KeyError on a mismatch + df.index._validate_index_level(level) dtype = df._columns[0].dtype if any(col_dtype != dtype for _, col_dtype in df._dtypes): raise ValueError( @@ -1271,10 +1393,22 @@ def unstack(df, level, fill_value=None, sort: bool = True): res._attrs = df.attrs return res else: - index = df.index.droplevel(level) + from cudf.core.indexed_frame import _check_duplicate_level_names + + specified = [level] if is_scalar(level) else list(level) + _check_duplicate_level_names( + [lv for lv in specified if not is_integer(lv)], + df.index.names, + ) if is_scalar(level): - columns = df.index.get_level_values(level) + # order rows/columns by the removed level's codes (pandas + # semantics: level order preserved, missing entries first), + # not by sorted level values + index_labels, index_idx, columns_labels, columns_idx = ( + _unstack_encode_by_codes(df.index, level) + ) else: + index = df.index.droplevel(level) new_names = [] ca_data = {} for lev in level: @@ -1285,8 +1419,34 @@ def unstack(df, level, fill_value=None, sort: bool = True): ColumnAccessor(ca_data, verify=False) ) columns.names = new_names - result = _pivot(df, index, columns) + columns_labels, columns_idx = columns._encode() + index_labels, index_idx = index._encode() + result = _pivot( + df._data, + index_labels, + index_idx, + columns_labels, + columns_idx, + promote_ints_on_missing=promote_ints_on_missing, + ) result._attrs = df.attrs + if is_scalar(level): + # pandas keeps unused categories of the removed level + # ("removed_level_full", pandas GH 17845) in + # result.columns.levels even though no columns are created + # for them. + _, level_idx = df.index._level_to_ca_label(level) + full_level = df.index.levels[level_idx].to_pandas() + pdi = result._data.to_pandas_index + if isinstance(pdi, pd.MultiIndex): + new_codes = full_level.get_indexer(pdi.get_level_values(-1)) + if (new_codes >= 0).all(): + result._data.to_pandas_index = pd.MultiIndex( + levels=[*pdi.levels[:-1], full_level], + codes=[*pdi.codes[:-1], new_codes], + names=pdi.names, + verify_integrity=False, + ) if result.index.nlevels == 1: result.index = result.index.get_level_values(result.index.names[0]) return result @@ -1600,7 +1760,7 @@ def pivot_table( to_unstack.append(i) else: to_unstack.append(name) - table = agged.unstack(to_unstack) + table = _unstack(agged, to_unstack, promote_ints_on_missing=False) if fill_value is not None: table = table.fillna(fill_value) diff --git a/python/cudf/cudf/pandas/fast_slow_proxy.py b/python/cudf/cudf/pandas/fast_slow_proxy.py index e228f5fd0589..662f05d565fd 100644 --- a/python/cudf/cudf/pandas/fast_slow_proxy.py +++ b/python/cudf/cudf/pandas/fast_slow_proxy.py @@ -1120,13 +1120,16 @@ def __get__(self, instance, owner) -> Any: raise e if _is_function_or_method(slow_attr): - self._attr = _MethodProxy( - fast_attr, - slow_attr, - _fsproxy_transfer_block=instance.get_transfer_blocking() - if instance is not None - else None, - ) + # Do NOT bake ``instance.get_transfer_blocking()`` into the + # _MethodProxy: ``self._attr`` is cached on the descriptor and + # shared by every instance of the proxy class, so a blocking + # state captured from whichever instance happens to resolve + # the attribute first would leak into every later call on any + # instance. Per-instance blocking is still enforced at call + # time: ``_fsproxy_fast`` raises RuntimeError for a blocked + # SLOW instance during ``_fast_arg`` conversion, forcing the + # slow path for that instance only. + self._attr = _MethodProxy(fast_attr, slow_attr) else: # for anything else, use a fast-slow attribute: self._attr, _ = _fast_slow_function_call( diff --git a/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py b/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py index 36bd33ad4e67..284b85ccc475 100644 --- a/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py +++ b/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py @@ -1715,67 +1715,8 @@ def pytest_unconfigure(config): "tests/frame/test_reductions.py::test_reduction_axis_none_returns_scalar[float64-True-mean]": "TODO: Add a reason for failure", "tests/frame/test_reductions.py::test_reduction_axis_none_returns_scalar[float64-True-median]": "TODO: Add a reason for failure", "tests/frame/test_reductions.py::test_reduction_axis_none_returns_scalar[float64-True-skew]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_stack_int_level_names[False]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_stack_int_level_names[True]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_stack_ints[False]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_stack_ints[True]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_stack_mixed_level[False]": "AssertionError: DataFrame.columns are different", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_stack_mixed_level[True]": "AssertionError: DataFrame.columns are different", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_stack_mixed_levels[False]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_stack_mixed_levels[True]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_stack_multi_preserve_categorical_dtype[False-labels0-data0-False]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_stack_multi_preserve_categorical_dtype[False-labels0-data0-True]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_stack_multi_preserve_categorical_dtype[False-labels1-data1-False]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_stack_multi_preserve_categorical_dtype[False-labels1-data1-True]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_stack_multi_preserve_categorical_dtype[True-labels0-data0-False]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_stack_multi_preserve_categorical_dtype[True-labels0-data0-True]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_stack_multi_preserve_categorical_dtype[True-labels1-data1-False]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_stack_multi_preserve_categorical_dtype[True-labels1-data1-True]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_stack_preserve_categorical_dtype[False-False]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_stack_preserve_categorical_dtype[False-True]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_stack_preserve_categorical_dtype[True-False]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_stack_preserve_categorical_dtype[True-True]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_stack_unstack[False]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_stack_unstack[True]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_unstack_bool": "AssertionError: DataFrame.iloc[:, 0] (column name='('col', 'c')') are different", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_unstack_multi_level_rows_and_cols": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_unstack_nan_index2": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_unstack_nan_index3": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_unstack_non_unique_index_names[False]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_unstack_non_unique_index_names[True]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_unstack_not_consolidated": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_unstack_swaplevel_sortlevel[0]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_unstack_swaplevel_sortlevel[baz]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_unstack_unused_levels": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_multi_level_stack_categorical[False]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_multi_level_stack_categorical[True]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack[False]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack[True]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_level_name[False]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_level_name[True]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_mixed_dtype[True]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_multiple_bug[False]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_multiple_bug[True]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_multiple_out_of_bounds[False]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_multiple_out_of_bounds[True]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_nan_in_multiindex_columns[False]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_nan_in_multiindex_columns[True]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_nan_level[False]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_nullable_dtype[False]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_nullable_dtype[True]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_order_with_unsorted_levels_multi_row_2[False]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_unstack_multiple[False]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_unstack_multiple[True]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_unstack_preserve_names[False]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_unstack_preserve_names[True]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_unstack_wrong_level_name[False-unstack]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_unstack_wrong_level_name[True-unstack]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_unstack_preserve_types": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_unstack_with_missing_int_cast_to_float": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::test_unstack_sort_false_nan[nan=first]": "AssertionError: Attributes of DataFrame.iloc[:, 0] (column name='('value', nan)') are different", - "tests/frame/test_stack_unstack.py::test_unstack_sort_false_nan[nan=last]": "AssertionError: Attributes of DataFrame.iloc[:, 3] (column name='('value', nan)') are different", - "tests/frame/test_stack_unstack.py::test_unstack_sort_false_nan[nan=second]": "AssertionError: Attributes of DataFrame.iloc[:, 1] (column name='('value', nan)') are different", - "tests/frame/test_stack_unstack.py::test_unstack_sort_false_nan[nan=third]": "AssertionError: Attributes of DataFrame.iloc[:, 2] (column name='('value', nan)') are different", + "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_unstack_not_consolidated": "Asserts DataFrame._mgr block layout (pandas internals)", + "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_unstack_with_missing_int_cast_to_float": "Asserts DataFrame._mgr block layout (pandas internals)", "tests/frame/test_subclass.py::TestDataFrameSubclassing::test_asof": "TODO: Add a reason for failure", "tests/frame/test_subclass.py::TestDataFrameSubclassing::test_equals_subclass": "TODO: Add a reason for failure", "tests/frame/test_subclass.py::TestDataFrameSubclassing::test_frame_subclassing_and_slicing": "TODO: Add a reason for failure", diff --git a/python/cudf/cudf/tests/reshape/test_unstack.py b/python/cudf/cudf/tests/reshape/test_unstack.py index 0edf16202d31..3a2462c4b49a 100644 --- a/python/cudf/cudf/tests/reshape/test_unstack.py +++ b/python/cudf/cudf/tests/reshape/test_unstack.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 import re @@ -14,20 +14,10 @@ "level", [ 0, - pytest.param( - 1, - marks=pytest.mark.xfail( - reason="Categorical column indexes not supported" - ), - ), + 1, 2, "foo", - pytest.param( - "bar", - marks=pytest.mark.xfail( - reason="Categorical column indexes not supported" - ), - ), + "bar", "baz", [], pytest.param( @@ -75,12 +65,7 @@ def test_unstack_multiindex(level): [ pd.Index(range(0, 5), name=None), pd.Index(range(0, 5), name="row_index"), - pytest.param( - pd.CategoricalIndex(["d", "e", "f", "g", "h"]), - marks=pytest.mark.xfail( - reason="Categorical column indexes not supported" - ), - ), + pd.CategoricalIndex(["d", "e", "f", "g", "h"]), ], ) @pytest.mark.parametrize( From fca8bb2aa5ebaaccaa2feb6c875f70d5c463d7c2 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Tue, 14 Jul 2026 04:26:21 +0000 Subject: [PATCH 2/6] Make bool-with-nulls to_pandas conversion unconditional Bool was the only dtype whose default-mode to_pandas emitted None for missing values; strings, categoricals and ints already produce nan and datetimes produce NaT, and pandas itself never places None in an upcast-to-object bool column. Dropping the mode.pandas_compatible gate makes the conversion consistent across dtypes and with pandas. This also makes concat of bool and float frames match pandas' float coercion; the corresponding strict xfail in test_concat.py now passes and is removed. --- python/cudf/cudf/core/column/column.py | 5 +++-- python/cudf/cudf/tests/reshape/test_concat.py | 13 +------------ 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/python/cudf/cudf/core/column/column.py b/python/cudf/cudf/core/column/column.py index d2505f416cf0..0907046b9fa1 100644 --- a/python/cudf/cudf/core/column/column.py +++ b/python/cudf/cudf/core/column/column.py @@ -1176,13 +1176,14 @@ def to_pandas( # TODO: Revisit using pa_array.to_pandas() once pandas 3.0 is supported np_array = pa_array.to_numpy(zero_copy_only=False, writable=True) if ( - cudf.get_option("mode.pandas_compatible") - and isinstance(self.dtype, np.dtype) + isinstance(self.dtype, np.dtype) and self.dtype.kind == "b" and self.has_nulls() ): # pandas represents missing values in an (upcast-to-object) # bool column as np.nan; pyarrow's to_numpy yields None. + # np.nan is also what every other cudf dtype converts + # missing values to (nan/NaT) when going to pandas. np_array[pa_array.is_null().to_numpy(zero_copy_only=False)] = ( np.nan ) diff --git a/python/cudf/cudf/tests/reshape/test_concat.py b/python/cudf/cudf/tests/reshape/test_concat.py index 606f23a5bba8..adaa385bd7ae 100644 --- a/python/cudf/cudf/tests/reshape/test_concat.py +++ b/python/cudf/cudf/tests/reshape/test_concat.py @@ -2600,7 +2600,7 @@ def test_concat_empty_dataframe(df_1_data, df_2_data): {}, ], ) -def test_concat_different_column_dataframe(request, df1_d, df2_d): +def test_concat_different_column_dataframe(df1_d, df2_d): pdf1 = pd.DataFrame(df1_d) pdf2 = pd.DataFrame(df2_d) @@ -2614,17 +2614,6 @@ def test_concat_different_column_dataframe(request, df1_d, df2_d): ) expect = pd.concat([pdf1, pdf2, pdf1], sort=False) - xfail_pair = df2_d == { - "a": [1, None, 3], - "b": [True, True, False], - "c": ["s3", None, "s4"], - } and isinstance(df1_d["b"], pd.Series) - request.applymarker( - pytest.mark.xfail( - xfail_pair, - reason="As of pandas 3.0, pandas coerces to float, cuDF coerces to bool", - ) - ) assert_eq(got, expect, check_dtype=False, check_index_type=True) From 82a6d7b36b5ce4ba81e6ff61abfb863eaa7baf9a Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Tue, 14 Jul 2026 04:32:08 +0000 Subject: [PATCH 3/6] Update np ufunc tests for nan-based bool null representation The ufunc tests masked expected bool results with None to match the old to_pandas conversion, with a comment asking whether it should be np.nan instead; it is now. --- python/cudf/cudf/tests/dataframe/test_np_ufuncs.py | 6 +++--- python/cudf/cudf/tests/series/test_np_ufuncs.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/python/cudf/cudf/tests/dataframe/test_np_ufuncs.py b/python/cudf/cudf/tests/dataframe/test_np_ufuncs.py index 430c54866da5..24c6e496cba4 100644 --- a/python/cudf/cudf/tests/dataframe/test_np_ufuncs.py +++ b/python/cudf/cudf/tests/dataframe/test_np_ufuncs.py @@ -123,9 +123,9 @@ def test_ufunc_dataframe(request, numpy_ufunc, has_nulls, indexed): np.less_equal, np.not_equal, ): - # cuDF .to_pandas for bools with nulls represents missing as None, - # should this be np.nan? - expect = expect.astype(object).mask(mask, None) + # cuDF .to_pandas represents missing values in bool + # columns as np.nan (upcast to object), like pandas + expect = expect.astype(object).mask(mask, np.nan) else: expect[mask] = np.nan assert_eq(got, expect, check_exact=False) diff --git a/python/cudf/cudf/tests/series/test_np_ufuncs.py b/python/cudf/cudf/tests/series/test_np_ufuncs.py index 01e7adefede7..446e0843382e 100644 --- a/python/cudf/cudf/tests/series/test_np_ufuncs.py +++ b/python/cudf/cudf/tests/series/test_np_ufuncs.py @@ -110,9 +110,9 @@ def test_ufunc_series(request, numpy_ufunc, has_nulls, indexed): np.less_equal, np.not_equal, ): - # cuDF .to_pandas for bools with nulls represents missing as None, - # should this be np.nan? - expect = expect.astype(object).mask(mask, None) + # cuDF .to_pandas represents missing values in bool + # columns as np.nan (upcast to object), like pandas + expect = expect.astype(object).mask(mask, np.nan) else: expect[mask] = np.nan assert_eq(got, expect, check_exact=False) From 7125055192a5c6938e82ff8c59bf5b23411866db Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Tue, 14 Jul 2026 08:33:20 +0000 Subject: [PATCH 4/6] Address review feedback * GroupBy.agg: preserve MultiIndex columns in the empty-columns branch. * ColumnAccessor.to_pandas_index: use missing-aware Index.equals for the lossless-cast round-trip guard. * sort_index(axis=1, level=...): validate out-of-range integer levels (matching pandas' IndexError), honor per-level ascending lists, and place missing labels per na_position via a null-aware stable multi-key sort. * MultiIndex._level_index_from_level: reject still-negative levels after normalization instead of silently indexing from the end; align the error message with pandas. * unstack: use the public level names for the result's column levels and keep restoring unused categories when the removed level contains NA (-1 codes for NA labels are pandas' canonical representation). * pivot_table: promote integer values to float64 like pandas when missing cells are left unfilled (fill_value=None); keep the integer dtype when fill_value is provided. Add a regression test. --- python/cudf/cudf/core/column_accessor.py | 4 +- python/cudf/cudf/core/groupby/groupby.py | 2 +- python/cudf/cudf/core/indexed_frame.py | 58 ++++++++++++++++--- python/cudf/cudf/core/multiindex.py | 12 ++-- python/cudf/cudf/core/reshape.py | 18 ++++-- .../cudf/tests/reshape/test_pivot_table.py | 31 +++++++++- 6 files changed, 104 insertions(+), 21 deletions(-) diff --git a/python/cudf/cudf/core/column_accessor.py b/python/cudf/cudf/core/column_accessor.py index 19da148086b1..c594ae8e0bad 100644 --- a/python/cudf/cudf/core/column_accessor.py +++ b/python/cudf/cudf/core/column_accessor.py @@ -386,7 +386,9 @@ def to_pandas_index(self) -> pd.Index: except (TypeError, ValueError): pass else: - if (cast_lvl.astype(lvl.dtype) == lvl).all(): + # missing-aware equality: ``==`` would treat + # NaN entries as unequal to themselves + if cast_lvl.astype(lvl.dtype).equals(lvl): lvl = cast_lvl changed = True new_levels.append(lvl) diff --git a/python/cudf/cudf/core/groupby/groupby.py b/python/cudf/cudf/core/groupby/groupby.py index 7c0ea6e247ea..15ed70d28da1 100644 --- a/python/cudf/cudf/core/groupby/groupby.py +++ b/python/cudf/cudf/core/groupby/groupby.py @@ -1292,7 +1292,7 @@ def agg(self, func=None, *args, engine=None, engine_kwargs=None, **kwargs): # RangeIndex) columns. data = ColumnAccessor( data, - multiindex=False, + multiindex=self.obj._data.multiindex, level_names=self.obj._data.level_names, rangeindex=self.obj._data.rangeindex, label_dtype=self.obj._data.label_dtype, diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index b3f68d61d8bb..0a8658697f51 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -5,6 +5,7 @@ from __future__ import annotations import copy +import functools import itertools import textwrap import warnings @@ -2929,20 +2930,61 @@ def sort_index( def _level_number(lvl): if isinstance(lvl, int): - return lvl + nlevels if lvl < 0 else lvl + norm = lvl + nlevels if lvl < 0 else lvl + if not 0 <= norm < nlevels: + raise IndexError( + f"Too many levels: Index has only " + f"{nlevels} levels, {lvl} is not a valid " + "level number" + ) + return norm return level_names.index(lvl) key_order = [_level_number(lvl) for lvl in level] + ascending_per_key: list[bool] = ( + [bool(flag) for flag in cast("Iterable[bool]", ascending)] + if is_list_like(ascending) + else [bool(ascending)] * len(key_order) + ) + if len(ascending_per_key) != len(key_order): + raise ValueError( + "level must have same length as ascending: " + f"{len(key_order)} != {len(ascending_per_key)}" + ) if sort_remaining: seen = set(key_order) - key_order.extend( - i for i in range(nlevels) if i not in seen + for i in range(nlevels): + if i not in seen: + key_order.append(i) + ascending_per_key.append(ascending_per_key[0]) + + def _is_na_label(value) -> bool: + return value is None or ( + isinstance(value, float) and value != value + ) + + def _label_sort_key( + label, pos: int, na_rank: int + ) -> tuple[int, Any]: + value = label[pos] + if _is_na_label(value): + return (na_rank, None) + return (1 - na_rank, value) + + # Stable multi-key sort: sort by the least significant key + # first. Missing labels are placed per ``na_position``, + # independently of the per-key direction. + labels = list(self._column_names) + for pos, asc in reversed( + list(zip(key_order, ascending_per_key, strict=True)) + ): + na_rank = int(asc == (na_position == "last")) + labels.sort( + key=functools.partial( + _label_sort_key, pos=pos, na_rank=na_rank + ), + reverse=not asc, ) - labels = sorted( - self._column_names, - key=lambda label: tuple(label[i] for i in key_order), - reverse=not ascending, - ) else: labels = sorted(self._column_names, reverse=not ascending) result_columns = (self._data[label] for label in labels) diff --git a/python/cudf/cudf/core/multiindex.py b/python/cudf/cudf/core/multiindex.py index c9c621ce7b98..b846f52ecf02 100644 --- a/python/cudf/cudf/core/multiindex.py +++ b/python/cudf/cudf/core/multiindex.py @@ -2109,14 +2109,14 @@ def _level_index_from_level(self, level) -> int: except ValueError: if not is_integer(level): raise KeyError(f"Level {level} not found") - if level < 0: - level += self.nlevels - if level >= self.nlevels: + norm = level + self.nlevels if level < 0 else level + if not 0 <= norm < self.nlevels: + # matches pandas MultiIndex._get_level_number raise IndexError( - f"Level {level} out of bounds. " - f"Index has {self.nlevels} levels." + f"Too many levels: Index has only {self.nlevels} " + f"levels, {level} is not a valid level number" ) from None - return level + return norm @_performance_tracking def get_indexer(self, target, method=None, limit=None, tolerance=None): diff --git a/python/cudf/cudf/core/reshape.py b/python/cudf/cudf/core/reshape.py index 4abef6d26b74..ce08dc3b5c61 100644 --- a/python/cudf/cudf/core/reshape.py +++ b/python/cudf/cudf/core/reshape.py @@ -1016,7 +1016,7 @@ def as_tuple(x): multiindex=True, level_names=( *col_accessor.level_names, - *columns_labels._column_names, + *columns_labels.names, ), verify=False, ) @@ -1439,8 +1439,12 @@ def _unstack( full_level = df.index.levels[level_idx].to_pandas() pdi = result._data.to_pandas_index if isinstance(pdi, pd.MultiIndex): - new_codes = full_level.get_indexer(pdi.get_level_values(-1)) - if (new_codes >= 0).all(): + level_values = pdi.get_level_values(-1) + new_codes = full_level.get_indexer(level_values) + # -1 codes for NA labels are pandas' canonical missing + # representation; only bail out when a non-NA label failed + # to map into the full level + if ((new_codes >= 0) | pd.isna(level_values)).all(): result._data.to_pandas_index = pd.MultiIndex( levels=[*pdi.levels[:-1], full_level], codes=[*pdi.codes[:-1], new_codes], @@ -1760,7 +1764,13 @@ def pivot_table( to_unstack.append(i) else: to_unstack.append(name) - table = _unstack(agged, to_unstack, promote_ints_on_missing=False) + table = _unstack( + agged, + to_unstack, + # pandas keeps the integer dtype when the missing cells are + # filled afterwards, and promotes to float64 when they are not + promote_ints_on_missing=fill_value is None, + ) if fill_value is not None: table = table.fillna(fill_value) diff --git a/python/cudf/cudf/tests/reshape/test_pivot_table.py b/python/cudf/cudf/tests/reshape/test_pivot_table.py index 368a8b3ffbed..347a73f84090 100644 --- a/python/cudf/cudf/tests/reshape/test_pivot_table.py +++ b/python/cudf/cudf/tests/reshape/test_pivot_table.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 import numpy as np @@ -94,3 +94,32 @@ def test_pivot_table_scalar_index_columns(index, columns): values="D", index=index, columns=columns, aggfunc="sum" ) assert_eq(result, expected) + + +@pytest.mark.parametrize("aggfunc", ["sum", "min", "max"]) +@pytest.mark.parametrize("fill_value", [None, 0]) +def test_pivot_table_sparse_int_fill_value(aggfunc, fill_value): + # pandas promotes integer values to float64 when the reshape leaves + # missing cells unfilled (fill_value=None), and keeps the integer + # dtype when they are filled + data = { + "i": ["r1", "r1", "r2"], + "c": ["a", "b", "a"], + "v": [1, 2, 3], + } + with cudf.option_context("mode.pandas_compatible", True): + result = cudf.DataFrame(data).pivot_table( + index="i", + columns="c", + values="v", + aggfunc=aggfunc, + fill_value=fill_value, + ) + expected = pd.DataFrame(data).pivot_table( + index="i", + columns="c", + values="v", + aggfunc=aggfunc, + fill_value=fill_value, + ) + assert_eq(expected, result) From e38ca69a9616be44de61df1b96d27cb84b4c6079 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Tue, 14 Jul 2026 10:58:29 +0000 Subject: [PATCH 5/6] Fix CI failures: scope down bool conversion and proxy changes, fix metadata regressions * Drop the bool-with-nulls to_pandas nan conversion entirely: pandas is not self-consistent here (read_orc/read_parquet ingest produces object columns holding None while unstack's upcast produces np.nan), so the change broke ORC/Parquet reader tests, to_pandas doctests, and eval/numexpr tests that compare against pandas' literal None. Restore the test_unstack_bool plugin entry with a real reason and revert the test expectation updates that depended on the conversion. * Revert the _MethodProxy transfer-block change: removing the baked block exposed ~200 pandas-test failures that only pass on main because an early mixed-object test poisons the class-level method cache into forcing the slow path for later tests. Restoring main's behavior keeps CI green; the contamination itself deserves a dedicated PR. Re-add the five plugin entries for tests that only fail under that ordering. * GroupBy.agg: only propagate the source's MultiIndex column metadata when the aggregation kept the source's tuple labels; relabeling aggregations emit new flat labels and crashed rebuilding the columns index (Length of names must match number of levels). * DataFrame binops: keep hierarchical columns when the operands' labels match but the equals check fails on level-dtype differences (e.g. Int8 vs int64 after level-dtype restoration). * Prune 49 plugin entries for tests genuinely fixed by this PR's reshape/metadata changes (verified passing in isolation). --- python/cudf/cudf/core/column/column.py | 12 ---- python/cudf/cudf/core/dataframe.py | 11 ++-- python/cudf/cudf/core/groupby/groupby.py | 31 ++++++++--- python/cudf/cudf/pandas/fast_slow_proxy.py | 17 +++--- .../pandas/scripts/pandas-testing-plugin.py | 55 ++----------------- .../cudf/tests/dataframe/test_np_ufuncs.py | 6 +- python/cudf/cudf/tests/reshape/test_concat.py | 13 ++++- .../cudf/cudf/tests/series/test_np_ufuncs.py | 6 +- 8 files changed, 60 insertions(+), 91 deletions(-) diff --git a/python/cudf/cudf/core/column/column.py b/python/cudf/cudf/core/column/column.py index 0907046b9fa1..a69e6e1dfcae 100644 --- a/python/cudf/cudf/core/column/column.py +++ b/python/cudf/cudf/core/column/column.py @@ -1175,18 +1175,6 @@ def to_pandas( # xref https://github.com/rapidsai/cudf/issues/21120 # TODO: Revisit using pa_array.to_pandas() once pandas 3.0 is supported np_array = pa_array.to_numpy(zero_copy_only=False, writable=True) - if ( - isinstance(self.dtype, np.dtype) - and self.dtype.kind == "b" - and self.has_nulls() - ): - # pandas represents missing values in an (upcast-to-object) - # bool column as np.nan; pyarrow's to_numpy yields None. - # np.nan is also what every other cudf dtype converts - # missing values to (nan/NaT) when going to pandas. - np_array[pa_array.is_null().to_numpy(zero_copy_only=False)] = ( - np.nan - ) return pd.Index( np_array, dtype=np_array.dtype, diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 7cab9984fb32..961573a44f2f 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -2583,6 +2583,11 @@ def _fill_same_ca_attributes( ) elif self._data._level_names == other._data._level_names: ca_attributes["level_names"] = self._data.level_names + if self._data.multiindex == other._data.multiindex: + # equal labels can still fail the ``equals`` check above + # on level-dtype differences (e.g. Int8 vs int64); the + # result keeps hierarchical columns like pandas + ca_attributes["multiindex"] = self._data.multiindex elif isinstance(other, (dict, Mapping)): # Need to fail early on host mapping types because we ultimately # convert everything to a dict. @@ -8249,12 +8254,6 @@ def stack( new_index_columns = [*repeated_index._columns, *tiled_index] index_names = [*self.index.names, *unique_named_levels.names] new_index = MultiIndex._from_data(dict(enumerate(new_index_columns))) - # Materialize the levels in order of first appearance (rather than the - # default sorted order) so that converting the result to pandas keeps - # the level order pandas' own ``stack`` produces. Otherwise a later - # ``unstack``/``to_pandas`` would lexicographically reorder the pivoted - # axis (e.g. ``"foo_10"`` before ``"foo_2"``). - new_index._maybe_materialize_codes_and_levels(sort=False) new_index.names = index_names # Attach codes/levels eagerly, matching how pandas' stack builds the # result MultiIndex, so a later unstack can restore the original diff --git a/python/cudf/cudf/core/groupby/groupby.py b/python/cudf/cudf/core/groupby/groupby.py index 15ed70d28da1..3839b8fa5190 100644 --- a/python/cudf/cudf/core/groupby/groupby.py +++ b/python/cudf/cudf/core/groupby/groupby.py @@ -1303,13 +1303,30 @@ def agg(self, func=None, *args, engine=None, engine_kwargs=None, **kwargs): and self.obj.ndim == 2 and self.obj._data.level_names != (None,) ): - data = ColumnAccessor( - data, - multiindex=self.obj._data.multiindex, - level_names=self.obj._data.level_names, - label_dtype=self.obj._data.label_dtype, - level_dtypes=self.obj._data.level_dtypes, - ) + if self.obj._data.multiindex and all( + isinstance(label, tuple) + and len(label) == self.obj._data.nlevels + for label in data + ): + # the aggregation kept the source's tuple labels: preserve + # the MultiIndex columns and their per-level metadata + data = ColumnAccessor( + data, + multiindex=True, + level_names=self.obj._data.level_names, + label_dtype=self.obj._data.label_dtype, + level_dtypes=self.obj._data.level_dtypes, + ) + else: + # relabeling aggregations (``agg(new=(col, func))``) emit + # new flat labels: the source's multi-level metadata does + # not describe them + data = ColumnAccessor( + data, + multiindex=False, + level_names=self.obj._data.level_names, + label_dtype=self.obj._data.label_dtype, + ) else: data = ColumnAccessor(data, multiindex=multilevel) if not multilevel and len(data) > 0: diff --git a/python/cudf/cudf/pandas/fast_slow_proxy.py b/python/cudf/cudf/pandas/fast_slow_proxy.py index 662f05d565fd..e228f5fd0589 100644 --- a/python/cudf/cudf/pandas/fast_slow_proxy.py +++ b/python/cudf/cudf/pandas/fast_slow_proxy.py @@ -1120,16 +1120,13 @@ def __get__(self, instance, owner) -> Any: raise e if _is_function_or_method(slow_attr): - # Do NOT bake ``instance.get_transfer_blocking()`` into the - # _MethodProxy: ``self._attr`` is cached on the descriptor and - # shared by every instance of the proxy class, so a blocking - # state captured from whichever instance happens to resolve - # the attribute first would leak into every later call on any - # instance. Per-instance blocking is still enforced at call - # time: ``_fsproxy_fast`` raises RuntimeError for a blocked - # SLOW instance during ``_fast_arg`` conversion, forcing the - # slow path for that instance only. - self._attr = _MethodProxy(fast_attr, slow_attr) + self._attr = _MethodProxy( + fast_attr, + slow_attr, + _fsproxy_transfer_block=instance.get_transfer_blocking() + if instance is not None + else None, + ) else: # for anything else, use a fast-slow attribute: self._attr, _ = _fast_slow_function_call( diff --git a/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py b/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py index 284b85ccc475..3a1cf1c5926a 100644 --- a/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py +++ b/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py @@ -1334,7 +1334,6 @@ def pytest_unconfigure(config): "tests/frame/methods/test_shift.py::TestDataFrameShift::test_shift_dt64values_axis1_invalid_fill[datetime64[us]-False]": "AssertionError: assert 1 == 2", "tests/frame/methods/test_shift.py::TestDataFrameShift::test_shift_dt64values_axis1_invalid_fill[timedelta64[us]-False]": "AssertionError: assert 1 == 2", "tests/frame/methods/test_shift.py::TestDataFrameShift::test_shift_dt64values_int_fill_deprecated": "TODO: Add a reason for failure", - "tests/frame/methods/test_sort_index.py::TestDataFrameSortIndex::test_sort_index_intervalindex": "TODO: Add a reason for failure", "tests/frame/methods/test_sort_index.py::TestDataFrameSortIndex::test_sort_index_nan": "TODO: Add a reason for failure", "tests/frame/methods/test_sort_values.py::TestDataFrameSortValues::test_sort_by_column_named_none": "AssertionError: DataFrame.index are different", "tests/frame/methods/test_sort_values.py::TestDataFrameSortValues::test_sort_values_by_empty_list": "TODO: Add a reason for failure", @@ -1425,9 +1424,6 @@ def pytest_unconfigure(config): "tests/frame/test_constructors.py::TestDataFrameConstructors::test_constructor_dict_cast": "TODO: Add a reason for failure", "tests/frame/test_constructors.py::TestDataFrameConstructors::test_constructor_dict_multiindex": "TODO: Add a reason for failure", "tests/frame/test_constructors.py::TestDataFrameConstructors::test_constructor_dict_nan_key[None]": "TODO: Add a reason for failure", - "tests/frame/test_constructors.py::TestDataFrameConstructors::test_constructor_dict_nan_key[nan0]": "AssertionError: Attributes of DataFrame.iloc[:, 1] (column name='nan') are different", - "tests/frame/test_constructors.py::TestDataFrameConstructors::test_constructor_dict_nan_key[nan1]": "AssertionError: Attributes of DataFrame.iloc[:, 1] (column name='nan') are different", - "tests/frame/test_constructors.py::TestDataFrameConstructors::test_constructor_dict_nan_key_and_columns": "TODO: Add a reason for failure", "tests/frame/test_constructors.py::TestDataFrameConstructors::test_constructor_dict_with_index": "TODO: Add a reason for failure", "tests/frame/test_constructors.py::TestDataFrameConstructors::test_constructor_dict_with_index_and_columns": "TODO: Add a reason for failure", "tests/frame/test_constructors.py::TestDataFrameConstructors::test_constructor_dict_with_none": "AssertionError: assert nan is None", @@ -1715,7 +1711,13 @@ def pytest_unconfigure(config): "tests/frame/test_reductions.py::test_reduction_axis_none_returns_scalar[float64-True-mean]": "TODO: Add a reason for failure", "tests/frame/test_reductions.py::test_reduction_axis_none_returns_scalar[float64-True-median]": "TODO: Add a reason for failure", "tests/frame/test_reductions.py::test_reduction_axis_none_returns_scalar[float64-True-skew]": "TODO: Add a reason for failure", + "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_unstack_bool": "cudf converts null bools to None where pandas' unstack upcasts to object with np.nan", "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_unstack_not_consolidated": "Asserts DataFrame._mgr block layout (pandas internals)", + "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_multiple_out_of_bounds[False]": "passes in isolation; fails under full-suite ordering (method-cache transfer-block contamination)", + "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_multiple_out_of_bounds[True]": "passes in isolation; fails under full-suite ordering (method-cache transfer-block contamination)", + "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_unstack_multiple[False]": "passes in isolation; fails under full-suite ordering (method-cache transfer-block contamination)", + "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_unstack_multiple[True]": "passes in isolation; fails under full-suite ordering (method-cache transfer-block contamination)", + "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_unstack_preserve_types": "passes in isolation; fails under full-suite ordering (method-cache transfer-block contamination)", "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_unstack_with_missing_int_cast_to_float": "Asserts DataFrame._mgr block layout (pandas internals)", "tests/frame/test_subclass.py::TestDataFrameSubclassing::test_asof": "TODO: Add a reason for failure", "tests/frame/test_subclass.py::TestDataFrameSubclassing::test_equals_subclass": "TODO: Add a reason for failure", @@ -1747,8 +1749,6 @@ def pytest_unconfigure(config): "tests/groupby/aggregate/test_aggregate.py::test_agg_str_with_kwarg_axis_1_raises[nunique]": "TODO: Add a reason for failure", "tests/groupby/aggregate/test_aggregate.py::test_groupby_aggregate_directory[size]": "TODO: Add a reason for failure", "tests/groupby/aggregate/test_aggregate.py::test_groupby_aggregate_empty_key_empty_return": "TODO: Add a reason for failure", - "tests/groupby/aggregate/test_aggregate.py::test_multiindex_custom_func[0]": "TODO: Add a reason for failure", - "tests/groupby/aggregate/test_aggregate.py::test_order_aggregate_multiple_funcs": "TODO: Add a reason for failure", "tests/groupby/aggregate/test_cython.py::test_cython_agg_EA_known_dtypes[data1-prod-large_int-False]": "TODO: Add a reason for failure", "tests/groupby/aggregate/test_cython.py::test_cython_agg_EA_known_dtypes[data1-prod-large_int-True]": "TODO: Add a reason for failure", "tests/groupby/aggregate/test_cython.py::test_cython_agg_EA_known_dtypes[data1-sum-large_int-False]": "TODO: Add a reason for failure", @@ -1831,7 +1831,6 @@ def pytest_unconfigure(config): "tests/groupby/test_apply.py::test_include_groups": "Failed: DID NOT RAISE ", "tests/groupby/test_apply.py::test_positional_slice_groups_datetimelike": "AssertionError: DataFrame are different", "tests/groupby/test_apply.py::test_time_field_bug": "TODO: Add a reason for failure", - "tests/groupby/test_categorical.py::test_describe_categorical_columns": "cudf's multi-level groupby aggregation and stack() drop the categorical column-index dtype", "tests/groupby/test_counting.py::TestCounting::test_ngroup_distinct": "TODO: Add a reason for failure", "tests/groupby/test_cumulative.py::test_groupby_cumprod_nan_influences_other_columns": "TODO: Add a reason for failure", "tests/groupby/test_cumulative.py::test_numpy_compat[cumprod]": "TODO: Add a reason for failure", @@ -1889,7 +1888,6 @@ def pytest_unconfigure(config): "tests/groupby/test_groupby.py::test_groupby_with_Time_Grouper[ns]": "TODO: Add a reason for failure", "tests/groupby/test_groupby.py::test_groupby_with_Time_Grouper[s]": "TODO: Add a reason for failure", "tests/groupby/test_groupby.py::test_groupby_with_Time_Grouper[us]": "TODO: Add a reason for failure", - "tests/groupby/test_groupby.py::test_groupby_with_hier_columns": "TODO: Add a reason for failure", "tests/groupby/test_groupby.py::test_groups_repr_truncates[1-{0: [0], ...}]": "TODO: Add a reason for failure", "tests/groupby/test_groupby.py::test_groups_repr_truncates[4-{0: [0], 1: [1], 2: [2], 3: [3], ...}]": "TODO: Add a reason for failure", "tests/groupby/test_groupby.py::test_groups_repr_truncates[5-{0: [0], 1: [1], 2: [2], 3: [3], 4: [4]}]": "TODO: Add a reason for failure", @@ -1898,7 +1896,6 @@ def pytest_unconfigure(config): "tests/groupby/test_groupby.py::test_ops_not_as_index[idxmin]": "TODO: Add a reason for failure", "tests/groupby/test_groupby.py::test_ops_not_as_index[size]": "TODO: Add a reason for failure", "tests/groupby/test_groupby.py::test_single_element_listlike_level_grouping[level_arg0-False]": "AssertionError: assert ['x', 'y'] == [('x',), ('y',)]", - "tests/groupby/test_groupby.py::test_wrap_aggregated_output_multindex": "TODO: Add a reason for failure", "tests/groupby/test_groupby_dropna.py::test_groupby_nan_included": "GroupBy.indices returns cupy arrays nested in a dict that cudf.pandas does not wrap, so assert_numpy_array_equal sees mismatched array classes", "tests/groupby/test_groupby_subclass.py::test_groupby_preserves_metadata": "TODO: Add a reason for failure", "tests/groupby/test_groupby_subclass.py::test_groupby_preserves_subclass[all-obj0]": "TODO: Add a reason for failure", @@ -2630,7 +2627,6 @@ def pytest_unconfigure(config): "tests/indexing/test_loc.py::TestLocSetitemWithExpansion::test_loc_setitem_with_expansion_nonunique_index[string-pyarrow-True]": 'AssertionError: Column name="0" are different', "tests/indexing/test_loc.py::TestLocSetitemWithExpansion::test_loc_setitem_with_expansion_nonunique_index[string-python-False]": 'AssertionError: Column name="0" are different', "tests/indexing/test_loc.py::TestLocSetitemWithExpansion::test_loc_setitem_with_expansion_nonunique_index[string-python-True]": 'AssertionError: Column name="0" are different', - "tests/indexing/test_loc.py::TestLocWithMultiIndex::test_loc_set_nan_in_categorical_series[Float64]": "TODO: Add a reason for failure", "tests/indexing/test_loc.py::test_loc_getitem_multiindex_tuple_level": "AssertionError: DataFrame Expected type , found instead", "tests/indexing/test_na_indexing.py::test_series_mask_boolean[True-list-mask0-values0-object]": "TODO: Add a reason for failure", "tests/indexing/test_na_indexing.py::test_series_mask_boolean[True-list-mask1-values0-object]": "TODO: Add a reason for failure", @@ -3068,7 +3064,6 @@ def pytest_unconfigure(config): "tests/reshape/concat/test_categorical.py::TestCategoricalConcat::test_categorical_index_upcast": "TODO: Add a reason for failure", "tests/reshape/concat/test_categorical.py::TestCategoricalConcat::test_concat_categorical_datetime": "TODO: Add a reason for failure", "tests/reshape/concat/test_concat.py::TestConcatenate::test_concat_copy": "TODO: Add a reason for failure", - "tests/reshape/concat/test_concat.py::TestConcatenate::test_concat_keys_specific_levels": "TODO: Add a reason for failure", "tests/reshape/concat/test_concat.py::TestConcatenate::test_concat_order": "TODO: Add a reason for failure", "tests/reshape/concat/test_concat.py::test_concat_empty_and_non_empty_frame_regression": "TODO: Add a reason for failure", "tests/reshape/concat/test_concat.py::test_concat_ignore_empty_object_float[None-datetime64[ns]]": "AssertionError: Attributes of DataFrame.iloc[:, 0] (column name='foo') are different", @@ -3197,10 +3192,8 @@ def pytest_unconfigure(config): "tests/reshape/merge/test_multi.py::TestMergeMulti::test_left_join_multi_index[True-False]": "AssertionError: DataFrame.iloc[:, 4] (column name='5th') are different", "tests/reshape/merge/test_multi.py::TestMergeMulti::test_left_join_multi_index[True-True]": "TODO: Add a reason for failure", "tests/reshape/test_crosstab.py::TestCrosstab::test_crosstab_duplicate_names": "TODO: Add a reason for failure", - "tests/reshape/test_crosstab.py::TestCrosstab::test_crosstab_multiple": "TODO: Add a reason for failure", "tests/reshape/test_crosstab.py::TestCrosstab::test_crosstab_no_overlap": "TODO: Add a reason for failure", "tests/reshape/test_crosstab.py::TestCrosstab::test_crosstab_with_categorial_columns": "TODO: Add a reason for failure", - "tests/reshape/test_crosstab.py::TestCrosstab::test_crosstab_with_empties": "TODO: Add a reason for failure", "tests/reshape/test_cut.py::test_bins[array]": "TODO: Add a reason for failure", "tests/reshape/test_cut.py::test_bins[list]": "TODO: Add a reason for failure", "tests/reshape/test_cut.py::test_bins_from_interval_index": "TODO: Add a reason for failure", @@ -3246,14 +3239,10 @@ def pytest_unconfigure(config): "tests/reshape/test_melt.py::TestWideToLong::test_raise_of_column_name_value": "TODO: Add a reason for failure", "tests/reshape/test_pivot.py::TestPivot::test_pivot_index_is_none": "AssertionError: DataFrame.index are different", "tests/reshape/test_pivot.py::TestPivotTable::test_categorical_pivot_index_ordering[False]": "TODO: Add a reason for failure", - "tests/reshape/test_pivot.py::TestPivotTable::test_daily": "TODO: Add a reason for failure", - "tests/reshape/test_pivot.py::TestPivotTable::test_monthly": "TODO: Add a reason for failure", - "tests/reshape/test_pivot.py::TestPivotTable::test_pivot_complex_aggfunc": "TODO: Add a reason for failure", "tests/reshape/test_pivot.py::TestPivotTable::test_pivot_datetime_tz": "ValueError: Length of names must match number of levels in MultiIndex.", "tests/reshape/test_pivot.py::TestPivotTable::test_pivot_index_with_nan[False]": "AssertionError: DataFrame.index are different", "tests/reshape/test_pivot.py::TestPivotTable::test_pivot_index_with_nan[True]": "AssertionError: DataFrame.index are different", "tests/reshape/test_pivot.py::TestPivotTable::test_pivot_multi_functions": "TODO: Add a reason for failure", - "tests/reshape/test_pivot.py::TestPivotTable::test_pivot_no_level_overlap": "TODO: Add a reason for failure", "tests/reshape/test_pivot.py::TestPivotTable::test_pivot_string_as_func": "TODO: Add a reason for failure", "tests/reshape/test_pivot.py::TestPivotTable::test_pivot_string_func_vs_func[f3-f_numpy3]": "TODO: Add a reason for failure", "tests/reshape/test_pivot.py::TestPivotTable::test_pivot_string_func_vs_func[f4-f_numpy4]": "TODO: Add a reason for failure", @@ -3264,7 +3253,6 @@ def pytest_unconfigure(config): "tests/reshape/test_pivot.py::TestPivotTable::test_pivot_table_nocols": "TODO: Add a reason for failure", "tests/reshape/test_pivot.py::TestPivotTable::test_pivot_table_not_series": "TODO: Add a reason for failure", "tests/reshape/test_pivot.py::TestPivotTable::test_pivot_table_with_iterator_values": "TODO: Add a reason for failure", - "tests/reshape/test_pivot.py::TestPivotTable::test_pivot_table_with_mixed_nested_tuples": "TODO: Add a reason for failure", "tests/reshape/test_pivot.py::TestPivotTable::test_pivot_with_categorical[False-False]": "TODO: Add a reason for failure", "tests/reshape/test_pivot.py::TestPivotTable::test_pivot_with_categorical[False-None]": "TODO: Add a reason for failure", "tests/reshape/test_pivot.py::TestPivotTable::test_pivot_with_categorical[False-True]": "TODO: Add a reason for failure", @@ -3779,33 +3767,6 @@ def pytest_unconfigure(config): "tests/strings/test_cat.py::test_str_cat_categorical[series-category-category-None-False]": "AssertionError: Attributes of Series are different", "tests/strings/test_cat.py::test_str_cat_categorical[series-category-object--False]": "AssertionError: Attributes of Series are different", "tests/strings/test_cat.py::test_str_cat_categorical[series-category-object-None-False]": "AssertionError: Attributes of Series are different", - "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[bool-dtype-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", - "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[categorical-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", - "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[datetime-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", - "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[datetime-tz-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", - "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[float32-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", - "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[float64-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", - "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[int16-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", - "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[int32-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", - "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[int64-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", - "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[int8-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", - "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[interval-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", - "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[multi-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", - "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[nullable_bool-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", - "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[nullable_float-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", - "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[nullable_int-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", - "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[nullable_uint-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", - "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[object-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", - "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[range-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", - "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[repeats-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", - "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[string-pyarrow-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", - "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[string-python-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", - "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[string-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", - "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[tuples-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", - "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[uint16-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", - "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[uint32-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", - "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[uint64-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", - "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[uint8-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", "tests/strings/test_extract.py::test_extract_expand_False_mixed_object": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", "tests/strings/test_extract.py::test_extract_expand_True[string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", "tests/strings/test_extract.py::test_extract_expand_True_mixed_object": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", @@ -3838,7 +3799,6 @@ def pytest_unconfigure(config): "tests/strings/test_extract.py::test_extract_expand_capture_groups_index[uint64-string=object]": "AssertionError: DataFrame.iloc[:, 1] (column name='number') are different", "tests/strings/test_extract.py::test_extract_expand_capture_groups_index[uint8-string=object]": "AssertionError: DataFrame.iloc[:, 1] (column name='number') are different", "tests/strings/test_extract.py::test_extract_expand_kwarg[string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", - "tests/strings/test_extract.py::test_extract_optional_groups[string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", "tests/strings/test_extract.py::test_extract_series[string=object-None]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", "tests/strings/test_extract.py::test_extract_series[string=object-series_name]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", "tests/strings/test_extract.py::test_extractall_column_names[string=object-(?P[AB])?(?P[123])-expected_names0]": "AssertionError: DataFrame.iloc[:, 0] (column name='letter') are different", @@ -4014,13 +3974,11 @@ def pytest_unconfigure(config): "tests/tslibs/test_to_offset.py::test_to_offset_uppercase_frequency_deprecated[2NS]": "TODO: Add a reason for failure", "tests/tslibs/test_to_offset.py::test_to_offset_uppercase_frequency_deprecated[2Us]": "TODO: Add a reason for failure", "tests/util/test_assert_frame_equal.py::test_allows_duplicate_labels": "TODO: Add a reason for failure", - "tests/util/test_assert_frame_equal.py::test_assert_frame_equal_extension_dtype_mismatch": "TODO: Add a reason for failure", "tests/util/test_assert_frame_equal.py::test_assert_frame_equal_nested_df_na[None]": "KeyError: 0", "tests/util/test_assert_frame_equal.py::test_assert_frame_equal_nested_df_na[nan]": "KeyError: 0", "tests/util/test_assert_frame_equal.py::test_frame_equal_index_dtype_mismatch[True-df11-df21-DataFrame\\\\.index level \\\\[0\\\\] are different]": "Failed: DID NOT RAISE ", "tests/util/test_assert_index_equal.py::test_index_equal_range_categories[True-True]": "TODO: Add a reason for failure", "tests/util/test_assert_series_equal.py::test_allows_duplicate_labels": "TODO: Add a reason for failure", - "tests/util/test_assert_series_equal.py::test_assert_series_equal_extension_dtype_mismatch": "TODO: Add a reason for failure", "tests/util/test_assert_series_equal.py::test_assert_series_equal_int_tol": "AssertionError: left is not an ExtensionArray", "tests/util/test_assert_series_equal.py::test_large_unequal_ints[Int64]": "Failed: DID NOT RAISE ", "tests/util/test_assert_series_equal.py::test_large_unequal_ints[int64]": "TODO: Add a reason for failure", @@ -4078,7 +4036,6 @@ def pytest_unconfigure(config): "tests/window/test_timeseries_window.py::TestRollingTS::test_rolling_on_decreasing_index[us]": "TODO: Add a reason for failure", "tests/window/test_win_type.py::test_cmov_window_corner[None]": "TODO: Add a reason for failure", "tests/window/test_win_type.py::test_win_type_not_implemented": "TODO: Add a reason for failure", - "tests/indexing/multiindex/test_loc.py::test_loc_getitem_duplicates_multiindex_empty_indexer[columns_indexer1]": "AssertionError: DataFrame.columns level [0] are different", } # Keep keys in alphabeical order diff --git a/python/cudf/cudf/tests/dataframe/test_np_ufuncs.py b/python/cudf/cudf/tests/dataframe/test_np_ufuncs.py index 24c6e496cba4..430c54866da5 100644 --- a/python/cudf/cudf/tests/dataframe/test_np_ufuncs.py +++ b/python/cudf/cudf/tests/dataframe/test_np_ufuncs.py @@ -123,9 +123,9 @@ def test_ufunc_dataframe(request, numpy_ufunc, has_nulls, indexed): np.less_equal, np.not_equal, ): - # cuDF .to_pandas represents missing values in bool - # columns as np.nan (upcast to object), like pandas - expect = expect.astype(object).mask(mask, np.nan) + # cuDF .to_pandas for bools with nulls represents missing as None, + # should this be np.nan? + expect = expect.astype(object).mask(mask, None) else: expect[mask] = np.nan assert_eq(got, expect, check_exact=False) diff --git a/python/cudf/cudf/tests/reshape/test_concat.py b/python/cudf/cudf/tests/reshape/test_concat.py index adaa385bd7ae..606f23a5bba8 100644 --- a/python/cudf/cudf/tests/reshape/test_concat.py +++ b/python/cudf/cudf/tests/reshape/test_concat.py @@ -2600,7 +2600,7 @@ def test_concat_empty_dataframe(df_1_data, df_2_data): {}, ], ) -def test_concat_different_column_dataframe(df1_d, df2_d): +def test_concat_different_column_dataframe(request, df1_d, df2_d): pdf1 = pd.DataFrame(df1_d) pdf2 = pd.DataFrame(df2_d) @@ -2614,6 +2614,17 @@ def test_concat_different_column_dataframe(df1_d, df2_d): ) expect = pd.concat([pdf1, pdf2, pdf1], sort=False) + xfail_pair = df2_d == { + "a": [1, None, 3], + "b": [True, True, False], + "c": ["s3", None, "s4"], + } and isinstance(df1_d["b"], pd.Series) + request.applymarker( + pytest.mark.xfail( + xfail_pair, + reason="As of pandas 3.0, pandas coerces to float, cuDF coerces to bool", + ) + ) assert_eq(got, expect, check_dtype=False, check_index_type=True) diff --git a/python/cudf/cudf/tests/series/test_np_ufuncs.py b/python/cudf/cudf/tests/series/test_np_ufuncs.py index 446e0843382e..01e7adefede7 100644 --- a/python/cudf/cudf/tests/series/test_np_ufuncs.py +++ b/python/cudf/cudf/tests/series/test_np_ufuncs.py @@ -110,9 +110,9 @@ def test_ufunc_series(request, numpy_ufunc, has_nulls, indexed): np.less_equal, np.not_equal, ): - # cuDF .to_pandas represents missing values in bool - # columns as np.nan (upcast to object), like pandas - expect = expect.astype(object).mask(mask, np.nan) + # cuDF .to_pandas for bools with nulls represents missing as None, + # should this be np.nan? + expect = expect.astype(object).mask(mask, None) else: expect[mask] = np.nan assert_eq(got, expect, check_exact=False) From 5fa0a74056cf8e3ec79752591326732070ddff60 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Wed, 15 Jul 2026 20:06:05 +0000 Subject: [PATCH 6/6] Fix pandas-tests CI: restore extract xfail entries, drop stack entries that pass in CI The 28 tests/strings/test_extract.py entries removed earlier still fail in CI (they only pass locally due to an environment artifact), so they are restored verbatim from main. The 5 test_stack_unstack.py entries kept with a 'fails under full-suite ordering' reason strict-XPASS in CI (the contamination is a local-only effect), so they are removed. --- .../pandas/scripts/pandas-testing-plugin.py | 33 ++++++++++++++++--- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py b/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py index 3a1cf1c5926a..23dabc8e8ce5 100644 --- a/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py +++ b/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py @@ -1713,11 +1713,6 @@ def pytest_unconfigure(config): "tests/frame/test_reductions.py::test_reduction_axis_none_returns_scalar[float64-True-skew]": "TODO: Add a reason for failure", "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_unstack_bool": "cudf converts null bools to None where pandas' unstack upcasts to object with np.nan", "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_unstack_not_consolidated": "Asserts DataFrame._mgr block layout (pandas internals)", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_multiple_out_of_bounds[False]": "passes in isolation; fails under full-suite ordering (method-cache transfer-block contamination)", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_multiple_out_of_bounds[True]": "passes in isolation; fails under full-suite ordering (method-cache transfer-block contamination)", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_unstack_multiple[False]": "passes in isolation; fails under full-suite ordering (method-cache transfer-block contamination)", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_unstack_multiple[True]": "passes in isolation; fails under full-suite ordering (method-cache transfer-block contamination)", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_unstack_preserve_types": "passes in isolation; fails under full-suite ordering (method-cache transfer-block contamination)", "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_unstack_with_missing_int_cast_to_float": "Asserts DataFrame._mgr block layout (pandas internals)", "tests/frame/test_subclass.py::TestDataFrameSubclassing::test_asof": "TODO: Add a reason for failure", "tests/frame/test_subclass.py::TestDataFrameSubclassing::test_equals_subclass": "TODO: Add a reason for failure", @@ -3767,6 +3762,33 @@ def pytest_unconfigure(config): "tests/strings/test_cat.py::test_str_cat_categorical[series-category-category-None-False]": "AssertionError: Attributes of Series are different", "tests/strings/test_cat.py::test_str_cat_categorical[series-category-object--False]": "AssertionError: Attributes of Series are different", "tests/strings/test_cat.py::test_str_cat_categorical[series-category-object-None-False]": "AssertionError: Attributes of Series are different", + "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[bool-dtype-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", + "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[categorical-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", + "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[datetime-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", + "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[datetime-tz-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", + "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[float32-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", + "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[float64-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", + "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[int16-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", + "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[int32-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", + "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[int64-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", + "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[int8-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", + "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[interval-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", + "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[multi-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", + "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[nullable_bool-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", + "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[nullable_float-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", + "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[nullable_int-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", + "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[nullable_uint-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", + "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[object-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", + "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[range-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", + "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[repeats-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", + "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[string-pyarrow-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", + "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[string-python-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", + "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[string-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", + "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[tuples-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", + "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[uint16-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", + "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[uint32-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", + "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[uint64-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", + "tests/strings/test_extract.py::test_extract_dataframe_capture_groups_index[uint8-string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", "tests/strings/test_extract.py::test_extract_expand_False_mixed_object": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", "tests/strings/test_extract.py::test_extract_expand_True[string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", "tests/strings/test_extract.py::test_extract_expand_True_mixed_object": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", @@ -3799,6 +3821,7 @@ def pytest_unconfigure(config): "tests/strings/test_extract.py::test_extract_expand_capture_groups_index[uint64-string=object]": "AssertionError: DataFrame.iloc[:, 1] (column name='number') are different", "tests/strings/test_extract.py::test_extract_expand_capture_groups_index[uint8-string=object]": "AssertionError: DataFrame.iloc[:, 1] (column name='number') are different", "tests/strings/test_extract.py::test_extract_expand_kwarg[string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", + "tests/strings/test_extract.py::test_extract_optional_groups[string=object]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", "tests/strings/test_extract.py::test_extract_series[string=object-None]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", "tests/strings/test_extract.py::test_extract_series[string=object-series_name]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", "tests/strings/test_extract.py::test_extractall_column_names[string=object-(?P[AB])?(?P[123])-expected_names0]": "AssertionError: DataFrame.iloc[:, 0] (column name='letter') are different",