diff --git a/python/cudf/cudf/core/column/numerical.py b/python/cudf/cudf/core/column/numerical.py index a1d14959e7ca..251514aa66ef 100644 --- a/python/cudf/cudf/core/column/numerical.py +++ b/python/cudf/cudf/core/column/numerical.py @@ -911,20 +911,23 @@ def as_numerical_column(self, dtype: DtypeObj) -> NumericalColumn: self.dtype ): # Short-circuit the cast if the dtypes are equivalent - # but not the same type object. + # but not the same type object. Do NOT mutate self._dtype: + # the column object may be shared with the caller's frame. if ( is_pandas_nullable_extension_dtype(dtype) and isinstance(self.dtype, np.dtype) and self.dtype.kind == "f" ): - # If the dtype is a pandas nullable extension type, we need to - # float column doesn't have any NaNs. + # NaNs must become nulls before viewing as a masked dtype. res = self.nans_to_nulls() - res._dtype = dtype - return res - else: - self._dtype = dtype - return self + return cast( + "NumericalColumn", + ColumnBase.create(res.plc_column, dtype), + ) + return cast( + "NumericalColumn", + ColumnBase.create(self.plc_column, dtype), + ) if self.dtype.kind == "f" and dtype.kind in "iu": if not is_pandas_nullable_extension_dtype(dtype) and ( self.nan_count > 0 diff --git a/python/cudf/cudf/core/column_accessor.py b/python/cudf/cudf/core/column_accessor.py index 5944e5486fe3..c594ae8e0bad 100644 --- a/python/cudf/cudf/core/column_accessor.py +++ b/python/cudf/cudf/core/column_accessor.py @@ -35,6 +35,32 @@ def _is_bool(val: Any) -> bool: return isinstance(val, (bool, np.bool_)) +def _is_nan_scalar(val: Any) -> bool: + return isinstance(val, float) and val != val + + +def _label_contains_nan(label: Any) -> bool: + if isinstance(label, tuple): + return any(_is_nan_scalar(lv) for lv in label) + return _is_nan_scalar(label) + + +def _canonicalize_nan_label(label: Any) -> Any: + """Map float NaN elements of a label to the np.nan singleton. + + Distinct float('nan') objects hash and compare unequal, so NaN-containing + labels round-tripped through a pandas Index (which materializes fresh NaN + objects on iteration) never match dict keys. Canonicalizing to the np.nan + singleton restores pandas' all-NaNs-are-equal label semantics (dict/tuple + comparison uses the per-element identity shortcut). + """ + if isinstance(label, tuple): + return tuple(np.nan if _is_nan_scalar(lv) else lv for lv in label) + if _is_nan_scalar(label): + return np.nan + return label + + class _NestedGetItemDict(dict): """A dictionary whose __getitem__ method accesses nested dicts. @@ -125,6 +151,11 @@ def __init__( self.rangeindex: bool = data.rangeindex self.label_dtype: DtypeObj | None = data.label_dtype self._level_dtypes = data._level_dtypes + if "to_pandas_index" in data.__dict__: + # carry over the primed/cached pandas index: it holds + # fidelity (e.g. explicit unsorted level order) that a + # rebuild from tuples would lose + self.to_pandas_index = data.__dict__["to_pandas_index"] elif isinstance(data, MutableMapping): # This code path is performance-critical for copies and should be # modified with care. @@ -162,7 +193,19 @@ def __iter__(self) -> Iterator: return iter(self._data) def __getitem__(self, key: Hashable) -> ColumnBase: - return self._data[key] + try: + return self._data[key] + except KeyError: + if _label_contains_nan(key): + # NaN labels lose object identity when round-tripped through + # a pandas Index; retry with NaNs canonicalized so all NaNs + # compare equal, matching pandas label semantics. + canon = _canonicalize_nan_label(key) + for existing in self._data: + c = _canonicalize_nan_label(existing) + if c is canon or c == canon: + return self._data[existing] + raise def __setitem__(self, key: Hashable, value: ColumnBase) -> None: self.set_by_label(key, value) @@ -322,6 +365,35 @@ def to_pandas_index(self) -> pd.Index: self.names, names=self.level_names, ) + if ( + self._level_dtypes is not None + and len(self._level_dtypes) == result.nlevels + ): + # ``from_tuples`` re-infers every level dtype from the + # materialized labels, degrading e.g. categorical levels + # to str, object levels to str once mixed-type labels are + # selected away, and int64 levels with missing entries to + # float64. Restore each preserved level dtype when the + # cast is lossless (round-trips to the inferred values). + new_levels = [] + changed = False + for lvl, level_dtype in zip( + result.levels, self._level_dtypes, strict=True + ): + if lvl.dtype != level_dtype: + try: + cast_lvl = lvl.astype(level_dtype) + except (TypeError, ValueError): + pass + else: + # missing-aware equality: ``==`` would treat + # NaN entries as unequal to themselves + if cast_lvl.astype(lvl.dtype).equals(lvl): + lvl = cast_lvl + changed = True + new_levels.append(lvl) + if changed: + result = result.set_levels(new_levels) else: # Determine if we can return a RangeIndex if self.rangeindex: diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index 02cdd9da3473..e159c32f568e 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -47,10 +47,12 @@ is_decimal128_dtype, is_dict_like, is_dtype_equal, + is_integer, is_list_like, is_scalar, ) from cudf.core import indexing_utils, reshape +from cudf.core.algorithms import factorize from cudf.core.column import ( CategoricalColumn, ColumnBase, @@ -89,6 +91,7 @@ ) from cudf.core.indexed_frame import ( IndexedFrame, + _check_duplicate_level_names, _FrameIndexer, _indices_from_labels, doc_reset_index_template, @@ -615,7 +618,12 @@ def _pd_index_level_dtypes(idx) -> tuple | None: dtype cannot be inferred from zero entries). """ if isinstance(idx, pd.MultiIndex): - return tuple(idx.get_level_values(i).dtype for i in range(idx.nlevels)) + # use the levels, not get_level_values: materializing a level that + # has missing entries (-1 codes) upcasts e.g. int64 to float64 + return tuple(level.dtype for level in idx.levels) + if isinstance(idx, cudf.MultiIndex): + # the per-row columns share their dtype with the levels + return tuple(dtype for _, dtype in idx._dtypes) return None @@ -837,7 +845,7 @@ def _array_to_column_accessor( columns_labels = columns else: columns_labels = pd.RangeIndex(data.shape[1]) - return ColumnAccessor( + ca = ColumnAccessor( { column_label: as_column(data[:, i], nan_as_null=nan_as_null) for column_label, i in zip( @@ -851,6 +859,11 @@ def _array_to_column_accessor( level_names=tuple(columns_labels.names), level_dtypes=_pd_index_level_dtypes(columns_labels), ) + if isinstance(columns_labels, pd.MultiIndex): + # prime the cache with the exact source MultiIndex (rebuilding from + # tuples would re-sort the levels) + ca.to_pandas_index = columns_labels + return ca @_performance_tracking @@ -1281,6 +1294,10 @@ def __init__( label_dtype=columns.dtype, level_dtypes=_pd_index_level_dtypes(columns), ) + if isinstance(columns, pd.MultiIndex): + # prime the cache with the exact source MultiIndex + # (rebuilding from tuples would re-sort the levels) + col_accessor.to_pandas_index = columns elif isinstance(data, Mapping): # Note: We excluded ColumnAccessor already above result = _mapping_to_column_accessor( @@ -1352,6 +1369,20 @@ def __init__( if dtype: self._data = self.astype(dtype)._data + final_pd_columns = ( + second_columns if second_columns is not None else columns + ) + if ( + self._data.multiindex + and isinstance(final_pd_columns, pd.MultiIndex) + and len(self._data) == len(final_pd_columns) + ): + # prime the cache with the exact source MultiIndex: rebuilding + # from tuples would re-sort the levels, losing e.g. an explicit + # unsorted level layout (level order affects pandas operations + # that work on codes, like legacy stack's sort) + self._data.to_pandas_index = final_pd_columns + @classmethod def _from_data( # type: ignore[override] cls, @@ -2554,6 +2585,11 @@ def _fill_same_ca_attributes( ) elif self._data._level_names == other._data._level_names: ca_attributes["level_names"] = self._data.level_names + if self._data.multiindex == other._data.multiindex: + # equal labels can still fail the ``equals`` check above + # on level-dtype differences (e.g. Int8 vs int64); the + # result keeps hierarchical columns like pandas + ca_attributes["multiindex"] = self._data.multiindex elif isinstance(other, (dict, Mapping)): # Need to fail early on host mapping types because we ultimately # convert everything to a dict. @@ -3237,6 +3273,7 @@ def columns(self, columns): rangeindex = False label_dtype = None level_names = None + level_dtypes = None if isinstance(columns, (pd.MultiIndex, cudf.MultiIndex)): multiindex = True if isinstance(columns, cudf.MultiIndex): @@ -3246,6 +3283,7 @@ def columns(self, columns): if pd_columns.nunique(dropna=False) != len(pd_columns): raise ValueError("Duplicate column names are not allowed") level_names = list(pd_columns.names) + level_dtypes = _pd_index_level_dtypes(pd_columns) elif isinstance(columns, (Index, ColumnBase, Series)): level_names = (getattr(columns, "name", None),) rangeindex = isinstance(columns, cudf.RangeIndex) @@ -3282,8 +3320,15 @@ def columns(self, columns): level_names=level_names, label_dtype=label_dtype, rangeindex=rangeindex, + level_dtypes=level_dtypes, verify=False, ) + if multiindex: + # prime the cache with the exact source MultiIndex: rebuilding + # from tuples would re-sort the levels, losing e.g. an explicit + # unsorted level layout (levels order affects pandas operations + # that work on codes, like legacy stack's sort) + self._data.to_pandas_index = pd_columns def _set_columns_like(self, other: ColumnAccessor) -> None: """ @@ -8096,6 +8141,11 @@ def stack( "https://github.com/pandas-dev/pandas/issues/53515" ) + _check_duplicate_level_names( + [lv for lv in level if not is_integer(lv)], + self._data.level_names, + ) + # Compute the columns to stack based on specified levels level_indices: list[int] = [] @@ -8110,10 +8160,24 @@ def stack( "mixture of both." ) else: - # Must be a list of positions, normalize negative positions - level_indices = [ - lv + self._data.nlevels if lv < 0 else lv for lv in level - ] + # Must be a list of positions; normalize negative positions + # and validate bounds to match pandas MultiIndex._get_level_number + nlevels = self._data.nlevels + for lv in level: + if lv < 0: + if lv + nlevels < 0: + raise IndexError( + f"Too many levels: Index has only {nlevels} " + f"levels, {lv} is not a valid level number" + ) + level_indices.append(lv + nlevels) + else: + if lv >= nlevels: + raise IndexError( + f"Too many levels: Index has only {nlevels} " + f"levels, not {lv + 1}" + ) + level_indices.append(lv) unnamed_levels_indices = [ i for i in range(self._data.nlevels) if i not in level_indices @@ -8121,48 +8185,127 @@ def stack( has_unnamed_levels = len(unnamed_levels_indices) > 0 column_name_idx = self._data.to_pandas_index - # Construct new index from the levels specified by `level` - named_levels = pd.MultiIndex.from_arrays( - [column_name_idx.get_level_values(lv) for lv in level_indices] + # pandas' Index.get_level_values resolves an integer argument by + # name first: if a level is *named* that integer, that level is + # returned regardless of position. All lookups below use positional + # indices, so strip the names to force positional resolution and + # re-attach the real names afterwards. + nameless_column_name_idx = column_name_idx.set_names( + [None] * column_name_idx.nlevels ) + # Construct new index from the levels specified by `level` + if isinstance(column_name_idx, pd.MultiIndex): + # build from codes/levels to keep the level dtypes: materializing + # via get_level_values/from_arrays turns missing entries into NaN + # and upcasts e.g. int64 levels to float64 + named_levels = pd.MultiIndex( + levels=[column_name_idx.levels[i] for i in level_indices], + codes=[column_name_idx.codes[i] for i in level_indices], + names=[column_name_idx.names[i] for i in level_indices], + verify_integrity=False, + ) + else: + named_levels = pd.MultiIndex.from_arrays( + [ + nameless_column_name_idx.get_level_values(lv).rename( + column_name_idx.names[lv] + ) + for lv in level_indices + ] + ) # Since `level` may only specify a subset of all levels, `unique()` is - # required to remove duplicates. In pandas, the order of the keys in - # the specified levels are always sorted. + # required to remove duplicates. In pandas legacy stack, the keys of + # the specified levels are sorted by their level *codes* when the + # columns have multiple levels (flat column labels keep their + # original order): level order is preserved even for unsorted levels + # and missing labels (code -1) come first. unique_named_levels = named_levels.unique() - if not future_stack: - unique_named_levels = unique_named_levels.sort_values() + if not future_stack and self._data.nlevels > 1: + unique_named_levels = unique_named_levels.take( + np.lexsort(tuple(reversed(unique_named_levels.codes))) + ) # Each index from the original dataframe should repeat by the number # of unique values in the named_levels repeated_index = self.index.repeat(len(unique_named_levels)) # Each column name should tile itself by len(df) times - cols = [ - as_column(unique_named_levels.get_level_values(i)) - for i in range(unique_named_levels.nlevels) - ] + nameless_unique_named_levels = unique_named_levels.set_names( + [None] * unique_named_levels.nlevels + ) + cols = [] + for i in range(unique_named_levels.nlevels): + if future_stack: + # pandas future stack materializes the level values (a + # level with missing entries becomes e.g. float64 with NaN) + cols.append( + as_column(nameless_unique_named_levels.get_level_values(i)) + ) + else: + # pandas legacy stack keeps the original level dtype and + # represents missing entries as nulls (-1 codes) + level_col = as_column(unique_named_levels.levels[i]) + level_codes = np.asarray(unique_named_levels.codes[i]).astype( + "int64" + ) + level_codes[level_codes == -1] = np.iinfo(SIZE_TYPE_DTYPE).min + cols.append( + level_col.take(as_column(level_codes), nullify=True) + ) with access_columns(*cols, mode="read", scope="internal"): plc_table = plc.reshape.tile( plc.Table([col.plc_column for col in cols]), self.shape[0], ) tiled_index = [ - ColumnBase.create(plc, dtype=dtype_from_pylibcudf_column(plc)) - for plc in plc_table.columns() + ColumnBase.create(plc_col, dtype=src_col.dtype) + for src_col, plc_col in zip( + cols, plc_table.columns(), strict=True + ) ] # Assemble the final index new_index_columns = [*repeated_index._columns, *tiled_index] index_names = [*self.index.names, *unique_named_levels.names] new_index = MultiIndex._from_data(dict(enumerate(new_index_columns))) - # Materialize the levels in order of first appearance (rather than the - # default sorted order) so that converting the result to pandas keeps - # the level order pandas' own ``stack`` produces. Otherwise a later - # ``unstack``/``to_pandas`` would lexicographically reorder the pivoted - # axis (e.g. ``"foo_10"`` before ``"foo_2"``). - new_index._maybe_materialize_codes_and_levels(sort=False) new_index.names = index_names + # Attach codes/levels eagerly, matching how pandas' stack builds the + # result MultiIndex, so a later unstack can restore the original + # row/column order (lazy materialization would sort the levels): + # the original index contributes its own levels/codes (repeated); + # a flat original index and the tiled stacked level(s) get + # appearance-order factorization. + new_levels = [] + new_codes = [] + n_tile = len(unique_named_levels) + if isinstance(self.index, MultiIndex): + src = self.index._maybe_materialize_codes_and_levels() + for src_level, src_code in zip( + src._levels, + src._codes, + strict=True, + ): + new_levels.append(src_level) + new_codes.append( + Index._from_column(src_code.astype(np.dtype(np.int64))) + .repeat(n_tile) + ._column + ) + else: + code, cats = factorize(self.index) + new_levels.append(cats) + new_codes.append( + Index._from_column(as_column(code).astype(np.dtype(np.int64))) + .repeat(n_tile) + ._column + ) + for tiled_col in tiled_index: + code, cats = factorize(Index._from_column(tiled_col)) + new_codes.append(as_column(code).astype(np.dtype(np.int64))) + new_levels.append(cats) + new_index._levels = new_levels + new_index._codes = new_codes # Compute the column indices that serves as the input for # `interleave_columns` @@ -8171,41 +8314,49 @@ def stack( ) if has_unnamed_levels: - unnamed_level_values = pd.MultiIndex.from_arrays( - list( - map( - column_name_idx.get_level_values, - unnamed_levels_indices, - ) - ) + # the columns axis has multiple levels here, so column_name_idx + # is always a pd.MultiIndex; build from codes/levels to keep the + # level dtypes and to resolve the levels positionally + unnamed_level_values = pd.MultiIndex( + levels=[ + column_name_idx.levels[i] for i in unnamed_levels_indices + ], + codes=[ + column_name_idx.codes[i] for i in unnamed_levels_indices + ], + names=[ + column_name_idx.names[i] for i in unnamed_levels_indices + ], + verify_integrity=False, ) def unnamed_group_generator(): if has_unnamed_levels: - for _, grpdf in column_idx_df.groupby(by=unnamed_level_values): + # sort=False iterates groups in first-appearance order, i.e. + # exactly ``unnamed_level_values.unique()`` order (also for + # NaN-containing tuple keys, which sorted groupby would + # reorder via codes), so the stacked columns can be zipped + # 1:1 with those keys when assembling the result. + for _, grpdf in column_idx_df.groupby( + by=unnamed_level_values, sort=False + ): # When stacking part of the levels, some combinations # of keys may not be present in this group but can be # present in others. Reindexing with the globally computed # `unique_named_levels` assigns -1 to these key # combinations, representing an all-null column that # is used in the subsequent libcudf call. - if future_stack: - yield grpdf.reindex( - unique_named_levels, axis=0, fill_value=-1 - ).values - else: - yield ( - grpdf.reindex( - unique_named_levels, axis=0, fill_value=-1 - ) - .sort_index() - .values - ) + # ``reindex`` returns rows in target order, so the + # legacy path needs no further sorting (the target was + # already sorted above). + yield grpdf.reindex( + unique_named_levels, axis=0, fill_value=-1 + ).values else: - if future_stack: + if future_stack or self._data.nlevels == 1: yield column_idx_df.values else: - yield column_idx_df.sort_index().values + yield column_idx_df.reindex(unique_named_levels).values # For each of the group constructed from the unnamed levels, # invoke `interleave_columns` to stack the values. @@ -8261,23 +8412,35 @@ def unnamed_group_generator(): unnamed_level_values = unnamed_level_values.get_level_values(0) unnamed_level_values = unnamed_level_values.unique() - data = ColumnAccessor( - dict( - zip( - unnamed_level_values, - [ - stacked[i] - for i in unnamed_level_values.argsort().argsort() - ] - if not future_stack - else [ - stacked[i] for i in unnamed_level_values.argsort() - ], - strict=True, + if isinstance(unnamed_level_values, pd.MultiIndex): + # build the labels from levels/codes to preserve scalar + # types: iterating a MultiIndex materializes e.g. an int64 + # level containing a missing entry as float + keys: Any = [ + tuple( + unnamed_level_values.levels[j][c] + if c != -1 + else np.nan + for j, c in enumerate(row) ) - ), + for row in zip(*unnamed_level_values.codes, strict=True) + ] + else: + keys = unnamed_level_values + + # ``stacked`` is in group first-appearance order (groupby with + # sort=False above), which is exactly the order of + # ``unnamed_level_values.unique()``: zip 1:1. + data = ColumnAccessor( + dict(zip(keys, stacked, strict=True)), isinstance(unnamed_level_values, pd.MultiIndex), unnamed_level_values.names, + label_dtype=( + None + if isinstance(unnamed_level_values, pd.MultiIndex) + else unnamed_level_values.dtype + ), + level_dtypes=_pd_index_level_dtypes(unnamed_level_values), ) result = DataFrame._from_data( @@ -8285,7 +8448,24 @@ def unnamed_group_generator(): ) if not future_stack and dropna: - return result.dropna(how="all") + # dropna would gather the index and discard the eagerly + # attached appearance-order codes/levels; compute the row mask + # explicitly so the codes can be subset alongside (pandas keeps + # the full pre-drop levels through dropna). + if isinstance(result, Series): + keep = result.notna() + else: + keep = ~result.isna().all(axis=1) + keep_col = keep._column + dropped = result._apply_boolean_mask( + BooleanMask(keep_col, len(result)) + ) + if isinstance(dropped.index, MultiIndex): + dropped.index._levels = new_levels + dropped.index._codes = [ + code.apply_boolean_mask(keep_col) for code in new_codes + ] + return dropped else: return result diff --git a/python/cudf/cudf/core/groupby/groupby.py b/python/cudf/cudf/core/groupby/groupby.py index b79dc5a85d3b..c98eb560d390 100644 --- a/python/cudf/cudf/core/groupby/groupby.py +++ b/python/cudf/cudf/core/groupby/groupby.py @@ -1314,7 +1314,7 @@ def agg(self, func=None, *args, engine=None, engine_kwargs=None, **kwargs): # RangeIndex) columns. data = ColumnAccessor( data, - multiindex=False, + multiindex=self.obj._data.multiindex, level_names=self.obj._data.level_names, rangeindex=self.obj._data.rangeindex, label_dtype=self.obj._data.label_dtype, @@ -1325,12 +1325,30 @@ def agg(self, func=None, *args, engine=None, engine_kwargs=None, **kwargs): and self.obj.ndim == 2 and self.obj._data.level_names != (None,) ): - data = ColumnAccessor( - data, - multiindex=False, - level_names=self.obj._data.level_names, - label_dtype=self.obj._data.label_dtype, - ) + if self.obj._data.multiindex and all( + isinstance(label, tuple) + and len(label) == self.obj._data.nlevels + for label in data + ): + # the aggregation kept the source's tuple labels: preserve + # the MultiIndex columns and their per-level metadata + data = ColumnAccessor( + data, + multiindex=True, + level_names=self.obj._data.level_names, + label_dtype=self.obj._data.label_dtype, + level_dtypes=self.obj._data.level_dtypes, + ) + else: + # relabeling aggregations (``agg(new=(col, func))``) emit + # new flat labels: the source's multi-level metadata does + # not describe them + data = ColumnAccessor( + data, + multiindex=False, + level_names=self.obj._data.level_names, + label_dtype=self.obj._data.label_dtype, + ) else: data = ColumnAccessor(data, multiindex=multilevel) if not multilevel and len(data) > 0: diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index b8cf52d668e8..bdc08e82fe88 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -5,6 +5,7 @@ from __future__ import annotations import copy +import functools import itertools import textwrap import warnings @@ -2921,7 +2922,71 @@ def sort_index( if ignore_index: out = out.reset_index(drop=True) else: - labels = sorted(self._column_names, reverse=not ascending) + if level is not None and self._data.multiindex: + if not is_list_like(level): + level = [level] + nlevels = self._data.nlevels + level_names = list(self._data.level_names) + + def _level_number(lvl): + if isinstance(lvl, int): + norm = lvl + nlevels if lvl < 0 else lvl + if not 0 <= norm < nlevels: + raise IndexError( + f"Too many levels: Index has only " + f"{nlevels} levels, {lvl} is not a valid " + "level number" + ) + return norm + return level_names.index(lvl) + + key_order = [_level_number(lvl) for lvl in level] + ascending_per_key: list[bool] = ( + [bool(flag) for flag in cast("Iterable[bool]", ascending)] + if is_list_like(ascending) + else [bool(ascending)] * len(key_order) + ) + if len(ascending_per_key) != len(key_order): + raise ValueError( + "level must have same length as ascending: " + f"{len(key_order)} != {len(ascending_per_key)}" + ) + if sort_remaining: + seen = set(key_order) + for i in range(nlevels): + if i not in seen: + key_order.append(i) + ascending_per_key.append(ascending_per_key[0]) + + def _is_na_label(value) -> bool: + return value is None or ( + isinstance(value, float) and value != value + ) + + def _label_sort_key( + label, pos: int, na_rank: int + ) -> tuple[int, Any]: + value = label[pos] + if _is_na_label(value): + return (na_rank, None) + return (1 - na_rank, value) + + # Stable multi-key sort: sort by the least significant key + # first. Missing labels are placed per ``na_position``, + # independently of the per-key direction. + labels = list(self._column_names) + for pos, asc in reversed( + list(zip(key_order, ascending_per_key, strict=True)) + ): + na_rank = int(asc == (na_position == "last")) + labels.sort( + key=functools.partial( + _label_sort_key, pos=pos, na_rank=na_rank + ), + reverse=not asc, + ) + else: + labels = sorted(self._column_names, reverse=not ascending) result_columns = (self._data[label] for label in labels) if ignore_index: ca = ColumnAccessor( diff --git a/python/cudf/cudf/core/multiindex.py b/python/cudf/cudf/core/multiindex.py index c9c621ce7b98..b846f52ecf02 100644 --- a/python/cudf/cudf/core/multiindex.py +++ b/python/cudf/cudf/core/multiindex.py @@ -2109,14 +2109,14 @@ def _level_index_from_level(self, level) -> int: except ValueError: if not is_integer(level): raise KeyError(f"Level {level} not found") - if level < 0: - level += self.nlevels - if level >= self.nlevels: + norm = level + self.nlevels if level < 0 else level + if not 0 <= norm < self.nlevels: + # matches pandas MultiIndex._get_level_number raise IndexError( - f"Level {level} out of bounds. " - f"Index has {self.nlevels} levels." + f"Too many levels: Index has only {self.nlevels} " + f"levels, {level} is not a valid level number" ) from None - return level + return norm @_performance_tracking def get_indexer(self, target, method=None, limit=None, tolerance=None): diff --git a/python/cudf/cudf/core/reshape.py b/python/cudf/cudf/core/reshape.py index e8736f789377..ce08dc3b5c61 100644 --- a/python/cudf/cudf/core/reshape.py +++ b/python/cudf/cudf/core/reshape.py @@ -11,7 +11,7 @@ import cudf from cudf.api.extensions import no_default -from cudf.api.types import is_list_like, is_scalar +from cudf.api.types import is_integer, is_list_like, is_scalar from cudf.core.column import ( ColumnBase, as_column, @@ -938,8 +938,11 @@ def get_dummies( def _pivot( col_accessor: ColumnAccessor, - index: Index | MultiIndex, - columns: Index | MultiIndex, + index_labels: Index | MultiIndex, + index_idx: ColumnBase, + columns_labels: Index | MultiIndex, + columns_idx: ColumnBase, + promote_ints_on_missing: bool = False, ) -> DataFrame: """ Reorganize the values of the DataFrame according to the given @@ -947,14 +950,22 @@ def _pivot( Parameters ---------- - col_accessor : DataFrame - index : Index - Index labels of the result - columns : Index - Column labels of the result + col_accessor : ColumnAccessor + Values to pivot into the result's columns. + index_labels : Index + Distinct index keys; row labels of the result. + index_idx : ColumnBase + Position of each source row's key within ``index_labels``. + columns_labels : Index + Distinct column keys; labels of the result's new column level(s). + columns_idx : ColumnBase + Position of each source row's key within ``columns_labels``. + promote_ints_on_missing : bool + Promote integer source columns to float64 when the reshape + introduces missing cells, as pandas' unstack does. Only the + unstack path wants this: pivot_table/crosstab fill missing + cells afterwards and keep the integer dtype. """ - columns_labels, columns_idx = columns._encode() - index_labels, index_idx = index._encode() column_labels = columns_labels.to_pandas().to_flat_index() result = {} @@ -964,12 +975,26 @@ def as_tuple(x): return x if isinstance(x, tuple) else (x,) nrows = len(index_labels) + promote_ints = promote_ints_on_missing and cudf.get_option( + "mode.pandas_compatible" + ) for col_label, col in col_accessor.items(): names = [ as_tuple(col_label) + as_tuple(name) for name in column_labels ] new_size = nrows * len(names) scatter_map = (columns_idx * np.int32(nrows)) + index_idx + if ( + promote_ints + and new_size > len(col) + and isinstance(col.dtype, np.dtype) + and col.dtype.kind in "iu" + ): + # pandas builds one 2-D values block per source column and + # promotes the whole block to float64 when the reshape + # introduces missing entries, so even gap-free result + # columns become float64 + col = col.astype(np.dtype(np.float64)) target_col = column_empty(row_count=new_size, dtype=col.dtype) target_col[scatter_map] = col result.update( @@ -984,16 +1009,82 @@ def as_tuple(x): ) ) - # the result of pivot always has a MultiIndex + # the result of pivot always has a MultiIndex; the leading level(s) + # come from the source frame's column labels, so preserve their names ca = ColumnAccessor( result, multiindex=True, - level_names=(None, *columns._column_names), + level_names=( + *col_accessor.level_names, + *columns_labels.names, + ), verify=False, ) return cudf.DataFrame._from_data(ca, index=index_labels) +def _unstack_encode_by_codes( + mi: MultiIndex, level +) -> tuple[Index | MultiIndex, ColumnBase, Index | MultiIndex, ColumnBase]: + """Encode unstack keys ordered by the MultiIndex level codes. + + libcudf's ``encode`` orders distinct keys by sorted value with nulls + last, but pandas' unstack orders keys by the index's level codes: the + level order is preserved and missing entries (code -1) come first. + Encoding the integer code columns instead of the level values yields + exactly that order. + """ + lvl_idx = mi._level_index_from_level(level) + mi._maybe_materialize_codes_and_levels() + names = mi.names + + def encode_side(sel: list[int]) -> tuple[Index | MultiIndex, ColumnBase]: + code_cols = [] + for i in sel: + code = mi._codes[i].astype(np.dtype(np.int64)).copy() # type: ignore[index] + # Normalize the NA sentinel (``MultiIndex.__init__`` stores + # ``iinfo(SIZE_TYPE_DTYPE).min``, lazy factorization stores -1) + # so the missing-key group encodes as one key that sorts first. + code[code < 0] = -1 + code_cols.append(code) + code_frame = cudf.DataFrame._from_data( + ColumnAccessor(dict(enumerate(code_cols)), verify=False) + ) + key_codes, idx = code_frame._encode() + labels_data = {} + out_levels = [] + out_codes = [] + for j, i in enumerate(sel): + kc = key_codes._columns[j].astype(np.dtype(np.int64)) + out_levels.append(mi._levels[i]) # type: ignore[index] + out_codes.append(kc) + gather_codes = kc.copy() + gather_codes[gather_codes == -1] = np.iinfo(SIZE_TYPE_DTYPE).min + # key by position: level names may be duplicated or None + labels_data[j] = mi._levels[i]._column.take( # type: ignore[index] + gather_codes, nullify=True + ) + if len(labels_data) == 1: + labels: Index | MultiIndex = cudf.Index._from_column( + next(iter(labels_data.values())), name=names[sel[0]] + ) + else: + mi_labels = cudf.MultiIndex._from_data(labels_data) + mi_labels.names = [names[i] for i in sel] + # carry the original level objects and the keys' codes so that + # a subsequent unstack/stack keeps ordering by the original + # levels, exactly like pandas (which reuses the level objects) + mi_labels._levels = out_levels + mi_labels._codes = out_codes + labels = mi_labels + return labels, idx + + remaining = [i for i in range(mi.nlevels) if i != lvl_idx] + index_labels, index_idx = encode_side(remaining) + columns_labels, columns_idx = encode_side([lvl_idx]) + return index_labels, index_idx, columns_labels, columns_idx + + def pivot( data: DataFrame, columns=None, index=no_default, values=no_default ) -> DataFrame: @@ -1125,8 +1216,15 @@ def pivot( if len(columns_index) != len(columns_index.drop_duplicates()): raise ValueError("Duplicate index-column pairs found. Cannot reshape.") + selection = data._data.select_by_label(cols_to_select) + if values is not no_default: + # pandas rebuilds the columns axis from ``values`` and drops the + # original columns-axis name(s) + selection._level_names = (None,) * selection.nlevels + columns_labels, columns_idx = column_data._encode() + index_labels, index_idx = index_data._encode() result = _pivot( - data._data.select_by_label(cols_to_select), index_data, column_data + selection, index_labels, index_idx, columns_labels, columns_idx ) result._attrs = data.attrs @@ -1229,6 +1327,23 @@ def unstack(df, level, fill_value=None, sort: bool = True): 2 7 dtype: int64 """ + return _unstack(df, level, fill_value=fill_value, sort=sort) + + +def _unstack( + df, + level, + fill_value=None, + sort: bool = True, + promote_ints_on_missing: bool = True, +): + """``unstack`` implementation. + + ``promote_ints_on_missing`` promotes integer source columns to float64 + when the reshape introduces missing cells, like pandas' unstack. + ``pivot_table`` (and thereby ``crosstab``) disables it because those fill + the missing cells afterwards and keep the integer dtype. + """ if not isinstance(df, cudf.DataFrame): raise ValueError("`df` should be a cudf Dataframe object.") @@ -1256,7 +1371,14 @@ def unstack(df, level, fill_value=None, sort: bool = True): if not is_scalar(level): if not level: return df + if len(level) == 1: + # pandas normalizes a length-1 list-like level to a scalar + level = level[0] if not isinstance(df.index, cudf.MultiIndex): + if not is_integer(level): + # pandas validates non-integer levels against the flat index + # name and raises KeyError on a mismatch + df.index._validate_index_level(level) dtype = df._columns[0].dtype if any(col_dtype != dtype for _, col_dtype in df._dtypes): raise ValueError( @@ -1271,10 +1393,22 @@ def unstack(df, level, fill_value=None, sort: bool = True): res._attrs = df.attrs return res else: - index = df.index.droplevel(level) + from cudf.core.indexed_frame import _check_duplicate_level_names + + specified = [level] if is_scalar(level) else list(level) + _check_duplicate_level_names( + [lv for lv in specified if not is_integer(lv)], + df.index.names, + ) if is_scalar(level): - columns = df.index.get_level_values(level) + # order rows/columns by the removed level's codes (pandas + # semantics: level order preserved, missing entries first), + # not by sorted level values + index_labels, index_idx, columns_labels, columns_idx = ( + _unstack_encode_by_codes(df.index, level) + ) else: + index = df.index.droplevel(level) new_names = [] ca_data = {} for lev in level: @@ -1285,8 +1419,38 @@ def unstack(df, level, fill_value=None, sort: bool = True): ColumnAccessor(ca_data, verify=False) ) columns.names = new_names - result = _pivot(df, index, columns) + columns_labels, columns_idx = columns._encode() + index_labels, index_idx = index._encode() + result = _pivot( + df._data, + index_labels, + index_idx, + columns_labels, + columns_idx, + promote_ints_on_missing=promote_ints_on_missing, + ) result._attrs = df.attrs + if is_scalar(level): + # pandas keeps unused categories of the removed level + # ("removed_level_full", pandas GH 17845) in + # result.columns.levels even though no columns are created + # for them. + _, level_idx = df.index._level_to_ca_label(level) + full_level = df.index.levels[level_idx].to_pandas() + pdi = result._data.to_pandas_index + if isinstance(pdi, pd.MultiIndex): + level_values = pdi.get_level_values(-1) + new_codes = full_level.get_indexer(level_values) + # -1 codes for NA labels are pandas' canonical missing + # representation; only bail out when a non-NA label failed + # to map into the full level + if ((new_codes >= 0) | pd.isna(level_values)).all(): + result._data.to_pandas_index = pd.MultiIndex( + levels=[*pdi.levels[:-1], full_level], + codes=[*pdi.codes[:-1], new_codes], + names=pdi.names, + verify_integrity=False, + ) if result.index.nlevels == 1: result.index = result.index.get_level_values(result.index.names[0]) return result @@ -1600,7 +1764,13 @@ def pivot_table( to_unstack.append(i) else: to_unstack.append(name) - table = agged.unstack(to_unstack) + table = _unstack( + agged, + to_unstack, + # pandas keeps the integer dtype when the missing cells are + # filled afterwards, and promotes to float64 when they are not + promote_ints_on_missing=fill_value is None, + ) if fill_value is not None: table = table.fillna(fill_value) diff --git a/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py b/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py index 7e8c65c7b680..0b45020dcb85 100644 --- a/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py +++ b/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py @@ -1302,7 +1302,6 @@ def pytest_unconfigure(config): "tests/frame/methods/test_shift.py::TestDataFrameShift::test_shift_dt64values_axis1_invalid_fill[datetime64[us]-False]": "AssertionError: assert 1 == 2", "tests/frame/methods/test_shift.py::TestDataFrameShift::test_shift_dt64values_axis1_invalid_fill[timedelta64[us]-False]": "AssertionError: assert 1 == 2", "tests/frame/methods/test_shift.py::TestDataFrameShift::test_shift_dt64values_int_fill_deprecated": "TODO: Add a reason for failure", - "tests/frame/methods/test_sort_index.py::TestDataFrameSortIndex::test_sort_index_intervalindex": "TODO: Add a reason for failure", "tests/frame/methods/test_sort_index.py::TestDataFrameSortIndex::test_sort_index_nan": "TODO: Add a reason for failure", "tests/frame/methods/test_sort_values.py::TestDataFrameSortValues::test_sort_by_column_named_none": "AssertionError: DataFrame.index are different", "tests/frame/methods/test_sort_values.py::TestDataFrameSortValues::test_sort_values_by_empty_list": "TODO: Add a reason for failure", @@ -1391,9 +1390,6 @@ def pytest_unconfigure(config): "tests/frame/test_constructors.py::TestDataFrameConstructors::test_constructor_dict_cast": "TODO: Add a reason for failure", "tests/frame/test_constructors.py::TestDataFrameConstructors::test_constructor_dict_multiindex": "TODO: Add a reason for failure", "tests/frame/test_constructors.py::TestDataFrameConstructors::test_constructor_dict_nan_key[None]": "TODO: Add a reason for failure", - "tests/frame/test_constructors.py::TestDataFrameConstructors::test_constructor_dict_nan_key[nan0]": "AssertionError: Attributes of DataFrame.iloc[:, 1] (column name='nan') are different", - "tests/frame/test_constructors.py::TestDataFrameConstructors::test_constructor_dict_nan_key[nan1]": "AssertionError: Attributes of DataFrame.iloc[:, 1] (column name='nan') are different", - "tests/frame/test_constructors.py::TestDataFrameConstructors::test_constructor_dict_nan_key_and_columns": "TODO: Add a reason for failure", "tests/frame/test_constructors.py::TestDataFrameConstructors::test_constructor_dict_with_index": "TODO: Add a reason for failure", "tests/frame/test_constructors.py::TestDataFrameConstructors::test_constructor_dict_with_index_and_columns": "TODO: Add a reason for failure", "tests/frame/test_constructors.py::TestDataFrameConstructors::test_constructor_dict_with_none": "AssertionError: assert nan is None", @@ -1675,67 +1671,9 @@ def pytest_unconfigure(config): "tests/frame/test_reductions.py::test_reduction_axis_none_returns_scalar[float64-True-mean]": "TODO: Add a reason for failure", "tests/frame/test_reductions.py::test_reduction_axis_none_returns_scalar[float64-True-median]": "TODO: Add a reason for failure", "tests/frame/test_reductions.py::test_reduction_axis_none_returns_scalar[float64-True-skew]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_stack_int_level_names[False]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_stack_int_level_names[True]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_stack_ints[False]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_stack_ints[True]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_stack_mixed_level[False]": "AssertionError: DataFrame.columns are different", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_stack_mixed_level[True]": "AssertionError: DataFrame.columns are different", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_stack_mixed_levels[False]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_stack_mixed_levels[True]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_stack_multi_preserve_categorical_dtype[False-labels0-data0-False]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_stack_multi_preserve_categorical_dtype[False-labels0-data0-True]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_stack_multi_preserve_categorical_dtype[False-labels1-data1-False]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_stack_multi_preserve_categorical_dtype[False-labels1-data1-True]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_stack_multi_preserve_categorical_dtype[True-labels0-data0-False]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_stack_multi_preserve_categorical_dtype[True-labels0-data0-True]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_stack_multi_preserve_categorical_dtype[True-labels1-data1-False]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_stack_multi_preserve_categorical_dtype[True-labels1-data1-True]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_stack_preserve_categorical_dtype[False-False]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_stack_preserve_categorical_dtype[False-True]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_stack_preserve_categorical_dtype[True-False]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_stack_preserve_categorical_dtype[True-True]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_stack_unstack[False]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_stack_unstack[True]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_unstack_bool": "AssertionError: DataFrame.iloc[:, 0] (column name='('col', 'c')') are different", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_unstack_multi_level_rows_and_cols": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_unstack_nan_index2": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_unstack_nan_index3": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_unstack_non_unique_index_names[False]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_unstack_non_unique_index_names[True]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_unstack_not_consolidated": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_unstack_swaplevel_sortlevel[0]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_unstack_swaplevel_sortlevel[baz]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_unstack_unused_levels": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_multi_level_stack_categorical[False]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_multi_level_stack_categorical[True]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack[False]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack[True]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_level_name[False]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_level_name[True]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_mixed_dtype[True]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_multiple_bug[False]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_multiple_bug[True]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_multiple_out_of_bounds[False]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_multiple_out_of_bounds[True]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_nan_in_multiindex_columns[False]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_nan_in_multiindex_columns[True]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_nan_level[False]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_nullable_dtype[False]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_nullable_dtype[True]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_order_with_unsorted_levels_multi_row_2[False]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_unstack_multiple[False]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_unstack_multiple[True]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_unstack_preserve_names[False]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_unstack_preserve_names[True]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_unstack_wrong_level_name[False-unstack]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_unstack_wrong_level_name[True-unstack]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_unstack_preserve_types": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_unstack_with_missing_int_cast_to_float": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::test_unstack_sort_false_nan[nan=first]": "AssertionError: Attributes of DataFrame.iloc[:, 0] (column name='('value', nan)') are different", - "tests/frame/test_stack_unstack.py::test_unstack_sort_false_nan[nan=last]": "AssertionError: Attributes of DataFrame.iloc[:, 3] (column name='('value', nan)') are different", - "tests/frame/test_stack_unstack.py::test_unstack_sort_false_nan[nan=second]": "AssertionError: Attributes of DataFrame.iloc[:, 1] (column name='('value', nan)') are different", - "tests/frame/test_stack_unstack.py::test_unstack_sort_false_nan[nan=third]": "AssertionError: Attributes of DataFrame.iloc[:, 2] (column name='('value', nan)') are different", + "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_unstack_bool": "cudf converts null bools to None where pandas' unstack upcasts to object with np.nan", + "tests/frame/test_stack_unstack.py::TestDataFrameReshape::test_unstack_not_consolidated": "Asserts DataFrame._mgr block layout (pandas internals)", + "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_unstack_with_missing_int_cast_to_float": "Asserts DataFrame._mgr block layout (pandas internals)", "tests/frame/test_subclass.py::TestDataFrameSubclassing::test_asof": "TODO: Add a reason for failure", "tests/frame/test_subclass.py::TestDataFrameSubclassing::test_equals_subclass": "TODO: Add a reason for failure", "tests/frame/test_subclass.py::TestDataFrameSubclassing::test_frame_subclassing_and_slicing": "TODO: Add a reason for failure", @@ -1766,8 +1704,6 @@ def pytest_unconfigure(config): "tests/groupby/aggregate/test_aggregate.py::test_agg_str_with_kwarg_axis_1_raises[nunique]": "TODO: Add a reason for failure", "tests/groupby/aggregate/test_aggregate.py::test_groupby_aggregate_directory[size]": "TODO: Add a reason for failure", "tests/groupby/aggregate/test_aggregate.py::test_groupby_aggregate_empty_key_empty_return": "TODO: Add a reason for failure", - "tests/groupby/aggregate/test_aggregate.py::test_multiindex_custom_func[0]": "TODO: Add a reason for failure", - "tests/groupby/aggregate/test_aggregate.py::test_order_aggregate_multiple_funcs": "TODO: Add a reason for failure", "tests/groupby/aggregate/test_cython.py::test_cython_agg_EA_known_dtypes[data1-prod-large_int-False]": "TODO: Add a reason for failure", "tests/groupby/aggregate/test_cython.py::test_cython_agg_EA_known_dtypes[data1-prod-large_int-True]": "TODO: Add a reason for failure", "tests/groupby/aggregate/test_cython.py::test_cython_agg_EA_known_dtypes[data1-sum-large_int-False]": "TODO: Add a reason for failure", @@ -1830,7 +1766,6 @@ def pytest_unconfigure(config): "tests/groupby/test_api.py::test_tab_completion": "TODO: Add a reason for failure", "tests/groupby/test_apply.py::test_apply_with_date_in_multiindex_does_not_convert_to_timestamp": "cudf stores datetime.date values as datetime64; the date type identity is lost on the GPU round trip", "tests/groupby/test_apply.py::test_positional_slice_groups_datetimelike": "the frame and its column Series are converted to pandas independently on fallback, losing the CoW block identity pandas' is_in_obj grouper check requires", - "tests/groupby/test_categorical.py::test_describe_categorical_columns": "cudf's multi-level groupby aggregation and stack() drop the categorical column-index dtype", "tests/groupby/test_cumulative.py::test_groupby_cumprod_nan_influences_other_columns": "TODO: Add a reason for failure", "tests/groupby/test_cumulative.py::test_numpy_compat[cumprod]": "TODO: Add a reason for failure", "tests/groupby/test_cumulative.py::test_numpy_compat[cumsum]": "TODO: Add a reason for failure", @@ -1886,7 +1821,6 @@ def pytest_unconfigure(config): "tests/groupby/test_groupby.py::test_groupby_with_Time_Grouper[ns]": "TODO: Add a reason for failure", "tests/groupby/test_groupby.py::test_groupby_with_Time_Grouper[s]": "TODO: Add a reason for failure", "tests/groupby/test_groupby.py::test_groupby_with_Time_Grouper[us]": "TODO: Add a reason for failure", - "tests/groupby/test_groupby.py::test_groupby_with_hier_columns": "TODO: Add a reason for failure", "tests/groupby/test_groupby.py::test_groups_repr_truncates[1-{0: [0], ...}]": "TODO: Add a reason for failure", "tests/groupby/test_groupby.py::test_groups_repr_truncates[4-{0: [0], 1: [1], 2: [2], 3: [3], ...}]": "TODO: Add a reason for failure", "tests/groupby/test_groupby.py::test_groups_repr_truncates[5-{0: [0], 1: [1], 2: [2], 3: [3], 4: [4]}]": "TODO: Add a reason for failure", @@ -1895,7 +1829,6 @@ def pytest_unconfigure(config): "tests/groupby/test_groupby.py::test_ops_not_as_index[idxmin]": "TODO: Add a reason for failure", "tests/groupby/test_groupby.py::test_ops_not_as_index[size]": "TODO: Add a reason for failure", "tests/groupby/test_groupby.py::test_single_element_listlike_level_grouping[level_arg0-False]": "AssertionError: assert ['x', 'y'] == [('x',), ('y',)]", - "tests/groupby/test_groupby.py::test_wrap_aggregated_output_multindex": "TODO: Add a reason for failure", "tests/groupby/test_groupby_dropna.py::test_groupby_nan_included": "GroupBy.indices returns cupy arrays nested in a dict that cudf.pandas does not wrap, so assert_numpy_array_equal sees mismatched array classes", "tests/groupby/test_groupby_subclass.py::test_groupby_preserves_metadata": "proxy pd.Index passed to the raw SubclassedSeries constructor loses its name: pandas' maybe_extract_name checks isinstance against the concrete Index class, which proxies cannot satisfy", "tests/groupby/test_grouping.py::TestGetGroup::test_get_group_grouped_by_tuple": "TODO: Add a reason for failure", @@ -2513,7 +2446,6 @@ def pytest_unconfigure(config): "tests/indexing/test_loc.py::TestLocSetitemWithExpansion::test_loc_setitem_with_expansion_nonunique_index[string-pyarrow-True]": 'AssertionError: Column name="0" are different', "tests/indexing/test_loc.py::TestLocSetitemWithExpansion::test_loc_setitem_with_expansion_nonunique_index[string-python-False]": 'AssertionError: Column name="0" are different', "tests/indexing/test_loc.py::TestLocSetitemWithExpansion::test_loc_setitem_with_expansion_nonunique_index[string-python-True]": 'AssertionError: Column name="0" are different', - "tests/indexing/test_loc.py::TestLocWithMultiIndex::test_loc_set_nan_in_categorical_series[Float64]": "TODO: Add a reason for failure", "tests/indexing/test_loc.py::test_loc_getitem_multiindex_tuple_level": "AssertionError: DataFrame Expected type , found instead", "tests/indexing/test_na_indexing.py::test_series_mask_boolean[True-list-mask0-values0-object]": "TODO: Add a reason for failure", "tests/indexing/test_na_indexing.py::test_series_mask_boolean[True-list-mask1-values0-object]": "TODO: Add a reason for failure", @@ -2904,7 +2836,6 @@ def pytest_unconfigure(config): "tests/reshape/concat/test_categorical.py::TestCategoricalConcat::test_categorical_index_upcast": "TODO: Add a reason for failure", "tests/reshape/concat/test_categorical.py::TestCategoricalConcat::test_concat_categorical_datetime": "TODO: Add a reason for failure", "tests/reshape/concat/test_concat.py::TestConcatenate::test_concat_copy": "TODO: Add a reason for failure", - "tests/reshape/concat/test_concat.py::TestConcatenate::test_concat_keys_specific_levels": "TODO: Add a reason for failure", "tests/reshape/concat/test_concat.py::TestConcatenate::test_concat_order": "TODO: Add a reason for failure", "tests/reshape/concat/test_concat.py::test_concat_empty_and_non_empty_frame_regression": "TODO: Add a reason for failure", "tests/reshape/concat/test_concat.py::test_concat_ignore_empty_object_float[None-datetime64[ns]]": "AssertionError: Attributes of DataFrame.iloc[:, 0] (column name='foo') are different", @@ -3032,10 +2963,8 @@ def pytest_unconfigure(config): "tests/reshape/merge/test_multi.py::TestMergeMulti::test_left_join_multi_index[True-False]": "AssertionError: DataFrame.iloc[:, 4] (column name='5th') are different", "tests/reshape/merge/test_multi.py::TestMergeMulti::test_left_join_multi_index[True-True]": "TODO: Add a reason for failure", "tests/reshape/test_crosstab.py::TestCrosstab::test_crosstab_duplicate_names": "TODO: Add a reason for failure", - "tests/reshape/test_crosstab.py::TestCrosstab::test_crosstab_multiple": "TODO: Add a reason for failure", "tests/reshape/test_crosstab.py::TestCrosstab::test_crosstab_no_overlap": "TODO: Add a reason for failure", "tests/reshape/test_crosstab.py::TestCrosstab::test_crosstab_with_categorial_columns": "TODO: Add a reason for failure", - "tests/reshape/test_crosstab.py::TestCrosstab::test_crosstab_with_empties": "TODO: Add a reason for failure", "tests/reshape/test_cut.py::test_bins[array]": "TODO: Add a reason for failure", "tests/reshape/test_cut.py::test_bins[list]": "TODO: Add a reason for failure", "tests/reshape/test_cut.py::test_bins_from_interval_index": "TODO: Add a reason for failure", @@ -3081,14 +3010,10 @@ def pytest_unconfigure(config): "tests/reshape/test_melt.py::TestWideToLong::test_raise_of_column_name_value": "TODO: Add a reason for failure", "tests/reshape/test_pivot.py::TestPivot::test_pivot_index_is_none": "AssertionError: DataFrame.index are different", "tests/reshape/test_pivot.py::TestPivotTable::test_categorical_pivot_index_ordering[False]": "TODO: Add a reason for failure", - "tests/reshape/test_pivot.py::TestPivotTable::test_daily": "TODO: Add a reason for failure", - "tests/reshape/test_pivot.py::TestPivotTable::test_monthly": "TODO: Add a reason for failure", - "tests/reshape/test_pivot.py::TestPivotTable::test_pivot_complex_aggfunc": "TODO: Add a reason for failure", "tests/reshape/test_pivot.py::TestPivotTable::test_pivot_datetime_tz": "ValueError: Length of names must match number of levels in MultiIndex.", "tests/reshape/test_pivot.py::TestPivotTable::test_pivot_index_with_nan[False]": "AssertionError: DataFrame.index are different", "tests/reshape/test_pivot.py::TestPivotTable::test_pivot_index_with_nan[True]": "AssertionError: DataFrame.index are different", "tests/reshape/test_pivot.py::TestPivotTable::test_pivot_multi_functions": "TODO: Add a reason for failure", - "tests/reshape/test_pivot.py::TestPivotTable::test_pivot_no_level_overlap": "TODO: Add a reason for failure", "tests/reshape/test_pivot.py::TestPivotTable::test_pivot_string_as_func": "TODO: Add a reason for failure", "tests/reshape/test_pivot.py::TestPivotTable::test_pivot_string_func_vs_func[f3-f_numpy3]": "TODO: Add a reason for failure", "tests/reshape/test_pivot.py::TestPivotTable::test_pivot_string_func_vs_func[f4-f_numpy4]": "TODO: Add a reason for failure", @@ -3099,7 +3024,6 @@ def pytest_unconfigure(config): "tests/reshape/test_pivot.py::TestPivotTable::test_pivot_table_nocols": "TODO: Add a reason for failure", "tests/reshape/test_pivot.py::TestPivotTable::test_pivot_table_not_series": "TODO: Add a reason for failure", "tests/reshape/test_pivot.py::TestPivotTable::test_pivot_table_with_iterator_values": "TODO: Add a reason for failure", - "tests/reshape/test_pivot.py::TestPivotTable::test_pivot_table_with_mixed_nested_tuples": "TODO: Add a reason for failure", "tests/reshape/test_pivot.py::TestPivotTable::test_pivot_with_categorical[False-False]": "TODO: Add a reason for failure", "tests/reshape/test_pivot.py::TestPivotTable::test_pivot_with_categorical[False-None]": "TODO: Add a reason for failure", "tests/reshape/test_pivot.py::TestPivotTable::test_pivot_with_categorical[False-True]": "TODO: Add a reason for failure", @@ -3843,13 +3767,11 @@ def pytest_unconfigure(config): "tests/tslibs/test_to_offset.py::test_to_offset_uppercase_frequency_deprecated[2NS]": "TODO: Add a reason for failure", "tests/tslibs/test_to_offset.py::test_to_offset_uppercase_frequency_deprecated[2Us]": "TODO: Add a reason for failure", "tests/util/test_assert_frame_equal.py::test_allows_duplicate_labels": "TODO: Add a reason for failure", - "tests/util/test_assert_frame_equal.py::test_assert_frame_equal_extension_dtype_mismatch": "TODO: Add a reason for failure", "tests/util/test_assert_frame_equal.py::test_assert_frame_equal_nested_df_na[None]": "KeyError: 0", "tests/util/test_assert_frame_equal.py::test_assert_frame_equal_nested_df_na[nan]": "KeyError: 0", "tests/util/test_assert_frame_equal.py::test_frame_equal_index_dtype_mismatch[True-df11-df21-DataFrame\\\\.index level \\\\[0\\\\] are different]": "Failed: DID NOT RAISE ", "tests/util/test_assert_index_equal.py::test_index_equal_range_categories[True-True]": "TODO: Add a reason for failure", "tests/util/test_assert_series_equal.py::test_allows_duplicate_labels": "TODO: Add a reason for failure", - "tests/util/test_assert_series_equal.py::test_assert_series_equal_extension_dtype_mismatch": "TODO: Add a reason for failure", "tests/util/test_assert_series_equal.py::test_assert_series_equal_int_tol": "AssertionError: left is not an ExtensionArray", "tests/util/test_assert_series_equal.py::test_large_unequal_ints[Int64]": "Failed: DID NOT RAISE ", "tests/util/test_assert_series_equal.py::test_large_unequal_ints[int64]": "TODO: Add a reason for failure", @@ -3907,7 +3829,6 @@ def pytest_unconfigure(config): "tests/window/test_timeseries_window.py::TestRollingTS::test_rolling_on_decreasing_index[us]": "TODO: Add a reason for failure", "tests/window/test_win_type.py::test_cmov_window_corner[None]": "TODO: Add a reason for failure", "tests/window/test_win_type.py::test_win_type_not_implemented": "TODO: Add a reason for failure", - "tests/indexing/multiindex/test_loc.py::test_loc_getitem_duplicates_multiindex_empty_indexer[columns_indexer1]": "AssertionError: DataFrame.columns level [0] are different", } # Keep keys in alphabeical order diff --git a/python/cudf/cudf/tests/reshape/test_pivot_table.py b/python/cudf/cudf/tests/reshape/test_pivot_table.py index 368a8b3ffbed..347a73f84090 100644 --- a/python/cudf/cudf/tests/reshape/test_pivot_table.py +++ b/python/cudf/cudf/tests/reshape/test_pivot_table.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 import numpy as np @@ -94,3 +94,32 @@ def test_pivot_table_scalar_index_columns(index, columns): values="D", index=index, columns=columns, aggfunc="sum" ) assert_eq(result, expected) + + +@pytest.mark.parametrize("aggfunc", ["sum", "min", "max"]) +@pytest.mark.parametrize("fill_value", [None, 0]) +def test_pivot_table_sparse_int_fill_value(aggfunc, fill_value): + # pandas promotes integer values to float64 when the reshape leaves + # missing cells unfilled (fill_value=None), and keeps the integer + # dtype when they are filled + data = { + "i": ["r1", "r1", "r2"], + "c": ["a", "b", "a"], + "v": [1, 2, 3], + } + with cudf.option_context("mode.pandas_compatible", True): + result = cudf.DataFrame(data).pivot_table( + index="i", + columns="c", + values="v", + aggfunc=aggfunc, + fill_value=fill_value, + ) + expected = pd.DataFrame(data).pivot_table( + index="i", + columns="c", + values="v", + aggfunc=aggfunc, + fill_value=fill_value, + ) + assert_eq(expected, result) diff --git a/python/cudf/cudf/tests/reshape/test_unstack.py b/python/cudf/cudf/tests/reshape/test_unstack.py index 0edf16202d31..3a2462c4b49a 100644 --- a/python/cudf/cudf/tests/reshape/test_unstack.py +++ b/python/cudf/cudf/tests/reshape/test_unstack.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 import re @@ -14,20 +14,10 @@ "level", [ 0, - pytest.param( - 1, - marks=pytest.mark.xfail( - reason="Categorical column indexes not supported" - ), - ), + 1, 2, "foo", - pytest.param( - "bar", - marks=pytest.mark.xfail( - reason="Categorical column indexes not supported" - ), - ), + "bar", "baz", [], pytest.param( @@ -75,12 +65,7 @@ def test_unstack_multiindex(level): [ pd.Index(range(0, 5), name=None), pd.Index(range(0, 5), name="row_index"), - pytest.param( - pd.CategoricalIndex(["d", "e", "f", "g", "h"]), - marks=pytest.mark.xfail( - reason="Categorical column indexes not supported" - ), - ), + pd.CategoricalIndex(["d", "e", "f", "g", "h"]), ], ) @pytest.mark.parametrize(