diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index c7697e616233..b9b29b937462 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -683,7 +683,7 @@ def _listlike_to_column_accessor( ser = ser.reindex(temp_index) temp_data[i] = ser._column - temp_frame = DataFrame._from_data( + combined = DataFrame._from_data( ColumnAccessor( temp_data, verify=False, @@ -691,9 +691,18 @@ def _listlike_to_column_accessor( ), index=temp_index, ) - transpose = temp_frame.T else: - transpose = cudf.concat(data, axis=1).T + combined = cudf.concat(data, axis=1) + # Aligning rows above can upcast integer columns that gained nulls + # (e.g. int -> float64), leaving columns with differing dtypes; + # transpose requires a single dtype, so promote to a common one + # first (matching pandas, which upcasts the whole frame). + if combined._num_columns > 1: + common_dtype = find_common_type( + [dtype for _, dtype in combined._dtypes] + ) + combined = combined.astype(common_dtype) + transpose = combined.T if columns is None: columns = pd.RangeIndex(transpose._num_columns) @@ -3230,6 +3239,7 @@ def _set_columns_like(self, other: ColumnAccessor) -> None: def reindex( self, labels=None, + *, index=None, columns=None, axis=None, @@ -3292,20 +3302,12 @@ def reindex( >>> new_index = ['Safari', 'Iceweasel', 'Comodo Dragon', 'IE10', ... 'Chrome'] >>> df.reindex(new_index) - http_status response_time - Safari 404 0.07 - Iceweasel NaN - Comodo Dragon NaN - IE10 404 0.08 - Chrome 200 0.02 - - .. pandas-compat:: - :meth:`pandas.DataFrame.reindex` - - Note: One difference from Pandas is that ``NA`` is used for rows - that do not match, rather than ``NaN``. One side effect of this is - that the column ``http_status`` retains an integer dtype in cuDF - where it is cast to float in Pandas. + http_status response_time + Safari 404.0 0.07 + Iceweasel NaN NaN + Comodo Dragon NaN NaN + IE10 404.0 0.08 + Chrome 200.0 0.02 We can fill in the missing values by passing a value to the keyword ``fill_value``. @@ -3342,6 +3344,11 @@ def reindex( if labels is None and index is None and columns is None: return self.copy(deep=copy) + if labels is not None and index is not None and columns is not None: + raise TypeError( + "Cannot specify all of 'labels', 'index', 'columns'." + ) + # pandas simply ignores the labels keyword if it is provided in # addition to index and columns, but it prohibits the axis arg. if (index is not None or columns is not None) and axis is not None: @@ -3543,7 +3550,18 @@ def set_index( and not isinstance(keys[0], (cudf.MultiIndex, pd.MultiIndex)) ): # Don't turn single level MultiIndex into an Index - idx = Index._from_column(data_to_add[0], name=names[0]) + freq = ( + getattr(keys[0], "freq", None) + if isinstance(keys[0], (cudf.Index, pd.Index)) + else None + ) + if freq is not None and data_to_add[0].dtype.kind == "M": + # Preserve the freq of a DatetimeIndex passed as the key. + idx = cudf.DatetimeIndex._from_column( + data_to_add[0], name=names[0], freq=freq + ) + else: + idx = Index._from_column(data_to_add[0], name=names[0]) else: idx = MultiIndex._from_data(dict(enumerate(data_to_add))) idx.names = names @@ -3926,7 +3944,16 @@ def _insert(self, loc, name, value, nan_as_null=None, ignore_index=True): self.index, how="right", sort=False ) - value = as_column(value, nan_as_null=nan_as_null) + if isinstance(value, (list, tuple)) and len(value) == 0: + # An empty list-like assigned as a DataFrame column has no + # inferable dtype and becomes float64, matching pandas (note + # pd.Series([]) is object, but DataFrame column assignment is + # float64). + value = as_column( + value, nan_as_null=nan_as_null, dtype=np.dtype(np.float64) + ) + else: + value = as_column(value, nan_as_null=nan_as_null) self._data.insert(name, value, loc=loc) @property @@ -7072,13 +7099,13 @@ def mode(self, axis=0, numeric_only=False, dropna=True): 3 bird 2 NaN By default, missing values are not considered, and the mode of wings - are both 0 and 2. The second row of species and legs contains ``NA``, + are both 0 and 2. The second row of species and legs contains ``NaN``, because they have only one mode, but the DataFrame has two rows. >>> df.mode() - species legs wings - 0 bird 2 0.0 - 1 NaN 2.0 + species legs wings + 0 bird 2.0 0.0 + 1 NaN NaN 2.0 Setting ``dropna=False``, ``NA`` values are considered and they can be the mode (like for wings). @@ -7091,9 +7118,9 @@ def mode(self, axis=0, numeric_only=False, dropna=True): computed, and columns of other types are ignored. >>> df.mode(numeric_only=True) - legs wings - 0 2 0.0 - 1 2.0 + legs wings + 0 2.0 0.0 + 1 NaN 2.0 .. pandas-compat:: :meth:`pandas.DataFrame.transpose` diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index c0274d9d18d6..ab86a89c753c 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -3954,6 +3954,20 @@ def _reindex( dtypes = {} df = self + # Original column dtypes, captured before any index handling + # mutates ``df``; used to infer the dtype of brand-new columns. + orig_col_dtypes = [dtype for _, dtype in self._dtypes] + frame_common_dtype = ( + orig_col_dtypes[0] + if ( + orig_col_dtypes + and all(dt == orig_col_dtypes[0] for dt in orig_col_dtypes) + and isinstance(orig_col_dtypes[0], np.dtype) + ) + else None + ) + row_reindex = index is not None + rows_added = False if index is not None: if not df.index.is_unique: raise ValueError( @@ -3992,8 +4006,9 @@ def _reindex( index=df.index, ) diff = index.difference(df.index) + rows_added = len(diff) > 0 df = lhs.join(rhs, how="left", sort=True) - if fill_value is not NA and len(diff) > 0: + if fill_value is not NA and rows_added: df.loc[diff] = fill_value # double-argsort to map back from sorted to unsorted positions df = df.take(index.argsort(ascending=True).argsort()) @@ -4036,24 +4051,93 @@ def _reindex( multiindex = False rangeindex = False - cols = { - name: ( - df._data[name].copy(deep=deep) - if name in df._data - else ( - column_empty( - dtype=dtypes.get(name, np.dtype(np.float64)), - row_count=len(index), - ).fillna(fill_value) - if fill_value is not NA - else column_empty( - dtype=dtypes.get(name, np.dtype(np.float64)), - row_count=len(index), + def _new_nulls_column(name): + # Build a brand-new column produced by reindex (entirely missing + # or fill values), choosing a dtype that matches pandas. + if fill_value is NA: + # All-null new column: keep a homogeneous float dtype, + # else float64 (numpy integer/bool cannot hold NaN). + if name in dtypes: + target = dtypes[name] + elif ( + frame_common_dtype is not None + and frame_common_dtype.kind == "f" + ): + target = frame_common_dtype + else: + target = np.dtype(np.float64) + # A numpy integer dtype cannot hold NA, so pandas upcasts an + # all-null reindexed column to float64. Match that here, + # mirroring the upcast applied to existing integer columns + # below so both paths agree. An empty result (row_count == 0) + # holds no NA, so the integer dtype is preserved -- matching + # pandas and cudf's prior behavior for e.g. reindex to an + # empty index. + if ( + isinstance(target, np.dtype) + and target.kind in "iu" + and len(index) > 0 + ): + target = np.dtype(np.float64) + return column_empty(dtype=target, row_count=len(index)) + # Non-null fill. A numeric scalar fill on a brand-new column of a + # homogeneous numpy frame promotes the frame dtype against the fill + # value (uint8 + 10 -> uint8, uint8 + 300 -> int64); otherwise the + # dtype is the fill value's own. Any other fill keeps the source / + # float64 default. ``fillna`` raises on incompatible fills so + # cudf.pandas falls back to pandas. + if ( + name not in dtypes + and is_scalar(fill_value) + and (scalar_col := as_column(fill_value, length=1)).dtype.kind + in "iuf" + ): + if row_reindex and frame_common_dtype is not None: + target = ( + frame_common_dtype + if scalar_col.can_cast_safely(frame_common_dtype) + else find_common_type( + [frame_common_dtype, scalar_col.dtype] + ) ) - ) + else: + target = scalar_col.dtype + else: + target = dtypes.get(name, np.dtype(np.float64)) + return column_empty(dtype=target, row_count=len(index)).fillna( + fill_value ) - for name in names - } + + # cudf cannot represent duplicate column names; pandas can. Raise + # so cudf.pandas falls back to pandas rather than silently + # collapsing duplicates. + names_list = list(names) + if len(names_list) != len(set(names_list)): + raise ValueError("Duplicate column names are not allowed") + + # pandas upcasts integer columns to float64 when default-NaN + # filling newly added rows on reindex (a numpy integer column + # cannot hold NA), so match that unconditionally. + upcast_int_nulls = rows_added and fill_value is NA + + cols = {} + for name in names_list: + if name in df._data: + col = df._data[name].copy(deep=deep) + if ( + upcast_int_nulls + # Only plain numpy integer columns cannot hold NA; + # nullable extension integers (masked ``IntX``, + # ``ArrowDtype``) natively represent NA, so pandas + # keeps their dtype rather than upcasting to float64. + and isinstance(col.dtype, np.dtype) + and col.dtype.kind in "iu" + and col.null_count + ): + col = col.astype(np.dtype(np.float64)) + else: + col = _new_nulls_column(name) + cols[name] = col result = self.__class__._from_data( data=ColumnAccessor( diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index 505074adfcd1..f02a5b827558 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -980,19 +980,11 @@ def reindex( d 40 dtype: int64 >>> series.reindex(['a', 'b', 'y', 'z']) - a 10 - b 20 - y - z - dtype: int64 - - .. pandas-compat:: - :meth:`pandas.Series.reindex` - - Note: One difference from Pandas is that ``NA`` is used for rows - that do not match, rather than ``NaN``. One side effect of this is - that the series retains an integer dtype in cuDF - where it is cast to float in Pandas. + a 10.0 + b 20.0 + y NaN + z NaN + dtype: float64 """ if index is None: @@ -3343,7 +3335,12 @@ def value_counts( else None ) res = res[res.index.notna()] - res = res.reindex(self.dtype.categories).fillna(0) + # Fill missing categories with a 0 count directly via + # ``reindex`` (rather than ``reindex(...).fillna(0)``) so + # the integer count dtype is preserved: a default-NA + # reindex upcasts integer columns to float64 in + # pandas-compatible mode. + res = res.reindex(self.dtype.categories, fill_value=0) res.index = res.index.astype(self.dtype) if nan_count is not None: res = cudf.concat([res, nan_count]) diff --git a/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py b/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py index d42a89acdec8..42d0be6597aa 100644 --- a/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py +++ b/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py @@ -1380,33 +1380,9 @@ def pytest_unconfigure(config): "tests/frame/methods/test_quantile.py::TestQuantileExtensionDtype::test_quantile_ea_with_na[Int64-Series]": "TODO: Add a reason for failure", "tests/frame/methods/test_rank.py::TestRank::test_rank": "AssertionError: numpy array are different", "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex": "TODO: Add a reason for failure", - "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_axis_style_raises": "TODO: Add a reason for failure", - "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_empty[cat_idx4-CategoricalIndex]": "AssertionError: DataFrame are different", - "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_empty[cat_idx4-Index]": "AssertionError: DataFrame are different", - "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_empty[cat_idx5-CategoricalIndex]": "AssertionError: DataFrame are different", - "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_empty[cat_idx5-Index]": "AssertionError: DataFrame are different", "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_empty_frame[kwargs3]": "AssertionError: Attributes of DataFrame.iloc[:, 0] (column name='a') are different", "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_fill_value": "TODO: Add a reason for failure", "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_index_name_matches_multiindex_level": "AssertionError: Attributes of DataFrame.iloc[:, 0] (column name='value') are different", - "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_int": "TODO: Add a reason for failure", - "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_positional_raises": "TODO: Add a reason for failure", - "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_single_column_ea_index_and_columns[Float32]": "TODO: Add a reason for failure", - "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_single_column_ea_index_and_columns[Float64]": "TODO: Add a reason for failure", - "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_single_column_ea_index_and_columns[Int16]": "TODO: Add a reason for failure", - "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_single_column_ea_index_and_columns[Int32]": "TODO: Add a reason for failure", - "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_single_column_ea_index_and_columns[Int64]": "TODO: Add a reason for failure", - "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_single_column_ea_index_and_columns[Int8]": "TODO: Add a reason for failure", - "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_single_column_ea_index_and_columns[UInt16]": "TODO: Add a reason for failure", - "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_single_column_ea_index_and_columns[UInt32]": "TODO: Add a reason for failure", - "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_single_column_ea_index_and_columns[UInt64]": "TODO: Add a reason for failure", - "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_single_column_ea_index_and_columns[UInt8]": "TODO: Add a reason for failure", - "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_uint_dtypes_fill_value[uint16]": "TODO: Add a reason for failure", - "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_uint_dtypes_fill_value[uint32]": "TODO: Add a reason for failure", - "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_uint_dtypes_fill_value[uint64]": "TODO: Add a reason for failure", - "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_uint_dtypes_fill_value[uint8]": "TODO: Add a reason for failure", - "tests/frame/methods/test_reindex.py::TestDataFrameSelectReindex::test_reindex_without_upcasting": "AssertionError: assert np.False_", - "tests/frame/methods/test_reindex.py::TestReindexSetIndex::test_dti_set_index_reindex_freq_with_tz": "AssertionError: assert None == ", - "tests/frame/methods/test_reindex.py::TestReindexSetIndex::test_setitem_reset_index_dtypes": "TODO: Add a reason for failure", "tests/frame/methods/test_rename.py::TestRename::test_rename": "TODO: Add a reason for failure", "tests/frame/methods/test_rename.py::TestRename::test_rename_axis_style_raises": "TODO: Add a reason for failure", "tests/frame/methods/test_rename.py::TestRename::test_rename_inplace": "TODO: Add a reason for failure", @@ -1598,7 +1574,7 @@ def pytest_unconfigure(config): "tests/frame/test_constructors.py::TestDataFrameConstructors::test_1d_object_array_does_not_copy": "TODO: Add a reason for failure", "tests/frame/test_constructors.py::TestDataFrameConstructors::test_2d_object_array_does_not_copy": "TODO: Add a reason for failure", "tests/frame/test_constructors.py::TestDataFrameConstructors::test_construct_from_dict_ea_series": "AttributeError: 'ndarray' object has no attribute '_data'. Did you mean: 'data'?", - "tests/frame/test_constructors.py::TestDataFrameConstructors::test_construct_with_two_categoricalindex_series": "TODO: Add a reason for failure", + "tests/frame/test_constructors.py::TestDataFrameConstructors::test_construct_with_two_categoricalindex_series": "cudf cannot represent a CategoricalIndex as columns; the constructed frame's columns become a plain (string) Index", "tests/frame/test_constructors.py::TestDataFrameConstructors::test_construction_nan_value_timedelta64_dtype": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", "tests/frame/test_constructors.py::TestDataFrameConstructors::test_constructor_dict": "TODO: Add a reason for failure", "tests/frame/test_constructors.py::TestDataFrameConstructors::test_constructor_dict_cast": "TODO: Add a reason for failure", @@ -3061,7 +3037,6 @@ def pytest_unconfigure(config): "tests/indexing/test_partial.py::TestEmptyFrameSetitemExpansion::test_empty_frame_setitem_index_name_retained": "AssertionError: DataFrame.index are different", "tests/indexing/test_partial.py::TestEmptyFrameSetitemExpansion::test_partial_set_empty_frame": "Failed: DID NOT RAISE ", "tests/indexing/test_partial.py::TestEmptyFrameSetitemExpansion::test_partial_set_empty_frame2": "TODO: Add a reason for failure", - "tests/indexing/test_partial.py::TestEmptyFrameSetitemExpansion::test_partial_set_empty_frame3": 'AssertionError: Column name="foo" are different', "tests/indexing/test_partial.py::TestEmptyFrameSetitemExpansion::test_partial_set_empty_frame5": "AssertionError: DataFrame.index are different", "tests/indexing/test_partial.py::TestEmptyFrameSetitemExpansion::test_partial_set_empty_frame_empty_consistencies": "AssertionError: Attributes of DataFrame.iloc[:, 1] (column name='y') are different", "tests/indexing/test_partial.py::TestEmptyFrameSetitemExpansion::test_partial_set_empty_frame_no_index": "object vs str dtype mismatch: cudf constructs zero-row columns as object to match pandas, but cudf operations produce the default string dtype for empty columns", @@ -4187,7 +4162,6 @@ def pytest_unconfigure(config): "tests/series/methods/test_reindex.py::test_reindex_fill_value": "AssertionError: Series are different", "tests/series/methods/test_reindex.py::test_reindex_inference": "AssertionError: Attributes of Series are different", "tests/series/methods/test_reindex.py::test_reindex_multiindex_automatic_level[a-True]": "AssertionError: Series are different", - "tests/series/methods/test_reindex.py::test_reindex_pad2": "TODO: Add a reason for failure", "tests/series/methods/test_replace.py::TestSeriesReplace::test_replace": "assert None is 0 -1.000000\n1 -1.000000\n2 -1.000000\n3 -1.000000\n4 1.799707\n5 1.144166\n6 ...", "tests/series/methods/test_replace.py::TestSeriesReplace::test_replace2": "assert 2020-01-01 -1\n2020-01-02 -1\n2020-01-03 -1\n2020-01-04 -1\n2020-01-05 ...", "tests/series/methods/test_replace.py::TestSeriesReplace::test_replace_Int_with_na[Int16]": "assert None is 0 \n1 \ndtype: Int16",