Align DataFrame.reindex dtype and validation behavior with pandas

python/cudf/cudf/core/dataframe.py

@@ -683,17 +683,26 @@ def _listlike_to_column_accessor(
                     ser = ser.reindex(temp_index)
                 temp_data[i] = ser._column
 
-            temp_frame = DataFrame._from_data(
+            combined = DataFrame._from_data(
                 ColumnAccessor(
                     temp_data,
                     verify=False,
                     rangeindex=True,
                 ),
                 index=temp_index,
             )
-            transpose = temp_frame.T
         else:
-            transpose = cudf.concat(data, axis=1).T
+            combined = cudf.concat(data, axis=1)
+        # Aligning rows above can upcast integer columns that gained nulls
+        # (e.g. int -> float64), leaving columns with differing dtypes;
+        # transpose requires a single dtype, so promote to a common one
+        # first (matching pandas, which upcasts the whole frame).
+        if combined._num_columns > 1:
+            common_dtype = find_common_type(
+                [dtype for _, dtype in combined._dtypes]
+            )
+            combined = combined.astype(common_dtype)
+        transpose = combined.T
 
         if columns is None:
             columns = pd.RangeIndex(transpose._num_columns)
@@ -3230,6 +3239,7 @@ def _set_columns_like(self, other: ColumnAccessor) -> None:
     def reindex(
         self,
         labels=None,
+        *,
         index=None,
         columns=None,
         axis=None,
@@ -3292,20 +3302,12 @@ def reindex(
         >>> new_index = ['Safari', 'Iceweasel', 'Comodo Dragon', 'IE10',
         ...              'Chrome']
         >>> df.reindex(new_index)
-                    http_status response_time
-        Safari                404          0.07
-        Iceweasel            <NA>           NaN
-        Comodo Dragon        <NA>           NaN
-        IE10                  404          0.08
-        Chrome                200          0.02
-
-        .. pandas-compat::
-            :meth:`pandas.DataFrame.reindex`
-
-            Note: One difference from Pandas is that ``NA`` is used for rows
-            that do not match, rather than ``NaN``. One side effect of this is
-            that the column ``http_status`` retains an integer dtype in cuDF
-            where it is cast to float in Pandas.
+                      http_status response_time
+        Safari              404.0          0.07
+        Iceweasel             NaN           NaN
+        Comodo Dragon         NaN           NaN
+        IE10                404.0          0.08
+        Chrome              200.0          0.02
 
         We can fill in the missing values by
         passing a value to the keyword ``fill_value``.
@@ -3342,6 +3344,11 @@ def reindex(
         if labels is None and index is None and columns is None:
             return self.copy(deep=copy)
 
+        if labels is not None and index is not None and columns is not None:
+            raise TypeError(
+                "Cannot specify all of 'labels', 'index', 'columns'."
+            )
+
         # pandas simply ignores the labels keyword if it is provided in
         # addition to index and columns, but it prohibits the axis arg.
         if (index is not None or columns is not None) and axis is not None:
@@ -3543,7 +3550,18 @@ def set_index(
             and not isinstance(keys[0], (cudf.MultiIndex, pd.MultiIndex))
         ):
             # Don't turn single level MultiIndex into an Index
-            idx = Index._from_column(data_to_add[0], name=names[0])
+            freq = (
+                getattr(keys[0], "freq", None)
+                if isinstance(keys[0], (cudf.Index, pd.Index))
+                else None
+            )
+            if freq is not None and data_to_add[0].dtype.kind == "M":
+                # Preserve the freq of a DatetimeIndex passed as the key.
+                idx = cudf.DatetimeIndex._from_column(
+                    data_to_add[0], name=names[0], freq=freq
+                )
+            else:
+                idx = Index._from_column(data_to_add[0], name=names[0])
         else:
             idx = MultiIndex._from_data(dict(enumerate(data_to_add)))
             idx.names = names
@@ -3926,7 +3944,16 @@ def _insert(self, loc, name, value, nan_as_null=None, ignore_index=True):
                     self.index, how="right", sort=False
                 )
 
-        value = as_column(value, nan_as_null=nan_as_null)
+        if isinstance(value, (list, tuple)) and len(value) == 0:
+            # An empty list-like assigned as a DataFrame column has no
+            # inferable dtype and becomes float64, matching pandas (note
+            # pd.Series([]) is object, but DataFrame column assignment is
+            # float64).
+            value = as_column(
+                value, nan_as_null=nan_as_null, dtype=np.dtype(np.float64)
+            )
+        else:
+            value = as_column(value, nan_as_null=nan_as_null)
         self._data.insert(name, value, loc=loc)
 
     @property
@@ -7072,13 +7099,13 @@ def mode(self, axis=0, numeric_only=False, dropna=True):
         3       bird     2   NaN
 
         By default, missing values are not considered, and the mode of wings
-        are both 0 and 2. The second row of species and legs contains ``NA``,
+        are both 0 and 2. The second row of species and legs contains ``NaN``,
         because they have only one mode, but the DataFrame has two rows.
 
         >>> df.mode()
-          species  legs  wings
-        0    bird     2    0.0
-        1     NaN  <NA>    2.0
+          species legs  wings
+        0    bird  2.0    0.0
+        1     NaN  NaN    2.0
 
         Setting ``dropna=False``, ``NA`` values are considered and they can be
         the mode (like for wings).
@@ -7091,9 +7118,9 @@ def mode(self, axis=0, numeric_only=False, dropna=True):
         computed, and columns of other types are ignored.
 
         >>> df.mode(numeric_only=True)
-           legs  wings
-        0     2    0.0
-        1  <NA>    2.0
+          legs  wings
+        0  2.0    0.0
+        1  NaN    2.0
 
         .. pandas-compat::
             :meth:`pandas.DataFrame.transpose`

python/cudf/cudf/core/indexed_frame.py

@@ -3954,6 +3954,20 @@ def _reindex(
             dtypes = {}
 
         df = self
+        # Original column dtypes, captured before any index handling
+        # mutates ``df``; used to infer the dtype of brand-new columns.
+        orig_col_dtypes = [dtype for _, dtype in self._dtypes]
+        frame_common_dtype = (
+            orig_col_dtypes[0]
+            if (
+                orig_col_dtypes
+                and all(dt == orig_col_dtypes[0] for dt in orig_col_dtypes)
+                and isinstance(orig_col_dtypes[0], np.dtype)
+            )
+            else None
+        )
+        row_reindex = index is not None
+        rows_added = False
         if index is not None:
             if not df.index.is_unique:
                 raise ValueError(
@@ -3992,8 +4006,9 @@ def _reindex(
                     index=df.index,
                 )
                 diff = index.difference(df.index)
+                rows_added = len(diff) > 0
                 df = lhs.join(rhs, how="left", sort=True)
-                if fill_value is not NA and len(diff) > 0:
+                if fill_value is not NA and rows_added:
                     df.loc[diff] = fill_value
                 # double-argsort to map back from sorted to unsorted positions
                 df = df.take(index.argsort(ascending=True).argsort())
@@ -4036,24 +4051,93 @@ def _reindex(
             multiindex = False
             rangeindex = False
 
-        cols = {
-            name: (
-                df._data[name].copy(deep=deep)
-                if name in df._data
-                else (
-                    column_empty(
-                        dtype=dtypes.get(name, np.dtype(np.float64)),
-                        row_count=len(index),
-                    ).fillna(fill_value)
-                    if fill_value is not NA
-                    else column_empty(
-                        dtype=dtypes.get(name, np.dtype(np.float64)),
-                        row_count=len(index),
+        def _new_nulls_column(name):
+            # Build a brand-new column produced by reindex (entirely missing
+            # or fill values), choosing a dtype that matches pandas.
+            if fill_value is NA:
+                # All-null new column: keep a homogeneous float dtype,
+                # else float64 (numpy integer/bool cannot hold NaN).
+                if name in dtypes:
+                    target = dtypes[name]
+                elif (
+                    frame_common_dtype is not None
+                    and frame_common_dtype.kind == "f"
+                ):
+                    target = frame_common_dtype
+                else:
+                    target = np.dtype(np.float64)
+                # A numpy integer dtype cannot hold NA, so pandas upcasts an
+                # all-null reindexed column to float64. Match that here,
+                # mirroring the upcast applied to existing integer columns
+                # below so both paths agree. An empty result (row_count == 0)
+                # holds no NA, so the integer dtype is preserved -- matching
+                # pandas and cudf's prior behavior for e.g. reindex to an
+                # empty index.
+                if (
+                    isinstance(target, np.dtype)
+                    and target.kind in "iu"
+                    and len(index) > 0
+                ):
+                    target = np.dtype(np.float64)
+                return column_empty(dtype=target, row_count=len(index))
+            # Non-null fill. A numeric scalar fill on a brand-new column of a
+            # homogeneous numpy frame promotes the frame dtype against the fill
+            # value (uint8 + 10 -> uint8, uint8 + 300 -> int64); otherwise the
+            # dtype is the fill value's own. Any other fill keeps the source /
+            # float64 default. ``fillna`` raises on incompatible fills so
+            # cudf.pandas falls back to pandas.
+            if (
+                name not in dtypes
+                and is_scalar(fill_value)
+                and (scalar_col := as_column(fill_value, length=1)).dtype.kind
+                in "iuf"
+            ):
+                if row_reindex and frame_common_dtype is not None:
+                    target = (
+                        frame_common_dtype
+                        if scalar_col.can_cast_safely(frame_common_dtype)
+                        else find_common_type(
+                            [frame_common_dtype, scalar_col.dtype]
+                        )
                     )
-                )
+                else:
+                    target = scalar_col.dtype
+            else:
+                target = dtypes.get(name, np.dtype(np.float64))
+            return column_empty(dtype=target, row_count=len(index)).fillna(
+                fill_value
             )
-            for name in names
-        }
+
+        # cudf cannot represent duplicate column names; pandas can. Raise
+        # so cudf.pandas falls back to pandas rather than silently
+        # collapsing duplicates.
+        names_list = list(names)
+        if len(names_list) != len(set(names_list)):
+            raise ValueError("Duplicate column names are not allowed")
+
+        # pandas upcasts integer columns to float64 when default-NaN
+        # filling newly added rows on reindex (a numpy integer column
+        # cannot hold NA), so match that unconditionally.
+        upcast_int_nulls = rows_added and fill_value is NA
+
+        cols = {}
+        for name in names_list:
+            if name in df._data:
+                col = df._data[name].copy(deep=deep)
+                if (
+                    upcast_int_nulls
+                    # Only plain numpy integer columns cannot hold NA;
+                    # nullable extension integers (masked ``IntX``,
+                    # ``ArrowDtype``) natively represent NA, so pandas
+                    # keeps their dtype rather than upcasting to float64.
+                    and isinstance(col.dtype, np.dtype)
+                    and col.dtype.kind in "iu"
+                    and col.null_count
+                ):
+                    col = col.astype(np.dtype(np.float64))
+            else:
+                col = _new_nulls_column(name)
+            cols[name] = col
 
         result = self.__class__._from_data(
             data=ColumnAccessor(

python/cudf/cudf/core/series.py

@@ -980,19 +980,11 @@ def reindex(
         d    40
         dtype: int64
         >>> series.reindex(['a', 'b', 'y', 'z'])
-        a      10
-        b      20
-        y    <NA>
-        z    <NA>
-        dtype: int64
-
-        .. pandas-compat::
-            :meth:`pandas.Series.reindex`
-
-            Note: One difference from Pandas is that ``NA`` is used for rows
-            that do not match, rather than ``NaN``. One side effect of this is
-            that the series retains an integer dtype in cuDF
-            where it is cast to float in Pandas.
+        a    10.0
+        b    20.0
+        y     NaN
+        z     NaN
+        dtype: float64
 
         """
         if index is None:
@@ -3343,7 +3335,12 @@ def value_counts(
                     else None
                 )
                 res = res[res.index.notna()]
-                res = res.reindex(self.dtype.categories).fillna(0)
+                # Fill missing categories with a 0 count directly via
+                # ``reindex`` (rather than ``reindex(...).fillna(0)``) so
+                # the integer count dtype is preserved: a default-NA
+                # reindex upcasts integer columns to float64 in
+                # pandas-compatible mode.
+                res = res.reindex(self.dtype.categories, fill_value=0)
                 res.index = res.index.astype(self.dtype)
                 if nan_count is not None:
                     res = cudf.concat([res, nan_count])