Skip to content

Allow StringColumn to always accept pandas string extention types #21251

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
mroeschke wants to merge 8 commits into from
Closed

Allow StringColumn to always accept pandas string extention types #21251

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
7df117f
Make is_dtype_obj_string stricter, simplify StringColumn.to_pandas, s…
mroeschke Jan 29, 2026
d9ac19c
Merge remote-tracking branch 'upstream/main' into ref/cudf/stringdtyp…
mroeschke Jan 29, 2026
9a49156
Add fix for intervaldtype, ensure to_pandas preserves prferred large_…
mroeschke Jan 29, 2026
4ad8d12
Add tests to xfail list
mroeschke Jan 29, 2026
84e376a
Merge remote-tracking branch 'upstream/main' into ref/cudf/stringdtyp…
mroeschke Jan 30, 2026
8a39492
Merge remote-tracking branch 'upstream/main' into ref/cudf/stringdtyp…
mroeschke Jan 30, 2026
a2ae7d1
accidentially used self.dtype
mroeschke Jan 30, 2026
be63cd1
Merge remote-tracking branch 'upstream/main' into ref/cudf/stringdtyp…
mroeschke Jan 30, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions python/cudf/cudf/core/column/column.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2932,6 +2932,8 @@ def as_column(
# pandas arrays define __arrow_array__ for better
# pyarrow.array conversion
arbitrary = arbitrary.array
if dtype is None and isinstance(arbitrary.dtype, pd.StringDtype):
dtype = arbitrary.dtype
result = as_column(
pa.array(arbitrary, from_pandas=True),
nan_as_null=nan_as_null,
Expand Down
50 changes: 17 additions & 33 deletions python/cudf/cudf/core/column/string.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,21 +117,8 @@ def _validate_args(
cls, plc_column: plc.Column, dtype: np.dtype
) -> tuple[plc.Column, np.dtype]:
plc_column, dtype = super()._validate_args(plc_column, dtype)
if (
not cudf.get_option("mode.pandas_compatible")
and dtype != CUDF_STRING_DTYPE
and dtype.kind != "U"

@galipremsagar galipremsagar Jan 31, 2026 •
edited
Loading

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think you will have to bring back this check(minus the pandas compatibility flag) to make np.dtype(object), np.dtype('str') and nullable dtypes work. Those are probably the causes for the new xfails.

) or (
cudf.get_option("mode.pandas_compatible")
and not is_dtype_obj_string(dtype)
):
raise ValueError(f"dtype must be {CUDF_STRING_DTYPE}")
if (
cudf.get_option("mode.pandas_compatible")
and isinstance(dtype, np.dtype)
and dtype.kind == "U"
):
dtype = CUDF_STRING_DTYPE
if not is_dtype_obj_string(dtype):
raise ValueError("dtype must be a valid cuDF string dtype")
return plc_column, dtype

@property
Expand Down Expand Up @@ -437,25 +424,22 @@ def to_pandas(
nullable: bool = False,
arrow_type: bool = False,
) -> pd.Index:
if (
cudf.get_option("mode.pandas_compatible")
and isinstance(self.dtype, pd.StringDtype)
and self.dtype.storage in ["pyarrow", "python"]
):
if self.dtype.storage == "pyarrow":
pandas_array = self.dtype.__from_arrow__(
if arrow_type or isinstance(self.dtype, pd.ArrowDtype):
return super().to_pandas(nullable=nullable, arrow_type=arrow_type)
elif nullable or isinstance(self.dtype, pd.StringDtype):
dtype_ref = (
self.dtype
if isinstance(self.dtype, pd.StringDtype)
else pd.StringDtype()
)
return pd.Index(
dtype_ref.__from_arrow__(
self.to_arrow().cast(pa.large_string())
)
elif self.dtype.na_value is np.nan:
pandas_array = pd.array(
self.to_arrow().to_pandas(), dtype=self.dtype
)
else:
return super().to_pandas(
nullable=nullable, arrow_type=arrow_type
)
return pd.Index(pandas_array, copy=False)
return super().to_pandas(nullable=nullable, arrow_type=arrow_type)
),
copy=False,
)
else:
return super().to_pandas(nullable=nullable, arrow_type=arrow_type)

def can_cast_safely(self, to_dtype: DtypeObj) -> bool:
if self.dtype == to_dtype:
Expand Down
11 changes: 8 additions & 3 deletions python/cudf/cudf/core/dtypes.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
SUPPORTED_NUMPY_TO_PYLIBCUDF_TYPES,
cudf_dtype_from_pa_type,
cudf_dtype_to_pa_type,
is_dtype_obj_string,
is_pandas_nullable_extension_dtype,
)

Expand Down Expand Up @@ -995,9 +996,13 @@ def __init__(
self._fields = {}
else:
self._subtype = cudf.dtype(subtype)
if isinstance(
self._subtype, cudf.CategoricalDtype
) or cudf.utils.dtypes.is_dtype_obj_string(self._subtype):
# TODO: Remove self._subtype.kind == "U" once cudf.dtype no longer accepts
# numpy string types
if (
isinstance(self._subtype, CategoricalDtype)
or is_dtype_obj_string(self._subtype)
or self._subtype.kind == "U"
):
raise TypeError(
"category, object, and string subtypes are not supported "
"for IntervalDtype"
Expand Down
37 changes: 37 additions & 0 deletions python/cudf/cudf/pandas/scripts/conftest-patch.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -723,6 +723,7 @@ def set_copy_on_write_option():
"tests/arrays/string_/test_string.py::test_min_max[string=string[pyarrow]-False-min]",
"tests/arrays/string_/test_string.py::test_min_max[string=string[python]-False-max]",
"tests/arrays/string_/test_string.py::test_min_max[string=string[python]-False-min]",
"tests/arrays/string_/test_string.py::test_numpy_array_ufunc[string=str[python]-Series]",
"tests/arrays/string_/test_string.py::test_repr[pyarrow_numpy]",
"tests/arrays/string_/test_string.py::test_repr[string=str[pyarrow]]",
"tests/arrays/string_/test_string.py::test_repr[string=str[python]]",
Expand Down Expand Up @@ -2168,6 +2169,26 @@ def set_copy_on_write_option():
"tests/extension/test_string.py::TestStringArray::test_unary_ufunc_dunder_equivalence[pyarrow-True-positive]",
"tests/extension/test_string.py::TestStringArray::test_unary_ufunc_dunder_equivalence[pyarrow_numpy-False-positive]",
"tests/extension/test_string.py::TestStringArray::test_unary_ufunc_dunder_equivalence[pyarrow_numpy-True-positive]",
"tests/extension/test_string.py::TestStringArray::test_unique[string=str[python]-False-<lambda>-Series]",
"tests/extension/test_string.py::TestStringArray::test_unique[string=str[python]-False-unique-Series]",
"tests/extension/test_string.py::TestStringArray::test_unique[string=str[python]-True-<lambda>-Series]",
"tests/extension/test_string.py::TestStringArray::test_unique[string=str[python]-True-unique-Series]",
"tests/extension/test_string.py::TestStringArray::test_unstack[string=str[python]-False-frame-index0]",
"tests/extension/test_string.py::TestStringArray::test_unstack[string=str[python]-False-frame-index1]",
"tests/extension/test_string.py::TestStringArray::test_unstack[string=str[python]-False-frame-index2]",
"tests/extension/test_string.py::TestStringArray::test_unstack[string=str[python]-False-frame-index3]",
"tests/extension/test_string.py::TestStringArray::test_unstack[string=str[python]-False-series-index0]",
"tests/extension/test_string.py::TestStringArray::test_unstack[string=str[python]-False-series-index1]",
"tests/extension/test_string.py::TestStringArray::test_unstack[string=str[python]-False-series-index2]",
"tests/extension/test_string.py::TestStringArray::test_unstack[string=str[python]-False-series-index3]",
"tests/extension/test_string.py::TestStringArray::test_unstack[string=str[python]-True-frame-index0]",
"tests/extension/test_string.py::TestStringArray::test_unstack[string=str[python]-True-frame-index1]",
"tests/extension/test_string.py::TestStringArray::test_unstack[string=str[python]-True-frame-index2]",
"tests/extension/test_string.py::TestStringArray::test_unstack[string=str[python]-True-frame-index3]",
"tests/extension/test_string.py::TestStringArray::test_unstack[string=str[python]-True-series-index0]",
"tests/extension/test_string.py::TestStringArray::test_unstack[string=str[python]-True-series-index1]",
"tests/extension/test_string.py::TestStringArray::test_unstack[string=str[python]-True-series-index2]",
"tests/extension/test_string.py::TestStringArray::test_unstack[string=str[python]-True-series-index3]",
"tests/extension/test_string.py::TestStringArray::test_value_counts_with_normalize[pyarrow-False]",
"tests/extension/test_string.py::TestStringArray::test_value_counts_with_normalize[pyarrow-True]",
"tests/extension/test_string.py::TestStringArray::test_value_counts_with_normalize[python-False]",
Expand Down Expand Up @@ -7004,6 +7025,22 @@ def set_copy_on_write_option():
"tests/series/test_constructors.py::TestSeriesConstructors::test_series_string_inference_scalar",
"tests/series/test_constructors.py::TestSeriesConstructors::test_series_string_inference_storage_definition",
"tests/series/test_constructors.py::TestSeriesConstructors::test_series_string_with_na_inference[None]",
"tests/series/test_cumulative.py::TestSeriesCumulativeOps::test_cum_methods_ea_strings[string=str[python]-data0-cumsum-True-expected_data0]",
"tests/series/test_cumulative.py::TestSeriesCumulativeOps::test_cum_methods_ea_strings[string=str[python]-data1-cumsum-False-expected_data1]",
"tests/series/test_cumulative.py::TestSeriesCumulativeOps::test_cum_methods_ea_strings[string=str[python]-data15-cummin-False-expected_data15]",
"tests/series/test_cumulative.py::TestSeriesCumulativeOps::test_cum_methods_ea_strings[string=str[python]-data17-cummin-False-expected_data17]",
"tests/series/test_cumulative.py::TestSeriesCumulativeOps::test_cum_methods_ea_strings[string=str[python]-data19-cummin-False-expected_data19]",
"tests/series/test_cumulative.py::TestSeriesCumulativeOps::test_cum_methods_ea_strings[string=str[python]-data2-cumsum-True-expected_data2]",
"tests/series/test_cumulative.py::TestSeriesCumulativeOps::test_cum_methods_ea_strings[string=str[python]-data25-cummax-False-expected_data25]",
"tests/series/test_cumulative.py::TestSeriesCumulativeOps::test_cum_methods_ea_strings[string=str[python]-data27-cummax-False-expected_data27]",
"tests/series/test_cumulative.py::TestSeriesCumulativeOps::test_cum_methods_ea_strings[string=str[python]-data29-cummax-False-expected_data29]",
"tests/series/test_cumulative.py::TestSeriesCumulativeOps::test_cum_methods_ea_strings[string=str[python]-data3-cumsum-False-expected_data3]",
"tests/series/test_cumulative.py::TestSeriesCumulativeOps::test_cum_methods_ea_strings[string=str[python]-data4-cumsum-True-expected_data4]",
"tests/series/test_cumulative.py::TestSeriesCumulativeOps::test_cum_methods_ea_strings[string=str[python]-data5-cumsum-False-expected_data5]",
"tests/series/test_cumulative.py::TestSeriesCumulativeOps::test_cum_methods_ea_strings[string=str[python]-data6-cumsum-True-expected_data6]",
"tests/series/test_cumulative.py::TestSeriesCumulativeOps::test_cum_methods_ea_strings[string=str[python]-data7-cumsum-False-expected_data7]",
"tests/series/test_cumulative.py::TestSeriesCumulativeOps::test_cum_methods_ea_strings[string=str[python]-data8-cumsum-True-expected_data8]",
"tests/series/test_cumulative.py::TestSeriesCumulativeOps::test_cum_methods_ea_strings[string=str[python]-data9-cumsum-False-expected_data9]",
"tests/series/test_formats.py::TestCategoricalRepr::test_categorical_series_repr_datetime",
"tests/series/test_formats.py::TestCategoricalRepr::test_categorical_series_repr_datetime_ordered",
"tests/series/test_formats.py::TestCategoricalRepr::test_categorical_series_repr_timedelta",
Expand Down
15 changes: 7 additions & 8 deletions python/cudf/cudf/utils/dtypes.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -634,23 +634,22 @@ def is_dtype_obj_categorical(obj):
return pd_types.is_categorical_dtype(obj)


def is_dtype_obj_string(obj):
"""Check whether the provided array or dtype is of the string dtype.
def is_dtype_obj_string(obj) -> bool:
"""Check whether the provided object is a cuDF string dtype.

Parameters
----------
obj : array-like or dtype
The array or dtype to check.
obj : Any
The object to check.

Returns
-------
bool
Whether or not the array or dtype is of the string dtype.
Whether or not the object is a cuDF string dtype.
"""
return (
obj is CUDF_STRING_DTYPE
or obj is np.dtype("str")
or (isinstance(obj, pd.StringDtype))
obj == CUDF_STRING_DTYPE
or isinstance(obj, pd.StringDtype)
or (
isinstance(obj, pd.ArrowDtype)
and (
Expand Down
Loading