diff --git a/python/cudf/cudf/core/accessors/string.py b/python/cudf/cudf/core/accessors/string.py index 3951691e361a..6c9e60564c77 100644 --- a/python/cudf/cudf/core/accessors/string.py +++ b/python/cudf/cudf/core/accessors/string.py @@ -120,6 +120,24 @@ def __init__(self, parent: Series | Index): ) super().__init__(parent=parent) + def _return_boolean(self, new_col: ColumnBase) -> Series | Index: + if ( + isinstance(self._column.dtype, pd.StringDtype) + and self._column.dtype.na_value is pd.NA + ): + # String predicates use nullable booleans regardless of storage. + new_col = new_col.astype(pd.BooleanDtype()) + return self._return_or_inplace(new_col) + + def _return_integer(self, new_col: ColumnBase) -> Series | Index: + if ( + isinstance(self._column.dtype, pd.StringDtype) + and self._column.dtype.na_value is pd.NA + ): + # Nullable string methods return pandas' nullable 64-bit integers. + new_col = new_col.astype(pd.Int64Dtype()) + return self._return_or_inplace(new_col) + def htoi(self) -> Series | Index: """ Returns integer value represented by each hex string. @@ -201,7 +219,7 @@ def len(self) -> Series | Index: 3 dtype: int32 """ - return self._return_or_inplace(self._column.count_characters()) + return self._return_integer(self._column.count_characters()) def byte_count(self) -> Series | Index: """ @@ -845,7 +863,7 @@ def contains( result_col = result_col.fillna(False) if na is not no_default: result_col = result_col.fillna(na) - return self._return_or_inplace(result_col) + return self._return_boolean(result_col) def like(self, pat: str, esc: str | None = None) -> Series | Index: """ @@ -1440,7 +1458,7 @@ def isdecimal(self) -> Series | Index: 3 False dtype: bool """ - return self._return_or_inplace( + return self._return_boolean( self._column.all_characters_of_type( plc.strings.char_types.StringCharacterTypes.DECIMAL ) @@ -1515,7 +1533,7 @@ def isalnum(self) -> Series | Index: 2 False dtype: bool """ - return self._return_or_inplace( + return self._return_boolean( self._column.all_characters_of_type( plc.strings.char_types.StringCharacterTypes.ALPHANUM ) @@ -1577,7 +1595,7 @@ def isalpha(self) -> Series | Index: 3 False dtype: bool """ - return self._return_or_inplace( + return self._return_boolean( self._column.all_characters_of_type( plc.strings.char_types.StringCharacterTypes.ALPHA ) @@ -1645,7 +1663,7 @@ def isdigit(self) -> Series | Index: 3 False dtype: bool """ - return self._return_or_inplace( + return self._return_boolean( self._column.all_characters_of_type( plc.strings.char_types.StringCharacterTypes.DIGIT ) @@ -1719,7 +1737,7 @@ def isnumeric(self) -> Series | Index: 3 False dtype: bool """ - return self._return_or_inplace( + return self._return_boolean( self._column.all_characters_of_type( plc.strings.char_types.StringCharacterTypes.NUMERIC ) @@ -1782,7 +1800,7 @@ def isupper(self) -> Series | Index: 3 False dtype: bool """ - return self._return_or_inplace( + return self._return_boolean( self._column.all_characters_of_type( plc.strings.char_types.StringCharacterTypes.UPPER, plc.strings.char_types.StringCharacterTypes.CASE_TYPES, @@ -1846,7 +1864,7 @@ def islower(self) -> Series | Index: 3 False dtype: bool """ - return self._return_or_inplace( + return self._return_boolean( self._column.all_characters_of_type( plc.strings.char_types.StringCharacterTypes.LOWER, plc.strings.char_types.StringCharacterTypes.CASE_TYPES, @@ -2115,7 +2133,7 @@ def istitle(self) -> Series | Index: 3 False dtype: bool """ - return self._return_or_inplace(self._column.is_title()) + return self._return_boolean(self._column.is_title()) def filter_alphanum( self, repl: str | None = None, keep: bool = True @@ -3652,7 +3670,7 @@ def count(self, pat: str, flags: int = 0) -> Series | Index: "unsupported value for `flags` parameter" ) pat = self._remove_named_capture_groups(pat) - return self._return_or_inplace(self._column.count_re(pat, flags)) + return self._return_integer(self._column.count_re(pat, flags)) def _findall( self, @@ -3925,7 +3943,7 @@ def isspace(self) -> Series | Index: 2 False dtype: bool """ - return self._return_or_inplace( + return self._return_boolean( self._column.all_characters_of_type( plc.strings.char_types.StringCharacterTypes.SPACE ) @@ -3936,9 +3954,7 @@ def _starts_ends_with( method: Callable[[plc.Column, plc.Column | plc.Scalar], plc.Column], pat: str | tuple[str, ...], ) -> Series | Index: - return self._return_or_inplace( - self._column.starts_ends_with(method, pat) - ) + return self._return_boolean(self._column.starts_ends_with(method, pat)) def endswith(self, pat: str | tuple[str, ...]) -> Series | Index: """ @@ -4117,9 +4133,7 @@ def _find( if end is None: end = -1 - return self._return_or_inplace( - self._column.find(method, sub, start, end) - ) + return self._return_integer(self._column.find(method, sub, start, end)) def find( self, sub: str, start: int = 0, end: int | None = None @@ -4273,7 +4287,7 @@ def index( if (result == -1).any(): raise ValueError("substring not found") else: - return result.astype(np.dtype(np.int64)) + return result def rindex( self, sub: str, start: int = 0, end: int | None = None @@ -4333,7 +4347,7 @@ def rindex( if (result == -1).any(): raise ValueError("substring not found") else: - return result.astype(np.dtype(np.int64)) + return result def match( self, @@ -4408,7 +4422,7 @@ def match( result = result.fillna(na) elif self._column._PANDAS_NA_VALUE in {np.nan, None}: result = result.fillna(False) - return self._return_or_inplace(result) + return self._return_boolean(result) def url_decode(self) -> Series | Index: """ diff --git a/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py b/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py index dbbb83bdbeff..d0a849042dfa 100644 --- a/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py +++ b/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py @@ -5503,7 +5503,6 @@ def pytest_unconfigure(config): "tests/strings/test_extract.py::test_extract_series[string=string[pyarrow]-series_name]": "Skipped: failing in pandas-tests sharded CI (PR #22992, run 28204832469)", "tests/strings/test_extract.py::test_extract_series[string=string[python]-None]": "Skipped: failing in pandas-tests sharded CI (PR #22992, run 28204832469)", "tests/strings/test_extract.py::test_extract_series[string=str[python]-None]": "Skipped: failing in pandas-tests sharded CI (PR #22992, run 28204832469)", - "tests/strings/test_find_replace.py::test_contains_compiled_regex_flags[string=string[pyarrow]]": "Skipped: failing in pandas-tests sharded CI (PR #22992, run 28204832469)", "tests/strings/test_find_replace.py::test_contains_compiled_regex[string=object]": "Skipped: failing in pandas-tests sharded CI (PR #22992, run 28204832469)", "tests/strings/test_find_replace.py::test_contains_compiled_regex[string=string[pyarrow]]": "Skipped: failing in pandas-tests sharded CI (PR #22992, run 28204832469)", "tests/strings/test_find_replace.py::test_contains_end_of_string[string=string[pyarrow]]": "Skipped: failing in pandas-tests sharded CI (PR #22992, run 28204832469)", @@ -5512,35 +5511,12 @@ def pytest_unconfigure(config): "tests/strings/test_find_replace.py::test_contains_lookarounds[string=str[pyarrow]-None-ab-expected_data4]": "Skipped: failing in pandas-tests sharded CI (PR #22992, run 28204832469)", "tests/strings/test_find_replace.py::test_contains_lookarounds[string=str[python]-na5-ab-expected_data4]": "Skipped: failing in pandas-tests sharded CI (PR #22992, run 28204832469)", "tests/strings/test_find_replace.py::test_contains_lookarounds[string=str[python]-None-ab-expected_data4]": "Skipped: failing in pandas-tests sharded CI (PR #22992, run 28204832469)", - "tests/strings/test_find_replace.py::test_contains_moar[string=string[pyarrow]]": "Skipped: failing in pandas-tests sharded CI (PR #22992, run 28204832469)", - "tests/strings/test_find_replace.py::test_contains_na_kwarg_for_nullable_string_dtype[string[pyarrow]-False-False-False]": "Skipped: failing in pandas-tests sharded CI (PR #22992, run 28204832469)", - "tests/strings/test_find_replace.py::test_contains_na_kwarg_for_nullable_string_dtype[string[pyarrow]-False-None-expected0]": "Skipped: failing in pandas-tests sharded CI (PR #22992, run 28204832469)", - "tests/strings/test_find_replace.py::test_contains_na_kwarg_for_nullable_string_dtype[string[pyarrow]-False-True-True]": "Skipped: failing in pandas-tests sharded CI (PR #22992, run 28204832469)", - "tests/strings/test_find_replace.py::test_contains_na_kwarg_for_nullable_string_dtype[string[pyarrow]-True-False-False]": "Skipped: failing in pandas-tests sharded CI (PR #22992, run 28204832469)", - "tests/strings/test_find_replace.py::test_contains_na_kwarg_for_nullable_string_dtype[string[pyarrow]-True-None-expected0]": "Skipped: failing in pandas-tests sharded CI (PR #22992, run 28204832469)", "tests/strings/test_find_replace.py::test_replace_end_of_string[string=string[pyarrow]]": "Skipped: failing in pandas-tests sharded CI (PR #22992, run 28204832469)", "tests/strings/test_find_replace.py::test_replace_end_of_string[string=str[pyarrow]]": "Skipped: failing in pandas-tests sharded CI (PR #22992, run 28204832469)", "tests/strings/test_find_replace.py::test_startswith[False-None-object-pat1]": "Skipped: failing in pandas-tests sharded CI (PR #22992, run 28204832469)", "tests/strings/test_find_replace.py::test_startswith[True-None-object-foo]": "Skipped: failing in pandas-tests sharded CI (PR #22992, run 28204832469)", - "tests/strings/test_string_array.py::test_string_array_boolean_array[string[pyarrow]-isdigit-expected0]": "Skipped: failing in pandas-tests sharded CI (PR #22992, run 28204832469)", - "tests/strings/test_string_array.py::test_string_array_boolean_array[string[pyarrow]-isnumeric-expected4]": "Skipped: failing in pandas-tests sharded CI (PR #22992, run 28204832469)", - "tests/strings/test_string_array.py::test_string_array[string[pyarrow]-contains]": "Skipped: failing in pandas-tests sharded CI (PR #22992, run 28204832469)", - "tests/strings/test_string_array.py::test_string_array[string[pyarrow]-endswith2]": "Skipped: failing in pandas-tests sharded CI (PR #22992, run 28204832469)", - "tests/strings/test_string_array.py::test_string_array[string[pyarrow]-endswith3]": "Skipped: failing in pandas-tests sharded CI (PR #22992, run 28204832469)", - "tests/strings/test_string_array.py::test_string_array[string[pyarrow]-endswith4]": "Skipped: failing in pandas-tests sharded CI (PR #22992, run 28204832469)", - "tests/strings/test_string_array.py::test_string_array[string[pyarrow]-isdecimal]": "Skipped: failing in pandas-tests sharded CI (PR #22992, run 28204832469)", - "tests/strings/test_string_array.py::test_string_array[string[pyarrow]-isdigit]": "Skipped: failing in pandas-tests sharded CI (PR #22992, run 28204832469)", - "tests/strings/test_string_array.py::test_string_array[string[pyarrow]-istitle]": "Skipped: failing in pandas-tests sharded CI (PR #22992, run 28204832469)", - "tests/strings/test_string_array.py::test_string_array[string[pyarrow]-len]": "Skipped: failing in pandas-tests sharded CI (PR #22992, run 28204832469)", - "tests/strings/test_string_array.py::test_string_array[string[pyarrow]-startswith0]": "Skipped: failing in pandas-tests sharded CI (PR #22992, run 28204832469)", - "tests/strings/test_string_array.py::test_string_array[string[pyarrow]-startswith1]": "Skipped: failing in pandas-tests sharded CI (PR #22992, run 28204832469)", - "tests/strings/test_string_array.py::test_string_array[string[pyarrow]-startswith3]": "Skipped: failing in pandas-tests sharded CI (PR #22992, run 28204832469)", - "tests/strings/test_string_array.py::test_string_array[string[python]-len]": "Skipped: failing in pandas-tests sharded CI (PR #22992, run 28204832469)", "tests/strings/test_strings.py::test_empty_str_methods[string=object]": "Skipped: failing in pandas-tests sharded CI (PR #22992, run 28204832469)", "tests/strings/test_strings.py::test_empty_str_methods[string=str[pyarrow]]": "Skipped: failing in pandas-tests sharded CI (PR #22992, run 28204832469)", - "tests/strings/test_strings.py::test_ismethods[string=string[pyarrow]-isalnum-expected1]": "Skipped: failing in pandas-tests sharded CI (PR #22992, run 28204832469)", - "tests/strings/test_strings.py::test_ismethods[string=string[pyarrow]-isnumeric-expected4]": "Skipped: failing in pandas-tests sharded CI (PR #22992, run 28204832469)", - "tests/strings/test_strings.py::test_len[string=string[python]]": "Skipped: failing in pandas-tests sharded CI (PR #22992, run 28204832469)", "tests/strings/test_strings.py::test_slice_replace[string=string[pyarrow]-None--2-z-expected5]": "Skipped: failing in pandas-tests sharded CI (PR #22992, run 28204832469)", "tests/strings/test_strings.py::test_slice_replace[string=string[python]--1-None-z-expected4]": "Skipped: failing in pandas-tests sharded CI (PR #22992, run 28204832469)", "tests/strings/test_strings.py::test_slice_replace[string=str[python]--10-3-z-expected7]": "Skipped: failing in pandas-tests sharded CI (PR #22992, run 28204832469)", diff --git a/python/cudf/cudf/tests/series/accessors/test_str.py b/python/cudf/cudf/tests/series/accessors/test_str.py index 6e02d0252486..c6796e5e9cbe 100644 --- a/python/cudf/cudf/tests/series/accessors/test_str.py +++ b/python/cudf/cudf/tests/series/accessors/test_str.py @@ -66,6 +66,88 @@ def test_getitem_out_of_bounds(): assert_eq(result, expected) +@pytest.mark.parametrize( + "dtype", + [ + pd.StringDtype(storage="python"), + pd.StringDtype(storage="pyarrow"), + pd.StringDtype(storage="pyarrow", na_value=np.nan), + pd.ArrowDtype(pa.string()), + ], +) +@pytest.mark.parametrize( + "method,args", + [ + ("contains", ("a",)), + ("startswith", ("a",)), + ("endswith", ("a",)), + ("isdigit", ()), + ("isnumeric", ()), + ("isalnum", ()), + ], +) +def test_string_predicate_extension_dtype(dtype, method, args): + ps = pd.Series(["a", None, "12"], dtype=dtype) + gs = cudf.from_pandas(ps) + + expected = getattr(ps.str, method)(*args) + result = getattr(gs.str, method)(*args) + + assert result.dtype == expected.dtype + assert_eq(result, expected) + + +@pytest.mark.parametrize("storage", ["python", "pyarrow"]) +@pytest.mark.parametrize( + "method,args", + [ + ("match", ("a",)), + ("isalpha", ()), + ("isdecimal", ()), + ("islower", ()), + ("isupper", ()), + ("istitle", ()), + ("isspace", ()), + ], +) +def test_string_additional_predicate_nullable_dtype(storage, method, args): + ps = pd.Series( + ["a", None, "12", "", " ", "ABC", "Abc"], + dtype=pd.StringDtype(storage=storage), + ) + gs = cudf.from_pandas(ps) + + expected = getattr(ps.str, method)(*args) + result = getattr(gs.str, method)(*args) + + assert result.dtype == expected.dtype + assert_eq(result, expected) + + +@pytest.mark.parametrize("storage", ["python", "pyarrow"]) +@pytest.mark.parametrize("data", [["aba", None, "abc"], [], [None, None]]) +@pytest.mark.parametrize( + "method,args", + [ + ("len", ()), + ("count", ("a",)), + ("find", ("a",)), + ("rfind", ("a",)), + ("index", ("a",)), + ("rindex", ("a",)), + ], +) +def test_string_numeric_nullable_dtype(storage, data, method, args): + ps = pd.Series(data, dtype=pd.StringDtype(storage=storage)) + gs = cudf.from_pandas(ps) + + expected = getattr(ps.str, method)(*args) + result = getattr(gs.str, method)(*args) + + assert result.dtype == expected.dtype + assert_eq(result, expected) + + @pytest.mark.parametrize("method", ["startswith", "endswith"]) @pytest.mark.parametrize("pat", [None, (1, 2), pd.Series([1])]) def test_startsendwith_invalid_pat(method, pat): diff --git a/python/cudf/cudf/tests/series/methods/test_value_counts.py b/python/cudf/cudf/tests/series/methods/test_value_counts.py index 4848f9462a78..a18e0e6890d1 100644 --- a/python/cudf/cudf/tests/series/methods/test_value_counts.py +++ b/python/cudf/cudf/tests/series/methods/test_value_counts.py @@ -207,6 +207,8 @@ def test_numeric_alpha_value_counts(): "UInt32", "UInt64", "boolean", + "string[python]", + "string[pyarrow]", ], ) def test_value_counts_empty_pandas_nullable(dtype, normalize, dropna): @@ -221,7 +223,14 @@ def test_value_counts_empty_pandas_nullable(dtype, normalize, dropna): @pytest.mark.parametrize( "dtype", - ["Float64", "Int64", "UInt32", "boolean"], + [ + "Float64", + "Int64", + "UInt32", + "boolean", + "string[python]", + "string[pyarrow]", + ], ) def test_value_counts_all_null_pandas_nullable(dtype, normalize, dropna): psr = pd.Series([pd.NA, pd.NA, pd.NA], dtype=dtype) @@ -238,6 +247,17 @@ def test_value_counts_all_null_pandas_nullable(dtype, normalize, dropna): ) +@pytest.mark.parametrize("dtype", ["string[python]", "string[pyarrow]"]) +def test_value_counts_pandas_nullable_string(dtype, normalize, dropna): + psr = pd.Series(["a", "a", "b", pd.NA], dtype=dtype, name="values") + gsr = cudf.from_pandas(psr) + + expected = psr.value_counts(dropna=dropna, normalize=normalize) + got = gsr.value_counts(dropna=dropna, normalize=normalize) + + assert_eq(expected, got, check_dtype=True, check_index_type=True) + + def test_value_counts_first_appearance_order(sort): # pandas returns groups in order of first appearance with sort=False # and keeps that order for equal counts with sort=True (GH 63155).