From 588722c936c1ef5ba9fb57483991f12d0c5b9ae2 Mon Sep 17 00:00:00 2001 From: Allen Xu Date: Thu, 16 Jul 2026 16:29:11 +0800 Subject: [PATCH 01/25] Fix lazy regex priority fallback in Glushkov engine Signed-off-by: Allen Xu --- cpp/src/strings/regex/glushkov_regcomp.cpp | 23 +++++++++------------- cpp/src/strings/regex/glushkov_regcomp.hpp | 2 ++ cpp/tests/strings/split_tests.cpp | 16 +++++++++++++++ 3 files changed, 27 insertions(+), 14 deletions(-) diff --git a/cpp/src/strings/regex/glushkov_regcomp.cpp b/cpp/src/strings/regex/glushkov_regcomp.cpp index cb740d0cb9af..577ee9da42d3 100644 --- a/cpp/src/strings/regex/glushkov_regcomp.cpp +++ b/cpp/src/strings/regex/glushkov_regcomp.cpp @@ -278,9 +278,9 @@ bool positions_chars_overlap(gkprog const& gp, uint32_t const p, uint32_t const * Glushkov's bit-order cannot represent. * * Two rules: - * Rule 1 – END before char: an ACCEPT item appears before the first CHAR_POS - * item in Thompson priority order → the pattern can empty-match in a way - * that priority_kill cannot handle correctly. + * Rule 1 – END before later char: an ACCEPT item appears before a CHAR_POS + * item in Thompson priority order → the accepted path has higher + * priority than a continuation that Glushkov cannot kill correctly. * Rule 2 – non-monotone gpos + char overlap: two CHAR_POS items appear with * the higher-priority one at a larger gpos (inverted bit order), AND * they can match a common character → priority_kill picks the wrong @@ -288,19 +288,14 @@ bool positions_chars_overlap(gkprog const& gp, uint32_t const p, uint32_t const */ bool frontier_has_priority_conflict(std::vector const& items, gkprog const& gp) { - // Rule 1: ACCEPT before any CHAR_POS, but only when the frontier also - // contains at least one CHAR_POS. An ACCEPT-only frontier (the normal - // "end of pattern" case) is not a priority conflict. - bool seen_char = false; - bool accept_before_char = false; + // Rule 1: ACCEPT before a later CHAR_POS. An ACCEPT-only frontier and a + // frontier ending in ACCEPT (the normal "end of pattern" case) are not + // priority conflicts. + bool seen_accept = false; for (auto const& item : items) { - if (item.kind == frontier_item::CHAR_POS) { - seen_char = true; - } else if (item.kind == frontier_item::ACCEPT && !seen_char) { - accept_before_char = true; - } + if (item.kind == frontier_item::ACCEPT) { seen_accept = true; } + if (item.kind == frontier_item::CHAR_POS && seen_accept) { return true; } } - if (accept_before_char && seen_char) { return true; } // Rule 2: non-monotone gpos pair with character overlap for (size_t i = 0; i < items.size(); ++i) { diff --git a/cpp/src/strings/regex/glushkov_regcomp.hpp b/cpp/src/strings/regex/glushkov_regcomp.hpp index 32c306520c25..6d7450aef070 100644 --- a/cpp/src/strings/regex/glushkov_regcomp.hpp +++ b/cpp/src/strings/regex/glushkov_regcomp.hpp @@ -102,6 +102,8 @@ struct gkprog { * - Pattern has more than GLUSHKOV_MAX_STATES character-consuming positions. * - Pattern is nullable (matches the empty string): priority semantics cannot * be faithfully represented without an ε-position for the empty match. + * - Pattern has a Thompson-priority frontier that Glushkov's bit ordering + * cannot represent faithfully. * * @param prog Compiled Thompson NFA (after reprog::finalize()). * @return Host-side Glushkov program, or nullptr on failure. diff --git a/cpp/tests/strings/split_tests.cpp b/cpp/tests/strings/split_tests.cpp index cacd98328844..4a61e24171a7 100644 --- a/cpp/tests/strings/split_tests.cpp +++ b/cpp/tests/strings/split_tests.cpp @@ -517,6 +517,22 @@ TEST_F(StringsSplitTest, SplitRecordRegex) } } +TEST_F(StringsSplitTest, SplitRecordRegexLazyQuantifier) +{ + auto const input = cudf::test::strings_column_wrapper({"\rbaab\r\ra"}); + auto const sv = cudf::strings_column_view(input); + auto const prog = + cudf::strings::regex_program::create("[^ \v\n\t\r\f]\\r+?\\n*", + cudf::strings::regex_flags::EXT_NEWLINE, + cudf::strings::capture_groups::NON_CAPTURE); + + using LCW = cudf::test::lists_column_wrapper; + LCW expected{LCW{"\rbaa", "\ra"}}; + auto const result = cudf::strings::split_record_re(sv, *prog); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->view(), expected); +} + TEST_F(StringsSplitTest, SplitRegexWithMaxSplit) { std::vector h_strings{" Héllo\tthesé", nullptr, "are\nsome ", "tést\rString", ""}; From 93e9edb8a1a8bbfacaa4bf79f0da912cc9c3f64e Mon Sep 17 00:00:00 2001 From: Allen Xu Date: Fri, 17 Jul 2026 15:01:00 +0800 Subject: [PATCH 02/25] Address regex review feedback Signed-off-by: Allen Xu --- cpp/src/strings/regex/glushkov_regcomp.cpp | 5 ++-- cpp/tests/strings/split_tests.cpp | 31 +++++++++++++++------- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/cpp/src/strings/regex/glushkov_regcomp.cpp b/cpp/src/strings/regex/glushkov_regcomp.cpp index 577ee9da42d3..a91f8108ba83 100644 --- a/cpp/src/strings/regex/glushkov_regcomp.cpp +++ b/cpp/src/strings/regex/glushkov_regcomp.cpp @@ -288,9 +288,8 @@ bool positions_chars_overlap(gkprog const& gp, uint32_t const p, uint32_t const */ bool frontier_has_priority_conflict(std::vector const& items, gkprog const& gp) { - // Rule 1: ACCEPT before a later CHAR_POS. An ACCEPT-only frontier and a - // frontier ending in ACCEPT (the normal "end of pattern" case) are not - // priority conflicts. + // Rule 1: ACCEPT before a later CHAR_POS. A frontier ending in ACCEPT (the + // normal "end of pattern" case) is not a priority conflict. bool seen_accept = false; for (auto const& item : items) { if (item.kind == frontier_item::ACCEPT) { seen_accept = true; } diff --git a/cpp/tests/strings/split_tests.cpp b/cpp/tests/strings/split_tests.cpp index 4a61e24171a7..ce9fd7a8b10e 100644 --- a/cpp/tests/strings/split_tests.cpp +++ b/cpp/tests/strings/split_tests.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -521,16 +521,29 @@ TEST_F(StringsSplitTest, SplitRecordRegexLazyQuantifier) { auto const input = cudf::test::strings_column_wrapper({"\rbaab\r\ra"}); auto const sv = cudf::strings_column_view(input); - auto const prog = - cudf::strings::regex_program::create("[^ \v\n\t\r\f]\\r+?\\n*", - cudf::strings::regex_flags::EXT_NEWLINE, - cudf::strings::capture_groups::NON_CAPTURE); + using LCW = cudf::test::lists_column_wrapper; - using LCW = cudf::test::lists_column_wrapper; - LCW expected{LCW{"\rbaa", "\ra"}}; - auto const result = cudf::strings::split_record_re(sv, *prog); + { + LCW expected({LCW{"\rbaa", "\ra"}}); + auto const prog = + cudf::strings::regex_program::create("[^ \v\n\t\r\f]\\r+?\\n*", + cudf::strings::regex_flags::EXT_NEWLINE, + cudf::strings::capture_groups::NON_CAPTURE); + auto const result = cudf::strings::split_record_re(sv, *prog); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->view(), expected); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->view(), expected); + } + + { + LCW expected({LCW{"\rbaa", "a"}}); + auto const prog = + cudf::strings::regex_program::create("[^ \v\n\t\r\f]\\r+\\n*", + cudf::strings::regex_flags::EXT_NEWLINE, + cudf::strings::capture_groups::NON_CAPTURE); + auto const result = cudf::strings::split_record_re(sv, *prog); + + CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->view(), expected); + } } TEST_F(StringsSplitTest, SplitRegexWithMaxSplit) From 0baa8ee6cf78468de13bb5318467bca55511142e Mon Sep 17 00:00:00 2001 From: Allen Xu Date: Fri, 17 Jul 2026 15:22:57 +0800 Subject: [PATCH 03/25] Clarify regex frontier terminology Signed-off-by: Allen Xu --- cpp/src/strings/regex/glushkov_regcomp.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/strings/regex/glushkov_regcomp.cpp b/cpp/src/strings/regex/glushkov_regcomp.cpp index a91f8108ba83..8d06aa980208 100644 --- a/cpp/src/strings/regex/glushkov_regcomp.cpp +++ b/cpp/src/strings/regex/glushkov_regcomp.cpp @@ -278,7 +278,7 @@ bool positions_chars_overlap(gkprog const& gp, uint32_t const p, uint32_t const * Glushkov's bit-order cannot represent. * * Two rules: - * Rule 1 – END before later char: an ACCEPT item appears before a CHAR_POS + * Rule 1 – ACCEPT before later char: an ACCEPT item appears before a CHAR_POS * item in Thompson priority order → the accepted path has higher * priority than a continuation that Glushkov cannot kill correctly. * Rule 2 – non-monotone gpos + char overlap: two CHAR_POS items appear with From 245bff594766d8a7f7fc6471e26edc88301bd26f Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Fri, 17 Jul 2026 08:42:40 -0500 Subject: [PATCH 04/25] Fix stale num_rows argument breaking mypy in cudf-polars duplicated-output path (#23303) `DataFrame.from_table` lost its `num_rows` parameter in #23234 (row counts are now inferred from the pylibcudf table, which carries them even for zero-column tables), but the duplicated-output path added in #23114 still passes `num_rows=0`. The two PRs merged around the same time, so this surfaced only after both landed: mypy now fails on every PR's `check-style` job (`engine/core.py:840: Unexpected keyword argument "num_rows"`), and the path would raise `TypeError` at runtime. `plc.copying.empty_like` already produces a 0-row table (including for zero-column inputs, verified), so dropping the argument preserves the intended "freshly-allocated empty same-schema frame" semantics exactly. Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Matthew Roeschke (https://github.com/mroeschke) - Lawrence Mitchell (https://github.com/wence-) URL: https://github.com/rapidsai/cudf/pull/23303 --- python/cudf_polars/cudf_polars/engine/core.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/cudf_polars/cudf_polars/engine/core.py b/python/cudf_polars/cudf_polars/engine/core.py index 1d92f2538066..80359eab925d 100644 --- a/python/cudf_polars/cudf_polars/engine/core.py +++ b/python/cudf_polars/cudf_polars/engine/core.py @@ -842,7 +842,6 @@ def drop_if_replicated( df.column_names, df.dtypes, df.stream, - num_rows=0, ) return df From 74d23c2fbcbb1a286e54644f980fb863ef8c3684 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Fri, 17 Jul 2026 13:22:34 -0500 Subject: [PATCH 05/25] Pin pyarrow<24 in the cudf and pylibcudf conda recipes (#23319) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The latest-deps conda CI jobs started failing on every PR (e.g. [this run on #23272](https://github.com/rapidsai/cudf/actions/runs/29585048860)) with pyarrow 25.0.0 in the environment — feather tests/doctests (`pyarrow.feather` deprecated as of 24), pylibcudf quantiles (`SortOptions(null_placement=)` deprecated in 25), ORC tests (out-of-ns-range timestamps now raise `ArrowInvalid` instead of silently overflowing), and the narwhals suite. cudf pins `pyarrow>=19.0.0,<24` (#22229) in `dependencies.yaml` for the conda, requirements, and pyproject outputs — but the hand-maintained conda recipes only declare `pyarrow>=19.0.0` with **no upper bound** (`conda/recipes/cudf/recipe.yaml` run dependency and `conda/recipes/pylibcudf/recipe.yaml` run constraint). The conda test environments don't list pyarrow directly (only the oldest-deps matrix pins `pyarrow==19.*`), so the env solve takes the bound from the built packages, and with the recipes unbounded the solver picked pyarrow 25.0.0. This is also how the narwhals job got pyarrow 25: its env installs the built cudf conda package in the same solve. This mirrors the `dependencies.yaml` bound into both recipes, which constrains every conda test environment that installs the built packages. Note: the wheel jobs were unaffected because the pip/pyproject metadata carries the `<24` bound. The remaining failure in the linked run (`conda-python-other-tests`) was a runner infra flake (`nvidia-smi`: "No devices were found") — retry only. For whenever the pin is actually lifted (#22229): the test-suite adaptations needed for pyarrow 24/25 (feather→`pyarrow.ipc` migration, per-sort-key `null_placement`, ORC timestamp-range handling, narwhals deselects) were worked out and verified in [9a3a288019](https://github.com/galipremsagar/cudf/commit/9a3a288019) (previous head of this branch). Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/23319 --- conda/recipes/cudf/recipe.yaml | 2 +- conda/recipes/pylibcudf/recipe.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conda/recipes/cudf/recipe.yaml b/conda/recipes/cudf/recipe.yaml index 24fb9413503c..39a06850ee5c 100644 --- a/conda/recipes/cudf/recipe.yaml +++ b/conda/recipes/cudf/recipe.yaml @@ -96,7 +96,7 @@ requirements: # lives in `dependencies.yaml::depends_on_numba_cuda_mlir`. - numba >=0.60.0,<0.65.0 - numpy >=2.0,<3.0 - - pyarrow>=19.0.0 + - pyarrow>=19.0.0,<24 # https://github.com/rapidsai/cudf/issues/22229 - libcudf =${{ version }} - pylibcudf =${{ version }} - ${{ pin_compatible("rmm", upper_bound="x.x") }} diff --git a/conda/recipes/pylibcudf/recipe.yaml b/conda/recipes/pylibcudf/recipe.yaml index 574fb95c44ed..d36d5b653174 100644 --- a/conda/recipes/pylibcudf/recipe.yaml +++ b/conda/recipes/pylibcudf/recipe.yaml @@ -95,7 +95,7 @@ requirements: - nvtx >=0.2.1 run_constraints: - numpy >=2.0,<3.0 - - pyarrow>=19.0.0 + - pyarrow>=19.0.0,<24 # https://github.com/rapidsai/cudf/issues/22229 ignore_run_exports: from_package: - cuda-cudart-dev From b7a252ce0523931c0b4d6c47e89692d190cfa9a3 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Fri, 17 Jul 2026 23:23:31 -0500 Subject: [PATCH 06/25] Fix cudf.pandas isinstance checks for unproxied pandas subclasses and wrap groupby resamplers (#23273) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Running pandas' own test suite under `cudf.pandas`, `tests/groupby/test_groupby_subclass.py` had 37 failing tests. This PR fixes 36 of them in `cudf.pandas` and removes the corresponding xfail entries from the pandas-testing plugin. Notably, it does **not** proxy the internal pandas subclasses (`pandas._testing.SubclassedDataFrame`/`SubclassedSeries`) — they stay raw and run entirely on the slow path; the fixes make the proxy layer coexist with such objects. ### `isinstance`/`issubclass` against proxy types now recognize raw pandas objects Instances of unproxied pandas subclasses operate on the slow path and produce raw pandas objects (e.g. `SubclassedDataFrame.groupby(...).sum()` returns a raw `SubclassedDataFrame`), but `isinstance(result, pd.DataFrame)` — where `pd.DataFrame` is the proxy type — returned `False`. `_FastSlowProxyMeta.__instancecheck__`/`__subclasscheck__` now fall back to checking against the proxy's slow type. The fallback is guarded by a new `_fsproxy_is_canonical` property (it only applies when the class is the registered proxy for its slow type), so a user-defined subclass of a proxy type does not match arbitrary instances of its parent's slow type. This fixes the 35 `test_groupby_preserves_subclass[*-obj0]` failures (DataFrame results were mis-routed into `assert_series_equal` because the `isinstance(result1, DataFrame)` branch check failed) and `corrwith-obj1` (the `isinstance(obj, Series)` skip guard was `False`, so `SeriesGroupBy.corrwith` ran instead of skipping). ### `groupby().resample()` no longer leaks raw pandas objects `DataFrameGroupBy.resample()` returns `DatetimeIndexResamplerGroupby` (or the Period/Timedelta variants), none of which were registered as intermediate proxy types — the resampler and everything derived from it (e.g. `.sum()`) escaped the proxy layer as raw pandas objects. Registered the five missing resampler classes (`PeriodIndexResampler`, `TimedeltaIndexResampler`, `DatetimeIndexResamplerGroupby`, `PeriodIndexResamplerGroupby`, `TimedeltaIndexResamplerGroupby`) as intermediate proxy types, fixing `test_groupby_resample_preserves_subclass[DataFrame]`. ### Remaining failure (inherent) `test_groupby_preserves_metadata` builds its expected value by passing a proxy `pd.Index` into the raw `SubclassedSeries` constructor. Real pandas' `maybe_extract_name` checks `isinstance(obj, Index)` against the concrete `Index` class, which a proxy instance cannot satisfy, so `name="c"` is lost while the actual groupby result correctly keeps it. This is not fixable without proxying the internal subclass or spoofing `__class__` on data proxies; the xfail entry now records this reason. ### Results - `tests/groupby/test_groupby_subclass.py`: 37 failed / 35 passed → 1 failed / 70 passed / 1 skipped - cudf.pandas unit tests pass: `test_fast_slow_proxy.py` (27 passed, 3 xfailed), `test_cudf_pandas.py` + `test_cudf_pandas_no_fallback.py` (419 passed, 3 xfailed) Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Matthew Roeschke (https://github.com/mroeschke) URL: https://github.com/rapidsai/cudf/pull/23273 --- python/cudf/cudf/pandas/_wrappers/pandas.py | 30 ++++++++++ python/cudf/cudf/pandas/fast_slow_proxy.py | 25 +++++++- .../pandas/scripts/pandas-testing-plugin.py | 59 +------------------ 3 files changed, 55 insertions(+), 59 deletions(-) diff --git a/python/cudf/cudf/pandas/_wrappers/pandas.py b/python/cudf/cudf/pandas/_wrappers/pandas.py index 341c65658e33..621163c4be2b 100644 --- a/python/cudf/cudf/pandas/_wrappers/pandas.py +++ b/python/cudf/cudf/pandas/_wrappers/pandas.py @@ -1337,6 +1337,36 @@ def Index__setattr__(self, name, value): pd.core.resample.DatetimeIndexResampler, ) +PeriodIndexResampler = make_intermediate_proxy_type( + "PeriodIndexResampler", + _Unusable, + pd.core.resample.PeriodIndexResampler, +) + +TimedeltaIndexResampler = make_intermediate_proxy_type( + "TimedeltaIndexResampler", + _Unusable, + pd.core.resample.TimedeltaIndexResampler, +) + +DatetimeIndexResamplerGroupby = make_intermediate_proxy_type( + "DatetimeIndexResamplerGroupby", + _Unusable, + pd.core.resample.DatetimeIndexResamplerGroupby, +) + +PeriodIndexResamplerGroupby = make_intermediate_proxy_type( + "PeriodIndexResamplerGroupby", + _Unusable, + pd.core.resample.PeriodIndexResamplerGroupby, +) + +TimedeltaIndexResamplerGroupby = make_intermediate_proxy_type( + "TimedeltaIndexResamplerGroupby", + _Unusable, + pd.core.resample.TimedeltaIndexResamplerGroupby, +) + StataReader = make_final_proxy_type( "StataReader", _Unusable, diff --git a/python/cudf/cudf/pandas/fast_slow_proxy.py b/python/cudf/cudf/pandas/fast_slow_proxy.py index e228f5fd0589..0297bce31b0d 100644 --- a/python/cudf/cudf/pandas/fast_slow_proxy.py +++ b/python/cudf/cudf/pandas/fast_slow_proxy.py @@ -586,18 +586,41 @@ def __dir__(self): except AttributeError: return type.__dir__(self) + @property + def _is_proxy_base_class(self) -> bool: + # True if this class is the base proxy class registered for its + # slow type, as opposed to e.g. a user-defined subclass of a + # proxy type (which shares ``_fsproxy_slow_type`` with its + # parent but must not match arbitrary instances of the slow + # type). + slow = getattr(self, "_fsproxy_slow_type", None) + return slow is not None and ( + get_final_type_map().get(slow) is self + or get_intermediate_type_map().get(slow) is self + ) + def __subclasscheck__(self, __subclass: type) -> bool: if super().__subclasscheck__(__subclass): return True if hasattr(__subclass, "_fsproxy_slow"): return issubclass(__subclass._fsproxy_slow, self._fsproxy_slow) + if self._is_proxy_base_class: + # An unproxied class (e.g. a user-defined subclass of the + # slow type such as ``pandas._testing.SubclassedDataFrame``) + # is a subclass of the proxy standing in for its parent. + return issubclass(__subclass, self._fsproxy_slow) return False def __instancecheck__(self, __instance: Any) -> bool: if super().__instancecheck__(__instance): return True - elif hasattr(type(__instance), "_fsproxy_slow"): + if hasattr(type(__instance), "_fsproxy_slow"): return issubclass(type(__instance), self) + if self._is_proxy_base_class: + # A raw (unproxied) slow object, e.g. produced by operations + # on an unproxied pandas subclass, is an instance of the + # proxy type standing in for its class. + return isinstance(__instance, self._fsproxy_slow) return False diff --git a/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py b/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py index 36bd33ad4e67..d74dafcb4a96 100644 --- a/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py +++ b/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py @@ -104,11 +104,6 @@ def pytest_unconfigure(config): NODEIDS_THAT_FAIL = { "tests/apply/test_frame_apply.py::test_apply[MockEngineDecorator]": "AssertionError", "tests/apply/test_frame_apply.py::test_apply[python]": "TODO: Add a reason for failure", - "tests/apply/test_frame_apply.py::test_apply_args[MockEngineDecorator-False-False-0]": "AssertionError", - "tests/apply/test_frame_apply.py::test_apply_args[MockEngineDecorator-False-False-1]": "AssertionError", - "tests/apply/test_frame_apply.py::test_apply_args[MockEngineDecorator-True-False-0]": "AssertionError", - "tests/apply/test_frame_apply.py::test_apply_args[MockEngineDecorator-True-False-1]": "AssertionError", - "tests/apply/test_frame_apply.py::test_apply_datetime_tz_issue[MockEngineDecorator]": "AssertionError", "tests/apply/test_frame_apply.py::test_apply_empty_infer_type[MockEngineDecorator-0-False-mean-columns]": "AssertionError: assert Index(['a', 'b', 'c'], dtype='str') is Index(['a', 'b', 'c'], dtype='str')", "tests/apply/test_frame_apply.py::test_apply_empty_infer_type[MockEngineDecorator-0-False-mean-index]": "assert RangeIndex(start=0, stop=0, step=1) is RangeIndex(start=0, stop=0, step=1)", "tests/apply/test_frame_apply.py::test_apply_empty_infer_type[MockEngineDecorator-0-True-mean-columns]": "AssertionError: assert Index(['a', 'b', 'c'], dtype='str') is Index(['a', 'b', 'c'], dtype='str')", @@ -118,15 +113,8 @@ def pytest_unconfigure(config): "tests/apply/test_frame_apply.py::test_apply_empty_infer_type[MockEngineDecorator-1-True-mean-columns]": "assert RangeIndex(start=0, stop=0, step=1) is RangeIndex(start=0, stop=0, step=1)", "tests/apply/test_frame_apply.py::test_apply_empty_infer_type[MockEngineDecorator-1-True-mean-index]": "AssertionError: assert Index(['a', 'b', 'c'], dtype='str') is Index(['a', 'b', 'c'], dtype='str')", "tests/apply/test_frame_apply.py::test_apply_function_runs_once": "TODO: Add a reason for failure", - "tests/apply/test_frame_apply.py::test_apply_getitem_axis_1[MockEngineDecorator]": "AssertionError", - "tests/apply/test_frame_apply.py::test_apply_no_suffix_index[MockEngineDecorator]": "AssertionError", "tests/apply/test_frame_apply.py::test_apply_raw_function_runs_once[MockEngineDecorator]": "assert [] == [1, 2, 3]", "tests/apply/test_frame_apply.py::test_apply_raw_function_runs_once[python]": "TODO: Add a reason for failure", - "tests/apply/test_frame_apply.py::test_frequency_is_original[MockEngineDecorator-2]": "AssertionError", - "tests/apply/test_frame_apply.py::test_frequency_is_original[MockEngineDecorator-3]": "AssertionError", - "tests/apply/test_frame_apply.py::test_frequency_is_original[MockEngineDecorator-5]": "AssertionError", - "tests/apply/test_frame_apply.py::test_result_type_series_result[MockEngineDecorator]": "AssertionError", - "tests/apply/test_frame_apply.py::test_result_type_series_result_other_index[MockEngineDecorator]": "AssertionError", "tests/apply/test_series_apply.py::test_apply[False]": "TODO: Add a reason for failure", "tests/apply/test_series_apply.py::test_apply[compat]": "TODO: Add a reason for failure", "tests/apply/test_series_apply.py::test_apply_map_evaluate_lambdas_the_same[compat-MockEngineDecorator-str]": "AssertionError: Attributes of Series are different", @@ -1141,7 +1129,6 @@ def pytest_unconfigure(config): "tests/frame/indexing/test_indexing.py::TestDataFrameIndexing::test_loc_bool_multiindex[True-boolean-indexer1]": "AssertionError: Did not see expected warning of class 'PerformanceWarning'", "tests/frame/indexing/test_indexing.py::TestDataFrameIndexing::test_setitem_boolean": "TODO: Add a reason for failure", "tests/frame/indexing/test_indexing.py::test_adding_new_conditional_column_with_string[object-False]": "TODO: Add a reason for failure", - "tests/frame/indexing/test_indexing.py::test_object_casting_indexing_wraps_datetimelike": "TODO: Add a reason for failure", "tests/frame/indexing/test_insert.py::TestDataFrameInsert::test_insert": "TODO: Add a reason for failure", "tests/frame/indexing/test_insert.py::TestDataFrameInsert::test_insert_delete_mixed_multiindex_columns": "AssertionError: DataFrame.columns level [1] are different", "tests/frame/indexing/test_mask.py::TestDataFrameMask::test_mask_inplace": "assert None is 0 1 2\n0 0.189053382 NaN NaN\n1 NaN 1....", @@ -1356,7 +1343,6 @@ def pytest_unconfigure(config): "tests/frame/methods/test_to_dict.py::test_to_dict_list_pd_scalars[val1]": "TODO: Add a reason for failure", "tests/frame/methods/test_to_numpy.py::TestToNumpy::test_to_numpy_copy": "TODO: Add a reason for failure", "tests/frame/methods/test_to_numpy.py::TestToNumpy::test_to_numpy_mixed_dtype_to_str": "TODO: Add a reason for failure", - "tests/frame/methods/test_to_records.py::TestDataFrameToRecords::test_to_records_dt64tz_column": "TODO: Add a reason for failure", "tests/frame/methods/test_transpose.py::TestTranspose::test_transpose_get_view_dt64tzget_view": "assert 3 == 1", "tests/frame/methods/test_truncate.py::TestDataFrameTruncate::test_truncate_multiindex[DataFrame]": "TODO: Add a reason for failure", "tests/frame/methods/test_value_counts.py::test_value_counts_with_missing_category": "TODO: Add a reason for failure", @@ -1399,7 +1385,6 @@ def pytest_unconfigure(config): "tests/frame/test_arithmetic.py::test_frame_with_zero_len_series_corner_cases[numexpr]": "TODO: Add a reason for failure", "tests/frame/test_arithmetic.py::test_frame_with_zero_len_series_corner_cases[python]": "TODO: Add a reason for failure", "tests/frame/test_constructors.py::TestDataFrameConstructorWithDatetimeTZ::test_columns_indexes_raise_on_sets": "TODO: Add a reason for failure", - "tests/frame/test_constructors.py::TestDataFrameConstructorWithDatetimeTZ::test_construction_from_ndarray_datetimelike": "TODO: Add a reason for failure", "tests/frame/test_constructors.py::TestDataFrameConstructorWithDatetimeTZ::test_construction_from_set_raises[frozenset]": "TODO: Add a reason for failure", "tests/frame/test_constructors.py::TestDataFrameConstructorWithDatetimeTZ::test_construction_from_set_raises[set]": "TODO: Add a reason for failure", "tests/frame/test_constructors.py::TestDataFrameConstructorWithDatetimeTZ::test_constructor_data_aware_dtype_naive['+01:15'-True]": "TODO: Add a reason for failure", @@ -1443,10 +1428,6 @@ def pytest_unconfigure(config): "tests/frame/test_constructors.py::TestDataFrameConstructors::test_constructor_mixed_dict_and_Series": "TODO: Add a reason for failure", "tests/frame/test_constructors.py::TestDataFrameConstructors::test_constructor_with_datetimes2": "TODO: Add a reason for failure", "tests/frame/test_constructors.py::TestDataFrameConstructors::test_datetime_date_tuple_columns_from_dict": "TODO: Add a reason for failure", - "tests/frame/test_constructors.py::TestDataFrameConstructors::test_datetimelike_values_with_object_dtype[DataFrame-M]": "TODO: Add a reason for failure", - "tests/frame/test_constructors.py::TestDataFrameConstructors::test_datetimelike_values_with_object_dtype[DataFrame-m]": "TODO: Add a reason for failure", - "tests/frame/test_constructors.py::TestDataFrameConstructors::test_datetimelike_values_with_object_dtype[Series-M]": "TODO: Add a reason for failure", - "tests/frame/test_constructors.py::TestDataFrameConstructors::test_datetimelike_values_with_object_dtype[Series-m]": "TODO: Add a reason for failure", "tests/frame/test_constructors.py::TestDataFrameConstructors::test_dict_keys_returns_rangeindex": "AssertionError: Index are different", "tests/frame/test_constructors.py::TestDataFrameConstructors::test_dict_nocopy[Float32-M8[ns]-False]": "TODO: Add a reason for failure", "tests/frame/test_constructors.py::TestDataFrameConstructors::test_dict_nocopy[Float32-bool0-False]": "TODO: Add a reason for failure", @@ -1959,43 +1940,7 @@ def pytest_unconfigure(config): "tests/groupby/test_groupby.py::test_single_element_listlike_level_grouping[level_arg0-False]": "AssertionError: assert ['x', 'y'] == [('x',), ('y',)]", "tests/groupby/test_groupby.py::test_wrap_aggregated_output_multindex": "TODO: Add a reason for failure", "tests/groupby/test_groupby_dropna.py::test_groupby_nan_included": "GroupBy.indices returns cupy arrays nested in a dict that cudf.pandas does not wrap, so assert_numpy_array_equal sees mismatched array classes", - "tests/groupby/test_groupby_subclass.py::test_groupby_preserves_metadata": "TODO: Add a reason for failure", - "tests/groupby/test_groupby_subclass.py::test_groupby_preserves_subclass[all-obj0]": "TODO: Add a reason for failure", - "tests/groupby/test_groupby_subclass.py::test_groupby_preserves_subclass[any-obj0]": "TODO: Add a reason for failure", - "tests/groupby/test_groupby_subclass.py::test_groupby_preserves_subclass[bfill-obj0]": "TODO: Add a reason for failure", - "tests/groupby/test_groupby_subclass.py::test_groupby_preserves_subclass[corrwith-obj0]": "TODO: Add a reason for failure", - "tests/groupby/test_groupby_subclass.py::test_groupby_preserves_subclass[corrwith-obj1]": "TODO: Add a reason for failure", - "tests/groupby/test_groupby_subclass.py::test_groupby_preserves_subclass[count-obj0]": "TODO: Add a reason for failure", - "tests/groupby/test_groupby_subclass.py::test_groupby_preserves_subclass[cumcount-obj0]": "TODO: Add a reason for failure", - "tests/groupby/test_groupby_subclass.py::test_groupby_preserves_subclass[cummax-obj0]": "TODO: Add a reason for failure", - "tests/groupby/test_groupby_subclass.py::test_groupby_preserves_subclass[cummin-obj0]": "TODO: Add a reason for failure", - "tests/groupby/test_groupby_subclass.py::test_groupby_preserves_subclass[cumprod-obj0]": "TODO: Add a reason for failure", - "tests/groupby/test_groupby_subclass.py::test_groupby_preserves_subclass[cumsum-obj0]": "TODO: Add a reason for failure", - "tests/groupby/test_groupby_subclass.py::test_groupby_preserves_subclass[diff-obj0]": "TODO: Add a reason for failure", - "tests/groupby/test_groupby_subclass.py::test_groupby_preserves_subclass[ffill-obj0]": "TODO: Add a reason for failure", - "tests/groupby/test_groupby_subclass.py::test_groupby_preserves_subclass[first-obj0]": "TODO: Add a reason for failure", - "tests/groupby/test_groupby_subclass.py::test_groupby_preserves_subclass[idxmax-obj0]": "TODO: Add a reason for failure", - "tests/groupby/test_groupby_subclass.py::test_groupby_preserves_subclass[idxmin-obj0]": "TODO: Add a reason for failure", - "tests/groupby/test_groupby_subclass.py::test_groupby_preserves_subclass[kurt-obj0]": "AttributeError: 'SubclassedDataFrame' object has no attribute 'dtype'. Did you mean: 'dtypes'?", - "tests/groupby/test_groupby_subclass.py::test_groupby_preserves_subclass[last-obj0]": "TODO: Add a reason for failure", - "tests/groupby/test_groupby_subclass.py::test_groupby_preserves_subclass[max-obj0]": "TODO: Add a reason for failure", - "tests/groupby/test_groupby_subclass.py::test_groupby_preserves_subclass[mean-obj0]": "TODO: Add a reason for failure", - "tests/groupby/test_groupby_subclass.py::test_groupby_preserves_subclass[median-obj0]": "TODO: Add a reason for failure", - "tests/groupby/test_groupby_subclass.py::test_groupby_preserves_subclass[min-obj0]": "TODO: Add a reason for failure", - "tests/groupby/test_groupby_subclass.py::test_groupby_preserves_subclass[ngroup-obj0]": "TODO: Add a reason for failure", - "tests/groupby/test_groupby_subclass.py::test_groupby_preserves_subclass[nunique-obj0]": "TODO: Add a reason for failure", - "tests/groupby/test_groupby_subclass.py::test_groupby_preserves_subclass[pct_change-obj0]": "TODO: Add a reason for failure", - "tests/groupby/test_groupby_subclass.py::test_groupby_preserves_subclass[prod-obj0]": "TODO: Add a reason for failure", - "tests/groupby/test_groupby_subclass.py::test_groupby_preserves_subclass[quantile-obj0]": "TODO: Add a reason for failure", - "tests/groupby/test_groupby_subclass.py::test_groupby_preserves_subclass[rank-obj0]": "TODO: Add a reason for failure", - "tests/groupby/test_groupby_subclass.py::test_groupby_preserves_subclass[sem-obj0]": "TODO: Add a reason for failure", - "tests/groupby/test_groupby_subclass.py::test_groupby_preserves_subclass[shift-obj0]": "TODO: Add a reason for failure", - "tests/groupby/test_groupby_subclass.py::test_groupby_preserves_subclass[size-obj0]": "TODO: Add a reason for failure", - "tests/groupby/test_groupby_subclass.py::test_groupby_preserves_subclass[skew-obj0]": "TODO: Add a reason for failure", - "tests/groupby/test_groupby_subclass.py::test_groupby_preserves_subclass[std-obj0]": "TODO: Add a reason for failure", - "tests/groupby/test_groupby_subclass.py::test_groupby_preserves_subclass[sum-obj0]": "TODO: Add a reason for failure", - "tests/groupby/test_groupby_subclass.py::test_groupby_preserves_subclass[var-obj0]": "TODO: Add a reason for failure", - "tests/groupby/test_groupby_subclass.py::test_groupby_resample_preserves_subclass[DataFrame]": "TODO: Add a reason for failure", + "tests/groupby/test_groupby_subclass.py::test_groupby_preserves_metadata": "proxy pd.Index passed to the raw SubclassedSeries constructor loses its name: pandas' maybe_extract_name checks isinstance against the concrete Index class, which proxies cannot satisfy", "tests/groupby/test_grouping.py::TestGetGroup::test_get_group_grouped_by_tuple": "TODO: Add a reason for failure", "tests/groupby/test_grouping.py::TestGetGroup::test_get_group_grouped_by_tuple_with_lambda": "TODO: Add a reason for failure", "tests/groupby/test_grouping.py::TestGrouping::test_groupby_apply_empty_with_group_keys_false": "AssertionError: DataFrame are different", @@ -2169,7 +2114,6 @@ def pytest_unconfigure(config): "tests/indexes/datetimes/test_date_range.py::TestDateRanges::test_range_tz_dateutil": "TODO: Add a reason for failure", "tests/indexes/datetimes/test_date_range.py::TestDateRanges::test_range_tz_pytz": "TODO: Add a reason for failure", "tests/indexes/datetimes/test_date_range.py::TestGenRangeGeneration::test_precision_finer_than_offset": "TODO: Add a reason for failure", - "tests/indexes/datetimes/test_datetime.py::TestDatetimeIndex::test_misc_coverage": "TODO: Add a reason for failure", "tests/indexes/datetimes/test_formats.py::TestDatetimeIndexRendering::test_dti_representation[ms]": "assert 'DatetimeInde...US/Eastern]')' == 'DatetimeInde...', freq=None)'", "tests/indexes/datetimes/test_formats.py::TestDatetimeIndexRendering::test_dti_representation[ns]": "assert 'DatetimeInde...US/Eastern]')' == 'DatetimeInde...', freq=None)'", "tests/indexes/datetimes/test_formats.py::TestDatetimeIndexRendering::test_dti_representation[s]": "assert 'DatetimeInde...US/Eastern]')' == 'DatetimeInde...', freq=None)'", @@ -2565,7 +2509,6 @@ def pytest_unconfigure(config): "tests/indexes/timedeltas/test_searchsorted.py::TestSearchSorted::test_searchsorted_invalid_argument_dtype[arg0]": "TODO: Add a reason for failure", "tests/indexes/timedeltas/test_setops.py::TestTimedeltaIndex::test_intersection_non_monotonic[None-rng2-expected2]": "TODO: Add a reason for failure", "tests/indexes/timedeltas/test_setops.py::TestTimedeltaIndex::test_union_freq_infer": "TODO: Add a reason for failure", - "tests/indexes/timedeltas/test_timedelta.py::TestTimedeltaIndex::test_misc_coverage": "TODO: Add a reason for failure", "tests/indexing/interval/test_interval.py::TestIntervalIndexInsideMultiIndex::test_reindex_behavior_with_interval_index[1010]": "TODO: Add a reason for failure", "tests/indexing/interval/test_interval.py::TestIntervalIndexInsideMultiIndex::test_reindex_behavior_with_interval_index[101]": "TODO: Add a reason for failure", "tests/indexing/interval/test_interval_new.py::TestIntervalIndex::test_loc_with_overlap[loc]": "TODO: Add a reason for failure", From 30cd09b7577ac7c7ddf699497625eb7eaa4b6706 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Sat, 18 Jul 2026 09:14:29 -0500 Subject: [PATCH 07/25] Fix GroupBy.apply result assembly, UDF closure side effects, and empty-frame dtypes (#23272) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Running pandas' own test suite under `cudf.pandas`, `tests/groupby/test_apply.py` had 28 failing tests. This PR fixes 26 of them (137 tests: 135 pass; the 2 remaining are inherent and keep documented plugin entries) and removes the fixed xfail entries from the pandas-testing plugin. ### cudf.pandas: UDF closure side effects were silently discarded `_transform_arg` rebuilt lists and dicts even when no element needed proxy conversion. The rebuilt container fails `_replace_closurevars`' identity check, so user functions were rebuilt around a *copy* of their closed-over mutable containers — a UDF like `lambda g: names.append(g.name)` appended into a throwaway copy on both the fast attempt and the pandas fallback, and the user's list stayed empty. The list/dict branches are now identity-preserving when unchanged, mirroring the existing object-ndarray branch. ### `GroupBy.apply` result assembly (pandas parity, all verified empirically on 3.0.3) - All-None `DataFrameGroupBy` results return an empty frame keeping the value columns and dtypes (pandas GH9684/GH57775). - Series results sharing an identical index stack into one row per group with columns given by the common index; a consistent Series name becomes the columns-axis name (GH6124). Series results with differing indexes concatenate lengthwise under the group keys (GH8467). This replaces row-count heuristics that mislabeled columns and mis-shaped results. - Transform results (chunks indexed like their input) restore the original row order regardless of `sort`, like pandas' `_concat_objects`; the final `sort_index` is removed since group-keyed results are already emitted in sorted key order and pandas preserves the UDF's within-group row order (GH52444). - `include_groups=True` raises `ValueError`, matching pandas 3.0. ### Supporting fixes - `DataFrame({"a": []})` defaults untyped empty sequences to float64 like pandas' constructor (numpy's empty-array default); `Series([])` stays object. - `reset_index` derives the result columns dtype via pandas' `Index.insert` provenance instead of re-inferring from the merged labels, and `Series.reset_index` resolves the value-column name before resetting (pandas' `to_frame(name).reset_index()` semantics). - `as_column` routes stdlib `datetime`/`timedelta` elements through the pandas object path, so mixed datetime+non-datetime lists raise `MixedTypeError` instead of silently coercing. - Removed a stale workaround in `test_groupby_apply_return_col_from_df` and a stale conditional xfail in `test_dataframe_assign_scalar` — both now match pandas exactly. ### Validation - pandas-tests `tests/groupby/test_apply.py`: 28 failed → 135/137 pass (2 inherent, documented). - pandas-tests reset_index/constructors/groupby neighbors: only pre-existing known failures. - cuDF classic: groupby suites clean including under `NO_EXTERNAL_ONLY_APIS=1`; dataframe/series/reshape/indexes/input_output sweep (70k+ tests) green. - `cudf.pandas` proxy unit tests pass. Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Matthew Roeschke (https://github.com/mroeschke) URL: https://github.com/rapidsai/cudf/pull/23272 --- .../internal/bench_fast_slow_proxy.py | 59 ++++++++++ python/cudf/cudf/core/column/column.py | 11 +- python/cudf/cudf/core/dataframe.py | 55 +++++---- python/cudf/cudf/core/groupby/groupby.py | 106 ++++++++++++------ python/cudf/cudf/core/indexed_frame.py | 13 +++ python/cudf/cudf/core/series.py | 14 ++- python/cudf/cudf/pandas/fast_slow_proxy.py | 46 +++++++- .../pandas/scripts/pandas-testing-plugin.py | 57 +--------- .../dataframe/methods/test_reductions.py | 18 +++ .../tests/dataframe/methods/test_rename.py | 7 +- .../cudf/tests/dataframe/test_constructors.py | 18 +++ python/cudf/cudf/tests/groupby/test_apply.py | 52 ++++++++- .../cudf_pandas_tests/test_fast_slow_proxy.py | 87 +++++++++++++- 13 files changed, 418 insertions(+), 125 deletions(-) create mode 100644 python/cudf/benchmarks/internal/bench_fast_slow_proxy.py diff --git a/python/cudf/benchmarks/internal/bench_fast_slow_proxy.py b/python/cudf/benchmarks/internal/bench_fast_slow_proxy.py new file mode 100644 index 000000000000..357f9038fc49 --- /dev/null +++ b/python/cudf/benchmarks/internal/bench_fast_slow_proxy.py @@ -0,0 +1,59 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Benchmarks of cudf.pandas proxy argument transformation.""" + +import pytest + +from cudf.pandas.fast_slow_proxy import _transform_arg, make_final_proxy_type + + +@pytest.fixture(scope="module") +def proxy_object(): + class Fast: + def __init__(self, x): + self.x = x + + def to_slow(self): + return Slow(self.x) + + class Slow: + def __init__(self, x): + self.x = x + + Pxy = make_final_proxy_type( + "Pxy", + Fast, + Slow, + fast_to_slow=lambda fast: fast.to_slow(), + slow_to_fast=lambda slow: Fast(slow.x), + ) + return Pxy(1) + + +@pytest.mark.parametrize("size", [10, 10_000]) +def bench_transform_arg_unchanged_list(benchmark, size): + # No element needs transforming: the identity-scan returns the + # original container without rebuilding it. + arg = list(range(size)) + benchmark(lambda: _transform_arg(arg, "_fsproxy_slow", set())) + + +@pytest.mark.parametrize("size", [10, 10_000]) +def bench_transform_arg_unchanged_dict(benchmark, size): + arg = {i: i for i in range(size)} + benchmark(lambda: _transform_arg(arg, "_fsproxy_slow", set())) + + +@pytest.mark.parametrize("size", [10, 10_000]) +def bench_transform_arg_list_with_proxy(benchmark, proxy_object, size): + # One proxy element forces the rebuild path. + arg = [*range(size - 1), proxy_object] + benchmark(lambda: _transform_arg(arg, "_fsproxy_slow", set())) + + +@pytest.mark.parametrize("size", [10, 10_000]) +def bench_transform_arg_dict_with_proxy(benchmark, proxy_object, size): + arg = {i: i for i in range(size - 1)} + arg["proxy"] = proxy_object + benchmark(lambda: _transform_arg(arg, "_fsproxy_slow", set())) diff --git a/python/cudf/cudf/core/column/column.py b/python/cudf/cudf/core/column/column.py index a69e6e1dfcae..2d728e59f334 100644 --- a/python/cudf/cudf/core/column/column.py +++ b/python/cudf/cudf/core/column/column.py @@ -3,6 +3,7 @@ from __future__ import annotations +import datetime import pickle import warnings from collections.abc import ( @@ -3864,9 +3865,17 @@ def as_column( length=length, ) elif ( - isinstance(element, (pd.Timestamp, pd.Timedelta, pd.Interval)) + isinstance( + element, + (datetime.datetime, datetime.timedelta, pd.Interval), + ) or element is pd.NaT ): + # datetime.datetime/timedelta cover their pd.Timestamp/ + # pd.Timedelta subclasses; routing stdlib datetimes through + # pandas keeps mixed datetime+non-datetime inputs on the + # object-dtype path (MixedTypeError) instead of silently + # coercing them. # TODO: Remove this after # https://github.com/apache/arrow/issues/26492 # is fixed. diff --git a/python/cudf/cudf/core/dataframe.py b/python/cudf/cudf/core/dataframe.py index b03f3f0c7fd7..02cdd9da3473 100644 --- a/python/cudf/cudf/core/dataframe.py +++ b/python/cudf/cudf/core/dataframe.py @@ -924,11 +924,13 @@ def _mapping_to_column_accessor( if ( dtype is None and len(column) == 0 - and isinstance(value, (list, tuple, range)) + and isinstance(value, (list, tuple, Iterator)) ): - # pandas' DataFrame constructor coerces untyped empty - # sequences to float64 (unlike Series([]), which stays - # object). + # pandas' DataFrame constructor defaults untyped empty + # sequences (list/tuple/iterator) to float64 (numpy's + # default for np.array([])), unlike Series([]) which + # defaults to object. An empty range stays int64 like + # pandas (as_column already handles it via from_range). column = column_empty(0, dtype=np.dtype(np.float64)) value_lengths.add(len(column)) col_data[key] = column @@ -4387,9 +4389,20 @@ def rename( result.index = out_index if columns: - result._data = result._data.rename_levels( - mapper=columns, level=level - ) + new_ca = result._data.rename_levels(mapper=columns, level=level) + # pandas' rename rebuilds the columns Index from the transformed + # labels (``Index(items, tupleize_cols=False)`` in + # ``_transform_index``), re-inferring dtypes rather than + # preserving the originals: renaming object-dtype columns to + # all-string labels yields ``str``, and MultiIndex level dtypes + # are likewise re-inferred. + if new_ca.multiindex: + new_ca._level_dtypes = None + else: + new_ca.label_dtype = pd.Index( + new_ca.names, tupleize_cols=False + ).dtype + result._data = new_ca return result @@ -6686,8 +6699,11 @@ def quantile( include=[np.number], exclude=["datetime64", "timedelta64"] ) - if columns is None: - columns = set(data_df._column_names) + if columns is not None: + requested = set(columns) + data_df = data_df[ + [k for k in data_df._column_names if k in requested] + ] if isinstance(q, numbers.Number): q_is_number = True @@ -6737,17 +6753,16 @@ def quantile( interpolation = interpolation or "linear" result = {} for k in data_df._column_names: - if k in columns: - ser = data_df[k] - res = ser.quantile( - qs, - interpolation=interpolation, - exact=exact, - quant_index=False, - )._column - if len(res) == 0: - res = column_empty(row_count=len(qs), dtype=ser.dtype) - result[k] = res + ser = data_df[k] + res = ser.quantile( + qs, + interpolation=interpolation, + exact=exact, + quant_index=False, + )._column + if len(res) == 0: + res = column_empty(row_count=len(qs), dtype=ser.dtype) + result[k] = res result_ca = ColumnAccessor( result, multiindex=data_df._data.multiindex, diff --git a/python/cudf/cudf/core/groupby/groupby.py b/python/cudf/cudf/core/groupby/groupby.py index 73e5ac65b4e5..23b985a98218 100644 --- a/python/cudf/cudf/core/groupby/groupby.py +++ b/python/cudf/cudf/core/groupby/groupby.py @@ -2445,6 +2445,16 @@ def _post_process_chunk_results( if not len(chunk_results): return self.obj.head(0) + if ( + isinstance(self.obj, DataFrame) + and not isinstance(chunk_results, ColumnBase) + and all(res is None for res in chunk_results) + ): + # pandas GH9684/GH57775: an all-None DataFrameGroupBy.apply + # returns an empty frame keeping the (non-grouping) columns and + # dtypes. (An all-None SeriesGroupBy.apply stays in the scalar + # branch below: pandas returns an object Series of Nones.) + return grouped_values.head(0).reset_index(drop=True) if isinstance(chunk_results, ColumnBase) or is_scalar( chunk_results[0] ): @@ -2473,41 +2483,49 @@ def _post_process_chunk_results( result.columns = result.columns.set_names( [chunk_results[0].name] ) - # When the UDF is like df.x + df.y, the result for each - # group is the same length as the original group - elif (total_rows := sum(len(chk) for chk in chunk_results)) in { - len(self.obj), - len(group_names), - }: - result = concat(chunk_results) - if total_rows == len(group_names): - result.index = group_names - # TODO: Is there a better way to determine what - # the column name should be, especially if we applied - # a nameless UDF. - result = result.to_frame( - name=grouped_values._column_names[0] - ) - else: - index_data = group_keys._data.copy(deep=True) - inner_name = grouped_values.index.name - index_data[None] = grouped_values.index._column - mi = MultiIndex._from_data(index_data) - # ColumnAccessor keys must be unique, so the inner - # level's name (which may duplicate a key name) is - # restored after construction. - mi.names = [*mi.names[:-1], inner_name] - result.index = mi - elif len(chunk_results) == len(group_names): - result = concat(chunk_results, axis=1).T + # pandas stacks Series results that share an identical index + # into a DataFrame with one row per group and columns given by + # the common index (DataFrameGroupBy._wrap_applied_output_series) + elif all( + chunk_results[0].index.equals(chk.index) + for chk in chunk_results[1:] + ): + # a consistent Series name becomes the columns-axis name + # (pandas GH6124). Chunks are renamed positionally before + # the axis=1 concat because cuDF rejects duplicate column + # names. + names = {chk.name for chk in chunk_results} + result = concat( + [chk.rename(i) for i, chk in enumerate(chunk_results)], + axis=1, + ).T result.index = group_names result.index.names = self.grouping.names + if len(names) == 1: + result._data._level_names = (names.pop(),) else: - raise TypeError( - "Error handling Groupby apply output with input of " - f"type {type(self.obj)} and output of " - f"type {type(chunk_results[0])}" + # pandas GH8467: Series results with differing indexes are + # concatenated along axis 0 into a Series with the group + # keys prepended as the outer index level(s), each key + # repeated by its chunk's actual length and the UDF-returned + # index kept as the inner level + # (GroupBy._concat_objects with ``not_indexed_same=True``). + # This also covers transform-like UDFs: chunks indexed like + # their input concatenate back to the grouped input's index. + lengths = [len(chk) for chk in chunk_results] + result = concat(chunk_results) + gather = as_column( + np.repeat(np.arange(len(group_names)), lengths) ) + index_data = { + i: col.take(gather) + for i, col in enumerate(group_names._columns) + } + inner_name = result.index.name + index_data[None] = result.index._column + mi = MultiIndex._from_data(index_data) + mi.names = [*self.grouping.names, inner_name] + result.index = mi else: result = concat(chunk_results) if self._group_keys: @@ -2525,6 +2543,19 @@ def _post_process_chunk_results( # construction. mi.names = [*mi.names[:-1], inner_name] result.index = mi + elif len(result) == len(grouped_values) and result.index.equals( + grouped_values.index + ): + # Every chunk result is indexed like its input chunk, i.e. + # the UDF acted as a transform. pandas restores the original + # row order in this case (GroupBy._concat_objects) regardless + # of ``sort``. The concatenated chunks are in key-sorted + # group order, so gather back through the inverse of the + # grouping permutation. + _, _, (positions,) = self._groups( + [self._range_column_from_obj] + ) + result = result.take(positions.argsort().values) return result @_performance_tracking @@ -2560,8 +2591,8 @@ def apply( where possible and will fall back to the iterative algorithm if necessary. include_groups : bool, default False - When True, will attempt to apply ``func`` to the groupings in - the case that they are columns of the DataFrame. + Only ``False`` is accepted (matching pandas 3.0, where + ``include_groups=True`` raises a ``ValueError``). kwargs : dict Optional keyword arguments to pass to the function. Currently not supported @@ -2641,6 +2672,9 @@ def mult(df): dtype: int64 """ + if include_groups: + # matches pandas 3.0 + raise ValueError("include_groups=True is no longer allowed.") if kwargs: raise NotImplementedError( "Passing kwargs to func is currently not supported." @@ -2721,8 +2755,10 @@ def mult(df): else: raise ValueError(f"Unsupported engine '{engine}'") - if self._sort: - result = result.sort_index() + # No final sort: group-keyed results are already produced in + # sorted group-key order, and pandas preserves the UDF's + # within-group row order (and a transform's original row order) + # regardless of ``sort`` (pandas GH52444). if self._as_index is False: result = result.reset_index() return result diff --git a/python/cudf/cudf/core/indexed_frame.py b/python/cudf/cudf/core/indexed_frame.py index 8a1db59fa71a..b8cf52d668e8 100644 --- a/python/cudf/cudf/core/indexed_frame.py +++ b/python/cudf/cudf/core/indexed_frame.py @@ -4842,11 +4842,24 @@ def _reset_index( new_column_data[name] = col # This is to match pandas where the new data columns are always # inserted to the left of existing data columns. + label_dtype = None + if not self._data.multiindex: + # pandas computes the result columns by Index.insert into the + # existing columns Index, which preserves its dtype (e.g. + # Index([None], dtype=object).insert(0, "a") stays object); + # rebuilding from the merged labels would re-infer (pandas 3.0 + # infers "str" for all-string labels). Emulate the insert + # provenance. + pd_columns = self._data.to_pandas_index + for new_name in reversed(list(new_column_data)): + pd_columns = pd_columns.insert(0, new_name) + label_dtype = pd_columns.dtype return ( ColumnAccessor( {**new_column_data, **self._data}, self._data.multiindex, self._data._level_names, + label_dtype=label_dtype, ), index, ) diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index b42c3c64018b..1784f90e527a 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -1093,16 +1093,22 @@ def reset_index( raise TypeError( "Cannot reset_index inplace on a Series to create a DataFrame" ) - data, index = self._reset_index( - level=level, drop=drop, allow_duplicates=allow_duplicates - ) if not drop: + # pandas semantics are ``self.to_frame(name).reset_index()``: + # resolve ``name`` first so the columns-dtype provenance in + # ``_reset_index`` sees the final value-column label. if name is no_default: name = 0 if self.name is None else self.name - data[name] = data.pop(self.name) + frame = self._to_frame(name, index=self.index) + data, index = frame._reset_index( + level=level, drop=drop, allow_duplicates=allow_duplicates + ) return self._constructor_expanddim._from_data( data, index, attrs=self.attrs ) + data, index = self._reset_index( + level=level, drop=drop, allow_duplicates=allow_duplicates + ) # For ``name`` behavior, see: # https://github.com/pandas-dev/pandas/issues/44575 # ``name`` has to be ignored when `drop=True` diff --git a/python/cudf/cudf/pandas/fast_slow_proxy.py b/python/cudf/cudf/pandas/fast_slow_proxy.py index 0297bce31b0d..20600c7fe9db 100644 --- a/python/cudf/cudf/pandas/fast_slow_proxy.py +++ b/python/cudf/cudf/pandas/fast_slow_proxy.py @@ -1418,7 +1418,26 @@ def _transform_arg( elif isinstance(arg, types.ModuleType) and attribute_name in arg.__dict__: return arg.__dict__[attribute_name] elif isinstance(arg, list): - return type(arg)(_transform_arg(a, attribute_name, seen) for a in arg) + transformed_list = [ + _transform_arg(a, attribute_name, seen) for a in arg + ] + if all( + new is old for new, old in zip(transformed_list, arg, strict=True) + ): + # No element needed transforming: return the original list (as + # the object-ndarray branch below already does) to preserve + # identity. A user function may close over a mutable container + # and mutate it for its side effects (e.g. + # ``names.append(group.name)`` inside groupby.apply); copying + # here would silently discard those side effects on both the + # fast attempt and the pandas fallback, and also defeats + # _replace_closurevars' unchanged-check so the original + # function object is never passed through. + return arg + # Pass an iterator rather than the materialized list so list + # subclasses see the same non-list iterable constructor argument + # they always received here. + return type(arg)(iter(transformed_list)) elif isinstance(arg, tuple): # This attempts to handle arbitrary subclasses of tuple by # assuming that if you've subclassed tuple with some special @@ -1455,9 +1474,19 @@ def _transform_arg( ) ) else: - return tuple( + transformed_tuple = [ _transform_arg(a, attribute_name, seen) for a in arg - ) + ] + if all( + new is old + for new, old in zip(transformed_tuple, arg, strict=True) + ): + # No element needed transforming: return the original + # tuple (immutable, so this is safe) so containers + # enclosing it also keep their identity (see the list + # branch above). + return arg + return tuple(transformed_tuple) elif hasattr(arg, "__getnewargs_ex__"): # Partial implementation of to reconstruct with # transformed pieces @@ -1490,12 +1519,21 @@ def _transform_arg( _transform_arg(a, attribute_name, seen) for a in args ) elif isinstance(arg, dict): - return { + transformed_dict = { _transform_arg(k, attribute_name, seen): _transform_arg( a, attribute_name, seen ) for k, a in arg.items() } + if len(transformed_dict) == len(arg) and all( + new_k is old_k and new_v is old_v + for (new_k, new_v), (old_k, old_v) in zip( + transformed_dict.items(), arg.items(), strict=True + ) + ): + # see the list branch above: preserve identity when unchanged + return arg + return transformed_dict elif isinstance(arg, np.ndarray) and arg.dtype == "O": transformed: list[Any] = [ _transform_arg(a, attribute_name, seen) for a in arg.flat diff --git a/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py b/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py index d74dafcb4a96..65c1740de95c 100644 --- a/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py +++ b/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py @@ -112,9 +112,6 @@ def pytest_unconfigure(config): "tests/apply/test_frame_apply.py::test_apply_empty_infer_type[MockEngineDecorator-1-False-mean-index]": "AssertionError: assert Index(['a', 'b', 'c'], dtype='str') is Index(['a', 'b', 'c'], dtype='str')", "tests/apply/test_frame_apply.py::test_apply_empty_infer_type[MockEngineDecorator-1-True-mean-columns]": "assert RangeIndex(start=0, stop=0, step=1) is RangeIndex(start=0, stop=0, step=1)", "tests/apply/test_frame_apply.py::test_apply_empty_infer_type[MockEngineDecorator-1-True-mean-index]": "AssertionError: assert Index(['a', 'b', 'c'], dtype='str') is Index(['a', 'b', 'c'], dtype='str')", - "tests/apply/test_frame_apply.py::test_apply_function_runs_once": "TODO: Add a reason for failure", - "tests/apply/test_frame_apply.py::test_apply_raw_function_runs_once[MockEngineDecorator]": "assert [] == [1, 2, 3]", - "tests/apply/test_frame_apply.py::test_apply_raw_function_runs_once[python]": "TODO: Add a reason for failure", "tests/apply/test_series_apply.py::test_apply[False]": "TODO: Add a reason for failure", "tests/apply/test_series_apply.py::test_apply[compat]": "TODO: Add a reason for failure", "tests/apply/test_series_apply.py::test_apply_map_evaluate_lambdas_the_same[compat-MockEngineDecorator-str]": "AssertionError: Attributes of Series are different", @@ -272,7 +269,6 @@ def pytest_unconfigure(config): "tests/computation/test_eval.py::TestAlignment::test_performance_warning_for_poor_alignment[True-numexpr-pandas]": "assert 0 == 1", "tests/computation/test_eval.py::TestAlignment::test_performance_warning_for_poor_alignment[True-numexpr-python]": "assert 0 == 1", "tests/computation/test_eval.py::TestEval::test_disallow_python_keywords": "TODO: Add a reason for failure", - "tests/computation/test_eval.py::TestOperations::test_query_inplace": "TODO: Add a reason for failure", "tests/computation/test_eval.py::test_eval_no_support_column_name[False]": "TODO: Add a reason for failure", "tests/computation/test_eval.py::test_eval_no_support_column_name[True]": "TODO: Add a reason for failure", "tests/computation/test_eval.py::test_method_calls_on_binop": "AssertionError: Attributes of Series are different", @@ -701,7 +697,6 @@ def pytest_unconfigure(config): "tests/dtypes/test_inference.py::TestTypeInference::test_categorical": "AssertionError: assert 'string' == 'categorical'", "tests/dtypes/test_inference.py::TestTypeInference::test_date": "TODO: Add a reason for failure", "tests/dtypes/test_inference.py::TestTypeInference::test_is_interval_array_subclass": "AssertionError: assert not True", - "tests/dtypes/test_inference.py::test_is_scipy_sparse[dok]": "TODO: Add a reason for failure", "tests/dtypes/test_missing.py::test_array_equivalent_series[val5]": "TODO: Add a reason for failure", "tests/dtypes/test_missing.py::test_array_equivalent_series[val6]": "TODO: Add a reason for failure", "tests/dtypes/test_missing.py::test_array_equivalent_series[val7]": "TODO: Add a reason for failure", @@ -1227,7 +1222,6 @@ def pytest_unconfigure(config): "tests/frame/methods/test_map.py::test_map_empty[-expected2]": "TODO: Add a reason for failure", "tests/frame/methods/test_map.py::test_map_empty[round-expected0]": "TODO: Add a reason for failure", "tests/frame/methods/test_map.py::test_map_empty[round-expected2]": "TODO: Add a reason for failure", - "tests/frame/methods/test_map.py::test_map_function_runs_once": "TODO: Add a reason for failure", "tests/frame/methods/test_map.py::test_map_na_ignore": "TODO: Add a reason for failure", "tests/frame/methods/test_map.py::test_map_str": "TODO: Add a reason for failure", "tests/frame/methods/test_matmul.py::TestMatMul::test_matmul": "TODO: Add a reason for failure", @@ -1306,7 +1300,6 @@ def pytest_unconfigure(config): "tests/frame/methods/test_reset_index.py::TestResetIndex::test_reset_index_level_missing[idx_lev0]": "TODO: Add a reason for failure", "tests/frame/methods/test_reset_index.py::TestResetIndex::test_reset_index_level_missing[idx_lev1]": "TODO: Add a reason for failure", "tests/frame/methods/test_reset_index.py::TestResetIndex::test_reset_index_multiindex_columns": "TODO: Add a reason for failure", - "tests/frame/methods/test_reset_index.py::TestResetIndex::test_reset_index_with_datetimeindex_cols[2012-12-31]": "TODO: Add a reason for failure", "tests/frame/methods/test_sample.py::TestSample::test_sample_random_state[DataFrame-np.array-arg0]": "TODO: Add a reason for failure", "tests/frame/methods/test_sample.py::TestSample::test_sample_random_state[Series-np.array-arg0]": "TODO: Add a reason for failure", "tests/frame/methods/test_set_axis.py::TestDataFrameSetAxis::test_set_axis_copy": "TODO: Add a reason for failure", @@ -1849,35 +1842,14 @@ def pytest_unconfigure(config): "tests/groupby/test_all_methods.py::test_not_c_contiguous_mask[var]": "assert not True", "tests/groupby/test_api.py::test_all_methods_categorized": "TODO: Add a reason for failure", "tests/groupby/test_api.py::test_tab_completion": "TODO: Add a reason for failure", - "tests/groupby/test_apply.py::test_apply_as_index_constant_lambda[False-expected0]": "AssertionError: DataFrame.columns are different", - "tests/groupby/test_apply.py::test_apply_datetime_issue[group_column_dtlike0]": "TODO: Add a reason for failure", - "tests/groupby/test_apply.py::test_apply_datetime_issue[group_column_dtlike1]": "TODO: Add a reason for failure", - "tests/groupby/test_apply.py::test_apply_frame_concat_series": "TODO: Add a reason for failure", - "tests/groupby/test_apply.py::test_apply_func_that_appends_group_to_list_without_copy": "TODO: Add a reason for failure", - "tests/groupby/test_apply.py::test_apply_index_key_error_bug[index_values0]": "TODO: Add a reason for failure", - "tests/groupby/test_apply.py::test_apply_index_key_error_bug[index_values1]": "TODO: Add a reason for failure", - "tests/groupby/test_apply.py::test_apply_nonmonotonic_float_index[arg0-idx0]": "TODO: Add a reason for failure", - "tests/groupby/test_apply.py::test_apply_nonmonotonic_float_index[arg2-idx2]": "TODO: Add a reason for failure", - "tests/groupby/test_apply.py::test_apply_with_date_in_multiindex_does_not_convert_to_timestamp": "TODO: Add a reason for failure", - "tests/groupby/test_apply.py::test_group_apply_once_per_group[GH10519]": "TODO: Add a reason for failure", - "tests/groupby/test_apply.py::test_group_apply_once_per_group[GH12155]": "TODO: Add a reason for failure", - "tests/groupby/test_apply.py::test_group_apply_once_per_group[GH20084]": "TODO: Add a reason for failure", - "tests/groupby/test_apply.py::test_group_apply_once_per_group[GH21417]": "TODO: Add a reason for failure", - "tests/groupby/test_apply.py::test_group_apply_once_per_group[GH2656]": "TODO: Add a reason for failure", - "tests/groupby/test_apply.py::test_group_apply_once_per_group[GH2936]": "TODO: Add a reason for failure", - "tests/groupby/test_apply.py::test_group_apply_once_per_group[GH7739 & GH10519]": "TODO: Add a reason for failure", - "tests/groupby/test_apply.py::test_groupby_apply_all_none": "TODO: Add a reason for failure", - "tests/groupby/test_apply.py::test_groupby_apply_store_copy": "KeyError: 0", - "tests/groupby/test_apply.py::test_include_groups": "Failed: DID NOT RAISE ", - "tests/groupby/test_apply.py::test_positional_slice_groups_datetimelike": "AssertionError: DataFrame are different", - "tests/groupby/test_apply.py::test_time_field_bug": "TODO: Add a reason for failure", + "tests/groupby/test_apply.py::test_apply_with_date_in_multiindex_does_not_convert_to_timestamp": "cudf stores datetime.date values as datetime64; the date type identity is lost on the GPU round trip", + "tests/groupby/test_apply.py::test_positional_slice_groups_datetimelike": "the frame and its column Series are converted to pandas independently on fallback, losing the CoW block identity pandas' is_in_obj grouper check requires", "tests/groupby/test_categorical.py::test_describe_categorical_columns": "cudf's multi-level groupby aggregation and stack() drop the categorical column-index dtype", "tests/groupby/test_counting.py::TestCounting::test_ngroup_distinct": "TODO: Add a reason for failure", "tests/groupby/test_cumulative.py::test_groupby_cumprod_nan_influences_other_columns": "TODO: Add a reason for failure", "tests/groupby/test_cumulative.py::test_numpy_compat[cumprod]": "TODO: Add a reason for failure", "tests/groupby/test_cumulative.py::test_numpy_compat[cumsum]": "TODO: Add a reason for failure", "tests/groupby/test_groupby.py::test_as_index_select_column": "TODO: Add a reason for failure", - "tests/groupby/test_groupby.py::test_group_name_available_in_inference_pass": "TODO: Add a reason for failure", "tests/groupby/test_groupby.py::test_group_on_two_row_multiindex_returns_one_tuple_key": "TODO: Add a reason for failure", "tests/groupby/test_groupby.py::test_groupby_column_index_name_lost[any]": "RuntimeError: Fast-to-slow transfer is blocked", "tests/groupby/test_groupby.py::test_groupby_column_index_name_lost[shift]": "TODO: Add a reason for failure", @@ -1967,7 +1939,6 @@ def pytest_unconfigure(config): "tests/groupby/test_timegrouper.py::TestGroupBy::test_groupby_datetime64_32_bit": "TODO: Add a reason for failure", "tests/groupby/test_timegrouper.py::TestGroupBy::test_groupby_with_timegrouper": "TODO: Add a reason for failure", "tests/groupby/test_timegrouper.py::TestGroupBy::test_scalar_call_versus_list_call": "TODO: Add a reason for failure", - "tests/groupby/test_timegrouper.py::TestGroupBy::test_timegrouper_apply_return_type_series": "TODO: Add a reason for failure", "tests/groupby/transform/test_transform.py::test_as_index_no_change[size-A]": "TODO: Add a reason for failure", "tests/groupby/transform/test_transform.py::test_as_index_no_change[size-keys1]": "TODO: Add a reason for failure", "tests/groupby/transform/test_transform.py::test_cython_transform_series[cumprod-args0-]": "TODO: Add a reason for failure", @@ -2073,7 +2044,6 @@ def pytest_unconfigure(config): "tests/indexes/datetimes/test_constructors.py::TestDatetimeIndex::test_constructor_coverage": "TODO: Add a reason for failure", "tests/indexes/datetimes/test_constructors.py::TestDatetimeIndex::test_constructor_datetime64_tzformat[W-SUN]": "TODO: Add a reason for failure", "tests/indexes/datetimes/test_constructors.py::TestDatetimeIndex::test_constructor_dtype_tz_mismatch_raises": "Failed: DID NOT RAISE ", - "tests/indexes/datetimes/test_constructors.py::TestDatetimeIndex::test_dti_from_tzaware_datetime[tz1]": "AssertionError: assert False", "tests/indexes/datetimes/test_constructors.py::TestTimeSeries::test_constructor_int64_nocopy": "TODO: Add a reason for failure", "tests/indexes/datetimes/test_constructors.py::TestTimeSeries::test_dti_constructor_small_int[int16]": "AssertionError: Index are different", "tests/indexes/datetimes/test_constructors.py::TestTimeSeries::test_dti_constructor_small_int[int32]": "AssertionError: Index are different", @@ -2168,7 +2138,6 @@ def pytest_unconfigure(config): "tests/indexes/datetimes/test_timezones.py::TestDatetimeIndexTimezones::test_date_range_localize2[s]": "assert nan == 3", "tests/indexes/datetimes/test_timezones.py::TestDatetimeIndexTimezones::test_date_range_localize2[us]": "assert nan == 3", "tests/indexes/datetimes/test_timezones.py::TestDatetimeIndexTimezones::test_dti_convert_tz_aware_datetime_datetime[tz0]": "AssertionError: assert zoneinfo.ZoneInfo(key='US/Eastern') is datetime.timezone.utc", - "tests/indexes/datetimes/test_timezones.py::TestDatetimeIndexTimezones::test_dti_convert_tz_aware_datetime_datetime[tz1]": "AssertionError: assert False", "tests/indexes/interval/test_astype.py::TestDatetimelikeSubtype::test_subtype_integer[index0-int64]": "TODO: Add a reason for failure", "tests/indexes/interval/test_astype.py::TestDatetimelikeSubtype::test_subtype_integer[index0-uint64]": "TODO: Add a reason for failure", "tests/indexes/interval/test_astype.py::TestDatetimelikeSubtype::test_subtype_integer[index1-int64]": "TODO: Add a reason for failure", @@ -2388,7 +2357,6 @@ def pytest_unconfigure(config): "tests/indexes/test_base.py::TestIndex::test_isin_nan_common_object[float0-float1]": "AssertionError: numpy array are different", "tests/indexes/test_base.py::TestIndex::test_isin_nan_common_object[float1-float0]": "AssertionError: numpy array are different", "tests/indexes/test_base.py::TestIndex::test_isin_nan_common_object[float1-float1]": "AssertionError: numpy array are different", - "tests/indexes/test_base.py::TestIndex::test_map_defaultdict": "TODO: Add a reason for failure", "tests/indexes/test_base.py::TestIndex::test_str_attribute_raises[index2]": "TODO: Add a reason for failure", "tests/indexes/test_base.py::TestIndex::test_str_bool_return": "TODO: Add a reason for failure", "tests/indexes/test_base.py::TestIndex::test_tab_completion[index1-False]": "TODO: Add a reason for failure", @@ -2457,7 +2425,6 @@ def pytest_unconfigure(config): "tests/indexes/test_common.py::TestCommon::test_to_frame[uint8-new_name]": "TODO: Add a reason for failure", "tests/indexes/test_common.py::test_ndarray_compat_properties[multi]": "TODO: Add a reason for failure", "tests/indexes/test_common.py::test_ndarray_compat_properties[tuples]": "TODO: Add a reason for failure", - "tests/indexes/test_index_new.py::TestIndexConstructorInference::test_constructor_datetimes_mixed_tzs": "AssertionError: Index are different", "tests/indexes/test_indexing.py::TestGetIndexer::test_get_indexer_base[multi]": "TODO: Add a reason for failure", "tests/indexes/test_indexing.py::TestGetIndexer::test_get_indexer_base[tuples]": "TODO: Add a reason for failure", "tests/indexes/test_indexing.py::TestTake::test_take_indexer_type": "TODO: Add a reason for failure", @@ -2810,25 +2777,12 @@ def pytest_unconfigure(config): "tests/io/parser/common/test_index.py::test_multi_index_blank_df[c_high-True-a,b\\nc,d-columns1-header1]": "Exception", "tests/io/parser/common/test_index.py::test_multi_index_blank_df[c_low-True-a,b\\nc,d-columns1-header1]": "Exception", "tests/io/parser/common/test_index.py::test_multi_index_blank_df[python-True-a,b\\nc,d-columns1-header1]": "Exception", - "tests/io/parser/dtypes/test_dtypes_basic.py::test_dtypes_defaultdict[c_high-float64]": "TODO: Add a reason for failure", - "tests/io/parser/dtypes/test_dtypes_basic.py::test_dtypes_defaultdict[c_high-float]": "TODO: Add a reason for failure", - "tests/io/parser/dtypes/test_dtypes_basic.py::test_dtypes_defaultdict[c_low-float64]": "TODO: Add a reason for failure", - "tests/io/parser/dtypes/test_dtypes_basic.py::test_dtypes_defaultdict[c_low-float]": "TODO: Add a reason for failure", - "tests/io/parser/dtypes/test_dtypes_basic.py::test_dtypes_defaultdict[python-float64]": "TODO: Add a reason for failure", - "tests/io/parser/dtypes/test_dtypes_basic.py::test_dtypes_defaultdict[python-float]": "TODO: Add a reason for failure", - "tests/io/parser/dtypes/test_dtypes_basic.py::test_dtypes_defaultdict_invalid[c_high]": "TODO: Add a reason for failure", - "tests/io/parser/dtypes/test_dtypes_basic.py::test_dtypes_defaultdict_invalid[c_low]": "TODO: Add a reason for failure", - "tests/io/parser/dtypes/test_dtypes_basic.py::test_dtypes_defaultdict_invalid[python]": "TODO: Add a reason for failure", - "tests/io/parser/dtypes/test_dtypes_basic.py::test_dtypes_defaultdict_mangle_dup_cols[c_high]": "TODO: Add a reason for failure", - "tests/io/parser/dtypes/test_dtypes_basic.py::test_dtypes_defaultdict_mangle_dup_cols[c_low]": "TODO: Add a reason for failure", - "tests/io/parser/dtypes/test_dtypes_basic.py::test_dtypes_defaultdict_mangle_dup_cols[python]": "TODO: Add a reason for failure", "tests/io/parser/test_compression.py::test_writes_tar_gz[c_high]": "TODO: Add a reason for failure", "tests/io/parser/test_compression.py::test_writes_tar_gz[c_low]": "TODO: Add a reason for failure", "tests/io/parser/test_compression.py::test_writes_tar_gz[python]": "TODO: Add a reason for failure", "tests/io/parser/test_parse_dates.py::test_nat_parse[c_high]": "TODO: Add a reason for failure", "tests/io/parser/test_parse_dates.py::test_nat_parse[c_low]": "TODO: Add a reason for failure", "tests/io/parser/test_parse_dates.py::test_nat_parse[python]": "TODO: Add a reason for failure", - "tests/io/parser/test_python_parser_only.py::test_on_bad_lines_callable_write_to_external_list[python]": "TODO: Add a reason for failure", "tests/io/parser/test_textreader.py::TestTextReader::test_integer_thousands_alt": "TODO: Add a reason for failure", "tests/io/pytables/test_timezones.py::test_append_with_timezones[0]": "TODO: Add a reason for failure", "tests/io/pytables/test_timezones.py::test_append_with_timezones[1]": "TODO: Add a reason for failure", @@ -3146,7 +3100,6 @@ def pytest_unconfigure(config): "tests/reshape/merge/test_merge.py::TestMerge::test_join_append_timedeltas": "AssertionError: Attributes of DataFrame.iloc[:, 0] (column name='d') are different", "tests/reshape/merge/test_merge.py::TestMerge::test_merge_datetime64tz_with_dst_transition": "ValueError: Inferred frequency None from passed values does not conform to passed frequency h", "tests/reshape/merge/test_merge.py::TestMerge::test_merge_left_empty_right_notempty": "TODO: Add a reason for failure", - "tests/reshape/merge/test_merge.py::TestMerge::test_merge_nan_right": "AssertionError: DataFrame.columns are different", "tests/reshape/merge/test_merge.py::TestMerge::test_merge_nocopy": "TODO: Add a reason for failure", "tests/reshape/merge/test_merge.py::TestMerge::test_merge_on_index_with_more_values[index14-expected_index14-outer]": "TODO: Add a reason for failure", "tests/reshape/merge/test_merge.py::TestMerge::test_merge_on_index_with_more_values[index14-expected_index14-right]": "TODO: Add a reason for failure", @@ -3556,11 +3509,6 @@ def pytest_unconfigure(config): "tests/series/methods/test_map.py::test_map_callable[MockEngineDecorator]": "AssertionError: assert Index([], dtype='object', name='bar') is Index([], dtype='object', name='bar')", "tests/series/methods/test_map.py::test_map_callable[None]": "AssertionError: assert Index([], dtype='object', name='bar') is Index([], dtype='object', name='bar')", "tests/series/methods/test_map.py::test_map_categorical_na_action[None-expected0]": "TODO: Add a reason for failure", - "tests/series/methods/test_map.py::test_map_counter": "TODO: Add a reason for failure", - "tests/series/methods/test_map.py::test_map_defaultdict": "TODO: Add a reason for failure", - "tests/series/methods/test_map.py::test_map_defaultdict_ignore_na": "TODO: Add a reason for failure", - "tests/series/methods/test_map.py::test_map_defaultdict_missing_key[None]": "TODO: Add a reason for failure", - "tests/series/methods/test_map.py::test_map_dict_subclass_with_missing": "TODO: Add a reason for failure", "tests/series/methods/test_map.py::test_map_dict_with_tuple_keys": "TODO: Add a reason for failure", "tests/series/methods/test_map.py::test_map_empty[bool-dtype]": "TODO: Add a reason for failure", "tests/series/methods/test_map.py::test_map_empty[categorical]": "TODO: Add a reason for failure", @@ -3864,7 +3812,6 @@ def pytest_unconfigure(config): "tests/test_col.py::test_cached_property": "AssertionError: assert False", "tests/test_col.py::test_custom_accessor": "AttributeError: 'Series' object has no attribute 'xyz'", "tests/test_common.py::test_serializable[obj0]": "TODO: Add a reason for failure", - "tests/test_common.py::test_standardize_mapping": "TODO: Add a reason for failure", "tests/test_common.py::test_temp_setattr[False]": "TODO: Add a reason for failure", "tests/test_common.py::test_temp_setattr[True]": "TODO: Add a reason for failure", "tests/test_downstream.py::test_dask_ufunc": "TODO: Add a reason for failure", diff --git a/python/cudf/cudf/tests/dataframe/methods/test_reductions.py b/python/cudf/cudf/tests/dataframe/methods/test_reductions.py index f713932c236c..92e769dfd362 100644 --- a/python/cudf/cudf/tests/dataframe/methods/test_reductions.py +++ b/python/cudf/cudf/tests/dataframe/methods/test_reductions.py @@ -89,6 +89,24 @@ def test_with_index(): assert_eq(pdf_q, gdf_q, check_index_type=False) +@pytest.mark.parametrize("method", ["single", "table"]) +def test_quantile_columns_subset(method): + # The cudf-specific ``columns`` argument restricts the computation in + # both the per-column and the row-selecting table methods. + q = [0, 0.5, 1] + + pdf = pd.DataFrame({"a": [4, 24, 13, 8, 7], "b": [1, 2, 3, 4, 5]}) + gdf = cudf.from_pandas(pdf) + + pdf_q = pdf[["a"]].quantile(q, interpolation="nearest") + gdf_q = gdf.quantile( + q, interpolation="nearest", method=method, columns=["a"] + ) + + assert list(gdf_q._column_names) == ["a"] + assert_eq(pdf_q, gdf_q, check_index_type=False) + + def test_with_multiindex(): q = [0, 0.5, 1] diff --git a/python/cudf/cudf/tests/dataframe/methods/test_rename.py b/python/cudf/cudf/tests/dataframe/methods/test_rename.py index 0548aea8ff54..4fab041655cd 100644 --- a/python/cudf/cudf/tests/dataframe/methods/test_rename.py +++ b/python/cudf/cudf/tests/dataframe/methods/test_rename.py @@ -79,11 +79,14 @@ def test_rename_reset_label_dtype(): assert_eq(result, expected) -def test_dataframe_rename_columns_keep_type(): +def test_dataframe_rename_columns_reinfers_label_dtype(): + # pandas rebuilds the columns Index from the transformed labels on + # rename, re-inferring its dtype (int8 -> int64) rather than + # preserving it. gdf = cudf.DataFrame([[1, 2, 3]]) gdf.columns = cudf.Index([4, 5, 6], dtype=np.int8) result = gdf.rename({4: 50}, axis="columns").columns - expected = pd.Index([50, 5, 6], dtype=np.int8) + expected = pd.Index([50, 5, 6], dtype=np.int64) assert_eq(result, expected) diff --git a/python/cudf/cudf/tests/dataframe/test_constructors.py b/python/cudf/cudf/tests/dataframe/test_constructors.py index 38073e9c4609..5f83236d9dee 100644 --- a/python/cudf/cudf/tests/dataframe/test_constructors.py +++ b/python/cudf/cudf/tests/dataframe/test_constructors.py @@ -1051,6 +1051,24 @@ def test_init_from_dict_of_empty_lists(): assert gdf["a"].dtype == np.dtype("float64") +def test_init_from_dict_of_empty_iterator(): + # Iterators drain into an untyped empty sequence, so they follow the + # same float64 default as empty lists. + pdf = pd.DataFrame({"a": iter([])}) + gdf = cudf.DataFrame({"a": iter([])}) + assert_eq(pdf, gdf) + assert gdf["a"].dtype == np.dtype("float64") + + +def test_init_from_dict_of_empty_range(): + # An empty range stays int64 like pandas (which converts range via + # np.arange), unlike untyped empty lists/tuples/iterators. + pdf = pd.DataFrame({"a": range(0)}) + gdf = cudf.DataFrame({"a": range(0)}) + assert_eq(pdf, gdf) + assert gdf["a"].dtype == np.dtype("int64") + + @pytest.mark.parametrize( "data,cols,index", [ diff --git a/python/cudf/cudf/tests/groupby/test_apply.py b/python/cudf/cudf/tests/groupby/test_apply.py index deb4fd92dea7..62dc09e3c2d0 100644 --- a/python/cudf/cudf/tests/groupby/test_apply.py +++ b/python/cudf/cudf/tests/groupby/test_apply.py @@ -580,9 +580,6 @@ def func(df): got = df.groupby("id").apply(func, include_groups=False) expect = pdf.groupby("id").apply(func, include_groups=False) - # pandas seems to erroneously add an extra MI level of ids - # TODO: Figure out how pandas groupby.apply determines the columns - expect = pd.DataFrame(expect.droplevel(1), columns=got.columns) assert_groupby_results_equal(expect, got) @@ -748,6 +745,55 @@ def test_groupby_apply_return_series_dataframe(func, args): assert_groupby_results_equal(expected, actual) +def test_groupby_apply_series_results_misaligned_lengths(): + # Series results whose per-group lengths differ from their input are + # concatenated with the group keys as the outer index level, each key + # repeated by its chunk's actual length, keeping the UDF-returned + # index as the inner level (pandas GH8467) -- even when the total + # output length coincides with len(df) (2 + 3 == 3 + 2 here). + pdf = pd.DataFrame({"k": [1, 1, 2, 2, 2], "v": [10, 20, 30, 40, 50]}) + gdf = cudf.from_pandas(pdf) + + def make_swap_sizes(series_type): + # group 1 has 2 rows -> 3 outputs; group 2 has 3 rows -> 2 outputs + def swap_sizes(g): + if len(g) == 2: + return series_type([1, 2, 3]) + return series_type([4, 5]) + + return swap_sizes + + expected = pdf.groupby("k").apply( + make_swap_sizes(pd.Series), include_groups=False + ) + actual = gdf.groupby("k").apply( + make_swap_sizes(cudf.Series), include_groups=False + ) + assert_eq(expected, actual) + + +def test_groupby_apply_series_results_fresh_index(): + # Per-group result lengths match the input, but the UDF rewrote the + # index: the UDF-returned index is kept as the inner level rather + # than the input rows' original labels. + pdf = pd.DataFrame({"k": [1, 1, 2, 2, 2], "v": [10, 20, 30, 40, 50]}) + gdf = cudf.from_pandas(pdf) + + def make_fresh_index(series_type): + def fresh_index(g): + return series_type(range(len(g))) + + return fresh_index + + expected = pdf.groupby("k").apply( + make_fresh_index(pd.Series), include_groups=False + ) + actual = gdf.groupby("k").apply( + make_fresh_index(cudf.Series), include_groups=False + ) + assert_eq(expected, actual) + + @pytest.mark.parametrize( "pdf", [pd.DataFrame(), pd.DataFrame({"a": []}), pd.Series([], dtype="float64")], diff --git a/python/cudf/cudf_pandas_tests/test_fast_slow_proxy.py b/python/cudf/cudf_pandas_tests/test_fast_slow_proxy.py index 8be51f2b6c90..30d2124edcbf 100644 --- a/python/cudf/cudf_pandas_tests/test_fast_slow_proxy.py +++ b/python/cudf/cudf_pandas_tests/test_fast_slow_proxy.py @@ -640,6 +640,90 @@ def test_transform_arg_preserves_object_ndarray_identity( assert type(result[1]) is type(expected) +@pytest.mark.parametrize("attribute_name", ["_fsproxy_fast", "_fsproxy_slow"]) +def test_transform_arg_preserves_list_and_dict_identity(attribute_name): + # A list or dict whose entries need no transformation must be + # returned as-is rather than rebuilt: a user function may close over + # a mutable container and mutate it for its side effects (e.g. + # ``names.append(group.name)`` inside groupby.apply), and rebuilding + # the closure around an equivalent copy would silently discard those + # mutations. + transform = partial( + _transform_arg, attribute_name=attribute_name, seen=set() + ) + for unchanged in ( + [], + [1, "a"], + {}, + {"k": 1, 2: "v"}, + [[1], {"k": (2,)}], + {"k": [1, {"nested": 2}]}, + ): + assert transform(unchanged) is unchanged + + +@pytest.mark.parametrize("attribute_name", ["_fsproxy_fast", "_fsproxy_slow"]) +def test_transform_arg_rebuilds_containers_holding_proxies( + attribute_name, final_proxy +): + transform = partial( + _transform_arg, attribute_name=attribute_name, seen=set() + ) + fast_x, slow_x, x = final_proxy + expected_type = type( + fast_x if attribute_name == "_fsproxy_fast" else slow_x + ) + + lst = [1, x] + result = transform(lst) + assert result is not lst + assert result[0] == 1 + assert type(result[1]) is expected_type + + dct = {"k": x} + result = transform(dct) + assert result is not dct + assert type(result["k"]) is expected_type + + # A proxy nested deeper down rebuilds every enclosing container. + nested = [{"k": x}] + result = transform(nested) + assert result is not nested + assert type(result[0]["k"]) is expected_type + + # Rebuilt lists preserve list subclasses. + class MyList(list): + pass + + my_list = MyList([1, x]) + result = transform(my_list) + assert result is not my_list + assert type(result) is MyList + assert type(result[1]) is expected_type + + +@pytest.mark.parametrize("attribute_name", ["_fsproxy_fast", "_fsproxy_slow"]) +def test_transform_arg_dict_subclass_identity_avoids_overrides( + attribute_name, +): + # The unchanged-identity detection must compare entries via ``items()`` + # (which the transformation itself already consumed) rather than + # ``__iter__``/``__getitem__``, which a mapping subclass may override + # with semantics that break the comparison. + class NoDunderDict(dict): + def __iter__(self): + raise AssertionError("__iter__ must not be used") + + def __getitem__(self, key): + raise AssertionError("__getitem__ must not be used") + + transform = partial( + _transform_arg, attribute_name=attribute_name, seen=set() + ) + unchanged = NoDunderDict({"k": 1}) + assert transform(unchanged) is unchanged + + def test_tuple_with_attrs_transform(): Bunch = tuple_with_attrs("Bunch", ["a", "b"], {"c", "d"}) Bunch2 = tuple_with_attrs("Bunch", ["a", "b"], {"c", "d"}) @@ -659,6 +743,7 @@ def test_tuple_with_attrs_transform(): cprime = transform(c) dprime = transform(d) assert a == aprime and a is not aprime - assert b == bprime and b is not bprime + # A plain tuple with no transformable elements keeps its identity. + assert b is bprime assert c == cprime and c is not cprime assert d == dprime and d is not dprime From 5d1ea3bfd53c39278a9bae60e3d2ec8ab3a659ea Mon Sep 17 00:00:00 2001 From: Niranda Perera Date: Mon, 20 Jul 2026 08:27:21 -0700 Subject: [PATCH 08/25] Rapidsmpf backref API changes for Host and Pinned MR (#23128) ## Description Adds PR changes to Host and Pinned MRs. Depends on https://github.com/rapidsai/rapidsmpf/pull/1106 ## Checklist - [x] I am familiar with the [Contributing Guidelines](https://github.com/rapidsai/cudf/blob/HEAD/CONTRIBUTING.md). - [x] New or existing tests cover these changes. - [x] The documentation is up to date with these changes. --------- Signed-off-by: niranda perera --- .../benchmarks/bench_shuffle.cpp | 8 ++++++-- .../streaming/bench_streaming_shuffle.cpp | 13 ++++++++----- .../benchmarks/streaming/ndsh/utils.cpp | 6 +++--- .../streaming/base_streaming_fixture.hpp | 2 +- .../tests/streaming/test_table_chunk.cpp | 19 ++++++++++++------- cpp/libcudf_streaming/tests/test_shuffler.cpp | 5 ++--- 6 files changed, 32 insertions(+), 21 deletions(-) diff --git a/cpp/libcudf_streaming/benchmarks/bench_shuffle.cpp b/cpp/libcudf_streaming/benchmarks/bench_shuffle.cpp index 0fbab9c2777b..ddbc7c11a38a 100644 --- a/cpp/libcudf_streaming/benchmarks/bench_shuffle.cpp +++ b/cpp/libcudf_streaming/benchmarks/bench_shuffle.cpp @@ -502,10 +502,14 @@ int main(int argc, char** argv) // We're only going to measure the last run, so disable initially. stats->disable(); + RAPIDSMPF_EXPECTS(args.pinned_mem_disable || rapidsmpf::is_pinned_memory_resources_supported(), + "pinned host memory is not supported on this system; pass `-L` to disable it.", + std::runtime_error); + auto pinned_pool_properties = + args.pinned_mem_disable ? rapidsmpf::PinnedMemoryDisabled : rapidsmpf::PinnedPoolProperties{}; auto br = rapidsmpf::BufferResource::create( rmm_mr, - args.pinned_mem_disable ? rapidsmpf::PinnedMemoryResource::Disabled - : rapidsmpf::PinnedMemoryResource::make_if_available(), + std::move(pinned_pool_properties), std::move(memory_limits), std::chrono::milliseconds{1}, std::make_shared(16, rmm::cuda_stream::flags::non_blocking), diff --git a/cpp/libcudf_streaming/benchmarks/streaming/bench_streaming_shuffle.cpp b/cpp/libcudf_streaming/benchmarks/streaming/bench_streaming_shuffle.cpp index 511b9a022135..1d7f1604498e 100644 --- a/cpp/libcudf_streaming/benchmarks/streaming/bench_streaming_shuffle.cpp +++ b/cpp/libcudf_streaming/benchmarks/streaming/bench_streaming_shuffle.cpp @@ -331,11 +331,14 @@ int main(int argc, char** argv) auto stats = rapidsmpf::Statistics::create(); - auto pinned_mr = args.pinned_mem_disable ? rapidsmpf::PinnedMemoryResource::Disabled - : rapidsmpf::PinnedMemoryResource::make_if_available(); - auto br = rapidsmpf::BufferResource::create( + RAPIDSMPF_EXPECTS(args.pinned_mem_disable || rapidsmpf::is_pinned_memory_resources_supported(), + "pinned host memory is not supported on this system; pass `-L` to disable it.", + std::runtime_error); + auto pinned_pool_properties = + args.pinned_mem_disable ? rapidsmpf::PinnedMemoryDisabled : rapidsmpf::PinnedPoolProperties{}; + auto br = rapidsmpf::BufferResource::create( rmm_mr, - pinned_mr, + std::move(pinned_pool_properties), std::move(memory_limits), std::nullopt, std::make_shared(16, rmm::cuda_stream::flags::non_blocking), @@ -425,7 +428,7 @@ int main(int argc, char** argv) if (args.enable_memory_profiler) { log->print(statistics->report({ .mr = stat_enabled_mr, - .pinned_mr = pinned_mr, + .pinned_mr = br->try_pinned_mr(), .header = "Statistics (of the last run):", })); } else { diff --git a/cpp/libcudf_streaming/benchmarks/streaming/ndsh/utils.cpp b/cpp/libcudf_streaming/benchmarks/streaming/ndsh/utils.cpp index 2261b9f4607f..49556081e849 100644 --- a/cpp/libcudf_streaming/benchmarks/streaming/ndsh/utils.cpp +++ b/cpp/libcudf_streaming/benchmarks/streaming/ndsh/utils.cpp @@ -195,10 +195,10 @@ std::pair, std::shared_ptr> cr "noting that this may significantly degrade spilling performance.", std::invalid_argument); + auto pinned_pool_properties = + arguments.no_pinned_host_memory ? PinnedMemoryDisabled : PinnedPoolProperties{}; auto br = BufferResource::create(std::move(mr), - arguments.no_pinned_host_memory - ? PinnedMemoryResource::Disabled - : PinnedMemoryResource::make_if_available(), + std::move(pinned_pool_properties), std::move(memory_limits), arguments.periodic_spill, std::make_shared( diff --git a/cpp/libcudf_streaming/tests/streaming/base_streaming_fixture.hpp b/cpp/libcudf_streaming/tests/streaming/base_streaming_fixture.hpp index 03576f816ef4..8a849bd22d98 100644 --- a/cpp/libcudf_streaming/tests/streaming/base_streaming_fixture.hpp +++ b/cpp/libcudf_streaming/tests/streaming/base_streaming_fixture.hpp @@ -46,7 +46,7 @@ class BaseStreamingFixture : public ::testing::Test { stream = cudf::get_default_stream(); br = rapidsmpf::BufferResource::create( - mr_cuda, rapidsmpf::PinnedMemoryResource::Disabled, std::move(memory_limits)); + mr_cuda, rapidsmpf::PinnedMemoryDisabled, std::move(memory_limits)); ctx = std::make_shared( std::move(options), GlobalEnvironment->comm_->logger(), br); } diff --git a/cpp/libcudf_streaming/tests/streaming/test_table_chunk.cpp b/cpp/libcudf_streaming/tests/streaming/test_table_chunk.cpp index 7d8347296583..35719072c67b 100644 --- a/cpp/libcudf_streaming/tests/streaming/test_table_chunk.cpp +++ b/cpp/libcudf_streaming/tests/streaming/test_table_chunk.cpp @@ -38,13 +38,18 @@ class StreamingTableChunk : public BaseStreamingFixture, auto stream_pool = std::make_shared(16, rmm::cuda_stream::flags::non_blocking); stream = cudf::get_default_stream(); - br = rapidsmpf::BufferResource::create( - mr_cuda, // device_mr - rapidsmpf::PinnedMemoryResource::make_if_available(), // pinned_mr - memory_limits, // memory_limits - std::chrono::milliseconds{1}, // periodic_spill_check - stream_pool, // stream_pool - rapidsmpf::Statistics::disabled() // statistics + // Enable pinned host memory only when supported; otherwise the non-pinned + // params still run and the PINNED_HOST cases skip in the test bodies. + auto pinned_pool_properties = rapidsmpf::is_pinned_memory_resources_supported() + ? rapidsmpf::PinnedPoolProperties{} + : rapidsmpf::PinnedMemoryDisabled; + br = rapidsmpf::BufferResource::create( + mr_cuda, // device_mr + std::move(pinned_pool_properties), // pinned_pool_properties + memory_limits, // memory_limits + std::chrono::milliseconds{1}, // periodic_spill_check + stream_pool, // stream_pool + rapidsmpf::Statistics::disabled() // statistics ); ctx = std::make_shared( options, GlobalEnvironment->comm_->logger(), br); diff --git a/cpp/libcudf_streaming/tests/test_shuffler.cpp b/cpp/libcudf_streaming/tests/test_shuffler.cpp index 8b4165f2219c..8a9274ed39c9 100644 --- a/cpp/libcudf_streaming/tests/test_shuffler.cpp +++ b/cpp/libcudf_streaming/tests/test_shuffler.cpp @@ -139,8 +139,7 @@ class MemoryLimits_NumPartition memory_limits = std::get<0>(GetParam()); total_num_partitions = std::get<1>(GetParam()); total_num_rows = std::get<2>(GetParam()); - br = rapidsmpf::BufferResource::create( - mr(), rapidsmpf::PinnedMemoryResource::Disabled, memory_limits); + br = rapidsmpf::BufferResource::create(mr(), rapidsmpf::PinnedMemoryDisabled, memory_limits); shuffler = std::make_unique(GlobalEnvironment->comm_, 0, // op_id @@ -267,7 +266,7 @@ TEST(Shuffler, SpillOnInsertAndExtraction) // exposed via `device_mr_adaptor()`, so the test can observe per-rank // allocation counts via `get_main_record().num_current_allocs()`. auto br = rapidsmpf::BufferResource::create(cudf::get_current_device_resource_ref(), - rapidsmpf::PinnedMemoryResource::Disabled, + rapidsmpf::PinnedMemoryDisabled, {{rapidsmpf::MemoryType::DEVICE, k_no_spill_limit}}, std::nullopt // disable periodic spill check ); From 79736b5a59688773f3ccfde4cc3ce8753408e363 Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Mon, 20 Jul 2026 14:42:39 -0500 Subject: [PATCH 09/25] Update upstream cuml tests run (#23335) Removes a few deleted test files. --- ci/test_cuml_compat.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/ci/test_cuml_compat.sh b/ci/test_cuml_compat.sh index 804e03e4b6f9..3dc3e16ed6c7 100755 --- a/ci/test_cuml_compat.sh +++ b/ci/test_cuml_compat.sh @@ -23,9 +23,7 @@ rapids-logger "pytest cuml cuDF-compat subset" timeout 15m python -m pytest \ --cache-clear \ - "${CUML_TESTS_DIR}/test_array.py" \ "${CUML_TESTS_DIR}/test_compose.py" \ - "${CUML_TESTS_DIR}/test_input_utils.py" \ "${CUML_TESTS_DIR}/test_kneighbors_classifier.py" \ "${CUML_TESTS_DIR}/test_kneighbors_regressor.py" \ "${CUML_TESTS_DIR}/test_label_encoder.py" \ From ef47279c2831cd5ca5ae65b5882cb7d0b0987299 Mon Sep 17 00:00:00 2001 From: Lawrence Mitchell Date: Mon, 20 Jul 2026 22:14:47 +0100 Subject: [PATCH 10/25] Avoid leaving multi-rank execution in a bad state in cudf-polars tests (#23340) If a cudf-polars query raises an exception during execution, it is possible that it can leave a dangling collective that then tears down one process. This is racy because it depends on tasks being cancelled in Python and then dropping C++ objects in a particular order. The one test in the cudf-polars tests suite that could do this is fixed on main, so backport the relevant changes (#23235), and un-xfail the test. Authors: - Lawrence Mitchell (https://github.com/wence-) - Mads R. B. Kristensen (https://github.com/madsbk) Approvers: - Richard (Rick) Zamora (https://github.com/rjzamora) - Matthew Murray (https://github.com/Matt711) - Bradley Dice (https://github.com/bdice) - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/23340 --- python/cudf_polars/tests/streaming/test_select.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/python/cudf_polars/tests/streaming/test_select.py b/python/cudf_polars/tests/streaming/test_select.py index 63e6eca1fef4..19b4c0b842c9 100644 --- a/python/cudf_polars/tests/streaming/test_select.py +++ b/python/cudf_polars/tests/streaming/test_select.py @@ -22,7 +22,6 @@ assert_gpu_result_equal, ) from cudf_polars.testing.engine_utils import warns_on_spmd -from cudf_polars.utils.versions import POLARS_VERSION_LT_141 @pytest.fixture @@ -181,13 +180,6 @@ def test_select_mean_with_decimals(engine): assert_gpu_result_equal(q, engine=engine) -@pytest.mark.xfail( - condition=not POLARS_VERSION_LT_141, - reason=( - "len() row count lost in zero-column streaming chunks " - "(https://github.com/rapidsai/cudf/issues/21428)" - ), -) def test_select_with_len(streaming_engine_factory): engine = streaming_engine_factory( StreamingOptions(max_rows_per_partition=3, fallback_mode="warn"), From 60e041d7bb53b4f5ee001f3ec838e006a353a322 Mon Sep 17 00:00:00 2001 From: Donald Tolley Date: Mon, 20 Jul 2026 15:02:46 -0700 Subject: [PATCH 11/25] Add fixed-size rolling window support to cudf-polars (#21964) ## Description Translate Polars `RollingFunction` expression nodes (`rolling_sum`, `rolling_min`, `rolling_max`, `rolling_mean`, `rolling_var`, `rolling_std`) into GPU-executable operations via libcudf's rolling window API. The new `FixedSizeRollingWindow` expression class uses a synthetic sequential integer orderby column with `grouped_range_rolling_window` to implement row-count-based windows through the existing range-based API. This supports configurable window size, center alignment, `min_periods`, and `ddof` for variance and standard deviation. Depends on pola-rs/polars#27108 which exposes `RollingFunction` in the Polars Python visitor. Closes #20307 xref #18633 ## Checklist - [x] I am familiar with the [Contributing Guidelines](https://github.com/rapidsai/cudf/blob/HEAD/CONTRIBUTING.md). - [x] New or existing tests cover these changes. - [ ] The documentation is up to date with these changes. --------- Co-authored-by: rjzamora --- python/cudf_polars/cudf_polars/dsl/expr.py | 7 +- .../cudf_polars/dsl/expressions/rolling.py | 100 ++++++++++++- .../cudf_polars/cudf_polars/dsl/translate.py | 37 +++++ .../tests/expressions/test_fixed_rolling.py | 133 ++++++++++++++++++ 4 files changed, 273 insertions(+), 4 deletions(-) create mode 100644 python/cudf_polars/tests/expressions/test_fixed_rolling.py diff --git a/python/cudf_polars/cudf_polars/dsl/expr.py b/python/cudf_polars/cudf_polars/dsl/expr.py index f5c0564e9f09..16285a6c195c 100644 --- a/python/cudf_polars/cudf_polars/dsl/expr.py +++ b/python/cudf_polars/cudf_polars/dsl/expr.py @@ -25,7 +25,11 @@ from cudf_polars.dsl.expressions.boolean import BooleanFunction from cudf_polars.dsl.expressions.datetime import TemporalFunction from cudf_polars.dsl.expressions.literal import Literal, LiteralColumn -from cudf_polars.dsl.expressions.rolling import GroupedWindow, RollingWindow +from cudf_polars.dsl.expressions.rolling import ( + FixedSizeRollingWindow, + GroupedWindow, + RollingWindow, +) from cudf_polars.dsl.expressions.selection import Filter, Gather from cudf_polars.dsl.expressions.slicing import Slice from cudf_polars.dsl.expressions.sorting import Sort, SortBy @@ -44,6 +48,7 @@ "ErrorExpr", "Expr", "Filter", + "FixedSizeRollingWindow", "Gather", "GroupedWindow", "Item", diff --git a/python/cudf_polars/cudf_polars/dsl/expressions/rolling.py b/python/cudf_polars/cudf_polars/dsl/expressions/rolling.py index 7f9c2868c04b..93eb388370fb 100644 --- a/python/cudf_polars/cudf_polars/dsl/expressions/rolling.py +++ b/python/cudf_polars/cudf_polars/dsl/expressions/rolling.py @@ -9,7 +9,7 @@ from collections import defaultdict from dataclasses import dataclass from functools import singledispatchmethod -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, ClassVar import pylibcudf as plc @@ -25,13 +25,13 @@ from cudf_polars.utils.versions import POLARS_VERSION_LT_136, POLARS_VERSION_LT_139 if TYPE_CHECKING: - from collections.abc import Sequence + from collections.abc import Callable, Sequence from rmm.pylibrmm.stream import Stream from cudf_polars.typing import ClosedInterval, Duration -__all__ = ["GroupedWindow", "RollingWindow", "to_request"] +__all__ = ["FixedSizeRollingWindow", "GroupedWindow", "RollingWindow", "to_request"] @dataclass(frozen=True) @@ -206,6 +206,100 @@ def do_evaluate( # noqa: D102 return Column(result, dtype=self.dtype) +class FixedSizeRollingWindow(Expr): + """ + Fixed-size integer-based rolling window aggregation. + + Handles expressions like ``pl.col("x").rolling_sum(window_size=3)``. + Uses ``pylibcudf.rolling.rolling_window`` with integer preceding + and following window sizes. + """ + + __slots__ = ( + "_agg_request", + "agg_name", + "fn_params", + "following", + "min_periods", + "preceding", + ) + _non_child = ( + "dtype", + "agg_name", + "preceding", + "following", + "min_periods", + "fn_params", + ) + + _aggregations: ClassVar[dict[str, Callable[..., plc.aggregation.Aggregation]]] = { + "sum": plc.aggregation.sum, + "min": plc.aggregation.min, + "max": plc.aggregation.max, + "mean": plc.aggregation.mean, + "var": plc.aggregation.variance, + "std": plc.aggregation.std, + } + + def __init__( + self, + dtype: DataType, + agg_name: str, + preceding: int, + following: int, + min_periods: int, + fn_params: tuple[Any, ...], + child: Expr, + ) -> None: + self.dtype = dtype + self.agg_name = agg_name + self.preceding = preceding + self.following = following + self.min_periods = min_periods + self.fn_params = fn_params + self.children = (child,) + self.is_pointwise = False + self._agg_request = self._make_agg_request() + if not plc.rolling.is_valid_rolling_aggregation( + child.dtype.plc_type, self._agg_request + ): + raise NotImplementedError( + f"Unsupported fixed-size rolling aggregation {agg_name}" + ) + + def _make_agg_request(self) -> plc.aggregation.Aggregation: + agg_fn = self._aggregations.get(self.agg_name) + if agg_fn is None: + raise NotImplementedError( + f"Unsupported fixed-size rolling aggregation: {self.agg_name}" + ) # pragma: no cover; translation validates aggregation names + return agg_fn(*self.fn_params) + + def do_evaluate( + self, df: DataFrame, *, context: ExecutionContext = ExecutionContext.FRAME + ) -> Column: + """Evaluate this expression given a dataframe for context.""" + if context != ExecutionContext.FRAME: + raise RuntimeError( + "Rolling aggregation inside groupby/over/rolling" + ) # pragma: no cover; translation raises first + (child,) = self.children + col = child.evaluate(df, context=context) + + result = plc.rolling.rolling_window( + col.obj, + self.preceding, + self.following, + self.min_periods, + self._agg_request, + stream=df.stream, + ) + if result.type() != self.dtype.plc_type: + result = plc.unary.cast(result, self.dtype.plc_type, stream=df.stream) + + return Column(result, dtype=self.dtype) + + class GroupedWindow(Expr): """ Compute a window ``.over(...)`` aggregation and broadcast to rows. diff --git a/python/cudf_polars/cudf_polars/dsl/translate.py b/python/cudf_polars/cudf_polars/dsl/translate.py index 24bd0ab66c01..967a2462d2dd 100644 --- a/python/cudf_polars/cudf_polars/dsl/translate.py +++ b/python/cudf_polars/cudf_polars/dsl/translate.py @@ -48,6 +48,8 @@ from cudf_polars.typing import NodeTraverser, Slice as Zlice +_HAS_ROLLING_FUNCTION = hasattr(plrs._expr_nodes, "RollingFunction") + __all__ = ["Translator", "translate_named_expr"] @@ -989,6 +991,41 @@ def _( options, *(translator.translate_expr(n=n, schema=schema) for n in node.input), ) + elif _HAS_ROLLING_FUNCTION and isinstance(name, plrs._expr_nodes.RollingFunction): + window_size, min_periods, weights, center, fn_params = options + if weights is not None: + raise NotImplementedError("Weighted rolling windows") + RF = plrs._expr_nodes.RollingFunction + agg_names = { + RF.Sum: "sum", + RF.Min: "min", + RF.Max: "max", + RF.Mean: "mean", + RF.Var: "var", + RF.Std: "std", + } + agg_name = agg_names.get(name) + if agg_name is None: + raise NotImplementedError(f"Unsupported rolling function: {name}") + # Convert center + window_size to preceding/following for libcudf. + # libcudf rolling_window semantics: element i uses elements + # [i - preceding + 1, i + following]. + if center: + following = (window_size - 1) // 2 + preceding = window_size - following + else: + preceding = window_size + following = 0 + # Polars produces null when count <= ddof for var/std, but + # libcudf produces NaN. Raise min_periods so that libcudf + # returns null instead. + if agg_name in ("var", "std"): + (ddof,) = fn_params + min_periods = max(min_periods, ddof + 1) + (child,) = (translator.translate_expr(n=n, schema=schema) for n in node.input) + return expr.FixedSizeRollingWindow( + dtype, agg_name, preceding, following, min_periods, fn_params, child + ) elif isinstance(name, str): children = (translator.translate_expr(n=n, schema=schema) for n in node.input) if name == "rechunk": diff --git a/python/cudf_polars/tests/expressions/test_fixed_rolling.py b/python/cudf_polars/tests/expressions/test_fixed_rolling.py new file mode 100644 index 000000000000..a4064c873298 --- /dev/null +++ b/python/cudf_polars/tests/expressions/test_fixed_rolling.py @@ -0,0 +1,133 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import pytest + +import polars as pl +from polars import polars as plrs # type: ignore[attr-defined] + +from cudf_polars.testing.asserts import ( + assert_gpu_result_equal, + assert_ir_translation_raises, +) + +pytestmark = pytest.mark.skipif( + not hasattr(plrs._expr_nodes, "RollingFunction"), + reason="RollingFunction not available in this polars version", +) + + +@pytest.fixture +def df(): + return pl.LazyFrame( + { + "x": [4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0], + } + ) + + +@pytest.mark.parametrize( + "rolling_fn", + ["rolling_sum", "rolling_min", "rolling_max", "rolling_mean"], +) +def test_fixed_rolling_basic(df, engine: pl.GPUEngine, rolling_fn): + q = df.select(getattr(pl.col("x"), rolling_fn)(window_size=3)) + assert_gpu_result_equal(q, engine=engine) + + +@pytest.mark.parametrize("window_size", [1, 2, 4, 8]) +def test_fixed_rolling_sum_window_sizes(df, engine: pl.GPUEngine, window_size): + q = df.select(pl.col("x").rolling_sum(window_size=window_size)) + assert_gpu_result_equal(q, engine=engine) + + +def test_fixed_rolling_sum_centered(df, engine: pl.GPUEngine): + q = df.select(pl.col("x").rolling_sum(window_size=3, center=True)) + assert_gpu_result_equal(q, engine=engine) + + +def test_fixed_rolling_sum_centered_even(df, engine: pl.GPUEngine): + q = df.select(pl.col("x").rolling_sum(window_size=4, center=True)) + assert_gpu_result_equal(q, engine=engine) + + +def test_fixed_rolling_sum_min_samples(df, engine: pl.GPUEngine): + q = df.select(pl.col("x").rolling_sum(window_size=3, min_samples=1)) + assert_gpu_result_equal(q, engine=engine) + + +def test_fixed_rolling_sum_with_nulls(engine: pl.GPUEngine): + df = pl.LazyFrame({"x": [1.0, None, 3.0, None, 5.0, 6.0]}) + q = df.select(pl.col("x").rolling_sum(window_size=3, min_samples=1)) + assert_gpu_result_equal(q, engine=engine) + + +def test_fixed_rolling_sum_all_null_window(engine: pl.GPUEngine): + df = pl.LazyFrame({"x": [None, None, None, 4.0, 5.0, 6.0]}) + q = df.select(pl.col("x").rolling_sum(window_size=3)) + assert_gpu_result_equal(q, engine=engine) + + +def test_fixed_rolling_mean_with_nulls(engine: pl.GPUEngine): + df = pl.LazyFrame({"x": [1.0, None, 3.0, 4.0, None, 6.0]}) + q = df.select(pl.col("x").rolling_mean(window_size=3, min_samples=1)) + assert_gpu_result_equal(q, engine=engine) + + +def test_fixed_rolling_var(df, engine: pl.GPUEngine): + q = df.select(pl.col("x").rolling_var(window_size=3)) + assert_gpu_result_equal(q, engine=engine) + + +def test_fixed_rolling_std(df, engine: pl.GPUEngine): + q = df.select(pl.col("x").rolling_std(window_size=3)) + assert_gpu_result_equal(q, engine=engine) + + +def test_fixed_rolling_var_ddof(df, engine: pl.GPUEngine): + q = df.select(pl.col("x").rolling_var(window_size=4, ddof=2)) + assert_gpu_result_equal(q, engine=engine) + + +def test_fixed_rolling_std_ddof(df, engine: pl.GPUEngine): + q = df.select(pl.col("x").rolling_std(window_size=4, ddof=0)) + assert_gpu_result_equal(q, engine=engine) + + +@pytest.mark.parametrize("dtype", [pl.Int32, pl.Int64, pl.Float32, pl.Float64]) +def test_fixed_rolling_sum_dtypes(engine: pl.GPUEngine, dtype): + df = pl.LazyFrame({"x": pl.Series([1, 2, 3, 4, 5, 6], dtype=dtype)}) + q = df.select(pl.col("x").rolling_sum(window_size=3)) + assert_gpu_result_equal(q, engine=engine) + + +# TODO: Remove once fixed-size rolling supports multi-partition streaming. +@pytest.mark.filterwarnings( + "ignore:This selection is not supported for multiple partitions\\.:UserWarning" +) +def test_fixed_rolling_large_window(engine: pl.GPUEngine): + data = list(range(500)) + df = pl.LazyFrame({"x": [float(v) for v in data]}) + q = df.select(pl.col("x").rolling_sum(window_size=250)) + assert_gpu_result_equal(q, engine=engine) + + +def test_fixed_rolling_weighted_raises(df, engine: pl.GPUEngine): + q = df.select(pl.col("x").rolling_mean(window_size=3, weights=[1.0, 2.0, 3.0])) + assert_ir_translation_raises(q, engine, NotImplementedError) + + +def test_fixed_rolling_invalid_dtype_raises(engine: pl.GPUEngine): + q = pl.LazyFrame({"x": ["a", "b", "c"]}).select( + pl.col("x").rolling_min(window_size=2) + ) + assert_ir_translation_raises(q, engine, NotImplementedError) + + +def test_fixed_rolling_unsupported_function_raises(engine: pl.GPUEngine): + q = pl.LazyFrame({"x": [1.0, 2.0, 3.0]}).select( + pl.col("x").rolling_quantile(0.5, window_size=2) + ) + assert_ir_translation_raises(q, engine, NotImplementedError) From 82aa0968c2a7523e134dcd922c26265443b235cc Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Mon, 20 Jul 2026 17:35:53 -0500 Subject: [PATCH 12/25] Reconcile Index.union dtypes before the empty-operand short-circuits (#23318) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes all 16 failing parametrizations of `tests/indexes/test_setops.py::test_union_dtypes` in the pandas test suite under `cudf.pandas` (entries removed from the pandas-testing plugin). pandas reconciles mismatched dtypes **before** short-circuiting on empty or equal operands, so the union result dtype must not depend on which operand is empty. `Index.union`'s empty-operand branches previously returned the non-empty (or left) operand's dtype unchanged, e.g. `Index([1]).union(Index([], dtype='float64'))` returned `int64` while the reversed call returned `float64`. Changes in `Index.union`: - Mismatched numeric dtypes (and same-kind datetime/timedelta unit mismatches) now promote to their `find_common_type` in the empty-operand branches, matching the non-empty merge path and pandas. Promotion is skipped when the common type degrades to `object` (mixed masked/arrow backend pairs), which would otherwise silently stringify numeric values. - `MixedTypeError` is now raised **in both modes** (no pandas-compatible-mode gating) for datetime/timedelta vs any other kind and for tz-naive vs tz-aware pairs. pandas produces `object` dtype for these, which cudf cannot represent, and classic cudf's non-empty merge path already raises `TypeError` for them (`MixedTypeError` subclasses `TypeError`) — previously the empty-operand branches silently returned the wrong dtype, and the tz-mixed merge silently stringified timestamps into an object index. Under `cudf.pandas` the raise triggers fallback to pandas. Categorical operands are excluded: the merge decategorizes them, so `dt_index.union(CategoricalIndex-of-datetimes)` legitimately succeeds and returns `datetime64`, matching pandas. - Both the raises and the promotion honor pandas' zero-length exemption (pandas-dev/pandas#60797): the dtype of a zero-length `RangeIndex` or object-dtype operand is ignored, so e.g. `Index([1, 2], dtype='int32').union(RangeIndex(0))` stays `int32` and `dt_index.union(RangeIndex(0))` succeeds. Also, `RangeIndex._try_reconstruct_range_index` now reconstructs 0- and 1-element set-op results as `RangeIndex` (pandas returns a `RangeIndex` whenever the result is representable as one), gated to exactly-`int64` results so dtypes materialized under `default_integer_bitwidth` are preserved. This deflakes the hypothesis-based `tests/indexes/ranges/test_setops.py::test_range_difference`, whose test helper crashes on any 1-element non-RangeIndex difference result (the pandas-tests hypothesis profile uses fresh random examples each run with no failure database, so this could flip CI runs at random). Verified with no new failures against baseline on `tests/indexes/{test_setops,datetimes/test_setops,timedeltas/test_setops,ranges/test_setops}.py` under `cudf.pandas`, plus the full classic `tests/indexes/` and concat/join suites. Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Matthew Roeschke (https://github.com/mroeschke) URL: https://github.com/rapidsai/cudf/pull/23318 --- python/cudf/cudf/core/index.py | 90 ++++++++++++++++--- .../pandas/scripts/pandas-testing-plugin.py | 19 ---- 2 files changed, 80 insertions(+), 29 deletions(-) diff --git a/python/cudf/cudf/core/index.py b/python/cudf/cudf/core/index.py index 9e3e4c6f6615..41454d544849 100644 --- a/python/cudf/cudf/core/index.py +++ b/python/cudf/cudf/core/index.py @@ -81,7 +81,7 @@ from collections.abc import Generator, Iterable from datetime import tzinfo - from cudf._typing import ColumnLike, Dtype + from cudf._typing import ColumnLike, Dtype, DtypeObj from cudf.core.dataframe import DataFrame from cudf.core.multiindex import MultiIndex from cudf.core.series import Series @@ -808,26 +808,83 @@ def union(self, other, sort: bool | None = None) -> Index: f"[None, False, True]; {sort} was passed." ) - if cudf.get_option("mode.pandas_compatible"): + # pandas ignores the dtype of a zero-length RangeIndex or + # object-dtype operand when reconciling dtypes for union + # (pandas-dev/pandas#60797), so such operands must neither + # trigger mixed-type errors nor dtype promotion. + dtype_ignored = any( + len(idx) == 0 + and (isinstance(idx, RangeIndex) or idx.dtype == np.dtype(object)) + for idx in (self, other) + ) + + if not dtype_ignored: # Cache dtype.kind to avoid repeated attribute access self_kind = self.dtype.kind other_kind = other.dtype.kind + if ( + (self_kind in "Mm" or other_kind in "Mm") + and self_kind != other_kind + and not isinstance(self.dtype, CategoricalDtype) + and not isinstance(other.dtype, CategoricalDtype) + ): + # datetime/timedelta + any other kind results in object + # dtype for union in pandas, which cudf cannot represent; + # the non-empty merge path already refuses these pairs. + # Categorical operands are excluded because the merge + # decategorizes them, matching pandas. + raise MixedTypeError("Cannot perform union with mixed types") + if ( + self_kind == "M" + and other_kind == "M" + and isinstance(self.dtype, pd.DatetimeTZDtype) + != isinstance(other.dtype, pd.DatetimeTZDtype) + ): + # tz-naive + tz-aware results in object dtype in pandas. + raise MixedTypeError("Cannot perform union with mixed types") + if (self_kind == "b" and other_kind != "b") or ( self_kind != "b" and other_kind == "b" ): # Bools + other types will result in mixed type. - # This is not yet consistent in pandas and specific to APIs. + # This is not yet consistent in pandas and specific to + # APIs. raise MixedTypeError("Cannot perform union with mixed types") if (self_kind == "i" and other_kind == "u") or ( self_kind == "u" and other_kind == "i" ): - # signed + unsigned types will result in - # mixed type for union in pandas. + # signed + unsigned types will result in mixed type for + # union in pandas, which cudf cannot represent. raise MixedTypeError("Cannot perform union with mixed types") + # pandas reconciles mismatched dtypes to their common type before + # short-circuiting on an empty operand, so the result dtype must + # not depend on which operand is empty. + promote_dtype: DtypeObj | None = None + if not dtype_ignored and self.dtype != other.dtype: + if is_dtype_obj_numeric( + self.dtype, include_decimal=False + ) and is_dtype_obj_numeric(other.dtype, include_decimal=False): + common_dtype = find_common_type([self.dtype, other.dtype]) + # Mixed-backend pairs can only reconcile to object; never + # promote numeric values through a lossy object cast. + if is_dtype_obj_numeric(common_dtype, include_decimal=False): + promote_dtype = common_dtype + elif ( + isinstance(self.dtype, np.dtype) + and isinstance(other.dtype, np.dtype) + and self.dtype.kind == other.dtype.kind + and self.dtype.kind in "Mm" + ): + # Same-kind datetime/timedelta dtypes only differ in + # unit; reconcile to the common resolution. + promote_dtype = find_common_type([self.dtype, other.dtype]) + if not len(other): res = self._get_reconciled_name_object(other) + if promote_dtype is not None: + res = res.astype(promote_dtype) if sort: return res.sort_values() # type: ignore[return-value] return res @@ -839,6 +896,8 @@ def union(self, other, sort: bool | None = None) -> Index: return res elif not len(self): res = other._get_reconciled_name_object(self) + if promote_dtype is not None: + res = res.astype(promote_dtype) if sort: return res.sort_values() return res @@ -3040,11 +3099,22 @@ def _try_reconstruct_range_index(self, index: Index) -> Self | Index: return index # Evenly spaced values can return a # RangeIndex instead of a materialized Index. - if not index._column.has_nulls() and len(index) > 1: - uniques = cupy.unique(cupy.diff(index.values)) - if len(uniques) == 1 and (diff := uniques[0].get()) != 0: - new_range = range(index[0], index[-1] + diff, diff) - return type(self)(new_range, name=index.name) + if not index._column.has_nulls(): + if len(index) > 1: + uniques = cupy.unique(cupy.diff(index.values)) + if len(uniques) == 1 and (diff := uniques[0].get()) != 0: + new_range = range(index[0], index[-1] + diff, diff) + return type(self)(new_range, name=index.name) + elif index.dtype == np.dtype(np.int64): + # 0- and 1-element results are always representable as a + # range, but only when int64: a narrower dtype means the + # result was deliberately materialized (e.g. under + # default_integer_bitwidth) and must be preserved. + if len(index) == 0: + return type(self)(range(0), name=index.name) + start = int(index[0]) + if start < np.iinfo(np.int64).max: + return type(self)(range(start, start + 1), name=index.name) return index def sort_values( diff --git a/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py b/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py index 65c1740de95c..ba05357c4863 100644 --- a/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py +++ b/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py @@ -2284,10 +2284,7 @@ def pytest_unconfigure(config): "tests/indexes/ranges/test_join.py::TestJoin::test_join_self[left]": "TODO: Add a reason for failure", "tests/indexes/ranges/test_join.py::TestJoin::test_join_self[outer]": "TODO: Add a reason for failure", "tests/indexes/ranges/test_join.py::TestJoin::test_join_self[right]": "TODO: Add a reason for failure", - "tests/indexes/ranges/test_range.py::TestRangeIndex::test_append[indices6-expected6]": "AssertionError: Index are different", - "tests/indexes/ranges/test_range.py::TestRangeIndex::test_append[indices7-expected7]": "AssertionError: Index are different", "tests/indexes/ranges/test_range.py::TestRangeIndex::test_cache": "TODO: Add a reason for failure", - "tests/indexes/ranges/test_range.py::test_append_one_nonempty_preserve_step": "AssertionError: Index are different", "tests/indexes/ranges/test_range.py::test_getitem_boolmask_all_false": "AssertionError: Index are different", "tests/indexes/ranges/test_range.py::test_getitem_boolmask_all_true": "AssertionError: Index are different", "tests/indexes/ranges/test_range.py::test_getitem_boolmask_returns_rangeindex": "AssertionError: Index are different", @@ -2448,22 +2445,6 @@ def pytest_unconfigure(config): "tests/indexes/test_setops.py::test_setop_with_categorical[empty-None-union]": "AssertionError: Index are different", "tests/indexes/test_setops.py::test_setop_with_categorical[nullable_bool-False-union]": "TODO: Add a reason for failure", "tests/indexes/test_setops.py::test_setop_with_categorical[nullable_bool-None-union]": "TODO: Add a reason for failure", - "tests/indexes/test_setops.py::test_union_dtypes[names0-datetime64[ns, CET]-float64-object]": "AssertionError: assert datetime64[ns, CET] == 'object'", - "tests/indexes/test_setops.py::test_union_dtypes[names0-datetime64[ns, CET]-int64-object]": "AssertionError: assert datetime64[ns, CET] == 'object'", - "tests/indexes/test_setops.py::test_union_dtypes[names0-datetime64[ns, CET]-uint64-object]": "AssertionError: assert datetime64[ns, CET] == 'object'", - "tests/indexes/test_setops.py::test_union_dtypes[names0-datetime64[ns]-float64-object]": "AssertionError: assert dtype(' Date: Mon, 20 Jul 2026 17:51:51 -0500 Subject: [PATCH 13/25] Support the limit parameter in GroupBy.ffill and bfill (#23302) The `limit` argument of `GroupBy.ffill`/`bfill` was accepted and silently ignored (documented as unsupported), returning unlimited fills. Implement it on top of the existing grouped `replace_nulls`: pass the group-relative row position (cumcount, masked null exactly where the value is null) through the same `replace_nulls` call, so each row learns the position of the value that sourced its fill; rows whose fill distance exceeds `limit` are re-nulled. A negative `limit` means unlimited, matching pandas. Fixes 24 pandas-tests (`test_group_fill_methods[ffill/bfill-1-*]`); their xfail entries are removed. Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/23302 --- python/cudf/cudf/core/groupby/groupby.py | 56 ++++++++++++++++--- .../pandas/scripts/pandas-testing-plugin.py | 24 -------- python/cudf/cudf/tests/groupby/test_ffill.py | 41 +++++++++++++- 3 files changed, 88 insertions(+), 33 deletions(-) diff --git a/python/cudf/cudf/core/groupby/groupby.py b/python/cudf/cudf/core/groupby/groupby.py index 23b985a98218..89da3067cb35 100644 --- a/python/cudf/cudf/core/groupby/groupby.py +++ b/python/cudf/cudf/core/groupby/groupby.py @@ -3424,14 +3424,50 @@ def _scan_fill( values = self.grouping.values from cudf.core.dataframe import DataFrame - result = self.obj._from_data( - dict( - zip( - values._column_names, - self._replace_nulls(values._columns, method), - strict=True, + value_columns = values._columns + if limit is not None and limit < 0: + # pandas treats a negative limit as unlimited + limit = None + if limit is None: + replaced = tuple(self._replace_nulls(value_columns, method)) + else: + # pandas accepts integer-valued floats + limit = int(limit) + # Group-relative row position, masked null exactly where the + # value is null: group-filling the positions with the same + # policy yields, per row, the position of the value that + # sourced its fill, making (own position - source position) + # the fill distance. The unmasked positions column is passed + # through the same replace_nulls call because the output rows + # come back in grouped order, not the original row order. + cum = self.cumcount()._column + pos_columns = tuple( + cum.set_mask(col.mask, col.null_count) if col.nullable else cum + for col in value_columns + ) + n = len(value_columns) + filled = tuple( + self._replace_nulls( + (*value_columns, *pos_columns, cum), method ) ) + grouped_cum = filled[-1] + limited = [] + for fcol, fpos in zip(filled[:n], filled[n : 2 * n], strict=True): + if method == plc.replace.ReplacePolicy.PRECEDING: + dist = grouped_cum - fpos + else: + dist = fpos - grouped_cum + # Rows within limit of their fill source stay valid + # (originally-valid rows are their own source, distance + # 0); nulls no fill reached have a null distance and stay + # null, so ``keep`` alone is a valid final null mask. + keep = (dist <= limit).fillna(False) + limited.append(fcol.set_mask(*keep.as_mask())) + replaced = tuple(limited) + + result = self.obj._from_data( + dict(zip(values._column_names, replaced, strict=True)) ) # Pandas' groupby.ffill/bfill builds the result columns via a ``take`` # on the input columns, which converts integer-valued column labels @@ -3463,7 +3499,9 @@ def ffill(self, limit: int | None = None): Parameters ---------- limit : int, default None - Unsupported + The maximum number of consecutive NA values within a group + filled forward from the most recent valid value. ``None`` + fills without limit. """ return self._scan_fill(plc.replace.ReplacePolicy.PRECEDING, limit) @@ -3473,7 +3511,9 @@ def bfill(self, limit: int | None = None): Parameters ---------- limit : int, default None - Unsupported + The maximum number of consecutive NA values within a group + filled backward from the next valid value. ``None`` fills + without limit. """ return self._scan_fill(plc.replace.ReplacePolicy.FOLLOWING, limit) diff --git a/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py b/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py index ba05357c4863..39e1b5a7d94d 100644 --- a/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py +++ b/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py @@ -1943,30 +1943,6 @@ def pytest_unconfigure(config): "tests/groupby/transform/test_transform.py::test_as_index_no_change[size-keys1]": "TODO: Add a reason for failure", "tests/groupby/transform/test_transform.py::test_cython_transform_series[cumprod-args0-]": "TODO: Add a reason for failure", "tests/groupby/transform/test_transform.py::test_cython_transform_series[cumsum-args1-]": "TODO: Add a reason for failure", - "tests/groupby/transform/test_transform.py::test_group_fill_methods[bfill-1-exp_vals3-1-2-False-False]": "AssertionError: DataFrame.iloc[:, 0] (column name='val') are different", - "tests/groupby/transform/test_transform.py::test_group_fill_methods[bfill-1-exp_vals3-1-2-False-True]": "AssertionError: DataFrame.iloc[:, 0] (column name='val') are different", - "tests/groupby/transform/test_transform.py::test_group_fill_methods[bfill-1-exp_vals3-1-2-True-False]": "AssertionError: Series are different", - "tests/groupby/transform/test_transform.py::test_group_fill_methods[bfill-1-exp_vals3-1-2-True-True]": "AssertionError: Series are different", - "tests/groupby/transform/test_transform.py::test_group_fill_methods[bfill-1-exp_vals3-1.0-2.0-False-False]": "AssertionError: DataFrame.iloc[:, 0] (column name='val') are different", - "tests/groupby/transform/test_transform.py::test_group_fill_methods[bfill-1-exp_vals3-1.0-2.0-False-True]": "AssertionError: DataFrame.iloc[:, 0] (column name='val') are different", - "tests/groupby/transform/test_transform.py::test_group_fill_methods[bfill-1-exp_vals3-1.0-2.0-True-False]": "AssertionError: Series are different", - "tests/groupby/transform/test_transform.py::test_group_fill_methods[bfill-1-exp_vals3-1.0-2.0-True-True]": "AssertionError: Series are different", - "tests/groupby/transform/test_transform.py::test_group_fill_methods[bfill-1-exp_vals3-foo-bar-False-False]": "AssertionError: DataFrame.iloc[:, 0] (column name='val') NA mask are different", - "tests/groupby/transform/test_transform.py::test_group_fill_methods[bfill-1-exp_vals3-foo-bar-False-True]": "AssertionError: DataFrame.iloc[:, 0] (column name='val') NA mask are different", - "tests/groupby/transform/test_transform.py::test_group_fill_methods[bfill-1-exp_vals3-foo-bar-True-False]": "AssertionError: Series NA mask are different", - "tests/groupby/transform/test_transform.py::test_group_fill_methods[bfill-1-exp_vals3-foo-bar-True-True]": "AssertionError: Series NA mask are different", - "tests/groupby/transform/test_transform.py::test_group_fill_methods[ffill-1-exp_vals1-1-2-False-False]": "AssertionError: DataFrame.iloc[:, 0] (column name='val') are different", - "tests/groupby/transform/test_transform.py::test_group_fill_methods[ffill-1-exp_vals1-1-2-False-True]": "AssertionError: DataFrame.iloc[:, 0] (column name='val') are different", - "tests/groupby/transform/test_transform.py::test_group_fill_methods[ffill-1-exp_vals1-1-2-True-False]": "AssertionError: Series are different", - "tests/groupby/transform/test_transform.py::test_group_fill_methods[ffill-1-exp_vals1-1-2-True-True]": "AssertionError: Series are different", - "tests/groupby/transform/test_transform.py::test_group_fill_methods[ffill-1-exp_vals1-1.0-2.0-False-False]": "AssertionError: DataFrame.iloc[:, 0] (column name='val') are different", - "tests/groupby/transform/test_transform.py::test_group_fill_methods[ffill-1-exp_vals1-1.0-2.0-False-True]": "AssertionError: DataFrame.iloc[:, 0] (column name='val') are different", - "tests/groupby/transform/test_transform.py::test_group_fill_methods[ffill-1-exp_vals1-1.0-2.0-True-False]": "AssertionError: Series are different", - "tests/groupby/transform/test_transform.py::test_group_fill_methods[ffill-1-exp_vals1-1.0-2.0-True-True]": "AssertionError: Series are different", - "tests/groupby/transform/test_transform.py::test_group_fill_methods[ffill-1-exp_vals1-foo-bar-False-False]": "AssertionError: DataFrame.iloc[:, 0] (column name='val') NA mask are different", - "tests/groupby/transform/test_transform.py::test_group_fill_methods[ffill-1-exp_vals1-foo-bar-False-True]": "AssertionError: DataFrame.iloc[:, 0] (column name='val') NA mask are different", - "tests/groupby/transform/test_transform.py::test_group_fill_methods[ffill-1-exp_vals1-foo-bar-True-False]": "AssertionError: Series NA mask are different", - "tests/groupby/transform/test_transform.py::test_group_fill_methods[ffill-1-exp_vals1-foo-bar-True-True]": "AssertionError: Series NA mask are different", "tests/groupby/transform/test_transform.py::test_groupby_transform_timezone_column[first]": "AssertionError: Attributes of DataFrame.iloc[:, 2] (column name='max_end_time') are different", "tests/groupby/transform/test_transform.py::test_groupby_transform_timezone_column[last]": "AssertionError: Attributes of DataFrame.iloc[:, 2] (column name='max_end_time') are different", "tests/groupby/transform/test_transform.py::test_groupby_transform_with_datetimes[idxmax-values1]": "AssertionError: Attributes of Series are different", diff --git a/python/cudf/cudf/tests/groupby/test_ffill.py b/python/cudf/cudf/tests/groupby/test_ffill.py index a3dbd9df7cf5..e45f2ca9eaad 100644 --- a/python/cudf/cudf/tests/groupby/test_ffill.py +++ b/python/cudf/cudf/tests/groupby/test_ffill.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 import pandas as pd @@ -61,3 +61,42 @@ def test_groupby_ffill_multi_value(): got = gdf.groupby(key_col).ffill() assert_groupby_results_equal(expect[value_cols], got[value_cols]) + + +@pytest.mark.parametrize("method", ["ffill", "bfill"]) +@pytest.mark.parametrize("limit", [0, 1, 2, -1, None]) +@pytest.mark.parametrize( + "values", + [ + [1.0, None, None, None, 2.0, None, None], + ["x", None, None, None, "y", None, None], + ], +) +def test_groupby_fill_limit(method, limit, values): + # interleaved keys: limit counts group-relative positions + keys = ["a", "b"] * 7 + data = {"key": keys, "val": [v for v in values for _ in range(2)]} + pdf = pd.DataFrame(data) + gdf = cudf.DataFrame(data) + + expect = getattr(pdf.groupby("key"), method)(limit=limit) + got = getattr(gdf.groupby("key"), method)(limit=limit) + + assert_groupby_results_equal(expect, got) + + +@pytest.mark.parametrize("method", ["ffill", "bfill"]) +def test_groupby_fill_limit_null_keys(method): + # null-key rows must stay null under dropna=True even with a limit + pdf = pd.DataFrame( + { + "key": [1.0, 1.0, None, 1.0, None, 1.0], + "val": [1.0, None, None, None, 2.0, None], + } + ) + gdf = cudf.DataFrame(pdf) + + expect = getattr(pdf.groupby("key", dropna=True), method)(limit=1) + got = getattr(gdf.groupby("key", dropna=True), method)(limit=1) + + assert_groupby_results_equal(expect, got) From 0929bb72bc20bfe392ea38d4ac266595933adedd Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Mon, 20 Jul 2026 18:34:09 -0500 Subject: [PATCH 14/25] Stop forward-filling in GroupBy.pct_change (#23301) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `pct_change` validated that `fill_method` must be `None` (pandas 3.0 removed the parameter) but still unconditionally forward-filled before shifting, baking in the pre-3.0 `fill_method="ffill"` default. pandas performs no filling: NaN appears wherever the value or the group-shifted value is NA. Shift the raw values instead. Fixes 4 pandas-tests (`test_pct_change[DataFrame/Series-±1-None]`); their xfail entries are removed. Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/23301 --- python/cudf/cudf/core/groupby/groupby.py | 13 +++++++----- .../pandas/scripts/pandas-testing-plugin.py | 4 ---- .../cudf/tests/groupby/test_pct_change.py | 20 ++++++++++++++++++- 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/python/cudf/cudf/core/groupby/groupby.py b/python/cudf/cudf/core/groupby/groupby.py index 89da3067cb35..42b794cd050a 100644 --- a/python/cudf/cudf/core/groupby/groupby.py +++ b/python/cudf/cudf/core/groupby/groupby.py @@ -3608,7 +3608,7 @@ def pct_change( ---------- periods : int, default 1 Periods to shift for forming percent change. - fill_method : str, default 'ffill' + fill_method : None Must be None. freq : str, optional Increment to use from time series API. @@ -3624,12 +3624,15 @@ def pct_change( if fill_method is not None: raise ValueError(f"fill_method must be None; got {fill_method=}.") - filled = self.ffill() - fill_grp = filled.groupby( + # pandas 3.0 removed fill_method: no filling is performed, so NaN + # appears wherever the value or the group-shifted value is NA. + values = self.grouping.values + values.index = self.obj.index + value_grp = values.groupby( self.grouping, sort=self._sort, dropna=self._dropna ) - shifted = fill_grp.shift(periods=periods, freq=freq) - return (filled / shifted) - 1 + shifted = value_grp.shift(periods=periods, freq=freq) + return (values / shifted) - 1 def _mimic_pandas_order( self, result: DataFrameOrSeries diff --git a/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py b/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py index 39e1b5a7d94d..1ffb60167185 100644 --- a/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py +++ b/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py @@ -1956,10 +1956,6 @@ def pytest_unconfigure(config): "tests/groupby/transform/test_transform.py::test_null_group_str_reducer[True-size]": "TODO: Add a reason for failure", "tests/groupby/transform/test_transform.py::test_null_group_str_transformer[False-cumcount]": "TODO: Add a reason for failure", "tests/groupby/transform/test_transform.py::test_null_group_str_transformer[True-cumcount]": "TODO: Add a reason for failure", - "tests/groupby/transform/test_transform.py::test_pct_change[DataFrame--1-None]": "AssertionError: DataFrame.iloc[:, 0] (column name='vals') are different", - "tests/groupby/transform/test_transform.py::test_pct_change[DataFrame-1-None]": "AssertionError: DataFrame.iloc[:, 0] (column name='vals') are different", - "tests/groupby/transform/test_transform.py::test_pct_change[Series--1-None]": "AssertionError: Series are different", - "tests/groupby/transform/test_transform.py::test_pct_change[Series-1-None]": "AssertionError: Series are different", "tests/groupby/transform/test_transform.py::test_transform_cumcount": "TODO: Add a reason for failure", "tests/groupby/transform/test_transform.py::test_transform_fast": "TODO: Add a reason for failure", "tests/groupby/transform/test_transform.py::test_transform_numeric_ret[count-a-expected0]": "TODO: Add a reason for failure", diff --git a/python/cudf/cudf/tests/groupby/test_pct_change.py b/python/cudf/cudf/tests/groupby/test_pct_change.py index 4ea328cb4ca8..6d62c5f48d45 100644 --- a/python/cudf/cudf/tests/groupby/test_pct_change.py +++ b/python/cudf/cudf/tests/groupby/test_pct_change.py @@ -1,7 +1,8 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import pandas as pd import pytest import cudf @@ -86,3 +87,20 @@ def test_groupby_pct_change_empty_columns(): expected = pdf.groupby("id").pct_change() assert_eq(expected, actual) + + +@pytest.mark.parametrize("periods", [-1, 1]) +def test_groupby_pct_change_no_fill(periods): + # pandas 3.0 removed fill_method: values are not forward-filled, so + # NaN propagates into the percent change + data = { + "key": ["a"] * 5 + ["b"] * 5, + "vals": [3.0, None, None, 1.0, 2.0] * 2, + } + pdf = pd.DataFrame(data) + gdf = cudf.DataFrame(data) + + expected = pdf.groupby("key")["vals"].pct_change(periods=periods) + actual = gdf.groupby("key")["vals"].pct_change(periods=periods) + + assert_eq(expected, actual) From 335b13eb02f16d9722263be3f2330256a6f01f26 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Mon, 20 Jul 2026 19:06:56 -0500 Subject: [PATCH 15/25] Return index labels from GroupBy idxmin/idxmax in agg and transform (#23298) libcudf's ARGMIN/ARGMAX return integer row positions; the gather to source-index labels lived only in the direct `idxmin`/`idxmax` methods, so `agg("idxmin")` and `transform("idxmin")` returned raw positions (and the wrong dtype for e.g. datetime indexes). Perform the position-to-label gather in the agg result assembly instead, raising pandas' `ValueError` for all-NA groups, and drop the now-redundant `_wrap_idxmin_idxmax` wrapper. MultiIndex sources keep the positional result (pandas maps them to tuple labels in an object column, which is not yet supported). Fixes 6 pandas-tests (`test_groupby_transform_with_datetimes[idxmin/idxmax]`, `test_null_group_str_reducer[*-idxmin/idxmax]`); their xfail entries are removed. Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/23298 --- python/cudf/cudf/core/groupby/groupby.py | 102 +++++++----------- .../pandas/scripts/pandas-testing-plugin.py | 6 -- python/cudf/cudf/tests/groupby/test_agg.py | 37 ++++++- 3 files changed, 72 insertions(+), 73 deletions(-) diff --git a/python/cudf/cudf/core/groupby/groupby.py b/python/cudf/cudf/core/groupby/groupby.py index 42b794cd050a..e6cdae9c37f4 100644 --- a/python/cudf/cudf/core/groupby/groupby.py +++ b/python/cudf/cudf/core/groupby/groupby.py @@ -1220,6 +1220,28 @@ def agg(self, func=None, *args, engine=None, engine_kwargs=None, **kwargs): create_dtype = get_dtype_of_same_kind( orig_dtype, ListDtype(orig_dtype) ) + if agg_kind in {"ARGMIN", "ARGMAX"} and not isinstance( + self.obj.index, MultiIndex + ): + # libcudf returns the integer row-position of the + # min/max element within each group (null if the + # group's values are all NA); pandas returns the + # *label* of that row from the source index and raises + # for all-NA groups. Gather from the raw position + # column before any dtype morphing (a masked gather + # map cannot feed ``take``). MultiIndex sources fall + # through and stay positional: pandas maps them to + # tuple labels in an object column, which is not + # currently supported. + pos_col = ColumnBase.create(plc_result, create_dtype) + if pos_col.has_nulls(): + how = "idxmin" if agg_kind == "ARGMIN" else "idxmax" + raise ValueError( + f"{how} with skipna=True encountered all NA " + "values in a group." + ) + data[key] = self.obj.index._column.take(pos_col) + continue # Override for specific aggregation types that need dtype adjustments if agg_kind in {"COUNT", "SIZE", "ARGMIN", "ARGMAX"}: if isinstance(orig_dtype, pd.StringDtype): @@ -1404,64 +1426,6 @@ def agg(self, func=None, *args, engine=None, engine_kwargs=None, **kwargs): return result - def _wrap_idxmin_idxmax( - self, result: DataFrame | Series, *, skipna: bool, how: str - ): - # libcudf's idxmin/idxmax return the integer row-position of the - # min/max element within each group (null if the group's values were - # all NA). pandas instead returns the *label* of that row taken from - # the source object's row index, so we validate skipna against the raw - # positions and then gather the corresponding index labels. - from cudf.core.multiindex import MultiIndex - from cudf.core.series import Series - - if not skipna: - # pandas does not support positional idxmin/idxmax with - # skipna=False (it cannot represent "the label of a NA"). - raise ValueError(f"{how} with skipna=False") - - key_names = set(self.grouping.names) - if result.ndim == 2: - value_items = [ - (name, col) - for name, col in result._column_labels_and_values - if name not in key_names - ] - else: - value_items = [(None, result._column)] - - if skipna and any(col.has_nulls() for _, col in value_items): - raise ValueError( - "Encountered all NA values in a group with skipna=True" - ) - - index = self.obj.index - if isinstance(index, MultiIndex): - # pandas maps the positions to tuple-valued MultiIndex labels - # stored in an object column, which is not currently supported. - # Leave the (positional) result untouched, as before. - return result - - def gather_labels(positions: ColumnBase) -> ColumnBase: - # ``gather`` cannot consume a null gather-map, so redirect null - # positions to an out-of-bounds index; ``take(nullify=True)`` then - # yields a null label for them while valid positions still gather - # their (possibly null) index label. - if positions.has_nulls(): - positions = positions.fillna(len(index)) - return index._column.take(positions, nullify=True) - - if result.ndim == 2: - for name, col in value_items: - result._data[name] = gather_labels(col) - else: - result = Series._from_column( - gather_labels(result._column), - index=result.index, - name=result.name, - ) - return result - def _reduce_numeric_only(self, op: str): raise NotImplementedError( f"numeric_only is not implemented for {type(self)}" @@ -3881,8 +3845,11 @@ def idxmin( numeric_only: bool = False, **kwargs: Any, ) -> DataFrame: - result = self._reduce("idxmin", numeric_only=numeric_only) - return self._wrap_idxmin_idxmax(result, skipna=skipna, how="idxmin") + if not skipna: + # pandas does not support positional idxmin with skipna=False + # (it cannot represent "the label of a NA"). + raise ValueError("idxmin with skipna=False") + return self._reduce("idxmin", numeric_only=numeric_only) def idxmax( self, @@ -3891,8 +3858,9 @@ def idxmax( numeric_only: bool = False, **kwargs: Any, ) -> DataFrame: - result = self._reduce("idxmax", numeric_only=numeric_only) - return self._wrap_idxmin_idxmax(result, skipna=skipna, how="idxmax") + if not skipna: + raise ValueError("idxmax with skipna=False") + return self._reduce("idxmax", numeric_only=numeric_only) def value_counts( self, @@ -4303,14 +4271,16 @@ def apply(self, func, *args, **kwargs): def idxmin( self, skipna: bool = True, min_count: int = 0, **kwargs: Any ) -> Series: - result = self._reduce("idxmin") - return self._wrap_idxmin_idxmax(result, skipna=skipna, how="idxmin") + if not skipna: + raise ValueError("idxmin with skipna=False") + return self._reduce("idxmin") def idxmax( self, skipna: bool = True, min_count: int = 0, **kwargs: Any ) -> Series: - result = self._reduce("idxmax") - return self._wrap_idxmin_idxmax(result, skipna=skipna, how="idxmax") + if not skipna: + raise ValueError("idxmax with skipna=False") + return self._reduce("idxmax") @property def dtype(self) -> pd.Series: diff --git a/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py b/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py index 1ffb60167185..e43cbe2c0150 100644 --- a/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py +++ b/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py @@ -1945,14 +1945,8 @@ def pytest_unconfigure(config): "tests/groupby/transform/test_transform.py::test_cython_transform_series[cumsum-args1-]": "TODO: Add a reason for failure", "tests/groupby/transform/test_transform.py::test_groupby_transform_timezone_column[first]": "AssertionError: Attributes of DataFrame.iloc[:, 2] (column name='max_end_time') are different", "tests/groupby/transform/test_transform.py::test_groupby_transform_timezone_column[last]": "AssertionError: Attributes of DataFrame.iloc[:, 2] (column name='max_end_time') are different", - "tests/groupby/transform/test_transform.py::test_groupby_transform_with_datetimes[idxmax-values1]": "AssertionError: Attributes of Series are different", - "tests/groupby/transform/test_transform.py::test_groupby_transform_with_datetimes[idxmin-values0]": "AssertionError: Attributes of Series are different", "tests/groupby/transform/test_transform.py::test_nan_in_cumsum_group_label": "AssertionError: Attributes of Series are different", - "tests/groupby/transform/test_transform.py::test_null_group_str_reducer[False-idxmax]": "TODO: Add a reason for failure", - "tests/groupby/transform/test_transform.py::test_null_group_str_reducer[False-idxmin]": "TODO: Add a reason for failure", "tests/groupby/transform/test_transform.py::test_null_group_str_reducer[False-size]": "TODO: Add a reason for failure", - "tests/groupby/transform/test_transform.py::test_null_group_str_reducer[True-idxmax]": "TODO: Add a reason for failure", - "tests/groupby/transform/test_transform.py::test_null_group_str_reducer[True-idxmin]": "TODO: Add a reason for failure", "tests/groupby/transform/test_transform.py::test_null_group_str_reducer[True-size]": "TODO: Add a reason for failure", "tests/groupby/transform/test_transform.py::test_null_group_str_transformer[False-cumcount]": "TODO: Add a reason for failure", "tests/groupby/transform/test_transform.py::test_null_group_str_transformer[True-cumcount]": "TODO: Add a reason for failure", diff --git a/python/cudf/cudf/tests/groupby/test_agg.py b/python/cudf/cudf/tests/groupby/test_agg.py index 411afd4abf23..43b295e499ec 100644 --- a/python/cudf/cudf/tests/groupby/test_agg.py +++ b/python/cudf/cudf/tests/groupby/test_agg.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 import decimal import itertools @@ -769,3 +769,38 @@ def test_sliced_child_dtype_accuracy(): col = result["b"]._column child = col._get_sliced_child() assert child.dtype == col.dtype.element_type + + +@pytest.mark.parametrize("op", ["idxmin", "idxmax"]) +@pytest.mark.parametrize( + "index", [[10, 20, 30, 40], ["w", "x", "y", "z"], None] +) +def test_groupby_idxminmax_returns_index_labels(op, index): + # pandas returns the row's index *label* (with the index dtype), not + # the positional row number, from agg/transform/the direct method + pdf = pd.DataFrame( + {"key": [1, 1, 2, 2], "val": [4.0, 3.0, 5.0, 6.0]}, index=index + ) + gdf = cudf.DataFrame(pdf) + + assert_groupby_results_equal( + getattr(pdf.groupby("key"), op)(), getattr(gdf.groupby("key"), op)() + ) + assert_groupby_results_equal( + pdf.groupby("key").agg(op), gdf.groupby("key").agg(op) + ) + assert_eq( + pdf.groupby("key")["val"].transform(op), + gdf.groupby("key")["val"].transform(op), + ) + + +@pytest.mark.parametrize("op", ["idxmin", "idxmax"]) +def test_groupby_idxminmax_all_na_group_raises(op): + pdf = pd.DataFrame({"key": [1, 1, 2, 2], "val": [4.0, 3.0, None, None]}) + gdf = cudf.DataFrame(pdf) + + with pytest.raises(ValueError): + getattr(pdf.groupby("key"), op)() + with pytest.raises(ValueError): + getattr(gdf.groupby("key"), op)() From 88a8e82fa9a7bd4b9707c5ff135d1a8e20c2e3dc Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Mon, 20 Jul 2026 20:45:11 -0500 Subject: [PATCH 16/25] Match pandas semantics for null reduction results and mode ordering (#23328) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Running `tests/reductions/test_reductions.py` from the pandas test suite under `cudf.pandas` showed 38 failures. This PR fixes the 34 that are fixable in cudf classic and documents the 4 inherent ones. The same root causes also fixed 16 more pandas tests across `frame/`, `arrays/boolean`, `arrays/timedeltas`, `extension/test_arrow.py`, `groupby/test_reductions.py`, and `frame/methods/test_replace.py`. ### Fixes - **`pd.NaT` singleton for temporal null reductions** (`_get_nan_for_dtype`): min/max/median etc. on empty or all-null datetime/timedelta columns returned a unit-qualified `np.datetime64('NaT')`/`np.timedelta64('NaT')`; pandas returns the `pd.NaT` singleton and its tests assert identity (`result is NaT`). - **`` for empty/all-null reductions of nullable dtypes** (`ColumnBase._reduce`): `Series([], dtype="Int64").mean()`/`.var()` returned `np.float64(nan)` because the all-null branch used the *result* dtype (float64); pandas returns `pd.NA`. `sum`/`product` identities (0/1) are unchanged, matching pandas. - **Kleene logic for `any(skipna=False)`** (`ColumnBase.any`): with nulls present it returned `True` unconditionally. For pandas nullable extension dtypes, a no-True result with nulls present is now `` (matching `all`); numpy dtypes keep the NaN-sentinel-truthy behavior. - **`pd.NA` guard before `np.isnan`** in `any`/`all` to avoid "boolean value of NA is ambiguous" now that `_reduce` can return `pd.NA`. - **`Series.mode(dropna=False)` null position**: pandas sorts mode results on the underlying representation, so NaT (`INT64_MIN` as i8) and the categorical null code (-1) sort *first* while float NaN and arrow/nullable `` sort *last*. cudf sorted nulls last everywhere. Nulls-first now applies to numpy datetime/timedelta, `DatetimeTZDtype`, and categorical dtypes only (arrow timestamp/duration keep nulls last, verified against pandas). - **`StringColumn.all()`**: no longer short-circuits `True` for *partially*-null columns with `skipna=False` — the result depends on the truthiness of the non-null strings (e.g. `all([NaN, ""], skipna=False)` is `False`); it now falls through so `cudf.pandas` computes the correct result. All-null columns still return `True`. - `test_timedelta_reductions` updated to assert `pd.NaT` identity like `test_datetime_reductions` already does. ### Pandas-testing plugin - Removed **50** now-passing xfail entries. - Replaced the "TODO" reasons on the 4 remaining `test_reductions.py` entries with real ones: `test_sum_overflow_float[float32-*]` (GPU tree-reduction accumulates float32 in a different order than numpy pairwise summation) and `test_any_all_object_dtype_missing[any-data0/1]` (None-vs-np.nan distinction is lost when object data becomes a nulled bool column). ### Testing - `tests/reductions/` in CI mode: 498 passed, 10 xfailed, no strict-XPASS. - Full pandas-tests suite in CI mode (206k tests): no failures attributable to this change; every removed xfail entry verified as strict-XPASS solo (not just under xdist, to rule out GPU-contention fallback). - Classic cudf sweep (series/dataframe/indexes/groupby/reshape/text/dtypes/general_functions/window, ~71k tests): failure set identical to unmodified baseline (the only failures are pre-existing groupby-JIT ones). - Raw (plugin-less) before/after comparison over all affected pandas-test files confirmed 11 newly-passing tests and no newly-failing ones. The `decimal128` stack/unstack failures that appeared in some xdist runs reproduce identically on unmodified cudf (order-dependent, pre-existing). Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Matthew Roeschke (https://github.com/mroeschke) URL: https://github.com/rapidsai/cudf/pull/23328 --- python/cudf/cudf/core/column/column.py | 33 +++++- python/cudf/cudf/core/column/string.py | 13 ++- python/cudf/cudf/core/series.py | 16 ++- .../pandas/scripts/pandas-testing-plugin.py | 58 +--------- .../cudf/tests/series/methods/test_mode.py | 32 +++++- .../tests/series/methods/test_reductions.py | 105 +++++++++++++++++- python/cudf/cudf/utils/dtypes.py | 11 +- 7 files changed, 197 insertions(+), 71 deletions(-) diff --git a/python/cudf/cudf/core/column/column.py b/python/cudf/cudf/core/column/column.py index 2d728e59f334..d6e57b96576c 100644 --- a/python/cudf/cudf/core/column/column.py +++ b/python/cudf/cudf/core/column/column.py @@ -1342,7 +1342,7 @@ def all( result = self._reduce( "all", skipna=True, min_count=min_count, **kwargs ) - if np.isnan(result): + if result is pd.NA or np.isnan(result): # Empty after dropping NaN/nulls - return np.bool_ result = np.bool_(True) @@ -1365,9 +1365,19 @@ def any( ) if self.size == 0: return False - if not skipna and (self.has_nulls() or self.nan_count > 0): + is_masked_dtype = is_pandas_nullable_extension_dtype(self.dtype) + if not skipna and ( + self.nan_count > 0 or (not is_masked_dtype and self.has_nulls()) + ): + # NaN values (and the NaN null sentinel of numpy dtypes) are + # truthy. For pandas nullable extension dtypes is not + # truthy; Kleene logic below decides between True and . return True - elif skipna and self.null_count == self.size: + if self.null_count == self.size: + if not skipna: + # All-null nullable column with skipna=False: Kleene + # any([NA, ...]) with no True values is . + return _get_nan_for_dtype(self.dtype) return False # For any(), we want NaN values to be treated as truthy. @@ -1375,10 +1385,20 @@ def any( result = self._reduce( "any", skipna=True, min_count=min_count, **kwargs ) - if np.isnan(result): + if result is pd.NA or np.isnan(result): # Empty after dropping NaN/nulls # If skipna=False, NaN values should be treated as truthy result = np.bool_(not skipna) + + # For pandas nullable extension dtypes with skipna=False, a False + # result in the presence of nulls is under Kleene logic. + if ( + not result + and not skipna + and self.null_count > 0 + and is_masked_dtype + ): + return _get_nan_for_dtype(self.dtype) return result def dropna(self) -> Self: @@ -2960,6 +2980,11 @@ def _reduce( return col_dtype.type(0) if op == "product": return col_dtype.type(1) + if is_pandas_nullable_extension_dtype(self.dtype): + # pandas returns for empty/all-null reductions of + # nullable dtypes even when the reduction result dtype is + # a plain numpy dtype (e.g. Int64.mean() -> float64). + return _get_nan_for_dtype(self.dtype) return _get_nan_for_dtype(col_dtype) # Perform the actual reduction diff --git a/python/cudf/cudf/core/column/string.py b/python/cudf/cudf/core/column/string.py index 13e9e958cc4e..b0e26ea8c3e1 100644 --- a/python/cudf/cudf/core/column/string.py +++ b/python/cudf/cudf/core/column/string.py @@ -231,12 +231,13 @@ def all( self, skipna: bool = True, min_count: int = 0, **kwargs: Any ) -> ScalarLike: """Check if all string values are truthy (non-empty).""" - if skipna and self.null_count == self.size: - return True - elif not skipna and self.has_nulls(): - # pandas 3 treats the NaN null sentinel as truthy, matching - # numpy semantics, so all(skipna=False) returns True when all - # values are null. + if self.null_count == self.size: + # With skipna=True nulls are dropped, so all-null is vacuously + # True. pandas 3 treats the NaN null sentinel as truthy, + # matching numpy semantics, so all(skipna=False) is True too. + # A partially-null column must NOT short-circuit here: the + # result depends on the truthiness of the non-null strings + # (e.g. all([NaN, ""], skipna=False) is False). return True raise NotImplementedError("`all` not implemented for `StringColumn`") diff --git a/python/cudf/cudf/core/series.py b/python/cudf/cudf/core/series.py index 1784f90e527a..92db077ed10f 100644 --- a/python/cudf/cudf/core/series.py +++ b/python/cudf/cudf/core/series.py @@ -2857,8 +2857,22 @@ def mode(self, dropna=True): if len(val_counts) > 0: val_counts = val_counts[val_counts == val_counts.iloc[0]] + # pandas sorts mode results on the underlying representation: + # NaT (INT64_MIN as i8) and the categorical null code (-1) sort + # before valid values, while float NaN and the of + # nullable/arrow dtypes (including arrow timestamps/durations) + # sort last. + na_position = ( + "first" + if ( + self.dtype.kind in "mM" + and isinstance(self.dtype, (np.dtype, pd.DatetimeTZDtype)) + ) + or isinstance(self.dtype, cudf.CategoricalDtype) + else "last" + ) return Series._from_column( - val_counts.index.sort_values()._column, + val_counts.index.sort_values(na_position=na_position)._column, name=self.name, attrs=self.attrs, ) diff --git a/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py b/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py index e43cbe2c0150..d7e5ae881310 100644 --- a/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py +++ b/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py @@ -148,8 +148,6 @@ def pytest_unconfigure(config): "tests/arithmetic/test_timedelta64.py::TestTimedeltaArraylikeMulDivOps::test_td64arr_mul_masked[Series-int64[pyarrow]]": "AssertionError: Attributes of Series are different", "tests/arrays/boolean/test_construction.py::test_coerce_to_array_from_boolean_array": "TODO: Add a reason for failure", "tests/arrays/boolean/test_construction.py::test_to_numpy_copy": "TODO: Add a reason for failure", - "tests/arrays/boolean/test_reduction.py::test_any_all[Series-values1-False-False-exp_any_noskip1-False]": "assert np.True_ is ", - "tests/arrays/boolean/test_reduction.py::test_any_all[Series-values2-False-True-exp_any_noskip2-exp_all_noskip2]": "assert np.True_ is ", "tests/arrays/boolean/test_reduction.py::test_reductions_return_types[False-count]": "AssertionError: vs ", "tests/arrays/boolean/test_reduction.py::test_reductions_return_types[True-count]": "AssertionError: vs ", "tests/arrays/categorical/test_analytics.py::TestCategoricalAnalytics::test_searchsorted[False]": "TODO: Add a reason for failure", @@ -243,10 +241,6 @@ def pytest_unconfigure(config): "tests/arrays/test_datetimes.py::TestDatetimeArray::test_date_range_lowercase_frequency_deprecated": "AssertionError: Index are different", "tests/arrays/test_period.py::test_registered": "TODO: Add a reason for failure", "tests/arrays/timedeltas/test_constructors.py::TestTimedeltaArrayConstructor::test_copy": "TODO: Add a reason for failure", - "tests/arrays/timedeltas/test_reductions.py::TestReductions::test_reductions_empty[False-max]": "TODO: Add a reason for failure", - "tests/arrays/timedeltas/test_reductions.py::TestReductions::test_reductions_empty[False-min]": "TODO: Add a reason for failure", - "tests/arrays/timedeltas/test_reductions.py::TestReductions::test_reductions_empty[True-max]": "TODO: Add a reason for failure", - "tests/arrays/timedeltas/test_reductions.py::TestReductions::test_reductions_empty[True-min]": "TODO: Add a reason for failure", "tests/base/test_conversion.py::test_array[index-arr3-_left]": "TODO: Add a reason for failure", "tests/base/test_conversion.py::test_array[index-arr4-_sparse_values]": "TODO: Add a reason for failure", "tests/base/test_conversion.py::test_array[series-arr4-_sparse_values]": "TODO: Add a reason for failure", @@ -753,10 +747,6 @@ def pytest_unconfigure(config): "tests/extension/test_arrow.py::TestArrowArray::test_reduce_frame[uint32-skew-True]": "cudf skew differs from pandas bias-corrected skew", "tests/extension/test_arrow.py::TestArrowArray::test_reduce_frame[uint64-skew-True]": "cudf skew differs from pandas bias-corrected skew", "tests/extension/test_arrow.py::TestArrowArray::test_reduce_frame[uint8-skew-True]": "cudf skew differs from pandas bias-corrected skew", - "tests/extension/test_arrow.py::TestArrowArray::test_reduce_series_boolean[timestamp[ms]-any-False]": "TODO: Add a reason for failure", - "tests/extension/test_arrow.py::TestArrowArray::test_reduce_series_boolean[timestamp[ns]-any-False]": "TODO: Add a reason for failure", - "tests/extension/test_arrow.py::TestArrowArray::test_reduce_series_boolean[timestamp[s]-any-False]": "TODO: Add a reason for failure", - "tests/extension/test_arrow.py::TestArrowArray::test_reduce_series_boolean[timestamp[us]-any-False]": "TODO: Add a reason for failure", "tests/extension/test_arrow.py::TestArrowArray::test_reduce_series_numeric[bool-kurt-False]": "TODO: Add a reason for failure", "tests/extension/test_arrow.py::TestArrowArray::test_reduce_series_numeric[decimal128(7, 3)-kurt-False]": "TODO: Add a reason for failure", "tests/extension/test_arrow.py::TestArrowArray::test_reduce_series_numeric[double-kurt-False]": "TODO: Add a reason for failure", @@ -801,7 +791,6 @@ def pytest_unconfigure(config): "tests/extension/test_arrow.py::test_arrow_floordiv_integral_invalid[pa_type3]": "Failed: DID NOT RAISE ", "tests/extension/test_arrow.py::test_arrow_string_addition_mixed_string_types": "AssertionError: ColumnBase are different", "tests/extension/test_arrow.py::test_astype_errors_ignore": "TODO: Add a reason for failure", - "tests/extension/test_arrow.py::test_boolean_reduce_series_all_null[any-False]": "TODO: Add a reason for failure", "tests/extension/test_arrow.py::test_comparison_not_propagating_arrow_error": "TODO: Add a reason for failure", "tests/extension/test_arrow.py::test_decimal_parse_raises": "Failed: DID NOT RAISE ", "tests/extension/test_arrow.py::test_dt_tz_localize_nonexistent[shift_backward-exp_date1]": "TODO: Add a reason for failure", @@ -1253,7 +1242,6 @@ def pytest_unconfigure(config): "tests/frame/methods/test_replace.py::TestDataFrameReplace::test_regex_replace_series_of_regexes": "assert a b c\n0 0 a a\n1 1 b b\n2 2 NaN NaN\n3 3 NaN d is a b c\n0 0 a ...", "tests/frame/methods/test_replace.py::TestDataFrameReplace::test_regex_replace_str_to_numeric": "assert a b c\n0 0 a a\n1 1 b b\n2 2 0 NaN\n3 3 0 d is a b c\n0 0 a a\n1 1 b ...", "tests/frame/methods/test_replace.py::TestDataFrameReplace::test_regex_replace_string_types[DataFrame-string=object-data1-to_replace1-expected1]": "AssertionError: DataFrame.iloc[:, 0] (column name='0') are different", - "tests/frame/methods/test_replace.py::TestDataFrameReplace::test_replace_NAT_with_None": "TODO: Add a reason for failure", "tests/frame/methods/test_replace.py::TestDataFrameReplace::test_replace_NA_with_None": "TODO: Add a reason for failure", "tests/frame/methods/test_replace.py::TestDataFrameReplace::test_replace_inplace": "assert A B C D\n2000-01-03 0.0 -0.522748 -0.413064 -2.441467\n2000-01-04 0.0 ...", "tests/frame/methods/test_replace.py::TestDataFrameReplace::test_replace_input_formats_listlike": "assert A B C\n0 -2.0 0 \n1 0.0 2 asdf\n2 -2.0 5 fd is A B C\n0 -2.0 0 \n1 0.0...", @@ -1670,8 +1658,6 @@ def pytest_unconfigure(config): "tests/frame/test_query_eval.py::TestDataFrameQueryPythonPython::test_query_numexpr_with_min_and_max_columns": "TODO: Add a reason for failure", "tests/frame/test_query_eval.py::TestDataFrameQueryWithMultiIndex::test_query_multiindex_get_index_resolvers": "TODO: Add a reason for failure", "tests/frame/test_reductions.py::TestDataFrameAnalytics::test_median": "AssertionError: Attributes of Series are different", - "tests/frame/test_reductions.py::TestDataFrameAnalytics::test_mode_dropna[False-expected3]": "AssertionError: DataFrame.iloc[:, 3] (column name='K') are different", - "tests/frame/test_reductions.py::TestDataFrameReductions::test_min_max_dt64_api_consistency_with_NaT": "TODO: Add a reason for failure", "tests/frame/test_reductions.py::TestDataFrameReductions::test_min_max_dt64_with_NaT_precision": "AssertionError: Series are different", "tests/frame/test_reductions.py::test_reduction_axis_none_returns_scalar[Float64-False-kurt]": "TODO: Add a reason for failure", "tests/frame/test_reductions.py::test_reduction_axis_none_returns_scalar[Float64-False-mean]": "TODO: Add a reason for failure", @@ -1929,8 +1915,6 @@ def pytest_unconfigure(config): "tests/groupby/test_reductions.py::test_basic_aggregations[int32]": "AssertionError: Attributes of Series are different", "tests/groupby/test_reductions.py::test_groupby_mean_no_overflow": "TODO: Add a reason for failure", "tests/groupby/test_reductions.py::test_groupby_sum_mincount_boolean[0]": "TODO: Add a reason for failure", - "tests/groupby/test_reductions.py::test_masked_kleene_logic[any-False-data2]": "AssertionError: Series NA mask are different", - "tests/groupby/test_reductions.py::test_masked_kleene_logic[any-False-data3]": "AssertionError: Series NA mask are different", "tests/groupby/test_reductions.py::test_mean_numeric_only_validates_bool": "Failed: DID NOT RAISE ", "tests/groupby/test_reductions.py::test_multifunc_skipna[True-prod-values3-float64-float64]": "cudf returns NA for an all-null prod; pandas returns the empty-product identity 1.0 (min_count/empty-reduction semantics, not skipna)", "tests/groupby/test_reductions.py::test_nunique_with_NaT[key1-data1-True-expected1]": "TODO: Add a reason for failure", @@ -2821,44 +2805,10 @@ def pytest_unconfigure(config): "tests/plotting/test_datetimelike.py::TestTSPlot::test_from_resampling_area_line_mixed_high_to_low[line-area]": "ValueError: You must pass a freq argument as current index has none.", "tests/plotting/test_datetimelike.py::TestTSPlot::test_from_weekly_resampling": "ValueError: You must pass a freq argument as current index has none.", "tests/plotting/test_datetimelike.py::TestTSPlot::test_to_weekly_resampling": "AssertionError: assert == ", - "tests/reductions/test_reductions.py::TestDatetime64SeriesReductions::test_minmax_nat_series[nat_ser0]": "TODO: Add a reason for failure", - "tests/reductions/test_reductions.py::TestDatetime64SeriesReductions::test_minmax_nat_series[nat_ser1]": "TODO: Add a reason for failure", - "tests/reductions/test_reductions.py::TestDatetime64SeriesReductions::test_minmax_nat_series[nat_ser2]": "TODO: Add a reason for failure", - "tests/reductions/test_reductions.py::TestIndexReductions::test_minmax_timedelta_empty_or_na[max]": "TODO: Add a reason for failure", - "tests/reductions/test_reductions.py::TestIndexReductions::test_minmax_timedelta_empty_or_na[min]": "TODO: Add a reason for failure", - "tests/reductions/test_reductions.py::TestReductions::test_nanminmax[index-datetime64[ns]-val2-max]": "TODO: Add a reason for failure", - "tests/reductions/test_reductions.py::TestReductions::test_nanminmax[index-datetime64[ns]-val2-min]": "TODO: Add a reason for failure", - "tests/reductions/test_reductions.py::TestReductions::test_nanminmax[series-datetime64[ns]-val2-max]": "TODO: Add a reason for failure", - "tests/reductions/test_reductions.py::TestReductions::test_nanminmax[series-datetime64[ns]-val2-min]": "TODO: Add a reason for failure", - "tests/reductions/test_reductions.py::TestReductions::test_nanops_empty_object[index-M8[ns]-max]": "TODO: Add a reason for failure", - "tests/reductions/test_reductions.py::TestReductions::test_nanops_empty_object[index-M8[ns]-min]": "TODO: Add a reason for failure", - "tests/reductions/test_reductions.py::TestReductions::test_nanops_empty_object[series-M8[ns]-max]": "TODO: Add a reason for failure", - "tests/reductions/test_reductions.py::TestReductions::test_nanops_empty_object[series-M8[ns]-min]": "TODO: Add a reason for failure", - "tests/reductions/test_reductions.py::TestSeriesMode::test_mode_category[False-expected11-expected21-expected31]": "TODO: Add a reason for failure", - "tests/reductions/test_reductions.py::TestSeriesMode::test_mode_datetime[False-expected11-expected21]": "TODO: Add a reason for failure", - "tests/reductions/test_reductions.py::TestSeriesMode::test_mode_timedelta[False-expected11-expected21]": "TODO: Add a reason for failure", - "tests/reductions/test_reductions.py::TestSeriesReductions::test_any_all_nullable_kleene_logic[any-False-data2-expected_data2-Float64]": "TypeError: boolean value of NA is ambiguous", - "tests/reductions/test_reductions.py::TestSeriesReductions::test_any_all_nullable_kleene_logic[any-False-data2-expected_data2-Int64]": "TypeError: boolean value of NA is ambiguous", - "tests/reductions/test_reductions.py::TestSeriesReductions::test_any_all_nullable_kleene_logic[any-False-data2-expected_data2-UInt64]": "TypeError: boolean value of NA is ambiguous", - "tests/reductions/test_reductions.py::TestSeriesReductions::test_any_all_nullable_kleene_logic[any-False-data2-expected_data2-boolean]": "TypeError: boolean value of NA is ambiguous", - "tests/reductions/test_reductions.py::TestSeriesReductions::test_any_all_nullable_kleene_logic[any-False-data3-expected_data3-Float64]": "TypeError: boolean value of NA is ambiguous", - "tests/reductions/test_reductions.py::TestSeriesReductions::test_any_all_nullable_kleene_logic[any-False-data3-expected_data3-Int64]": "TypeError: boolean value of NA is ambiguous", - "tests/reductions/test_reductions.py::TestSeriesReductions::test_any_all_nullable_kleene_logic[any-False-data3-expected_data3-UInt64]": "TypeError: boolean value of NA is ambiguous", - "tests/reductions/test_reductions.py::TestSeriesReductions::test_any_all_nullable_kleene_logic[any-False-data3-expected_data3-boolean]": "TypeError: boolean value of NA is ambiguous", - "tests/reductions/test_reductions.py::TestSeriesReductions::test_any_all_object_dtype_missing[any-data0]": "assert np.True_ == False", - "tests/reductions/test_reductions.py::TestSeriesReductions::test_any_all_object_dtype_missing[any-data1]": "assert np.True_ == False", - "tests/reductions/test_reductions.py::TestSeriesReductions::test_any_all_string_dtype[string=str[pyarrow]]": "assert not np.True_", - "tests/reductions/test_reductions.py::TestSeriesReductions::test_any_all_string_dtype[string=str[python]]": "assert not np.True_", - "tests/reductions/test_reductions.py::TestSeriesReductions::test_empty_timeseries_reductions_return_nat[False-M8[ns]]": "TODO: Add a reason for failure", - "tests/reductions/test_reductions.py::TestSeriesReductions::test_empty_timeseries_reductions_return_nat[False-m8[ns]]": "AssertionError: assert np.timedelta64('NaT','ns') is NaT", - "tests/reductions/test_reductions.py::TestSeriesReductions::test_empty_timeseries_reductions_return_nat[True-M8[ns]]": "TODO: Add a reason for failure", - "tests/reductions/test_reductions.py::TestSeriesReductions::test_empty_timeseries_reductions_return_nat[True-m8[ns]]": "AssertionError: assert np.timedelta64('NaT','ns') is NaT", - "tests/reductions/test_reductions.py::TestSeriesReductions::test_ops_consistency_on_empty_nullable[Int64-mean]": "TODO: Add a reason for failure", - "tests/reductions/test_reductions.py::TestSeriesReductions::test_ops_consistency_on_empty_nullable[Int64-var]": "TODO: Add a reason for failure", - "tests/reductions/test_reductions.py::TestSeriesReductions::test_ops_consistency_on_empty_nullable[boolean-mean]": "TODO: Add a reason for failure", - "tests/reductions/test_reductions.py::TestSeriesReductions::test_ops_consistency_on_empty_nullable[boolean-var]": "TODO: Add a reason for failure", - "tests/reductions/test_reductions.py::TestSeriesReductions::test_sum_overflow_float[float32-False]": "TODO: Add a reason for failure", - "tests/reductions/test_reductions.py::TestSeriesReductions::test_sum_overflow_float[float32-True]": "TODO: Add a reason for failure", + "tests/reductions/test_reductions.py::TestSeriesReductions::test_any_all_object_dtype_missing[any-data0]": "cudf cannot distinguish None (falsy) from np.nan (truthy) in object-dtype data; both become nulls, and the NaN-truthy semantics win", + "tests/reductions/test_reductions.py::TestSeriesReductions::test_any_all_object_dtype_missing[any-data1]": "cudf cannot distinguish None (falsy) from np.nan (truthy) in object-dtype data; both become nulls, and the NaN-truthy semantics win", + "tests/reductions/test_reductions.py::TestSeriesReductions::test_sum_overflow_float[float32-False]": "libcudf's parallel tree reduction accumulates float32 sums in a different order than numpy's pairwise summation, so low-order bits differ (cudf's result is closer to the exact sum)", + "tests/reductions/test_reductions.py::TestSeriesReductions::test_sum_overflow_float[float32-True]": "libcudf's parallel tree reduction accumulates float32 sums in a different order than numpy's pairwise summation, so low-order bits differ (cudf's result is closer to the exact sum)", "tests/reductions/test_stat_reductions.py::TestDatetimeLikeStatReductions::test_dt64_mean['Asia/Tokyo'-index]": "AssertionError: assert Timestamp('2001-01-05 15:00:00') == Timestamp('2001-01-06 00:00:00+0900', tz='Asia/Tokyo')", "tests/reductions/test_stat_reductions.py::TestDatetimeLikeStatReductions::test_dt64_mean['Asia/Tokyo'-series]": "AssertionError: assert Timestamp('2001-01-05 15:00:00') == Timestamp('2001-01-06 00:00:00+0900', tz='Asia/Tokyo')", "tests/reductions/test_stat_reductions.py::TestDatetimeLikeStatReductions::test_dt64_mean['US/Eastern'-index]": "AssertionError: assert Timestamp('2001-01-06 05:00:00') == Timestamp('2001-01-06 00:00:00-0500', tz='US/Eastern')", diff --git a/python/cudf/cudf/tests/series/methods/test_mode.py b/python/cudf/cudf/tests/series/methods/test_mode.py index fb546d19ca08..280b87441de1 100644 --- a/python/cudf/cudf/tests/series/methods/test_mode.py +++ b/python/cudf/cudf/tests/series/methods/test_mode.py @@ -1,7 +1,9 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 import numpy as np +import pandas as pd +import pyarrow as pa import pytest import cudf @@ -83,3 +85,31 @@ def test_series_mode(gs, dropna): expected = expected.astype("float64") assert_eq(expected, actual, check_dtype=False) + + +@pytest.mark.parametrize( + "pdata", + [ + pd.Series([1, 1, 2, None, None], dtype="datetime64[ns]"), + pd.Series([1, 1, 2, None, None], dtype="timedelta64[ns]"), + pd.Series( + pd.Categorical( + [1, 1, 2, 3, 3, np.nan, np.nan], + categories=[3, 2, 1], + ordered=True, + ) + ), + pd.Series(pd.Categorical([1, 2, np.nan, np.nan])), + pd.Series([1, 1, None, None], dtype="Int64"), + pd.Series([1.0, 1.0, np.nan, np.nan]), + pd.Series([1, 1, None, None], dtype=pd.ArrowDtype(pa.timestamp("ns"))), + pd.Series([1, 1, None, None], dtype=pd.ArrowDtype(pa.duration("ns"))), + ], +) +def test_mode_null_position_matches_pandas(pdata, dropna): + # pandas sorts mode results on the underlying representation: NaT + # (INT64_MIN as i8) and the categorical null code (-1) sort before + # valid values, while float NaN and the of nullable and arrow + # dtypes (including arrow timestamps/durations) sort last. + gs = cudf.from_pandas(pdata) + assert_eq(pdata.mode(dropna=dropna), gs.mode(dropna=dropna)) diff --git a/python/cudf/cudf/tests/series/methods/test_reductions.py b/python/cudf/cudf/tests/series/methods/test_reductions.py index e1fab510b431..8c76d017dfc2 100644 --- a/python/cudf/cudf/tests/series/methods/test_reductions.py +++ b/python/cudf/cudf/tests/series/methods/test_reductions.py @@ -903,7 +903,7 @@ def test_timedelta_reductions(data, op, timedelta_types_as_str): actual = getattr(sr, op)() expected = getattr(psr, op)() - if np.isnat(expected.to_numpy()) and np.isnat(actual): + if expected is pd.NaT and actual is pd.NaT: assert True else: assert_eq(expected.to_numpy(), actual) @@ -1124,6 +1124,109 @@ def test_datetime_reductions(data, reduction_methods, datetime_types_as_str): assert_eq(expected, actual) +@pytest.mark.parametrize( + "dtype", + [ + "datetime64[ns]", + "datetime64[ms]", + "timedelta64[ns]", + "timedelta64[s]", + ], +) +@pytest.mark.parametrize("data", [[], [None, None]]) +@pytest.mark.parametrize("op", ["min", "max"]) +def test_temporal_reduction_all_null_returns_nat_singleton(dtype, data, op): + # Reductions with no valid values return the pd.NaT singleton + # (identity, matching pandas), not a unit-qualified numpy NaT. + psr = pd.Series(data, dtype=dtype) + sr = cudf.Series(psr) + expected = getattr(psr, op)() + assert expected is pd.NaT + assert getattr(sr, op)() is expected + assert getattr(cudf.Index(sr), op)() is getattr(pd.Index(psr), op)() + + +@pytest.mark.parametrize("dtype", ["Int64", "UInt32", "Float64", "boolean"]) +@pytest.mark.parametrize("data", [[], [None, None]]) +@pytest.mark.parametrize("op", ["mean", "var", "std", "min", "max"]) +def test_masked_reduction_all_null_returns_na(dtype, data, op): + # pandas returns for empty/all-null reductions of nullable + # dtypes even when the reduction result dtype is a plain numpy dtype + # (e.g. Int64.mean() -> float64). + psr = pd.Series(data, dtype=dtype) + sr = cudf.Series(psr) + expected = getattr(psr, op)() + assert expected is pd.NA + assert getattr(sr, op)() is expected + + +@pytest.mark.parametrize("dtype", ["boolean", "Int64", "UInt64", "Float64"]) +@pytest.mark.parametrize( + "data", + [ + [0, 0, 0], + [1, 1, 1], + [pd.NA, pd.NA, pd.NA], + [0, pd.NA, 0], + [1, pd.NA, 1], + [1, pd.NA, 0], + ], +) +@pytest.mark.parametrize("op", ["any", "all"]) +def test_any_all_masked_kleene_logic(dtype, data, op, skipna): + # Kleene logic for nullable dtypes (pandas GH-37506/GH-41967): with + # skipna=False a result that would flip if the nulls were filled is + # ; with skipna=True nulls are simply dropped. + psr = pd.Series(pd.array(data, dtype=dtype)) + sr = cudf.Series(psr) + expected = getattr(psr, op)(skipna=skipna) + result = getattr(sr, op)(skipna=skipna) + if expected is pd.NA: + assert result is pd.NA + else: + assert result == expected + + +@pytest.mark.parametrize("data", [[np.nan, 0.0], [np.nan, 1.0]]) +def test_any_all_numpy_float_nan_sentinel_truthy(data): + # For numpy dtypes the NaN null sentinel is truthy with skipna=False + # (numpy semantics), unlike the Kleene of nullable dtypes. + sr = cudf.Series(data) + psr = pd.Series(data) + for skipna in [True, False]: + assert sr.any(skipna=skipna) == psr.any(skipna=skipna) + assert sr.all(skipna=skipna) == psr.all(skipna=skipna) + + +def test_any_all_masked_with_true_nan_values(): + # A nullable float column holding actual NaN values (not ): NaN + # is a truthy value, so all() over the remaining values is True and + # any(skipna=False) short-circuits to True. + psr = pd.Series( + pd.arrays.FloatingArray( + np.array([np.nan, np.nan]), np.array([False, False]) + ) + ) + sr = cudf.Series([np.nan, np.nan], nan_as_null=False, dtype="Float64") + assert sr.all() == psr.all() + assert sr.all(skipna=False) == psr.all(skipna=False) + assert sr.any(skipna=False) == psr.any(skipna=False) + + +def test_string_any_all_skipna_false_partial_nulls(): + # cudf strings follow the pandas-3 "str" dtype (NaN null sentinel): + # with skipna=False the sentinel is truthy, so any() is True, but + # all() depends on the truthiness of the non-null strings and is not + # computed natively (it must not blanket-return True for + # partially-null columns). + psr = pd.Series(["", None], dtype=pd.StringDtype(na_value=np.nan)) + sr = cudf.Series(["", None]) + assert sr.any(skipna=False) == psr.any(skipna=False) + assert not psr.all(skipna=False) + with pytest.raises(NotImplementedError): + sr.all(skipna=False) + + @pytest.mark.parametrize("op", ["min", "max"]) def test_categorical_maxima(op): ser = cudf.Series( diff --git a/python/cudf/cudf/utils/dtypes.py b/python/cudf/cudf/utils/dtypes.py index 75e9b0ffbf39..8322926040f0 100644 --- a/python/cudf/cudf/utils/dtypes.py +++ b/python/cudf/cudf/utils/dtypes.py @@ -273,12 +273,15 @@ def _get_nan_for_dtype(dtype: DtypeObj) -> ScalarLike: """Return the appropriate NaN/NaT value for the given dtype. Returns the null value for the dtype (e.g., np.float64('nan'), - np.datetime64('NaT'), or the dtype's ``na_value`` for pandas - nullable extension dtypes). + pd.NaT, or the dtype's ``na_value`` for pandas nullable extension + dtypes). """ if dtype.kind in "mM": - time_unit, _ = np.datetime_data(dtype) - return dtype.type("nat", time_unit) + # pandas datetime/timedelta reductions return the pd.NaT + # singleton for null results, and callers rely on identity + # (``result is pd.NaT``), not a unit-qualified + # np.datetime64/np.timedelta64 "NaT". + return pd.NaT elif dtype.kind == "f": if is_pandas_nullable_extension_dtype(dtype): return dtype.na_value From 82edf97f8293def566e7d0545b2de3e1c6985b17 Mon Sep 17 00:00:00 2001 From: Yunsong Wang <12716979+PointKernel@users.noreply.github.com> Date: Mon, 20 Jul 2026 19:25:24 -0700 Subject: [PATCH 17/25] Fix incorrect DECIMAL128 groupby sums on Blackwell with CUDA 13.0 (#23229) Closes #23150. This PR fixes incorrect `DECIMAL128` groupby sums on Blackwell with CUDA 13.0, where nvcc miscompiles the native 128-bit `atomicCAS` in `atomic_add(__int128_t*, __int128_t)` on `sm_100` and newer. The native path is now disabled for that configuration only. Falling back to the existing two-word path alone did not help: its per-word `cuda::atomic_ref` CAS loops hit the same miscompilation. The fallback now uses two native 64-bit `atomicAdd` operations with carry propagation, avoiding CAS lowering entirely. As before, the two words are not one indivisible transaction, which is fine for commutative sum updates. CUDA 13.3 and Hopper are unaffected and keep the native path. Verified on B200 with CUDA 13.0: the new regression test fails without the fix and passes with it, and the issue's pylibcudf reproducer matches the Polars CPU reference over repeated runs. Authors: - Yunsong Wang (https://github.com/PointKernel) Approvers: - Basit Ayantunde (https://github.com/lamarrr) - David Wendt (https://github.com/davidwendt) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cudf/pull/23229 --- .../cudf/detail/utilities/device_atomics.cuh | 40 +++++--------- cpp/tests/groupby/sum_tests.cpp | 54 ++++++++++++++++++- 2 files changed, 67 insertions(+), 27 deletions(-) diff --git a/cpp/include/cudf/detail/utilities/device_atomics.cuh b/cpp/include/cudf/detail/utilities/device_atomics.cuh index 5821548a5702..f84a18118da5 100644 --- a/cpp/include/cudf/detail/utilities/device_atomics.cuh +++ b/cpp/include/cudf/detail/utilities/device_atomics.cuh @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -356,9 +356,9 @@ __device__ __forceinline__ uint64_t calculate_carry_64(uint64_t old_val, /** * @brief Atomic addition for __int128_t with architecture-specific optimization * - * Uses native 128-bit CAS on Hopper+ GPUs (compute capability 9.0+) for optimal - * performance. Falls back to two 64-bit atomic CAS operations with carry propagation - * on older GPU architectures. + * Uses native 128-bit CAS on Hopper+ GPUs for optimal performance, except for CUDA 13.0 on + * Blackwell. Other configurations fall back to two 64-bit atomic additions with carry + * propagation. * * @param address Pointer to the __int128_t value * @param val Value to add @@ -366,7 +366,10 @@ __device__ __forceinline__ uint64_t calculate_carry_64(uint64_t old_val, */ __forceinline__ __device__ __int128_t atomic_add(__int128_t* address, __int128_t val) { -#if __CUDA_ARCH__ >= 900 + // CUDA 13.0 miscompiles the native 128-bit CAS on Blackwell; the fix is confirmed in 13.3, so + // gate off all 13.x before 13.3. See https://github.com/rapidsai/cudf/issues/23150. +#if __CUDA_ARCH__ >= 900 && \ + !(__CUDA_ARCH__ >= 1000 && __CUDACC_VER_MAJOR__ == 13 && __CUDACC_VER_MINOR__ < 3) __int128_t expected, desired; do { @@ -380,30 +383,15 @@ __forceinline__ __device__ __int128_t atomic_add(__int128_t* address, __int128_t __uint128_t const add_val_unsigned = static_cast<__uint128_t>(val); // Split the 128-bit add value into two 64-bit parts - uint64_t const add_low = static_cast(add_val_unsigned); - uint64_t const add_high = static_cast(add_val_unsigned >> 64); + auto const add_low = static_cast(add_val_unsigned); + auto const add_high = static_cast(add_val_unsigned >> 64); - uint64_t carry = 0; uint64_t old_parts[2]; - auto atomic_add_word = [&](int i, uint64_t current_add) { - uint64_t expected_part, new_part; - cuda::atomic_ref atomic_part{target_ptr[i]}; + old_parts[0] = atomicAdd(reinterpret_cast(target_ptr), add_low); + auto const carry = calculate_carry_64(old_parts[0], add_low, 0); + old_parts[1] = atomicAdd(reinterpret_cast(target_ptr + 1), add_high + carry); - do { - expected_part = atomic_part.load(); - new_part = expected_part + current_add + carry; - } while ( - !atomic_part.compare_exchange_weak(expected_part, new_part, cuda::memory_order_relaxed)); - - old_parts[i] = expected_part; - carry = calculate_carry_64(expected_part, current_add, carry); - }; - - atomic_add_word(0, add_low); - atomic_add_word(1, add_high); - - __uint128_t const old_val_unsigned = - (static_cast<__uint128_t>(old_parts[1]) << 64) | old_parts[0]; + auto const old_val_unsigned = (static_cast<__uint128_t>(old_parts[1]) << 64) | old_parts[0]; return static_cast<__int128_t>(old_val_unsigned); #endif } diff --git a/cpp/tests/groupby/sum_tests.cpp b/cpp/tests/groupby/sum_tests.cpp index b6b16669a866..ef72f3f40742 100644 --- a/cpp/tests/groupby/sum_tests.cpp +++ b/cpp/tests/groupby/sum_tests.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -283,3 +283,55 @@ TEST_F(GroupByDecimal128ShmemAlignmentTest, Decimal128SumAfterInt32Sum) auto const expected = fp128{{2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2}, scale}; CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(expected, sorted_result->get_column(0)); } + +// Regression test for https://github.com/rapidsai/cudf/issues/23150. +// Blackwell returned incorrect sums when aggregating three DECIMAL128 columns. +TEST_F(GroupByDecimal128ShmemAlignmentTest, MultiColumnDecimal128Sum) +{ + using namespace numeric; + using fp128 = cudf::test::fixed_point_column_wrapper<__int128_t>; + + constexpr int num_cols = 3; + constexpr cudf::size_type num_rows = 1'000'000; + constexpr int num_groups = 4; + auto const scale = scale_type{-2}; + + // A large base makes each group's running sum cross the 2^64 low-word boundary, exercising + // carry propagation in the fallback, and every fifth row is negated to exercise borrow. + constexpr __int128_t base = static_cast<__int128_t>(1) << 50; + + std::vector keys_data(num_rows); + std::vector> vals_data(num_cols, std::vector<__int128_t>(num_rows)); + std::vector> sums(num_cols, std::vector<__int128_t>(num_groups, 0)); + for (cudf::size_type i = 0; i < num_rows; ++i) { + auto const k = i % num_groups; + keys_data[i] = k; + for (int c = 0; c < num_cols; ++c) { + auto v = base + static_cast<__int128_t>((100 + i % 7) * 100 + (13 * c + i) % 100); + if (i % 5 == 0) { v = -v; } + vals_data[c][i] = v; + sums[c][k] += v; + } + } + + auto const keys = + cudf::test::fixed_width_column_wrapper(keys_data.begin(), keys_data.end()); + std::vector vals; + std::vector requests(num_cols); + for (int c = 0; c < num_cols; ++c) { + vals.emplace_back(vals_data[c].begin(), vals_data[c].end(), scale); + requests[c].values = vals[c]; + requests[c].aggregations.push_back(cudf::make_sum_aggregation()); + } + + cudf::groupby::groupby gb(cudf::table_view({keys})); + auto [result_keys, results] = gb.aggregate(requests); + + auto const sort_order = cudf::sorted_order(result_keys->view()); + for (int c = 0; c < num_cols; ++c) { + auto const sorted = + cudf::gather(cudf::table_view({results[c].results[0]->view()}), *sort_order); + auto const expected = fp128(sums[c].begin(), sums[c].end(), scale); + CUDF_TEST_EXPECT_COLUMNS_EQUIVALENT(expected, sorted->get_column(0)); + } +} From 50e9c31047dfe0a2268e2c9cd362041e7f9077c4 Mon Sep 17 00:00:00 2001 From: David Wendt <45795991+davidwendt@users.noreply.github.com> Date: Mon, 20 Jul 2026 22:33:02 -0400 Subject: [PATCH 18/25] Fix null_precedence access in preprocessed_table::create (#23238) Change `transform_lists_of_structs` call to pass `new_null_precedence_lhs[col_idx]` instead of `null_precedence[col_idx]`. The two-table `preprocessed_table::create` overload loops over the decomposed columns (post-struct-flattening) but indexes into the original `null_precedence` span. When a struct column like `struct` is decomposed into 3 linear columns, the `col_idx` is [1,2] but `null_precedence` has only 1 element which causes an OOB access. This change now also matches the single-table version. This error was produced in a debug build test run since the `cuda::std::span` operator performs an `assert` there. Authors: - David Wendt (https://github.com/davidwendt) Approvers: - Tianyu Liu (https://github.com/kingcrimsontianyu) - Muhammad Haseeb (https://github.com/mhaseeb123) - Yunsong Wang (https://github.com/PointKernel) URL: https://github.com/rapidsai/cudf/pull/23238 --- cpp/src/row_operator/row_operators.cu | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/src/row_operator/row_operators.cu b/cpp/src/row_operator/row_operators.cu index 4b21d9c980cb..698b184abef8 100644 --- a/cpp/src/row_operator/row_operators.cu +++ b/cpp/src/row_operator/row_operators.cu @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -753,7 +753,7 @@ preprocessed_table::create(table_view const& lhs, transform_lists_of_structs( lhs_col, rhs_col, - null_precedence.empty() ? null_order::BEFORE : null_precedence[col_idx], + null_precedence.empty() ? null_order::BEFORE : new_null_precedence_lhs[col_idx], stream, cudf::get_current_device_resource_ref()); From fbd7bde9a2f83a95817be3a09867efee629f6e8e Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Mon, 20 Jul 2026 22:08:38 -0500 Subject: [PATCH 19/25] Route GroupBy.transform size/cumcount/ngroup to their group-level implementations (#23297) `transform` funneled every string aggregation through `agg(func)`, but `size`, `cumcount`, and `ngroup` count group rows rather than aggregating each value column: pandas returns a single Series for them (unnamed for DataFrameGroupBy, keeping the source name for SeriesGroupBy; `ngroup` previously raised `AttributeError` in cudf). Route them to the existing `GroupBy.size/cumcount/ngroup` methods, and neutralize `as_index` inside `transform`, matching pandas where `as_index` has no effect on transform results (pandas GH#49834). Fixes 8 pandas-tests (`test_as_index_no_change[size-*]`, `test_null_group_str_reducer[*-size]`, `test_null_group_str_transformer[*-cumcount]`, `test_transform_cumcount`, `test_transform_transformation_func[cumcount]`); their xfail entries are removed. Note: `test_transform_numeric_ret[size-cols1-expected1]` needs both this fix and #23296; its xfail entry stays until both are merged. Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Matthew Roeschke (https://github.com/mroeschke) - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/23297 --- ci/test_narwhals.sh | 7 ++- python/cudf/cudf/core/groupby/groupby.py | 30 +++++++++++-- .../pandas/scripts/pandas-testing-plugin.py | 17 ------- .../cudf/cudf/tests/groupby/test_transform.py | 44 ++++++++++++++++++- 4 files changed, 75 insertions(+), 23 deletions(-) diff --git a/ci/test_narwhals.sh b/ci/test_narwhals.sh index 13c30f178279..26a123cc3f7c 100755 --- a/ci/test_narwhals.sh +++ b/ci/test_narwhals.sh @@ -61,8 +61,13 @@ test_get_level \ " # test_dtypes: narwhals' dtype mapping changed with polars 1.40 (reports Object where the test expects Int8). +# test_len_over_2369: cudf now routes transform("size") to the group-level size +# (https://github.com/rapidsai/cudf/issues/18491 is fixed), so narwhals' own +# xfail for the cudf constructor is stale and the test strict-XPASSes; deselect +# until narwhals removes the xfail. TESTS_THAT_NEED_NARWHALS_FIX_FOR_CUDF=" \ test_dtypes or \ +test_len_over_2369[cudf] or \ test_to_numpy[cudf] or \ test_fill_null_strategies_with_limit_as_none[cudf] or \ test_fill_null_series_limit_as_none[cudf] or \ @@ -163,7 +168,6 @@ rapids-logger "Run narwhals tests for cuDF Pandas" # test_is_finite_expr & test_is_finite_series: https://github.com/rapidsai/cudf/issues/18257 # test_maybe_convert_dtypes_pandas: https://github.com/rapidsai/cudf/issues/14149 # test_log_dtype_pandas: cudf is promoting the type to float64 -# test_len_over_2369: It fails during fallback. The error is 'DataFrame' object has no attribute 'to_frame' # test_all_ignore_nulls, test_allh_kleene, and test_anyh_kleene: https://github.com/rapidsai/cudf/issues/19417 # test_offset_by_date_pandas: https://github.com/rapidsai/cudf/issues/19418 # test_select_boolean_cols and test_select_boolean_cols_multi_group_by: https://github.com/rapidsai/cudf/issues/19421 @@ -173,7 +177,6 @@ test_is_finite_expr or \ test_is_finite_series or \ test_maybe_convert_dtypes_pandas or \ test_log_dtype_pandas or \ -test_len_over_2369 or \ test_all_ignore_nulls or \ test_allh_kleene or \ test_anyh_kleene or \ diff --git a/python/cudf/cudf/core/groupby/groupby.py b/python/cudf/cudf/core/groupby/groupby.py index e6cdae9c37f4..b79dc5a85d3b 100644 --- a/python/cudf/cudf/core/groupby/groupby.py +++ b/python/cudf/cudf/core/groupby/groupby.py @@ -2748,7 +2748,12 @@ def _broadcast(self, values: Series) -> Series: values = values._align_to_index( self.grouping.keys, how="right", allow_non_unique=True ) - values.index = self.obj.index + # Even when no alignment is needed (every group is a single row, + # so the aggregated index already equals the group keys), the + # result must be indexed like the input rows, not the group + # labels (pandas GH#9941: transform returns an obj-indexed + # result). + values.index = self.obj.index return values @_performance_tracking @@ -2809,8 +2814,27 @@ def transform( raise TypeError( "Aggregation must be a named aggregation or a callable" ) + gb = self + if not self._as_index: + # as_index has no effect on transform in pandas (GH#49834): + # the key-column reset that agg/size apply for as_index=False + # must not leak into the broadcast result. + gb = copy.copy(self) + gb._as_index = True + if func == "size": + # size counts group rows rather than aggregating each value + # column, so pandas broadcasts GroupBy.size() as a single + # Series (unnamed for DataFrameGroupBy, keeping the source + # name for SeriesGroupBy) instead of going per-column. + return gb._broadcast(gb.size()) + if func == "cumcount": + # cumcount numbers the rows of each group: always an unnamed + # Series over the original index, never a per-column result. + return gb.cumcount() + if func == "ngroup": + return gb.ngroup() try: - result = self.agg(func) + result = gb.agg(func) except TypeError as e: raise NotImplementedError( "Currently, `transform()` supports only aggregations." @@ -2822,7 +2846,7 @@ def transform( "Unexpected result length for scan transform" ) return result - return self._broadcast(result) + return gb._broadcast(result) def rolling(self, *args, **kwargs): """ diff --git a/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py b/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py index d7e5ae881310..7e8c65c7b680 100644 --- a/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py +++ b/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py @@ -1831,7 +1831,6 @@ def pytest_unconfigure(config): "tests/groupby/test_apply.py::test_apply_with_date_in_multiindex_does_not_convert_to_timestamp": "cudf stores datetime.date values as datetime64; the date type identity is lost on the GPU round trip", "tests/groupby/test_apply.py::test_positional_slice_groups_datetimelike": "the frame and its column Series are converted to pandas independently on fallback, losing the CoW block identity pandas' is_in_obj grouper check requires", "tests/groupby/test_categorical.py::test_describe_categorical_columns": "cudf's multi-level groupby aggregation and stack() drop the categorical column-index dtype", - "tests/groupby/test_counting.py::TestCounting::test_ngroup_distinct": "TODO: Add a reason for failure", "tests/groupby/test_cumulative.py::test_groupby_cumprod_nan_influences_other_columns": "TODO: Add a reason for failure", "tests/groupby/test_cumulative.py::test_numpy_compat[cumprod]": "TODO: Add a reason for failure", "tests/groupby/test_cumulative.py::test_numpy_compat[cumsum]": "TODO: Add a reason for failure", @@ -1920,27 +1919,11 @@ def pytest_unconfigure(config): "tests/groupby/test_reductions.py::test_nunique_with_NaT[key1-data1-True-expected1]": "TODO: Add a reason for failure", "tests/groupby/test_reductions.py::test_nunique_with_timegrouper": "TODO: Add a reason for failure", "tests/groupby/test_reductions.py::test_sum_skipna_object[False]": "Inherent cudf.pandas None-vs-NaN difference for object-dtype null (skipna logic is correct)", - "tests/groupby/test_timegrouper.py::TestGroupBy::test_groupby_datetime64_32_bit": "TODO: Add a reason for failure", "tests/groupby/test_timegrouper.py::TestGroupBy::test_groupby_with_timegrouper": "TODO: Add a reason for failure", "tests/groupby/test_timegrouper.py::TestGroupBy::test_scalar_call_versus_list_call": "TODO: Add a reason for failure", - "tests/groupby/transform/test_transform.py::test_as_index_no_change[size-A]": "TODO: Add a reason for failure", - "tests/groupby/transform/test_transform.py::test_as_index_no_change[size-keys1]": "TODO: Add a reason for failure", "tests/groupby/transform/test_transform.py::test_cython_transform_series[cumprod-args0-]": "TODO: Add a reason for failure", "tests/groupby/transform/test_transform.py::test_cython_transform_series[cumsum-args1-]": "TODO: Add a reason for failure", - "tests/groupby/transform/test_transform.py::test_groupby_transform_timezone_column[first]": "AssertionError: Attributes of DataFrame.iloc[:, 2] (column name='max_end_time') are different", - "tests/groupby/transform/test_transform.py::test_groupby_transform_timezone_column[last]": "AssertionError: Attributes of DataFrame.iloc[:, 2] (column name='max_end_time') are different", "tests/groupby/transform/test_transform.py::test_nan_in_cumsum_group_label": "AssertionError: Attributes of Series are different", - "tests/groupby/transform/test_transform.py::test_null_group_str_reducer[False-size]": "TODO: Add a reason for failure", - "tests/groupby/transform/test_transform.py::test_null_group_str_reducer[True-size]": "TODO: Add a reason for failure", - "tests/groupby/transform/test_transform.py::test_null_group_str_transformer[False-cumcount]": "TODO: Add a reason for failure", - "tests/groupby/transform/test_transform.py::test_null_group_str_transformer[True-cumcount]": "TODO: Add a reason for failure", - "tests/groupby/transform/test_transform.py::test_transform_cumcount": "TODO: Add a reason for failure", - "tests/groupby/transform/test_transform.py::test_transform_fast": "TODO: Add a reason for failure", - "tests/groupby/transform/test_transform.py::test_transform_numeric_ret[count-a-expected0]": "TODO: Add a reason for failure", - "tests/groupby/transform/test_transform.py::test_transform_numeric_ret[count-cols1-expected1]": "TODO: Add a reason for failure", - "tests/groupby/transform/test_transform.py::test_transform_numeric_ret[size-a-expected0]": "TODO: Add a reason for failure", - "tests/groupby/transform/test_transform.py::test_transform_numeric_ret[size-cols1-expected1]": "TODO: Add a reason for failure", - "tests/groupby/transform/test_transform.py::test_transform_transformation_func[cumcount]": "TODO: Add a reason for failure", "tests/indexes/base_class/test_reshape.py::TestReshape::test_insert_missing[Decimal]": "TODO: Add a reason for failure", "tests/indexes/categorical/test_astype.py::TestAstype::test_categorical_date_roundtrip[False]": "TODO: Add a reason for failure", "tests/indexes/categorical/test_astype.py::TestAstype::test_categorical_date_roundtrip[True]": "TODO: Add a reason for failure", diff --git a/python/cudf/cudf/tests/groupby/test_transform.py b/python/cudf/cudf/tests/groupby/test_transform.py index 357cfb71b1ce..81009acc33cf 100644 --- a/python/cudf/cudf/tests/groupby/test_transform.py +++ b/python/cudf/cudf/tests/groupby/test_transform.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 import itertools @@ -77,3 +77,45 @@ def test_groupby_transform_maintain_index(by): assert_groupby_results_equal( pdf.groupby(by).transform("max"), gdf.groupby(by).transform("max") ) + + +def test_transform_size_returns_series(): + # pandas broadcasts GroupBy.size() as a single Series: unnamed for + # DataFrameGroupBy, keeping the source name for SeriesGroupBy. + pdf = pd.DataFrame({"A": [1, 1, 2], "B": [4.0, 5.0, 6.0]}) + gdf = cudf.DataFrame(pdf) + + assert_eq( + pdf.groupby("A").transform("size"), + gdf.groupby("A").transform("size"), + ) + assert_eq( + pdf.groupby("A")["B"].transform("size"), + gdf.groupby("A")["B"].transform("size"), + ) + + +@pytest.mark.parametrize("as_index", [True, False]) +def test_transform_as_index_no_change(as_index): + # as_index has no effect on transform (pandas GH#49834) + pdf = pd.DataFrame({"A": [1, 1, 2], "B": [4, 5, 6]}) + gdf = cudf.DataFrame(pdf) + + expect = pdf.groupby("A", as_index=as_index).transform("size") + got = gdf.groupby("A", as_index=as_index).transform("size") + + assert_eq(expect, got) + + +def test_transform_cumcount_series(dropna): + # transform("cumcount") is an unnamed Series over the original index, + # never a per-value-column result + pdf = pd.DataFrame( + {"A": [1, 1, None, 2], "B": [4.0, 5.0, 6.0, 7.0]}, + index=[3, 2, 1, 0], + ) + gdf = cudf.DataFrame(pdf) + + expect = pdf.groupby("A", dropna=dropna).transform("cumcount") + got = gdf.groupby("A", dropna=dropna).transform("cumcount") + assert_eq(expect, got) From f1eb7d8023a2466049ef49eca560e9168cbf06c1 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Tue, 21 Jul 2026 09:15:22 -0500 Subject: [PATCH 20/25] Recognize named-aggregation lambdas as scans in GroupBy (#23300) A lambda like `lambda x: x.cumsum()` resolves through `make_aggregation`'s `op(Aggregation)` protocol, where `Aggregation.cumsum` is an alias of `sum`: the scan/reduction distinction exists only in the aggregation name. `_is_all_scan_aggregate` identified scans via `__name__` (`""`), so such transforms silently computed group totals and broadcast them instead of scanning. Probe callables with a name-recording stand-in mirroring the `op(Aggregation)` protocol; true UDFs raise inside the probe and fall back to `__name__` as before. Fixes 2 pandas-tests (`test_cython_transform_series[cumsum/cumprod-]`); their xfail entries are removed. Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/23300 --- python/cudf/cudf/core/groupby/groupby.py | 23 ++++++++++++++++++- .../pandas/scripts/pandas-testing-plugin.py | 2 -- .../cudf/cudf/tests/groupby/test_transform.py | 12 ++++++++++ 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/python/cudf/cudf/core/groupby/groupby.py b/python/cudf/cudf/core/groupby/groupby.py index b79dc5a85d3b..b826500b77fa 100644 --- a/python/cudf/cudf/core/groupby/groupby.py +++ b/python/cudf/cudf/core/groupby/groupby.py @@ -215,7 +215,28 @@ def _is_all_scan_aggregate(all_aggs: list[list[str]]) -> bool: } def get_name(agg): - return agg.__name__ if callable(agg) else agg + if not callable(agg): + return agg + if agg is not list: + # A ``lambda x: x.cumsum()``-style aggregation carries its + # scan-ness only in the aggregation name it resolves to + # (``Aggregation.cumsum`` is an alias of ``sum``; libcudf + # separates scan from reduction by the *call*, not the + # aggregation object). Probe the callable with a + # name-recording stand-in mirroring ``make_aggregation``'s + # ``op(Aggregation)`` protocol; true UDFs raise inside the + # probe and fall back to ``__name__``. + class _NameProbe: + def __getattr__(self, name): + return lambda *args, **kwargs: name + + try: + name = agg(_NameProbe()) + except Exception: + return agg.__name__ + if isinstance(name, str): + return name + return agg.__name__ all_scan = all( get_name(agg_name) in groupby_scans diff --git a/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py b/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py index 7e8c65c7b680..f09d75e33b39 100644 --- a/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py +++ b/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py @@ -1921,8 +1921,6 @@ def pytest_unconfigure(config): "tests/groupby/test_reductions.py::test_sum_skipna_object[False]": "Inherent cudf.pandas None-vs-NaN difference for object-dtype null (skipna logic is correct)", "tests/groupby/test_timegrouper.py::TestGroupBy::test_groupby_with_timegrouper": "TODO: Add a reason for failure", "tests/groupby/test_timegrouper.py::TestGroupBy::test_scalar_call_versus_list_call": "TODO: Add a reason for failure", - "tests/groupby/transform/test_transform.py::test_cython_transform_series[cumprod-args0-]": "TODO: Add a reason for failure", - "tests/groupby/transform/test_transform.py::test_cython_transform_series[cumsum-args1-]": "TODO: Add a reason for failure", "tests/groupby/transform/test_transform.py::test_nan_in_cumsum_group_label": "AssertionError: Attributes of Series are different", "tests/indexes/base_class/test_reshape.py::TestReshape::test_insert_missing[Decimal]": "TODO: Add a reason for failure", "tests/indexes/categorical/test_astype.py::TestAstype::test_categorical_date_roundtrip[False]": "TODO: Add a reason for failure", diff --git a/python/cudf/cudf/tests/groupby/test_transform.py b/python/cudf/cudf/tests/groupby/test_transform.py index 81009acc33cf..6d4902813b61 100644 --- a/python/cudf/cudf/tests/groupby/test_transform.py +++ b/python/cudf/cudf/tests/groupby/test_transform.py @@ -119,3 +119,15 @@ def test_transform_cumcount_series(dropna): expect = pdf.groupby("A", dropna=dropna).transform("cumcount") got = gdf.groupby("A", dropna=dropna).transform("cumcount") assert_eq(expect, got) + + +def test_transform_scan_lambda(): + # a named-aggregation lambda resolving to a scan must scan per group, + # not broadcast the group total + pdf = pd.DataFrame({"key": [0, 0, 1, 1], "val": [1.0, 2.0, 3.0, 4.0]}) + gdf = cudf.DataFrame(pdf) + + expect = pdf.groupby("key")["val"].transform(lambda x: x.cumsum()) + got = gdf.groupby("key")["val"].transform(lambda x: x.cumsum()) + + assert_eq(expect, got) From bf8107f16acdb829fe1fd2f3da9a9a347dcc308e Mon Sep 17 00:00:00 2001 From: paul-aiyedun <53453937+paul-aiyedun@users.noreply.github.com> Date: Tue, 21 Jul 2026 08:46:13 -0700 Subject: [PATCH 21/25] Add cuDF JAR build support for all Maven classifiers (#23261) * Split the JAR build into three composable stages (static libcudf build, per-classifier JAR packaging, and Maven-repo gather) so each stage is independently runnable in CI and locally. * Link against a static libcudf built from source per CUDA version rather than a conda shared libcudf. * Emit the Maven classifier based on the host architecture the build runs on, introducing a new `-arm64` suffix to distinguish aarch64 JARs from their x86_64 counterparts. * Add `test_java_build_local.sh` as a one-command local reproducer of the full CI matrix for the host arch, with per-step timings and GPU compute-capability auto-detection. Contributes to https://github.com/rapidsai/cudf/issues/22204 Authors: - https://github.com/paul-aiyedun Approvers: - Mike Sarahan (https://github.com/msarahan) - Tim Liu (https://github.com/NvTimLiu) URL: https://github.com/rapidsai/cudf/pull/23261 --- .github/workflows/build.yaml | 70 +++++ dependencies.yaml | 24 ++ java/ci/README.md | 109 ++++++- java/ci/argparse.sh | 36 +++ java/ci/assemble_maven_repo.sh | 180 +++++++++++ java/ci/build_cudf_java_jar.sh | 246 +++++++++++++++ java/ci/build_cudf_java_jar_in_container.sh | 121 +++++++ java/ci/build_static_libcudf.sh | 138 ++++++++ java/ci/build_static_libcudf_in_container.sh | 95 ++++++ java/ci/test_java_build_local.sh | 315 +++++++++++++++++++ java/pom.xml | 7 + 11 files changed, 1327 insertions(+), 14 deletions(-) create mode 100644 java/ci/argparse.sh create mode 100755 java/ci/assemble_maven_repo.sh create mode 100755 java/ci/build_cudf_java_jar.sh create mode 100755 java/ci/build_cudf_java_jar_in_container.sh create mode 100755 java/ci/build_static_libcudf.sh create mode 100755 java/ci/build_static_libcudf_in_container.sh create mode 100755 java/ci/test_java_build_local.sh diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 97c3add68349..1e166c0a97ac 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -471,3 +471,73 @@ jobs: with: push: true cuda: '["12.9", "13.3"]' + # Build the cuDF Java JAR for every Maven classifier. + java-build: + needs: [telemetry-setup] + strategy: + fail-fast: false + matrix: + include: + - { cuda: "12.9", cuda_major: "12", arch: "x86_64", runner: "linux-amd64-cpu16" } + - { cuda: "13.3", cuda_major: "13", arch: "x86_64", runner: "linux-amd64-cpu16" } + - { cuda: "12.9", cuda_major: "12", arch: "aarch64", runner: "linux-arm64-cpu16" } + - { cuda: "13.3", cuda_major: "13", arch: "aarch64", runner: "linux-arm64-cpu16" } + runs-on: ${{ matrix.runner }} + permissions: + contents: read + steps: + - name: Checkout code repo + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + ref: ${{ inputs.sha }} + fetch-depth: 0 + persist-credentials: false + - name: Build static libcudf + run: | + ./java/ci/build_static_libcudf.sh \ + --output-dir "${RUNNER_TEMP}/libcudf" \ + --cuda-version "${{ matrix.cuda }}" + - name: Build cuDF Java JAR + run: | + ./java/ci/build_cudf_java_jar.sh \ + --libcudf-dir "${RUNNER_TEMP}/libcudf" \ + --output-dir "${RUNNER_TEMP}/jars" \ + --cuda-version "${{ matrix.cuda }}" + - name: Upload per-entry JAR artifact + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: cudf_java_cuda${{ matrix.cuda_major }}_${{ matrix.arch }} + # Ship only the JARs and POMs. Exclude the per-classifier Maven build scratch dir. + path: | + ${{ runner.temp }}/jars + !${{ runner.temp }}/jars/.mvn-temp-target + if-no-files-found: error + # Assemble the per-classifier JARs into one Maven-repository layout. + java-gather: + needs: [java-build] + runs-on: linux-amd64-cpu4 + permissions: + contents: read + steps: + - name: Checkout code repo + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + ref: ${{ inputs.sha }} + persist-credentials: false + - name: Download per-entry JAR artifacts + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + pattern: cudf_java_cuda* + path: ${{ runner.temp }}/jars + merge-multiple: true + - name: Assemble Maven repository layout + run: | + ./java/ci/assemble_maven_repo.sh \ + --jars-dir "${RUNNER_TEMP}/jars" \ + --output-dir "${RUNNER_TEMP}/maven-repo" + - name: Upload combined Maven repository artifact + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: cudf_java_maven_repo + path: ${{ runner.temp }}/maven-repo + if-no-files-found: error diff --git a/dependencies.yaml b/dependencies.yaml index ca2933cac67b..227a11857df0 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -129,6 +129,19 @@ files: - depends_on_cudf_polars - depends_on_ray - depends_on_cudf_streaming + build_java: + # Toolchain for building static libcudf from source and packaging the cuDF + # Java JAR. `depends_on_libcudf` is excluded because the Java build links + # against a static libcudf built from source, not a conda shared libcudf. + output: none + includes: + - build_base + - build_all + - build_cpp + - cuda + - cuda_static + - cuda_version + - build_java test_java: output: none includes: @@ -632,6 +645,17 @@ dependencies: - croaring==4.4.2 - flatbuffers==24.3.25 - librdkafka<2.15.0a0 + build_java: + common: + - output_types: conda + packages: + - boost + # cuda_profiler_api.h is used by the JNI layer (CudaJni.cpp) but is + # not pulled in by the base `cuda` dev packages. + - cuda-profiler-api + - make + - maven + - openjdk=8.* depends_on_libnvcomp: common: - output_types: conda diff --git a/java/ci/README.md b/java/ci/README.md index 3f3060ef5b45..219e08e5eaff 100644 --- a/java/ci/README.md +++ b/java/ci/README.md @@ -1,11 +1,100 @@ # Build Jar artifact of cuDF -## Build the docker image +## Recommended: self-contained release build scripts -### Prerequisite +The scripts under `java/ci/` build the cuDF Java JAR for every Maven classifier the +same way locally and in CI (GitHub Actions is only a thin wrapper that adds +artifact upload/download). Each script pulls the RAPIDS `ci-conda` build image, +runs the build in a throwaway container, and writes its output to a host +directory. No local `docker build` is required, and no GPU is required to build. -1. Docker should be installed. -2. [nvidia-docker](https://github.com/NVIDIA/nvidia-docker) should be installed. +### Prerequisites + +1. Docker is installed and the current user can run `docker`. +2. Network access to pull `rapidsai/ci-conda:-latest`. + +### Local one-command shortcut + +For local testing only, `java/ci/test_java_build_local.sh` runs Steps 1-3 end-to-end for both CUDA 12 and CUDA 13 on the host architecture. + +```bash +./java/ci/test_java_build_local.sh --work-dir /tmp/java-build-test +``` + +### Step 1 - Build the static libcudf install tree + +```bash +./java/ci/build_static_libcudf.sh --output-dir /tmp/libcudf-cuda12 --cuda-version 12.9 +``` + +This produces a static libcudf install tree (`lib/libcudf.a` plus its static +dependencies) under the given output directory. Build outputs are host-user-owned +so plain `rm -rf` works. + +### Step 2 - Package the cuDF Java JAR for one classifier + +```bash +./java/ci/build_cudf_java_jar.sh \ + --libcudf-dir /tmp/libcudf-cuda12 \ + --output-dir /tmp/jars \ + --cuda-version 12.9 +``` + +This compiles the JNI layer against the static libcudf from Step 1 and emits a +single classifier JAR (e.g. `cudf-26.08.0-SNAPSHOT-cuda12.jar`) plus its POM +into a classifier-named subdirectory under `--output-dir`: + +``` +/tmp/jars/cuda12/ + cudf-26.08.0-SNAPSHOT-cuda12.jar + cudf-26.08.0-SNAPSHOT.pom +``` + +The classifier is derived from `--cuda-version` (major) + host arch (`uname +-m`): `cuda12` / `cuda13` on `x86_64`, `cuda12-arm64` / `cuda13-arm64` on +`aarch64`. Producing the ARM classifiers requires a real `aarch64` host. +Repeat Step 2 for each classifier, pointing `--libcudf-dir` at the matching +static libcudf tree and using the same `--output-dir` (each classifier lands +in its own subdirectory). Concurrent invocations for different classifiers +are safe because each nests its own bind-mount over `/repo/java/target` +inside the container. + +### Step 3 - Assemble the Maven repository layout + +```bash +./java/ci/assemble_maven_repo.sh \ + --jars-dir /tmp/jars \ + --output-dir /tmp/maven-repo +``` + +This walks every subdirectory of `--jars-dir` (each subdir name IS the +classifier), gathers the per-classifier JAR and shared POM, derives the +artifact version from the JAR filenames (requiring a single unique version +across subdirs), and lays them out as: + +``` +/tmp/maven-repo/ai/rapids/cudf/26.08.0-SNAPSHOT/ + cudf-26.08.0-SNAPSHOT-cuda12.jar + cudf-26.08.0-SNAPSHOT-cuda13.jar + cudf-26.08.0-SNAPSHOT.pom +``` + +The set of classifiers is whatever subdirectories are present under +`--jars-dir`. For a local `x86_64`-only run, populate `/tmp/jars/cuda12/` +and `/tmp/jars/cuda13/`. For the full four-way release build, add +`/tmp/jars/cuda12-arm64/` and `/tmp/jars/cuda13-arm64/`. + +In GitHub Actions (`.github/workflows/build.yaml`), the `java-build` matrix job +runs Steps 1-2 per (CUDA x arch) entry and uploads each classifier subdir as a +per-entry artifact. The separate `java-gather` job downloads them (with +`merge-multiple: true`, so all subdirs land in a single parent dir), runs +Step 3, and uploads the combined `cudf_java_maven_repo` artifact. + +## Legacy: manual Dockerfile.rocky build (obsolete) + +> The `java/ci/Dockerfile.rocky` + `java/ci/build-in-docker.sh` flow below is the +> old build path. It is retained for reference but superseded by the +> self-contained scripts above. ### Build the docker image @@ -20,25 +109,19 @@ The following CUDA versions are supported w/ CUDA Enhanced Compatibility: Change the --build-arg CUDA_VERSION to what you need. You can replace the tag "cudf-build:12.9.1-devel-rocky8" with another name you like. -## Start the docker then build - -### Start the docker +### Start the docker then build Run below command to start a docker container with GPU. ```bash nvidia-docker run -it cudf-build:12.9.1-devel-rocky8 bash ``` -### Download the cuDF source code - You can download the cuDF repo in the docker container or you can mount it into the container. Here I choose to download again in the container. ```bash git clone --recursive https://github.com/rapidsai/cudf.git -b main ``` -### Build cuDF jar with devtoolset - ```bash cd cudf export WORKSPACE=`pwd` @@ -46,6 +129,4 @@ source java/ci/env.sh ${sclCMD} "java/ci/build-in-docker.sh" ``` -### The output - -You can find the cuDF jar in java/target/ like cudf-26.10.0-SNAPSHOT-cuda12.jar. +You can find the cuDF jar in java/target/ like cudf-26.08.0-SNAPSHOT-cuda12.jar. diff --git a/java/ci/argparse.sh b/java/ci/argparse.sh new file mode 100644 index 000000000000..9a06d35c9df8 --- /dev/null +++ b/java/ci/argparse.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Shared argparse helpers for the java/ci/ host orchestrator scripts. +# Meant to be sourced, not executed: +# . "${SCRIPT_DIR}/argparse.sh" + +# require_value +# Check: exit 1 when is empty (i.e. the flag was passed +# without its argument, or was the last token on the command line). +require_value() { + local flag=$1 + local value=$2 + if [[ -z ${value} ]]; then + echo "Error: ${flag} requires a value" >&2 + exit 1 + fi +} + +# require_arg +# Check: assert that a required flag was actually supplied by the +# caller. Prints the script's print_help (if defined) then exits 1 on failure. +# Preserves the existing behavior of showing help after a "required flag missing" +# error. +require_arg() { + local flag=$1 + local value=$2 + if [[ -z ${value} ]]; then + echo "Error: ${flag} is required." >&2 + if declare -F print_help > /dev/null; then + print_help + fi + exit 1 + fi +} diff --git a/java/ci/assemble_maven_repo.sh b/java/ci/assemble_maven_repo.sh new file mode 100755 index 000000000000..b1d3e65a457c --- /dev/null +++ b/java/ci/assemble_maven_repo.sh @@ -0,0 +1,180 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Decoupled gather step: assemble per-classifier cuDF Java JARs into a single +# Maven-repository-layout directory. +# +# Input: --jars-dir contains one subdirectory per classifier, each holding +# exactly one cudf--.jar and a cudf-.pom. Subdir +# names ARE the classifier names, and the artifact version is derived from +# the JAR filenames (all subdirs must agree). +# +# Output layout: +# /ai/rapids/cudf//cudf--.jar +# /ai/rapids/cudf//cudf-.pom + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# shellcheck disable=SC1091 +. "${SCRIPT_DIR}/argparse.sh" + +GROUP_PATH="ai/rapids" +ARTIFACT_ID="cudf" + +JARS_DIR="" +OUTPUT_DIR="" + +print_help() { + cat << EOF + +Usage: assemble_maven_repo.sh --jars-dir --output-dir + +Gathers per-classifier cuDF Java JARs into a single Maven-repository-layout tree. + +REQUIRED: + -j, --jars-dir Parent directory containing one subdirectory per + classifier (each holding cudf--.jar + and cudf-.pom). Subdir name is the classifier. + -o, --output-dir Directory to receive the combined Maven-repository layout. + +OPTIONS: + -h, --help Show this help message. + +EXAMPLE: + assemble_maven_repo.sh --jars-dir /tmp/jars --output-dir /tmp/maven-repo + # given /tmp/jars/{cuda12,cuda13}/ inputs, produces: + # /tmp/maven-repo/ai/rapids/cudf//cudf--cuda12.jar + # /tmp/maven-repo/ai/rapids/cudf//cudf--cuda13.jar + # /tmp/maven-repo/ai/rapids/cudf//cudf-.pom + +EOF +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case $1 in + -h|--help) + print_help + exit 0 + ;; + -j|--jars-dir) + require_value "$1" "$2" + JARS_DIR=$2 + shift 2 + ;; + -o|--output-dir) + require_value "$1" "$2" + OUTPUT_DIR=$2 + shift 2 + ;; + *) + echo "Error: Unknown argument $1" + print_help + exit 1 + ;; + esac + done +} + +parse_args "$@" + +require_arg --jars-dir "${JARS_DIR}" +require_arg --output-dir "${OUTPUT_DIR}" + +if [[ ! -d ${JARS_DIR} ]]; then + echo "Error: --jars-dir '${JARS_DIR}' does not exist." + exit 1 +fi + +if [[ -e ${OUTPUT_DIR} && -n "$(ls -A "${OUTPUT_DIR}" 2>/dev/null)" ]]; then + echo "Error: --output-dir '${OUTPUT_DIR}' must be empty or nonexistent" >&2 + exit 1 +fi + +ASSEMBLE_FINISHED=0 +cleanup_partial_output() { + if [[ ${ASSEMBLE_FINISHED} -eq 0 && -e ${OUTPUT_DIR} ]]; then + echo "Assembly did not complete; removing partial output at ${OUTPUT_DIR}" >&2 + rm -rf "${OUTPUT_DIR}" + fi +} +trap cleanup_partial_output EXIT + +echo "Assembling Maven repository layout" +echo " jars dir: ${JARS_DIR}" +echo " output dir: ${OUTPUT_DIR}" + +# Walk every classifier subdirectory. Each subdir must contain exactly one +# cudf-*-.jar. The version is derived from the filename and must +# match across all subdirs. +FIRST_VERSION="" +CLASSIFIERS_SEEN="" + +for subdir in "${JARS_DIR}"/*/; do + classifier=$(basename "${subdir}") + + jar="" + for candidate in "${subdir}"cudf-*-"${classifier}".jar; do + if [[ -f "${candidate}" ]]; then + if [[ -n "${jar}" ]]; then + echo "Error: multiple JARs in ${subdir} match cudf-*-${classifier}.jar" >&2 + exit 1 + fi + jar=${candidate} + fi + done + + if [[ -z "${jar}" ]]; then + echo "Error: no cudf-*-${classifier}.jar found in ${subdir}" >&2 + exit 1 + fi + + # Filename is cudf--.jar. Peel prefix and suffix. + base=$(basename "${jar}" .jar) + version=$(echo "${base}" | sed -e 's/^cudf-//' -e "s/-${classifier}$//") + + if [[ -z "${FIRST_VERSION}" ]]; then + FIRST_VERSION=${version} + elif [[ "${version}" != "${FIRST_VERSION}" ]]; then + echo "Error: inconsistent versions across subdirs: ${FIRST_VERSION} vs ${version} (${subdir})" >&2 + exit 1 + fi + + DEST_DIR="${OUTPUT_DIR}/${GROUP_PATH}/${ARTIFACT_ID}/${version}" + mkdir -p "${DEST_DIR}" + cp -f "${jar}" "${DEST_DIR}/" + echo " + $(basename "${jar}")" + CLASSIFIERS_SEEN="${CLASSIFIERS_SEEN} ${classifier}" +done + +if [[ -z "${FIRST_VERSION}" ]]; then + echo "Error: no classifier subdirs found under ${JARS_DIR}" >&2 + exit 1 +fi + +# POM is identical across subdirs; copy the first one found. +POM_SRC="" +for subdir in "${JARS_DIR}"/*/; do + candidate=${subdir}cudf-${FIRST_VERSION}.pom + if [[ -f "${candidate}" ]]; then + POM_SRC=${candidate} + break + fi +done + +if [[ -z "${POM_SRC}" ]]; then + echo "Error: no cudf-${FIRST_VERSION}.pom found under ${JARS_DIR}" >&2 + exit 1 +fi + +DEST_DIR="${OUTPUT_DIR}/${GROUP_PATH}/${ARTIFACT_ID}/${FIRST_VERSION}" +cp -f "${POM_SRC}" "${DEST_DIR}/cudf-${FIRST_VERSION}.pom" +echo " + cudf-${FIRST_VERSION}.pom" + +echo "Maven repository assembled successfully at ${OUTPUT_DIR}" +echo "Classifiers present:${CLASSIFIERS_SEEN}" + +ASSEMBLE_FINISHED=1 diff --git a/java/ci/build_cudf_java_jar.sh b/java/ci/build_cudf_java_jar.sh new file mode 100755 index 000000000000..084b00c68478 --- /dev/null +++ b/java/ci/build_cudf_java_jar.sh @@ -0,0 +1,246 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Self-contained packaging of the cuDF Java JAR for a single classifier. +# +# Consumes a prebuilt static libcudf install tree (from build_static_libcudf.sh), +# compiles the JNI layer against it inside a throwaway RAPIDS ci-conda container, +# and emits the single classifier JAR (plus its POM) to a per-classifier +# subdirectory under --output-dir. This script is layout-agnostic: it produces +# one classifier's artifacts and knows nothing about the combined +# Maven-repository layout (see java/ci/assemble_maven_repo.sh). + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(git -C "${SCRIPT_DIR}" rev-parse --show-toplevel)" + +# shellcheck disable=SC1091 +. "${SCRIPT_DIR}/argparse.sh" + +LIBCUDF_DIR="" +OUTPUT_DIR="" +CUDA_VERSION="" +CMAKE_CUDA_ARCHITECTURES="" +PARALLEL_LEVEL="$(nproc)" + +print_help() { + cat << EOF + +Usage: build_cudf_java_jar.sh --libcudf-dir --output-dir \\ + --cuda-version [OPTIONS] + +Packages the cuDF Java JAR for a single classifier inside a RAPIDS ci-conda +container, linking against a prebuilt static libcudf. Always builds for the +host architecture (uname -m). The build image is fixed to +rapidsai/ci-conda:-latest (version derived from the VERSION +file). + +The classifier is derived from --cuda-version (major) + host arch (uname -m), +mirroring the pom.xml Groovy logic: "cuda" for x86_64, +"cuda-arm64" for aarch64. The classifier JAR + POM are written to +//. Concurrent invocations targeting different +classifiers are safe. + +REQUIRED: + -l, --libcudf-dir Static libcudf install tree produced by + build_static_libcudf.sh. + -o, --output-dir Host parent directory. The script creates and writes + to //, which must not already + exist. + -c, --cuda-version CUDA version to build for (e.g. "12.9" or "12.9.1"). + Must match --cuda-version of the static libcudf tree; + determines the cuda12/cuda13 classifier. + +OPTIONS: + -A, --cmake-cuda-architectures + Override the CUDA architecture list (e.g. "80" or + "80;90"). When unset, uses cuDF's default RAPIDS + architecture list. Must match the value passed to + build_static_libcudf.sh when producing the static + libcudf tree in --libcudf-dir, or device linking of + libcudfjni.so against libcudf.a will fail. + -j, --parallel Build parallelism (default: nproc = ${PARALLEL_LEVEL}). + -h, --help Show this help message. + +EXAMPLES: + build_cudf_java_jar.sh -l /tmp/libcudf-cuda12 -o /tmp/jars -c 12.9 + build_cudf_java_jar.sh -l /tmp/libcudf-cuda13 -o /tmp/jars -c 13.3 -A 80 + # writes: + # /tmp/jars/cuda12/cudf--cuda12.jar + # /tmp/jars/cuda12/cudf-.pom + +EOF +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case $1 in + -h|--help) + print_help + exit 0 + ;; + -l|--libcudf-dir) + require_value "$1" "$2" + LIBCUDF_DIR=$2 + shift 2 + ;; + -o|--output-dir) + require_value "$1" "$2" + OUTPUT_DIR=$2 + shift 2 + ;; + -c|--cuda-version) + require_value "$1" "$2" + CUDA_VERSION=$2 + shift 2 + ;; + -A|--cmake-cuda-architectures) + require_value "$1" "$2" + CMAKE_CUDA_ARCHITECTURES=$2 + shift 2 + ;; + -j|--parallel) + require_value "$1" "$2" + PARALLEL_LEVEL=$2 + shift 2 + ;; + *) + echo "Error: Unknown argument $1" + print_help + exit 1 + ;; + esac + done +} + +parse_args "$@" + +require_arg --libcudf-dir "${LIBCUDF_DIR}" +require_arg --output-dir "${OUTPUT_DIR}" +require_arg --cuda-version "${CUDA_VERSION}" + +if [[ ! -d ${LIBCUDF_DIR} ]]; then + echo "Error: --libcudf-dir '${LIBCUDF_DIR}' does not exist." + exit 1 +fi + +# Derive the Maven classifier from --cuda-version major + host arch, mirroring +# the pom.xml Groovy logic: "cuda" for x86_64, "cuda-arm64" for +# aarch64. +CUDA_MAJOR="$(echo "${CUDA_VERSION}" | cut -d. -f1)" +HOST_ARCH="$(uname -m)" +case "${HOST_ARCH}" in + x86_64) + CLASSIFIER="cuda${CUDA_MAJOR}" + ;; + aarch64|arm64) + CLASSIFIER="cuda${CUDA_MAJOR}-arm64" + ;; + *) + echo "Error: Unsupported host arch '${HOST_ARCH}' (expected x86_64 or aarch64)" >&2 + exit 1 + ;; +esac + +RAPIDS_VERSION="$(head -1 "${REPO_ROOT}/VERSION" | cut -d. -f1,2)" +IMAGE="rapidsai/ci-conda:${RAPIDS_VERSION}-latest" + +mkdir -p "${OUTPUT_DIR}" +OUTPUT_DIR="$(cd "${OUTPUT_DIR}" && pwd)" +LIBCUDF_DIR="$(cd "${LIBCUDF_DIR}" && pwd)" + +CLASSIFIER_OUT="${OUTPUT_DIR}/${CLASSIFIER}" +if [[ -e ${CLASSIFIER_OUT} ]]; then + echo "Error: classifier output '${CLASSIFIER_OUT}' already exists. Remove it before re-running." >&2 + exit 1 +fi +mkdir -p "${CLASSIFIER_OUT}" + +# Per-classifier scratch dir for Maven's java/target/. Nested bind-mount over +# /repo/java/target inside the container isolates concurrent invocations +# (each classifier gets its own target/). The `.mvn-temp-target/` prefix +# keeps this dir invisible to the default `*/` globbing in +# assemble_maven_repo.sh's classifier discovery loop. +# +# Recreate the scratch on every launch: the in-container mvn cannot clean a +# bind-mount point (rmdir on /repo/java/target fails with EBUSY), so the +# host wrapper is responsible for guaranteeing a clean starting target/. +TARGET_SCRATCH="${OUTPUT_DIR}/.mvn-temp-target/${CLASSIFIER}" +rm -rf "${TARGET_SCRATCH}" +mkdir -p "${TARGET_SCRATCH}" + +echo "Packaging cuDF Java JAR" +echo " image: ${IMAGE}" +echo " cuda version: ${CUDA_VERSION}" +echo " classifier: ${CLASSIFIER}" +echo " parallel: ${PARALLEL_LEVEL}" +echo " libcudf dir: ${LIBCUDF_DIR}" +echo " output dir: ${CLASSIFIER_OUT}" +echo " target dir: ${TARGET_SCRATCH}" +if [[ -n ${CMAKE_CUDA_ARCHITECTURES} ]]; then + echo " cmake cuda archs: ${CMAKE_CUDA_ARCHITECTURES}" +fi + +DOCKER_ARGS=( + --rm + --volume "${REPO_ROOT}:/repo" + --volume "${LIBCUDF_DIR}:/libcudf:ro" + --volume "${CLASSIFIER_OUT}:/output" + --volume "${TARGET_SCRATCH}:/repo/java/target" + --workdir /repo + --env RAPIDS_CUDA_VERSION="${CUDA_VERSION}" + --env PARALLEL_LEVEL="${PARALLEL_LEVEL}" + --env HOST_UID="$(id -u)" + --env HOST_GID="$(id -g)" +) + +if [[ -n ${CMAKE_CUDA_ARCHITECTURES} ]]; then + DOCKER_ARGS+=(--env CMAKE_CUDA_ARCHITECTURES="${CMAKE_CUDA_ARCHITECTURES}") +fi + +docker run "${DOCKER_ARGS[@]}" "${IMAGE}" \ + bash /repo/java/ci/build_cudf_java_jar_in_container.sh + +# Post-run: assert exactly one main classifier JAR + one POM, and that the +# JAR's classifier suffix matches the subdir name we chose (catches pom drift). +PRODUCED_JAR="" +for candidate in "${CLASSIFIER_OUT}"/cudf-*-"${CLASSIFIER}".jar; do + if [[ -f "${candidate}" ]]; then + if [[ -n "${PRODUCED_JAR}" ]]; then + echo "Error: multiple JARs matching cudf-*-${CLASSIFIER}.jar found in ${CLASSIFIER_OUT}" + ls -1 "${CLASSIFIER_OUT}" + exit 1 + fi + PRODUCED_JAR=${candidate} + fi +done + +if [[ -z "${PRODUCED_JAR}" ]]; then + echo "Error: no cudf-*-${CLASSIFIER}.jar found in ${CLASSIFIER_OUT}" + ls -1 "${CLASSIFIER_OUT}" || true + exit 1 +fi + +PRODUCED_POM="" +for candidate in "${CLASSIFIER_OUT}"/cudf-*.pom; do + if [[ -f "${candidate}" ]]; then + if [[ -n "${PRODUCED_POM}" ]]; then + echo "Error: multiple POMs found in ${CLASSIFIER_OUT}" + ls -1 "${CLASSIFIER_OUT}" + exit 1 + fi + PRODUCED_POM=${candidate} + fi +done + +if [[ -z "${PRODUCED_POM}" ]]; then + echo "Error: no cudf-*.pom found in ${CLASSIFIER_OUT}" + ls -1 "${CLASSIFIER_OUT}" || true + exit 1 +fi + +echo "cuDF Java JAR build succeeded:" +echo " $(basename "${PRODUCED_JAR}")" +echo " $(basename "${PRODUCED_POM}")" diff --git a/java/ci/build_cudf_java_jar_in_container.sh b/java/ci/build_cudf_java_jar_in_container.sh new file mode 100755 index 000000000000..98fce46c4f4a --- /dev/null +++ b/java/ci/build_cudf_java_jar_in_container.sh @@ -0,0 +1,121 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# In-container packaging of the cuDF Java JAR for a single classifier. +# +# This script runs inside the rapidsai/ci-conda container launched by +# java/ci/build_cudf_java_jar.sh. It generates the build_java conda toolchain +# environment, compiles the JNI layer against a prebuilt static libcudf +# (mounted at /libcudf), and packages the cuDF Java JAR. The resulting +# classifier JAR and its POM are copied to /output. /output and +# /repo/java/target are chowned to HOST_UID:HOST_GID on exit so the host user +# owns the outputs. +# +# Inputs (environment variables): +# RAPIDS_CUDA_VERSION CUDA version, e.g. 12.9 or 12.9.1 (required). +# PARALLEL_LEVEL Build parallelism (default: nproc). +# CMAKE_CUDA_ARCHITECTURES Optional override for -DCMAKE_CUDA_ARCHITECTURES. +# HOST_UID / HOST_GID Chown target for /output and /repo/java/target +# (both required). + +set -e + +OUTPUT_DIR=/output +REPO_ROOT=/repo +CUDF_INSTALL_DIR=/libcudf + +. /opt/conda/etc/profile.d/conda.sh + +if [[ -z ${RAPIDS_CUDA_VERSION} ]]; then + echo "Error: RAPIDS_CUDA_VERSION must be set" >&2 + exit 1 +fi + +if [[ -z ${HOST_UID} || -z ${HOST_GID} ]]; then + echo "Error: HOST_UID and HOST_GID must both be set" >&2 + exit 1 +fi + +_chown_outputs_on_exit() { + chown -R "${HOST_UID}:${HOST_GID}" "${OUTPUT_DIR}" "${REPO_ROOT}/java/target" 2>/dev/null || true +} +trap _chown_outputs_on_exit EXIT + +if [[ -z ${PARALLEL_LEVEL} ]]; then + PARALLEL_LEVEL=$(nproc) +fi + +CUDA_MAJOR_MINOR=$(echo "${RAPIDS_CUDA_VERSION}" | cut -d. -f1,2) + +rapids-logger "Configuring conda strict channel priority" +conda config --set channel_priority strict + +rapids-logger "Generating build_java conda environment (cuda=${CUDA_MAJOR_MINOR}, arch=$(arch))" +ENV_YAML_DIR="$(mktemp -d)" +rapids-dependency-file-generator \ + --output conda \ + --file-key build_java \ + --matrix "cuda=${CUDA_MAJOR_MINOR};arch=$(arch)" | tee "${ENV_YAML_DIR}/env.yaml" + +rapids-mamba-retry env create --yes -f "${ENV_YAML_DIR}/env.yaml" -n build_java +conda activate build_java + +rapids-print-env + +if [[ -z ${CUDACXX} ]]; then + export CUDACXX="${CONDA_PREFIX}/bin/nvcc" +fi +if [[ -z ${LIBCUDF_KERNEL_CACHE_PATH} ]]; then + export LIBCUDF_KERNEL_CACHE_PATH=/tmp/rapids-kernel-cache +fi + +BUILD_ARG=( + -B + "-Dmaven.repo.local=/tmp/.m2" + "-Dparallel.level=${PARALLEL_LEVEL}" + "-DskipTests=true" + "-DCUDF_USE_PER_THREAD_DEFAULT_STREAM=ON" + "-DCUDF_JNI_LIBCUDF_STATIC=ON" + "-DUSE_GDS=OFF" +) + +if [[ -n ${CMAKE_CUDA_ARCHITECTURES} ]]; then + BUILD_ARG+=("-DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES}") +fi + +cd "${REPO_ROOT}/java" + +CUDF_VERSION="$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout "${BUILD_ARG[@]}")" +rapids-logger "Packaging cuDF Java JAR version ${CUDF_VERSION} (libcudf: ${CUDF_INSTALL_DIR})" + +# The `clean` goal is intentionally omitted: /repo/java/target is a +# bind-mount point, so when `mvn clean` attempts to remove the directory, +# it fails with EBUSY. The host wrapper (build_cudf_java_jar.sh) recreates +# the scratch dir before each container launch to guarantee target/ starts empty. +CUDF_INSTALL_DIR="${CUDF_INSTALL_DIR}" mvn package "${BUILD_ARG[@]}" + +MAIN_JAR="" +for candidate in target/cudf-"${CUDF_VERSION}"-*.jar; do + case "${candidate}" in + *-tests.jar|*-sources.jar|*-javadoc.jar) + continue + ;; + esac + if [[ -f ${candidate} ]]; then + MAIN_JAR=${candidate} + break + fi +done + +if [[ -z ${MAIN_JAR} ]]; then + echo "Error: no cuDF classifier JAR produced under target/" + ls -l target/ || true + exit 1 +fi + +mkdir -p "${OUTPUT_DIR}" +cp -f "${MAIN_JAR}" "${OUTPUT_DIR}/" +cp -f pom.xml "${OUTPUT_DIR}/cudf-${CUDF_VERSION}.pom" + +rapids-logger "Emitted $(basename "${MAIN_JAR}") + cudf-${CUDF_VERSION}.pom to ${OUTPUT_DIR}" diff --git a/java/ci/build_static_libcudf.sh b/java/ci/build_static_libcudf.sh new file mode 100755 index 000000000000..2fa2e435471c --- /dev/null +++ b/java/ci/build_static_libcudf.sh @@ -0,0 +1,138 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Self-contained build of a static libcudf install tree. +# +# Pulls the RAPIDS ci-conda image, builds libcudf with BUILD_SHARED_LIBS=OFF +# inside a throwaway container, and installs the static libcudf tree (libcudf.a +# plus its static dependencies) into a directory on the host. No GPU is required +# to build. + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(git -C "${SCRIPT_DIR}" rev-parse --show-toplevel)" + +# shellcheck disable=SC1091 +. "${SCRIPT_DIR}/argparse.sh" + +OUTPUT_DIR="" +CUDA_VERSION="" +CMAKE_CUDA_ARCHITECTURES="" +PARALLEL_LEVEL="$(nproc)" + +print_help() { + cat << EOF + +Usage: build_static_libcudf.sh --output-dir --cuda-version [OPTIONS] + +Builds a static libcudf install tree inside a RAPIDS ci-conda container and +writes it to a directory on the host. Always builds for the host architecture +(uname -m). The build image is fixed to rapidsai/ci-conda:-latest +(version derived from the VERSION file). + +REQUIRED: + -o, --output-dir Host directory to receive the static install tree + (libcudf.a and its static dependencies). + -c, --cuda-version CUDA version to build for (e.g. "12.9" or "12.9.1"). + +OPTIONS: + -A, --cmake-cuda-architectures + Override the CUDA architecture list (e.g. "80" or + "80;90"). When unset, uses cuDF's default RAPIDS + architecture list. When packaging the cuDF Java JAR + against this static libcudf tree, pass the same value + to build_cudf_java_jar.sh --cmake-cuda-architectures + or device linking of libcudfjni.so against libcudf.a + will fail. + -j, --parallel Build parallelism (default: nproc = ${PARALLEL_LEVEL}). + -h, --help Show this help message. + +EXAMPLES: + build_static_libcudf.sh --output-dir /tmp/libcudf-cuda12 --cuda-version "12.9" + build_static_libcudf.sh -o /tmp/libcudf-cuda13 -c 13.3 -A "80" + +EOF +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case $1 in + -h|--help) + print_help + exit 0 + ;; + -o|--output-dir) + require_value "$1" "$2" + OUTPUT_DIR=$2 + shift 2 + ;; + -c|--cuda-version) + require_value "$1" "$2" + CUDA_VERSION=$2 + shift 2 + ;; + -A|--cmake-cuda-architectures) + require_value "$1" "$2" + CMAKE_CUDA_ARCHITECTURES=$2 + shift 2 + ;; + -j|--parallel) + require_value "$1" "$2" + PARALLEL_LEVEL=$2 + shift 2 + ;; + *) + echo "Error: Unknown argument $1" + print_help + exit 1 + ;; + esac + done +} + +parse_args "$@" + +require_arg --output-dir "${OUTPUT_DIR}" +require_arg --cuda-version "${CUDA_VERSION}" + +RAPIDS_VERSION="$(head -1 "${REPO_ROOT}/VERSION" | cut -d. -f1,2)" +IMAGE="rapidsai/ci-conda:${RAPIDS_VERSION}-latest" + +mkdir -p "${OUTPUT_DIR}" +OUTPUT_DIR="$(cd "${OUTPUT_DIR}" && pwd)" + +echo "Building static libcudf" +echo " image: ${IMAGE}" +echo " cuda version: ${CUDA_VERSION}" +echo " parallel: ${PARALLEL_LEVEL}" +echo " output dir: ${OUTPUT_DIR}" +if [[ -n ${CMAKE_CUDA_ARCHITECTURES} ]]; then + echo " cmake cuda archs: ${CMAKE_CUDA_ARCHITECTURES}" +fi + +DOCKER_ARGS=( + --rm + --volume "${REPO_ROOT}:/repo" + --volume "${OUTPUT_DIR}:/output" + --workdir /repo + --env RAPIDS_CUDA_VERSION="${CUDA_VERSION}" + --env PARALLEL_LEVEL="${PARALLEL_LEVEL}" + --env HOST_UID="$(id -u)" + --env HOST_GID="$(id -g)" +) + +if [[ -n ${CMAKE_CUDA_ARCHITECTURES} ]]; then + DOCKER_ARGS+=(--env CMAKE_CUDA_ARCHITECTURES="${CMAKE_CUDA_ARCHITECTURES}") +fi + +docker run "${DOCKER_ARGS[@]}" "${IMAGE}" \ + bash /repo/java/ci/build_static_libcudf_in_container.sh + +if [[ -f "${OUTPUT_DIR}/lib/libcudf.a" || -f "${OUTPUT_DIR}/lib64/libcudf.a" ]]; then + echo "Static libcudf build succeeded: ${OUTPUT_DIR}" +else + echo "Error: expected libcudf.a not found under ${OUTPUT_DIR}/lib or ${OUTPUT_DIR}/lib64" + exit 1 +fi diff --git a/java/ci/build_static_libcudf_in_container.sh b/java/ci/build_static_libcudf_in_container.sh new file mode 100755 index 000000000000..e77637e747b2 --- /dev/null +++ b/java/ci/build_static_libcudf_in_container.sh @@ -0,0 +1,95 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# In-container build of a static libcudf install tree. +# +# This script runs inside the rapidsai/ci-conda container launched by +# java/ci/build_static_libcudf.sh. It generates the build_java conda toolchain +# environment, builds libcudf with BUILD_SHARED_LIBS=OFF, and installs the +# resulting static libcudf (plus its static dependencies) into /output. Then +# chowns /output to HOST_UID:HOST_GID so the host user owns the outputs. +# +# Inputs (environment variables): +# RAPIDS_CUDA_VERSION CUDA version, e.g. 12.9 or 12.9.1 (required). +# PARALLEL_LEVEL Build parallelism (default: nproc). +# CMAKE_CUDA_ARCHITECTURES Optional override for -DCMAKE_CUDA_ARCHITECTURES. +# HOST_UID / HOST_GID Chown target for /output (both required). + +set -e + +INSTALL_PREFIX=/output +REPO_ROOT=/repo +BUILD_DIR=/tmp/libcudf-build + +. /opt/conda/etc/profile.d/conda.sh + +if [[ -z ${RAPIDS_CUDA_VERSION} ]]; then + echo "Error: RAPIDS_CUDA_VERSION must be set" >&2 + exit 1 +fi + +if [[ -z ${HOST_UID} || -z ${HOST_GID} ]]; then + echo "Error: HOST_UID and HOST_GID must both be set" >&2 + exit 1 +fi + +if [[ -z ${PARALLEL_LEVEL} ]]; then + PARALLEL_LEVEL=$(nproc) +fi + +CUDA_MAJOR_MINOR=$(echo "${RAPIDS_CUDA_VERSION}" | cut -d. -f1,2) + +rapids-logger "Configuring conda strict channel priority" +conda config --set channel_priority strict + +rapids-logger "Generating build_java conda environment (cuda=${CUDA_MAJOR_MINOR}, arch=$(arch))" +ENV_YAML_DIR="$(mktemp -d)" +rapids-dependency-file-generator \ + --output conda \ + --file-key build_java \ + --matrix "cuda=${CUDA_MAJOR_MINOR};arch=$(arch)" | tee "${ENV_YAML_DIR}/env.yaml" + +rapids-mamba-retry env create --yes -f "${ENV_YAML_DIR}/env.yaml" -n build_java +conda activate build_java + +rapids-print-env + +if [[ -z ${CUDACXX} ]]; then + export CUDACXX="${CONDA_PREFIX}/bin/nvcc" +fi +if [[ -z ${LIBCUDF_KERNEL_CACHE_PATH} ]]; then + export LIBCUDF_KERNEL_CACHE_PATH=/tmp/rapids-kernel-cache +fi + +CMAKE_ARGS=( + -S "${REPO_ROOT}/cpp" + -B "${BUILD_DIR}" + -GNinja + -DCMAKE_INSTALL_PREFIX="${INSTALL_PREFIX}" + -DBUILD_SHARED_LIBS=OFF + -DBUILD_TESTS=OFF + -DUSE_NVTX=ON + -DCUDF_LARGE_STRINGS_DISABLED=ON + -DCUDF_USE_ARROW_STATIC=ON + -DCUDF_ENABLE_ARROW_S3=OFF + -DCUDF_USE_PER_THREAD_DEFAULT_STREAM=ON + -DRMM_LOGGING_LEVEL=OFF + -DCUDF_KVIKIO_REMOTE_IO=OFF +) + +if [[ -n ${CMAKE_CUDA_ARCHITECTURES} ]]; then + CMAKE_ARGS+=("-DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES}") +fi + +rapids-logger "Configuring static libcudf" +cmake "${CMAKE_ARGS[@]}" + +rapids-logger "Building static libcudf with ${PARALLEL_LEVEL} jobs" +cmake --build "${BUILD_DIR}" --parallel "${PARALLEL_LEVEL}" + +rapids-logger "Installing static libcudf to ${INSTALL_PREFIX}" +cmake --install "${BUILD_DIR}" + +rapids-logger "Chowning ${INSTALL_PREFIX} to ${HOST_UID}:${HOST_GID}" +chown -R "${HOST_UID}:${HOST_GID}" "${INSTALL_PREFIX}" diff --git a/java/ci/test_java_build_local.sh b/java/ci/test_java_build_local.sh new file mode 100755 index 000000000000..1d67ad62771d --- /dev/null +++ b/java/ci/test_java_build_local.sh @@ -0,0 +1,315 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Local end-to-end verification of the build workflow: builds the static +# libcudf install tree and the classifier JAR for both CUDA 12 and CUDA 13, +# then runs the decoupled gather step to assemble the combined +# Maven-repository layout. Mirrors what the java-build matrix + java-gather +# jobs in .github/workflows/build.yaml do in CI. +# +# Runs on either x86_64 or aarch64 hosts. Each invocation covers only the +# host architecture: on x86_64 it produces the "cuda12" and "cuda13" +# classifier JARs; on aarch64 it produces "cuda12-arm64" and "cuda13-arm64". +# The arm64 suffix is added automatically by the child build scripts (via +# pom.xml's classifier logic keyed off `uname -m`). To cover all four +# release classifiers, run this script once on each architecture. +# +# Both static libcudf builds run in parallel, and both JAR builds run in +# parallel (each uses a nested bind-mount over /repo/java/target inside the +# container to isolate Maven output). + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# shellcheck disable=SC1091 +. "${SCRIPT_DIR}/argparse.sh" + +WORK_DIR="" +PARALLEL_LEVEL="$(nproc)" +CMAKE_CUDA_ARCHITECTURES="" + +# CUDA versions built for both classifiers. Keep in sync with the java-build +# matrix in .github/workflows/build.yaml. +CUDA12_VERSION="12.9" +CUDA13_VERSION="13.3" + +# Wall-clock timing variables. +STEP_NAMES=() +STEP_ELAPSED=() + +print_help() { + cat << EOF + +Usage: test_java_build_local.sh --work-dir [OPTIONS] + +Runs the full build+gather pipeline for both CUDA 12 and CUDA 13 on the host +architecture (x86_64 or aarch64) and assembles the combined Maven-repository +layout. + +REQUIRED: + -w, --work-dir Scratch directory for build outputs. Subtrees created: + /libcudf-cuda12 static libcudf (CUDA 12) + /libcudf-cuda13 static libcudf (CUDA 13) + /jars/ per-classifier JAR + POM + /maven-repo combined Maven layout + where is "cuda12" / "cuda13" on x86_64 + and "cuda12-arm64" / "cuda13-arm64" on aarch64. + +OPTIONS: + -j, --parallel Total build parallelism (default: nproc = ${PARALLEL_LEVEL}). + Each concurrent JAR build gets --parallel/2 to avoid + RAM pressure from two parallel nvcc runs. + -A, --cmake-cuda-architectures + CUDA architecture list (e.g. "80" or "80;90") passed to + both build scripts. The literal value "all" is a + sentinel meaning "do not pass --cmake-cuda-architectures + to child scripts" — child scripts then fall back to + cuDF's default RAPIDS full architecture list (slow). + Default: auto-detect the local GPU's compute + capability via nvidia-smi (e.g. Ampere -> "80"). If + nvidia-smi is missing or returns nothing, falls back + to "all". + -h, --help Show this help message. + +EXAMPLES: + # Fast run (auto-detect local GPU arch): + ./java/ci/test_java_build_local.sh --work-dir /tmp/java-build-test + + # Explicit override: + ./java/ci/test_java_build_local.sh --work-dir /tmp/java-build-test \\ + --cmake-cuda-architectures 80 + + # Full RAPIDS arch list (slow, e.g. GPU-less host): + ./java/ci/test_java_build_local.sh --work-dir /tmp/java-build-test \\ + --cmake-cuda-architectures all + +EOF +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case $1 in + -h|--help) + print_help + exit 0 + ;; + -w|--work-dir) + require_value "$1" "$2" + WORK_DIR=$2 + shift 2 + ;; + -j|--parallel) + require_value "$1" "$2" + PARALLEL_LEVEL=$2 + shift 2 + ;; + -A|--cmake-cuda-architectures) + require_value "$1" "$2" + CMAKE_CUDA_ARCHITECTURES=$2 + shift 2 + ;; + *) + echo "Error: Unknown argument $1" + print_help + exit 1 + ;; + esac + done +} + +log_step() { + echo + echo "============================================================" + echo "== $1" + echo "============================================================" +} + +format_elapsed() { + local s=$1 + printf '%dm %02ds' $((s/60)) $((s%60)) +} + +record_step_end() { + local name=$1 + local start=$2 + local elapsed=$((SECONDS - start)) + STEP_NAMES+=("${name}") + STEP_ELAPSED+=("${elapsed}") + echo + echo "== ${name} completed in $(format_elapsed "${elapsed}")" +} + +parse_args "$@" + +require_arg --work-dir "${WORK_DIR}" + +mkdir -p "${WORK_DIR}" +WORK_DIR="$(cd "${WORK_DIR}" && pwd)" +LOG_DIR="${WORK_DIR}/logs" +mkdir -p "${LOG_DIR}" + +# Remove outputs from any prior run so cmake/mvn does not see stale artifacts. +rm -rf "${WORK_DIR}/libcudf-cuda12" \ + "${WORK_DIR}/libcudf-cuda13" \ + "${WORK_DIR}/jars" \ + "${WORK_DIR}/maven-repo" + +# Auto-detect the local GPU's compute capability when the flag was not passed. +# "all" is the sentinel value that means "do not forward this flag to child +# scripts". Child scripts fall back to cuDF's default RAPIDS architecture +# list (slow but correct on GPU-less hosts). +if [[ -z ${CMAKE_CUDA_ARCHITECTURES} ]]; then + DETECTED="" + if command -v nvidia-smi > /dev/null 2>&1; then + DETECTED=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader 2>/dev/null | head -1 | tr -d '.' | tr -d ' ') + fi + if [[ -n ${DETECTED} ]]; then + CMAKE_CUDA_ARCHITECTURES=${DETECTED} + echo "Auto-detected local GPU compute capability: ${CMAKE_CUDA_ARCHITECTURES}" + else + CMAKE_CUDA_ARCHITECTURES="all" + echo "No local GPU detected; falling back to cuDF's default RAPIDS architecture list" + fi +fi + +echo "cuDF Java local build verification" +echo " host arch: $(uname -m)" +echo " work dir: ${WORK_DIR}" +echo " parallel: ${PARALLEL_LEVEL}" +echo " cuda12 version: ${CUDA12_VERSION}" +echo " cuda13 version: ${CUDA13_VERSION}" +echo " cmake cuda architectures: ${CMAKE_CUDA_ARCHITECTURES}" +echo " logs: ${LOG_DIR}/{static,jar}_cuda{12,13}.log" + +# When forwarding to child scripts, "all" means "don't pass the flag". +CHILD_CMAKE_ARGS=() +if [[ ${CMAKE_CUDA_ARCHITECTURES} != "all" ]]; then + CHILD_CMAKE_ARGS=(--cmake-cuda-architectures "${CMAKE_CUDA_ARCHITECTURES}") +fi + +# Both Step 1 and Step 2 launch two concurrent builds. Each build gets half +# of PARALLEL_LEVEL so together they stay within PARALLEL_LEVEL. +STEP_PARALLEL=$((PARALLEL_LEVEL / 2)) +if [[ ${STEP_PARALLEL} -lt 1 ]]; then + STEP_PARALLEL=1 +fi + +# Step 1: static libcudf builds in parallel. +log_step "Step 1: building static libcudf for CUDA 12 and CUDA 13 in parallel" +STEP1_START=${SECONDS} + +"${SCRIPT_DIR}/build_static_libcudf.sh" \ + --output-dir "${WORK_DIR}/libcudf-cuda12" \ + --cuda-version "${CUDA12_VERSION}" \ + --parallel "${STEP_PARALLEL}" \ + "${CHILD_CMAKE_ARGS[@]}" \ + > "${LOG_DIR}/static_cuda12.log" 2>&1 & +STATIC_CUDA12_PID=$! + +"${SCRIPT_DIR}/build_static_libcudf.sh" \ + --output-dir "${WORK_DIR}/libcudf-cuda13" \ + --cuda-version "${CUDA13_VERSION}" \ + --parallel "${STEP_PARALLEL}" \ + "${CHILD_CMAKE_ARGS[@]}" \ + > "${LOG_DIR}/static_cuda13.log" 2>&1 & +STATIC_CUDA13_PID=$! + +echo " cuda12 static pid: ${STATIC_CUDA12_PID} (tail -f ${LOG_DIR}/static_cuda12.log)" +echo " cuda13 static pid: ${STATIC_CUDA13_PID} (tail -f ${LOG_DIR}/static_cuda13.log)" + +STATIC_CUDA12_RC=0 +STATIC_CUDA13_RC=0 +if ! wait "${STATIC_CUDA12_PID}"; then + STATIC_CUDA12_RC=1 +fi +if ! wait "${STATIC_CUDA13_PID}"; then + STATIC_CUDA13_RC=1 +fi + +if [[ ${STATIC_CUDA12_RC} -ne 0 ]]; then + echo "Error: static libcudf CUDA 12 build failed." + echo "See ${LOG_DIR}/static_cuda12.log" +fi +if [[ ${STATIC_CUDA13_RC} -ne 0 ]]; then + echo "Error: static libcudf CUDA 13 build failed." + echo "See ${LOG_DIR}/static_cuda13.log" +fi +if [[ ${STATIC_CUDA12_RC} -ne 0 || ${STATIC_CUDA13_RC} -ne 0 ]]; then + exit 1 +fi + +record_step_end "Step 1: static libcudf (parallel run)" "${STEP1_START}" + +# Step 2: JAR builds in parallel. Each build's container nests a bind-mount +# over /repo/java/target so concurrent Maven runs don't clobber each other. +log_step "Step 2: packaging cuDF Java JARs for cuda12 and cuda13 in parallel" +STEP2_START=${SECONDS} + +"${SCRIPT_DIR}/build_cudf_java_jar.sh" \ + --libcudf-dir "${WORK_DIR}/libcudf-cuda12" \ + --output-dir "${WORK_DIR}/jars" \ + --cuda-version "${CUDA12_VERSION}" \ + --parallel "${STEP_PARALLEL}" \ + "${CHILD_CMAKE_ARGS[@]}" \ + > "${LOG_DIR}/jar_cuda12.log" 2>&1 & +JAR_CUDA12_PID=$! + +"${SCRIPT_DIR}/build_cudf_java_jar.sh" \ + --libcudf-dir "${WORK_DIR}/libcudf-cuda13" \ + --output-dir "${WORK_DIR}/jars" \ + --cuda-version "${CUDA13_VERSION}" \ + --parallel "${STEP_PARALLEL}" \ + "${CHILD_CMAKE_ARGS[@]}" \ + > "${LOG_DIR}/jar_cuda13.log" 2>&1 & +JAR_CUDA13_PID=$! + +echo " cuda12 jar pid: ${JAR_CUDA12_PID} (tail -f ${LOG_DIR}/jar_cuda12.log)" +echo " cuda13 jar pid: ${JAR_CUDA13_PID} (tail -f ${LOG_DIR}/jar_cuda13.log)" + +JAR_CUDA12_RC=0 +JAR_CUDA13_RC=0 +if ! wait "${JAR_CUDA12_PID}"; then + JAR_CUDA12_RC=1 +fi +if ! wait "${JAR_CUDA13_PID}"; then + JAR_CUDA13_RC=1 +fi + +if [[ ${JAR_CUDA12_RC} -ne 0 ]]; then + echo "Error: cuDF Java JAR CUDA 12 build failed." + echo "See ${LOG_DIR}/jar_cuda12.log" +fi +if [[ ${JAR_CUDA13_RC} -ne 0 ]]; then + echo "Error: cuDF Java JAR CUDA 13 build failed." + echo "See ${LOG_DIR}/jar_cuda13.log" +fi +if [[ ${JAR_CUDA12_RC} -ne 0 || ${JAR_CUDA13_RC} -ne 0 ]]; then + exit 1 +fi + +record_step_end "Step 2: JAR builds (parallel run)" "${STEP2_START}" + +# Step 3: gather into a combined Maven-repository layout. +log_step "Step 3: assembling combined Maven-repository layout" +STEP3_START=${SECONDS} + +"${SCRIPT_DIR}/assemble_maven_repo.sh" \ + --jars-dir "${WORK_DIR}/jars" \ + --output-dir "${WORK_DIR}/maven-repo" + +record_step_end "Step 3: assemble Maven repo" "${STEP3_START}" + +# Derive the assembled version from the output tree for display purposes. +CUDF_VERSION=$(basename "$(ls -d "${WORK_DIR}/maven-repo/ai/rapids/cudf"/*/ | head -1)") + +log_step "Success" +echo "Combined Maven repository:" +echo " ${WORK_DIR}/maven-repo/ai/rapids/cudf/${CUDF_VERSION}/" +echo +echo "Timings:" +for i in "${!STEP_NAMES[@]}"; do + printf ' %-45s %s\n' "${STEP_NAMES[$i]}" "$(format_elapsed "${STEP_ELAPSED[$i]}")" +done +printf ' %-45s %s\n' "Total wall time" "$(format_elapsed "${SECONDS}")" diff --git a/java/pom.xml b/java/pom.xml index e6a96ad343da..18213018233c 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -649,6 +649,13 @@ def cm = cudaPattern.matcher(nvccout) if (cm.find()) { def classifier = 'cuda' + cm.group(1) + // Emit "cuda" on x86_64 and "cuda-arm64" on + // aarch64 so a single pom produces a distinct Maven + // classifier per architecture. + def osArch = System.getProperty('os.arch') + if (osArch == 'aarch64' || osArch == 'arm64') { + classifier = classifier + '-arm64' + } project.properties['cuda.classifier'] = classifier } else { throw new RuntimeException('could not find CUDA version') From f9fa84253b3686b80860e11c1cbcdf3b702b6a8b Mon Sep 17 00:00:00 2001 From: Lawrence Mitchell Date: Tue, 21 Jul 2026 17:35:04 +0100 Subject: [PATCH 22/25] Add timeout stacktrace utility for cudf-polars tests, remove pytest-timeout (#23332) ## Description Timing out an individual test is actually not useful since it is not safe to cancel a test at an arbitrary point in execution: that might leave a collective dangling. Moreover, many of the Polars tests run quite close to the, arbitrarily chosen, timeout limit on CI runners if they are heavily loaded. Since the signal we actually want is the state of a hanging process, instead just add a test-suite level timeout with a utility to print the state of the hanging processes. closes https://github.com/rapidsai/cudf/issues/22948 ## Checklist - [x] I am familiar with the [Contributing Guidelines](https://github.com/rapidsai/cudf/blob/HEAD/CONTRIBUTING.md). - [x] New or existing tests cover these changes. - [x] The documentation is up to date with these changes. --- ci/run_cudf_polars_polars_tests.sh | 13 +- ci/run_cudf_polars_pytests.sh | 7 +- ci/test_wheel_cudf_polars.sh | 3 + ci/timeout_with_stack.py | 353 ++++++++++++++++++ .../all_cuda-129_arch-aarch64.yaml | 3 +- .../all_cuda-129_arch-x86_64.yaml | 3 +- .../all_cuda-133_arch-aarch64.yaml | 3 +- .../all_cuda-133_arch-x86_64.yaml | 3 +- dependencies.yaml | 7 +- python/cudf_polars/pyproject.toml | 3 +- python/cudf_polars/tests/conftest.py | 3 - .../tests/expressions/test_rolling.py | 1 - .../cudf_polars/tests/streaming/test_scan.py | 1 - .../cudf_polars/tests/streaming/test_sort.py | 1 - 14 files changed, 385 insertions(+), 19 deletions(-) create mode 100644 ci/timeout_with_stack.py diff --git a/ci/run_cudf_polars_polars_tests.sh b/ci/run_cudf_polars_polars_tests.sh index 5302942db173..3778e1a9035a 100755 --- a/ci/run_cudf_polars_polars_tests.sh +++ b/ci/run_cudf_polars_polars_tests.sh @@ -4,6 +4,8 @@ set -euo pipefail +TIMEOUT_TOOL_PATH="$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/timeout_with_stack.py + # Support invoking run_cudf_polars_pytests.sh outside the script directory # Assumption, polars has been cloned in the root of the repo. cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../polars/ @@ -60,16 +62,18 @@ DESELECTED_TESTS_STR=$(printf -- " --deselect %s" "${DESELECTED_TESTS[@]}") # Don't quote the `DESELECTED_...` variable because `pytest` can't handle # multiple quoted arguments inline # shellcheck disable=SC2086 +# Fail fast (-x) because failed tests pollute the state echo "Run polars tests with injected in-memory GPU engine" -python -m pytest \ +python "${TIMEOUT_TOOL_PATH}" --enable-python 3600 \ + python -m pytest \ --import-mode=importlib \ --cache-clear \ + -x \ -m "" \ -p cudf_polars.testing.inject_gpu_engine \ -n 4 \ --dist=worksteal \ --tb=native \ - --timeout=240 \ --durations 10 --durations-min 10 \ -ra \ $DESELECTED_TESTS_STR \ @@ -81,9 +85,11 @@ python -m pytest \ echo "Run polars tests with injected SPMD GPU engine, small blocksize" CUDF_POLARS__EXECUTOR__TARGET_PARTITION_SIZE=805306368 \ CUDF_POLARS__EXECUTOR__FALLBACK_MODE=silent \ - python -m pytest \ +python "${TIMEOUT_TOOL_PATH}" --enable-python 3600 \ + python -m pytest \ --import-mode=importlib \ --cache-clear \ + -x \ -v \ -m "" \ -p cudf_polars.testing.inject_gpu_engine \ @@ -91,7 +97,6 @@ CUDF_POLARS__EXECUTOR__FALLBACK_MODE=silent \ -n 4 \ --dist=worksteal \ --tb=native \ - --timeout=240 \ --durations 10 --durations-min 10 \ -ra \ $DESELECTED_TESTS_STR \ diff --git a/ci/run_cudf_polars_pytests.sh b/ci/run_cudf_polars_pytests.sh index 82d1ccd4879f..3fac6910c5a8 100755 --- a/ci/run_cudf_polars_pytests.sh +++ b/ci/run_cudf_polars_pytests.sh @@ -1,11 +1,14 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail # It is essential to cd into python/cudf_polars as `pytest-xdist` + `coverage` seem to work only at this directory level. # Support invoking run_cudf_polars_pytests.sh outside the script directory +TIMEOUT_TOOL_PATH="$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/timeout_with_stack.py + cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cudf_polars/ -python -m pytest --cache-clear "$@" tests +python "${TIMEOUT_TOOL_PATH}" --enable-python 3600 \ + python -m pytest --cache-clear "$@" tests diff --git a/ci/test_wheel_cudf_polars.sh b/ci/test_wheel_cudf_polars.sh index 09315bbe981c..d9df9c05d4eb 100755 --- a/ci/test_wheel_cudf_polars.sh +++ b/ci/test_wheel_cudf_polars.sh @@ -89,12 +89,14 @@ for version in "${VERSIONS[@]}"; do COVERAGE_ARGS=(--no-cov) fi + # Fail fast (-x) rather than trying to continue because failed tests pollute the state ./ci/run_cudf_polars_pytests.sh \ -vv \ "${COVERAGE_ARGS[@]}" \ --numprocesses=4 \ --dist=worksteal \ --durations 10 --durations-min 10 \ + -x \ -ra \ --junitxml="${RAPIDS_TESTS_DIR}/junit-cudf-polars-${version}.xml" @@ -106,6 +108,7 @@ for version in "${VERSIONS[@]}"; do EXITCODE=1 FAILED+=("${version}") rapids-logger "Tests failed for polars ${version}.*" + break else PASSED+=("${version}") rapids-logger "Tests passed for polars ${version}.*" diff --git a/ci/timeout_with_stack.py b/ci/timeout_with_stack.py new file mode 100644 index 000000000000..02f62408fe9e --- /dev/null +++ b/ci/timeout_with_stack.py @@ -0,0 +1,353 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +""" +Module for running commands with timeout and capturing stack traces. + +This module provides functionality to run commands with a timeout, capture stack traces +of processes that exceed the timeout, and properly terminate process trees. + +See Also +-------- +subprocess.Popen : For running subprocesses without timeout. +psutil.Process : For process management and information. + +Examples +-------- +>>> from timeout_with_stack import run_with_timeout +>>> exit_code = run_with_timeout(["sleep", "10"], timeout=5) +>>> print(f"Process exited with code: {exit_code}") +""" + +from __future__ import annotations + +import argparse +import os +import shutil +import signal +import subprocess +import sys +import time +from contextlib import suppress +from enum import IntEnum +from typing import TYPE_CHECKING + +import psutil + +if TYPE_CHECKING: + from collections.abc import Sequence + from types import FrameType + + +class StackType(IntEnum): + """Enum representing the type of stack trace to capture.""" + + C = 0 + Python = 1 + + +def get_child_pids(pid: int) -> list[int]: + """ + Get all child PIDs of a given process. + + This function retrieves all child process IDs (PIDs) of a given process, + including recursively nested child processes. + + Parameters + ---------- + pid + The process ID of the parent process. + + Returns + ------- + A list of child process IDs. Returns an empty list if the parent process + does not exist. + + See Also + -------- + psutil.Process.children : For getting child processes. + + Examples + -------- + >>> from timeout_with_stack import get_child_pids + >>> child_pids = get_child_pids(1234) + >>> print(f"Child PIDs: {child_pids}") + """ + try: + parent = psutil.Process(pid) + children = parent.children(recursive=True) + return [p.pid for p in children] + except psutil.NoSuchProcess: + return [] + + +def capture_stack_trace(pid: int, stack_type=StackType.C) -> None: + """ + Capture stack trace for a given process. + + This function captures the stack trace of a process using GDB. It prints the + stack trace to stdout, which can be useful for debugging hanging or long-running + processes. + + Parameters + ---------- + pid + The process ID of the process to capture stack trace for. + stack_type + The stack type to extract, either C or Python. + + See Also + -------- + capture_all_stacks : For capturing stack traces of a process and its children. + + Examples + -------- + >>> from timeout_with_stack import capture_stack_trace + >>> capture_stack_trace(1234) + """ + if stack_type is StackType.C: + bt_command = "thread apply all bt" + print(f"\nCapturing C stack trace for process {pid}:") + else: + bt_command = "thread apply all py-bt" + print(f"\nCapturing Python stack trace for process {pid}:") + gdb = shutil.which("gdb") + if gdb is None: + print(f"Skipping stack trace for process {pid}: gdb not found") + return + + try: + proc = subprocess.run( + [ + gdb, + "--quiet", + "--pid", + str(pid), + "-ex", + "set pagination off", + "-ex", + "set confirm off", + "-ex", + bt_command, + "-ex", + "quit", + ], + capture_output=True, + text=True, + check=False, + timeout=120, + ) + except subprocess.TimeoutExpired: + print(f"Timed out capturing stack trace for process {pid}") + return + + print(proc.stdout) + if proc.stderr: + print(proc.stderr, file=sys.stderr) + + +def capture_all_stacks(pid: int, *, enable_python: bool = False) -> None: + """ + Capture stack traces for parent and all child processes. + + This function captures stack traces for both the parent process and all its + child processes. It first captures the parent's stack trace, then recursively + captures stack traces for all child processes. + + Parameters + ---------- + pid + The process ID of the parent process. + enable_python + Whether to capture Python stack traces. + + See Also + -------- + capture_stack_trace : For capturing stack trace of a single process. + get_child_pids : For getting child process IDs. + + Examples + -------- + >>> from timeout_with_stack import capture_all_stacks + >>> capture_all_stacks(1234, enable_python=True) + """ + # Capture parent process stack + if enable_python: + capture_stack_trace(pid, stack_type=StackType.Python) + capture_stack_trace(pid, stack_type=StackType.C) + + # Get and capture all child processes + child_pids = get_child_pids(pid) + for child_pid in child_pids: + if enable_python: + capture_stack_trace(child_pid, stack_type=StackType.Python) + capture_stack_trace(child_pid, stack_type=StackType.C) + + +def terminate_process_tree(pid: int) -> None: + """ + Terminate a process and all its children. + + This function terminates a process and all its child processes. It first + attempts to gracefully terminate all processes, then forcefully kills any + remaining processes after a timeout. + + Parameters + ---------- + pid + The process ID of the parent process to terminate. + + See Also + -------- + psutil.Process.terminate : For gracefully terminating a process. + psutil.Process.kill : For forcefully killing a process. + + Examples + -------- + >>> from timeout_with_stack import terminate_process_tree + >>> terminate_process_tree(1234) + """ + try: + parent = psutil.Process(pid) + children = parent.children(recursive=True) + + # Terminate children first + for child in children: + with suppress(psutil.NoSuchProcess): + child.terminate() + + # Create a copy of children list + terminated_children = list(children) + + # Wait for all children to terminate + for child in terminated_children: + with suppress(psutil.TimeoutExpired): + child.wait(timeout=3) + + # Kill any remaining children + for child in terminated_children: + with suppress(psutil.NoSuchProcess): + child.kill() + + # Terminate parent + parent.terminate() + try: + parent.wait(timeout=3) + except psutil.TimeoutExpired: + parent.kill() + + except psutil.NoSuchProcess: + pass + + +def install_signal_handler(pid: int, *signals: signal.Signals) -> None: + """ + Install signal handler that terminates the given pid on receiving any of signals + """ + + def handler(signum: int, frame: FrameType | None) -> None: + print(f"Received {signum=}, terminating process tree") + terminate_process_tree(pid) + sys.exit(signum) + + for sig in signals: + signal.signal(sig, handler) + + +def run_with_timeout( + cmd: Sequence[str], timeout: float, *, enable_python: bool = False +) -> int: + """ + Run a command with a timeout and capture stack traces if it exceeds the timeout. + + This function runs a command with a specified timeout. If the command exceeds + the timeout, it captures stack traces of the process and its children before + terminating them. It handles keyboard interrupts gracefully. + + Parameters + ---------- + cmd + The command and its arguments to run. + timeout + Maximum time in seconds to allow the command to run. + enable_python + Whether to capture Python stack traces. + + Returns + ------- + Return code of the command, or 124 if timeout occurred, or signal.SIGINT + if interrupted by keyboard. + + See Also + -------- + subprocess.Popen : For running subprocesses without timeout. + capture_all_stacks : For capturing stack traces of processes. + + Examples + -------- + >>> from timeout_with_stack import run_with_timeout + >>> exit_code = run_with_timeout(["sleep", "10"], timeout=5, enable_python=True) + >>> print(f"Process exited with code: {exit_code}") + """ + # Start the process with a new process group + # Note: preexec_fn is used here as we need to create a new process group + # for proper termination of child processes + process = subprocess.Popen( + cmd, + preexec_fn=os.setsid, + ) + install_signal_handler( + process.pid, + signal.SIGTERM, + signal.SIGABRT, + signal.SIGHUP, + signal.SIGQUIT, + ) + start_time = time.time() + + try: + while time.time() - start_time < timeout: + if process.poll() is not None: + return process.returncode + time.sleep(0.1) + + print(f"\nProcess timed out after {timeout} seconds") + print("Capturing stack traces for all processes...") + + # Capture stacks for parent and all children + capture_all_stacks(process.pid, enable_python=enable_python) + + # Terminate the entire process tree + print("\nTerminating process tree...") + terminate_process_tree(process.pid) + except KeyboardInterrupt: + print("\nReceived keyboard interrupt") + terminate_process_tree(process.pid) + return signal.SIGINT + else: + return 124 + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Run a command with timeout and capture stack traces" + ) + parser.add_argument("timeout", type=float, help="Timeout in seconds") + parser.add_argument( + "--enable-python", + action="store_true", + help="Enable Python stack trace capture", + ) + parser.add_argument( + "command", nargs=argparse.REMAINDER, help="Command to run" + ) + + args = parser.parse_args() + + if not args.command: + parser.error("No command specified") + + exit_code = run_with_timeout( + args.command, args.timeout, enable_python=args.enable_python + ) + sys.exit(exit_code) diff --git a/conda/environments/all_cuda-129_arch-aarch64.yaml b/conda/environments/all_cuda-129_arch-aarch64.yaml index bb5c331a5331..f07b03fe38d2 100644 --- a/conda/environments/all_cuda-129_arch-aarch64.yaml +++ b/conda/environments/all_cuda-129_arch-aarch64.yaml @@ -35,6 +35,7 @@ dependencies: - flatbuffers==24.3.25 - fsspec>=0.6.0 - gcc_linux-aarch64=14.* +- gdb - hypothesis>=6.131.7 - identify>=2.5.20 - include-what-you-use==0.24.0 @@ -76,6 +77,7 @@ dependencies: - pandoc - polars>=1.35,<1.43 - pre-commit +- psutil - pyarrow>=19.0.0,<24 - pytables - pytest-benchmark @@ -83,7 +85,6 @@ dependencies: - pytest-cov - pytest-httpserver - pytest-rerunfailures!=16.0.0 -- pytest-timeout - pytest-xdist - pytest<9.1.0 - python-calamine diff --git a/conda/environments/all_cuda-129_arch-x86_64.yaml b/conda/environments/all_cuda-129_arch-x86_64.yaml index bf90421d131b..879c41b52895 100644 --- a/conda/environments/all_cuda-129_arch-x86_64.yaml +++ b/conda/environments/all_cuda-129_arch-x86_64.yaml @@ -35,6 +35,7 @@ dependencies: - flatbuffers==24.3.25 - fsspec>=0.6.0 - gcc_linux-64=14.* +- gdb - hypothesis>=6.131.7 - identify>=2.5.20 - include-what-you-use==0.24.0 @@ -76,6 +77,7 @@ dependencies: - pandoc - polars>=1.35,<1.43 - pre-commit +- psutil - pyarrow>=19.0.0,<24 - pytables - pytest-benchmark @@ -83,7 +85,6 @@ dependencies: - pytest-cov - pytest-httpserver - pytest-rerunfailures!=16.0.0 -- pytest-timeout - pytest-xdist - pytest<9.1.0 - python-calamine diff --git a/conda/environments/all_cuda-133_arch-aarch64.yaml b/conda/environments/all_cuda-133_arch-aarch64.yaml index 13e08205f2d5..2eb46543b345 100644 --- a/conda/environments/all_cuda-133_arch-aarch64.yaml +++ b/conda/environments/all_cuda-133_arch-aarch64.yaml @@ -35,6 +35,7 @@ dependencies: - flatbuffers==24.3.25 - fsspec>=0.6.0 - gcc_linux-aarch64=14.* +- gdb - hypothesis>=6.131.7 - identify>=2.5.20 - include-what-you-use==0.24.0 @@ -76,6 +77,7 @@ dependencies: - pandoc - polars>=1.35,<1.43 - pre-commit +- psutil - pyarrow>=19.0.0,<24 - pytables - pytest-benchmark @@ -83,7 +85,6 @@ dependencies: - pytest-cov - pytest-httpserver - pytest-rerunfailures!=16.0.0 -- pytest-timeout - pytest-xdist - pytest<9.1.0 - python-calamine diff --git a/conda/environments/all_cuda-133_arch-x86_64.yaml b/conda/environments/all_cuda-133_arch-x86_64.yaml index f59ceb96e161..c9ba9fd97378 100644 --- a/conda/environments/all_cuda-133_arch-x86_64.yaml +++ b/conda/environments/all_cuda-133_arch-x86_64.yaml @@ -35,6 +35,7 @@ dependencies: - flatbuffers==24.3.25 - fsspec>=0.6.0 - gcc_linux-64=14.* +- gdb - hypothesis>=6.131.7 - identify>=2.5.20 - include-what-you-use==0.24.0 @@ -76,6 +77,7 @@ dependencies: - pandoc - polars>=1.35,<1.43 - pre-commit +- psutil - pyarrow>=19.0.0,<24 - pytables - pytest-benchmark @@ -83,7 +85,6 @@ dependencies: - pytest-cov - pytest-httpserver - pytest-rerunfailures!=16.0.0 -- pytest-timeout - pytest-xdist - pytest<9.1.0 - python-calamine diff --git a/dependencies.yaml b/dependencies.yaml index 227a11857df0..3955fd771045 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -1196,13 +1196,18 @@ dependencies: packages: - rich - pytest-httpserver - - pytest-timeout - zstandard + # Used by ci/timeout_with_stack.py utility + - psutil # The polars test suite constructs pandas objects for interop and # dask-cuda pulls pandas in transitively (unbounded), so constrain it # here to exclude the 3.0.4 release that segfaults on pd.Timedelta # (https://github.com/pandas-dev/pandas/issues/66086). - *pandas + - output_types: conda + packages: + # Used by timeout_with_stack.py utility + - gdb test_python_narwhals: common: - output_types: [conda, requirements, pyproject] diff --git a/python/cudf_polars/pyproject.toml b/python/cudf_polars/pyproject.toml index 3ee317f37c53..4e942917ef77 100644 --- a/python/cudf_polars/pyproject.toml +++ b/python/cudf_polars/pyproject.toml @@ -45,9 +45,9 @@ classifiers = [ test = [ "dask-cuda==26.10.*,>=0.0.0a0", "pandas>=3.0.0,<3.0.4a0", + "psutil", "pytest-cov", "pytest-httpserver", - "pytest-timeout", "pytest-xdist", "pytest<9.1.0", "rich", @@ -91,7 +91,6 @@ filterwarnings = [ "error", "ignore:Port .* is already in use.:UserWarning", ] -timeout = 45 xfail_strict = true [tool.coverage.report] diff --git a/python/cudf_polars/tests/conftest.py b/python/cudf_polars/tests/conftest.py index 218592d52a39..22e034b37d78 100644 --- a/python/cudf_polars/tests/conftest.py +++ b/python/cudf_polars/tests/conftest.py @@ -339,9 +339,6 @@ def engine_raise_on_fail() -> pl.GPUEngine: def timeout_seconds() -> int: """ Conservative timeout for APIs that accept a timeout parameter. - - Since pytest-timeout is installed, ensure this value is less than timeout - in python/cudf_polars/pyproject.toml. """ return 30 diff --git a/python/cudf_polars/tests/expressions/test_rolling.py b/python/cudf_polars/tests/expressions/test_rolling.py index e80bb25679c1..570497147d99 100644 --- a/python/cudf_polars/tests/expressions/test_rolling.py +++ b/python/cudf_polars/tests/expressions/test_rolling.py @@ -358,7 +358,6 @@ def test_rank_over_with_null_values( @pytest.mark.parametrize("method", ["ordinal", "dense", "min", "max", "average"]) @pytest.mark.parametrize("descending", [False, True]) @pytest.mark.parametrize("order_by", [None, ["g2", pl.col("x2") * 2]]) -@pytest.mark.timeout(120) def test_rank_over_with_null_group_keys( engine: pl.GPUEngine, df: pl.LazyFrame, diff --git a/python/cudf_polars/tests/streaming/test_scan.py b/python/cudf_polars/tests/streaming/test_scan.py index 13e88ead7731..15ba081e69fb 100644 --- a/python/cudf_polars/tests/streaming/test_scan.py +++ b/python/cudf_polars/tests/streaming/test_scan.py @@ -73,7 +73,6 @@ def df(): ("parquet", pl.scan_parquet), ], ) -@pytest.mark.timeout(90) def test_parallel_scan( tmp_path: Path, df: pl.DataFrame, diff --git a/python/cudf_polars/tests/streaming/test_sort.py b/python/cudf_polars/tests/streaming/test_sort.py index 71e1535988e3..9707ba9886b6 100644 --- a/python/cudf_polars/tests/streaming/test_sort.py +++ b/python/cudf_polars/tests/streaming/test_sort.py @@ -87,7 +87,6 @@ def large_frames(): ) -@pytest.mark.timeout(120) def test_sort(df, engine): q = df.sort(by=["y", "z"]) assert_gpu_result_equal(q, engine=engine) From c352e0f3ede0fb28817e1f4377d32215f9494cfd Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Tue, 21 Jul 2026 11:42:59 -0500 Subject: [PATCH 23/25] Stop mutating the source column dtype in equivalent-type numerical casts (#23364) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Split out of #23255 (1/6). `NumericalColumn.as_numerical_column` short-circuits casts between equivalent dtypes (same pylibcudf type, e.g. `float64` → `Float64`), but implemented the shortcut by assigning the target dtype onto `self._dtype` in place. The column object is shared with the caller's Series/DataFrame, so the *source* object silently changed dtype as a side effect of the cast. This returns a fresh column over the same pylibcudf data instead (`nans_to_nulls` first for float → masked casts), and adds a classic regression test. Fixes 5 pandas-tests (`test_stack_nullable_dtype[*]`, `test_loc_set_nan_in_categorical_series[Float64]`, `test_assert_series_equal_extension_dtype_mismatch`, `test_assert_frame_equal_extension_dtype_mismatch`); their xfail entries are removed. Attribution verified by running the node ids against an isolated build containing only this fix (they pass) and a clean build (they fail). Independent of the other #23255 split PRs; can merge in any order. Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/23364 --- python/cudf/cudf/core/column/numerical.py | 19 ++++++++------- .../pandas/scripts/pandas-testing-plugin.py | 5 ---- .../cudf/tests/series/methods/test_astype.py | 24 +++++++++++++++++++ 3 files changed, 35 insertions(+), 13 deletions(-) diff --git a/python/cudf/cudf/core/column/numerical.py b/python/cudf/cudf/core/column/numerical.py index a1d14959e7ca..251514aa66ef 100644 --- a/python/cudf/cudf/core/column/numerical.py +++ b/python/cudf/cudf/core/column/numerical.py @@ -911,20 +911,23 @@ def as_numerical_column(self, dtype: DtypeObj) -> NumericalColumn: self.dtype ): # Short-circuit the cast if the dtypes are equivalent - # but not the same type object. + # but not the same type object. Do NOT mutate self._dtype: + # the column object may be shared with the caller's frame. if ( is_pandas_nullable_extension_dtype(dtype) and isinstance(self.dtype, np.dtype) and self.dtype.kind == "f" ): - # If the dtype is a pandas nullable extension type, we need to - # float column doesn't have any NaNs. + # NaNs must become nulls before viewing as a masked dtype. res = self.nans_to_nulls() - res._dtype = dtype - return res - else: - self._dtype = dtype - return self + return cast( + "NumericalColumn", + ColumnBase.create(res.plc_column, dtype), + ) + return cast( + "NumericalColumn", + ColumnBase.create(self.plc_column, dtype), + ) if self.dtype.kind == "f" and dtype.kind in "iu": if not is_pandas_nullable_extension_dtype(dtype) and ( self.nan_count > 0 diff --git a/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py b/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py index f09d75e33b39..5f3155adb71b 100644 --- a/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py +++ b/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py @@ -1721,8 +1721,6 @@ def pytest_unconfigure(config): "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_nan_in_multiindex_columns[False]": "TODO: Add a reason for failure", "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_nan_in_multiindex_columns[True]": "TODO: Add a reason for failure", "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_nan_level[False]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_nullable_dtype[False]": "TODO: Add a reason for failure", - "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_nullable_dtype[True]": "TODO: Add a reason for failure", "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_order_with_unsorted_levels_multi_row_2[False]": "TODO: Add a reason for failure", "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_unstack_multiple[False]": "TODO: Add a reason for failure", "tests/frame/test_stack_unstack.py::TestStackUnstackMultiLevel::test_stack_unstack_multiple[True]": "TODO: Add a reason for failure", @@ -2511,7 +2509,6 @@ def pytest_unconfigure(config): "tests/indexing/test_loc.py::TestLocSetitemWithExpansion::test_loc_setitem_with_expansion_nonunique_index[string-pyarrow-True]": 'AssertionError: Column name="0" are different', "tests/indexing/test_loc.py::TestLocSetitemWithExpansion::test_loc_setitem_with_expansion_nonunique_index[string-python-False]": 'AssertionError: Column name="0" are different', "tests/indexing/test_loc.py::TestLocSetitemWithExpansion::test_loc_setitem_with_expansion_nonunique_index[string-python-True]": 'AssertionError: Column name="0" are different', - "tests/indexing/test_loc.py::TestLocWithMultiIndex::test_loc_set_nan_in_categorical_series[Float64]": "TODO: Add a reason for failure", "tests/indexing/test_loc.py::test_loc_getitem_multiindex_tuple_level": "AssertionError: DataFrame Expected type , found instead", "tests/indexing/test_na_indexing.py::test_series_mask_boolean[True-list-mask0-values0-object]": "TODO: Add a reason for failure", "tests/indexing/test_na_indexing.py::test_series_mask_boolean[True-list-mask1-values0-object]": "TODO: Add a reason for failure", @@ -3841,13 +3838,11 @@ def pytest_unconfigure(config): "tests/tslibs/test_to_offset.py::test_to_offset_uppercase_frequency_deprecated[2NS]": "TODO: Add a reason for failure", "tests/tslibs/test_to_offset.py::test_to_offset_uppercase_frequency_deprecated[2Us]": "TODO: Add a reason for failure", "tests/util/test_assert_frame_equal.py::test_allows_duplicate_labels": "TODO: Add a reason for failure", - "tests/util/test_assert_frame_equal.py::test_assert_frame_equal_extension_dtype_mismatch": "TODO: Add a reason for failure", "tests/util/test_assert_frame_equal.py::test_assert_frame_equal_nested_df_na[None]": "KeyError: 0", "tests/util/test_assert_frame_equal.py::test_assert_frame_equal_nested_df_na[nan]": "KeyError: 0", "tests/util/test_assert_frame_equal.py::test_frame_equal_index_dtype_mismatch[True-df11-df21-DataFrame\\\\.index level \\\\[0\\\\] are different]": "Failed: DID NOT RAISE ", "tests/util/test_assert_index_equal.py::test_index_equal_range_categories[True-True]": "TODO: Add a reason for failure", "tests/util/test_assert_series_equal.py::test_allows_duplicate_labels": "TODO: Add a reason for failure", - "tests/util/test_assert_series_equal.py::test_assert_series_equal_extension_dtype_mismatch": "TODO: Add a reason for failure", "tests/util/test_assert_series_equal.py::test_assert_series_equal_int_tol": "AssertionError: left is not an ExtensionArray", "tests/util/test_assert_series_equal.py::test_large_unequal_ints[Int64]": "Failed: DID NOT RAISE ", "tests/util/test_assert_series_equal.py::test_large_unequal_ints[int64]": "TODO: Add a reason for failure", diff --git a/python/cudf/cudf/tests/series/methods/test_astype.py b/python/cudf/cudf/tests/series/methods/test_astype.py index a3eaa0020cfa..3071da934e13 100644 --- a/python/cudf/cudf/tests/series/methods/test_astype.py +++ b/python/cudf/cudf/tests/series/methods/test_astype.py @@ -1626,3 +1626,27 @@ def test_astype_aware_to_naive_raises(): cudf_ser.astype("datetime64[ns]") with pytest.raises(TypeError): pd_ser.astype("datetime64[ns]") + + +@pytest.mark.parametrize( + "data, src_dtype, masked_dtype", + [ + ([1.0, 2.0, float("nan")], "float64", pd.Float64Dtype()), + ([1, 2, 3], "int64", pd.Int64Dtype()), + ], +) +def test_astype_masked_equivalent_dtype_no_source_mutation( + data, src_dtype, masked_dtype +): + # casting to the equivalent masked dtype takes a short-circuit path; + # it must not mutate the source column's dtype in place (the column + # is shared with the source Series/frame) + ser = cudf.Series(data, dtype=src_dtype) + result = ser.astype(masked_dtype) + + assert ser.dtype == np.dtype(src_dtype) + assert result.dtype == masked_dtype + assert_eq( + result.to_pandas(), + pd.Series(data, dtype=src_dtype).astype(masked_dtype), + ) From cf27be6524d51cb3c761c9ac159320ea0cdb827e Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Tue, 21 Jul 2026 13:02:19 -0500 Subject: [PATCH 24/25] Mirror cudf.pandas class-level monkeypatches onto the real type (#23001) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Split out of #22927 per review. ## Problem A class-level attribute write on a cudf.pandas proxy type — e.g. `monkeypatch.setattr(pd.ExcelFile, "parse", fn)` — was only applied to the proxy class. Code that runs under `disable_module_accelerator()` (such as the pandas fallback path of `pd.read_excel`) resolves attributes from the *real* class, so a patch applied only to the proxy was invisible to it. ## Fix Add `_FastSlowProxyMeta.__setattr__`/`__delattr__` to mirror runtime class-level patches onto the underlying "slow" (real) type: - `__setattr__` mirrors the assignment after translating the assigned value into "slow" space. A plain value is forwarded as-is. Re-assigning the proxy's *pristine* attribute for a name (which is exactly what `monkeypatch.setattr` / `mock.patch.object` save and re-assign on undo) translates to the slow type's pristine attribute for that name — restored if the slow type had one of its own, or removed if it didn't (leaving any inherited implementation visible). Proxy machinery (e.g. a saved `pd.ExcelFile.parse`, a `_MethodProxy`) unwraps to the slow object it delegates to. - `__delattr__` mirrors a deletion as a deletion. Nothing is restored on delete. The pristine state is a per-type map `name -> (pristine proxy attribute, pristine slow class-dict entry)` snapshotted once, when `make_*_proxy_type` finishes building the type (the same point mirroring is enabled via `_fsproxy_mirror_slow_overrides`). It is a fixed translation table, not runtime patch tracking: there is no stash of "what to put back", and undo works for any code that follows the standard save/patch/re-assign pattern (pytest `monkeypatch`, `unittest.mock.patch.object`, manual saves) because the saved value itself identifies the pristine state. The translation is what makes mirroring safe at all — the values readable off a proxy type live in proxy space, and forwarding e.g. the saved `columns` property or `eval`/`query` functions verbatim onto `pandas.DataFrame` would install cudf machinery on the real class (for `columns` this infinitely recurses on the fallback path). cudf.pandas's own custom methods (`DataFrame.eval`/`query`) are installed via the new `_setattr_fsproxy_no_mirror` helper, which registers them as part of the proxy's pristine state without forwarding them to pandas. ## Tests Adds unit tests in `cudf_pandas_tests/test_fast_slow_proxy.py` covering: set/delete mirroring, monkeypatch round-trips (new attr, existing attr, nested, `delattr`), `mock.patch.object` (which saves the raw descriptor without resolving it), properties, plain data attributes, `staticmethod`/`classmethod` descriptor preservation, methods the slow type only inherits, and the no-mirror helper; plus an end-to-end test in `test_cudf_pandas.py` that patches/unpatches `DataFrame.columns`/`eval` and `Series.str` and checks real pandas is restored and functional. Removes 14 now-passing xfails from the pandas-tests plugin (13× `read_excel` engine-selection tests that monkeypatch the engine, plus a monkeypatch-registered custom accessor). The attribution of these 14 to the proxy fix (vs the Excel-reader fixes remaining in #22927) was verified locally by running each removed xfail with the proxy fix in isolation. Authors: - GALI PREM SAGAR (https://github.com/galipremsagar) - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Vyas Ramasubramani (https://github.com/vyasr) URL: https://github.com/rapidsai/cudf/pull/23001 --- python/cudf/cudf/pandas/_wrappers/pandas.py | 7 +- python/cudf/cudf/pandas/fast_slow_proxy.py | 189 ++++++++++ .../pandas/scripts/pandas-testing-plugin.py | 14 - .../cudf_pandas_tests/test_cudf_pandas.py | 32 ++ .../cudf_pandas_tests/test_fast_slow_proxy.py | 339 ++++++++++++++++++ 5 files changed, 565 insertions(+), 16 deletions(-) diff --git a/python/cudf/cudf/pandas/_wrappers/pandas.py b/python/cudf/cudf/pandas/_wrappers/pandas.py index 621163c4be2b..6cba72851da4 100644 --- a/python/cudf/cudf/pandas/_wrappers/pandas.py +++ b/python/cudf/cudf/pandas/_wrappers/pandas.py @@ -47,6 +47,7 @@ _FastSlowAttribute, _FunctionProxy, _maybe_wrap_result, + _setattr_fsproxy_no_mirror, _State, _Unusable, is_proxy_object, @@ -1691,8 +1692,10 @@ def _df_query_method(self, *args, local_dict=None, global_dict=None, **kwargs): ) -DataFrame.eval = _df_eval_method -DataFrame.query = _df_query_method +# These custom implementations are installed by cudf.pandas itself and must +# not be mirrored onto (and clobber) the real ``pandas.DataFrame``. +_setattr_fsproxy_no_mirror(DataFrame, "eval", _df_eval_method) +_setattr_fsproxy_no_mirror(DataFrame, "query", _df_query_method) _JsonReader = make_intermediate_proxy_type( "_JsonReader", diff --git a/python/cudf/cudf/pandas/fast_slow_proxy.py b/python/cudf/cudf/pandas/fast_slow_proxy.py index 20600c7fe9db..60fa07dc37b4 100644 --- a/python/cudf/cudf/pandas/fast_slow_proxy.py +++ b/python/cudf/cudf/pandas/fast_slow_proxy.py @@ -362,6 +362,11 @@ def _fsproxy_state(self) -> _State: final_type_map[fast_type] = cls final_type_map[slow_type] = cls + # Proxy type fully constructed: snapshot its pristine state and, from + # here on, mirror class-level attribute writes (genuine runtime + # monkeypatches) onto the underlying "slow" (real) type. + _enable_fsproxy_mirroring(cls) + return cls @@ -511,6 +516,11 @@ def _fsproxy_fast_to_slow(self): intermediate_type_map[fast_type] = cls intermediate_type_map[slow_type] = cls + # Proxy type fully constructed: snapshot its pristine state and, from + # here on, mirror class-level attribute writes (genuine runtime + # monkeypatches) onto the underlying "slow" (real) type. + _enable_fsproxy_mirroring(cls) + return cls @@ -560,6 +570,55 @@ def get_registered_functions(): return dict() +_SLOW_ABSENT = object() + + +def _enable_fsproxy_mirroring(cls: type) -> None: + """Finalize a proxy type for class-level patch mirroring. + + Snapshots the proxy type's pristine public class attributes together + with the slow type's pristine class-dict entries for the same names, + then enables mirroring of class-level attribute writes/deletions onto + the slow type (see ``_FastSlowProxyMeta.__setattr__``/``__delattr__``). + + The snapshot is a fixed translation table, not runtime patch tracking: + re-assigning the proxy's pristine attribute for ``name`` (which is what + ``monkeypatch``/``mock.patch`` save and re-assign on undo) translates + to restoring the slow type's pristine attribute for ``name``. + """ + slow = cls._fsproxy_slow_type # type: ignore[attr-defined] + pristine = { + name: (value, slow.__dict__.get(name, _SLOW_ABSENT)) + for name, value in cls.__dict__.items() + if not name.startswith("_") + } + type.__setattr__(cls, "_fsproxy_pristine_attrs", pristine) + type.__setattr__(cls, "_fsproxy_mirror_slow_overrides", True) + + +def _setattr_fsproxy_no_mirror(cls: type, name: str, value: Any) -> None: + """Install a cudf.pandas-internal attribute on a proxy type. + + ``_FastSlowProxyMeta.__setattr__`` mirrors class-level attribute writes + onto the underlying "slow" (real) type so that runtime monkeypatches stay + visible to the pandas fallback path. cudf.pandas itself installs a handful + of custom methods (e.g. ``DataFrame.query``/``DataFrame.eval``) onto the + proxy classes that must *not* clobber pandas' genuine implementations; use + this helper for those (rare) assignments, after the proxy type has been + fully constructed by ``make_*_proxy_type``. The attribute is registered as + part of the proxy's pristine state so that a later save/patch/re-assign + cycle restores the slow type's own attribute rather than forwarding the + cudf-internal object to it. + """ + type.__setattr__(cls, name, value) + if not name.startswith("_"): + slow = cls._fsproxy_slow_type # type: ignore[attr-defined] + cls._fsproxy_pristine_attrs[name] = ( # type: ignore[attr-defined] + value, + slow.__dict__.get(name, _SLOW_ABSENT), + ) + + class _FastSlowProxyMeta(type): """ Metaclass used to dynamically find class attributes and @@ -578,6 +637,136 @@ def _fsproxy_slow(self) -> type: def _fsproxy_fast(self) -> type: return self._fsproxy_fast_type + def __new__(mcls, *args, **kwargs): + cls = super().__new__(mcls, *args, **kwargs) + # Per-proxy-type switch controlling whether class-level attribute + # writes are mirrored onto the underlying "slow" (real) type (see + # ``__setattr__``/``__delattr__``). It starts disabled so that the + # attributes installed while the proxy type is being built are not + # forwarded to the real type; ``make_*_proxy_type`` enables it once + # construction is complete. Initialized in ``__new__`` rather than + # ``__init__`` because cooperating metaclasses may perform + # class-level attribute writes from their own ``__new__`` — e.g. + # ``ABCMeta.__new__`` assigns ``__abstractmethods__``, dispatching + # to ``__setattr__`` below before ``__init__`` ever runs. + type.__setattr__(cls, "_fsproxy_mirror_slow_overrides", False) + return cls + + def __setattr__(cls, name, value): + # Class-level attribute assignments on a proxy type (e.g. + # ``monkeypatch.setattr(pd.ExcelFile, "parse", fn)``) must also be + # mirrored onto the underlying "slow" (real) type. Code that runs + # under ``disable_module_accelerator()`` (e.g. the pandas fallback + # path of ``pd.read_excel``) resolves attributes from the real + # class, not the proxy, so a patch applied only to the proxy would + # otherwise be invisible to that code. The assigned value is first + # translated into its slow-space equivalent: re-assigning the + # proxy's pristine attribute translates to the slow type's pristine + # attribute, and proxy machinery is unwrapped to the slow object it + # delegates to, so save/patch/re-assign cycles round-trip on the + # real type as well. + type.__setattr__(cls, name, value) + if not cls._fsproxy_mirror_slow_overrides: + # The proxy type is still being constructed (or this is a + # non-pandas proxy): only mirror user/runtime monkeypatches, + # never the custom methods cudf.pandas installs on the proxy + # classes itself. + return + if name.startswith("_"): + return + slow = cls._fsproxy_slow_type + try: + # Mirroring is best-effort: translating a wrapped proxy instance + # can require a fast-to-slow conversion, which may itself fail; + # never let that escape an otherwise-successful assignment. + pristine = cls._fsproxy_pristine_attrs + entry = pristine.get(name) + if entry is not None and value is entry[0]: + # The proxy's pristine attribute for ``name`` is being + # re-assigned (e.g. ``monkeypatch``/``mock.patch`` undo + # re-setting the saved class-dict entry). Its slow-space + # equivalent is the slow type's pristine attribute. + if entry[1] is _SLOW_ABSENT: + # The slow type never defined ``name`` itself: the proxy + # mirrors the slow type's *dir*, so it has pristine + # attributes for methods the slow type only inherits + # (e.g. ``DataFrame.head`` lives on ``NDFrame``). + # Mirroring a patch for such a name added a shadowing + # entry to the slow type's dict; undoing the patch must + # remove that entry again so the inherited + # implementation becomes visible. It may legitimately be + # missing (the mirror is best-effort), hence the guard. + if name in slow.__dict__: + delattr(slow, name) + else: + setattr(slow, name, entry[1]) + return + # Otherwise translate the assigned value into "slow" space + # before mirroring it: a value read off a proxy type (e.g. the + # original that a caller saves before patching and re-assigns + # to undo) is proxy machinery wrapping a slow-side object, and + # mirroring it verbatim would install that machinery on the + # real type. Unwrap it to the slow object it delegates to, so + # save/patch/re-assign cycles round-trip on the real type; + # values with no determinable slow-side equivalent are not + # mirrored at all. + if isinstance(value, _FastSlowAttribute): + # The proxy's own delegating descriptor (a *pristine* one is + # already handled by identity above; this covers a + # descriptor obtained some other way): its slow equivalent + # is the method it wraps, if it ever resolved one. + attr = value._attr + if not isinstance(attr, _MethodProxy): + return + value = attr + if isinstance(value, _FunctionProxy): + unwrapped = value._fsproxy_slow + if entry is not None and entry[1] is not _SLOW_ABSENT: + # Re-assigning a saved ``cls.method`` (a ``_MethodProxy`` + # over the *resolved* slow attribute): if it resolves + # back to the slow type's pristine attribute, restore + # the pristine class-dict entry itself so + # ``classmethod``/``staticmethod`` descriptors are not + # degraded to their bound/plain-function forms. + descriptor = entry[1] + try: + resolved = ( + descriptor.__get__(None, slow) + if hasattr(type(descriptor), "__get__") + else descriptor + ) + if unwrapped is resolved or unwrapped == resolved: + setattr(slow, name, descriptor) + return + except Exception: + pass + setattr(slow, name, unwrapped) + elif isinstance(value, _FastSlowProxy): + setattr(slow, name, value._fsproxy_slow) + elif isinstance(value, _FastSlowProxyMeta): + slow_type = getattr(value, "_fsproxy_slow_type", None) + if slow_type is not None: + setattr(slow, name, slow_type) + else: + setattr(slow, name, value) + except Exception: + pass + + def __delattr__(cls, name): + # Mirror class-level attribute *deletions* onto the underlying "slow" + # (real) type as well, for the same reason as ``__setattr__``: after + # ``del cls.name`` the attribute is gone from the proxy, so it must + # also be gone from the real type seen by fallback code. + type.__delattr__(cls, name) + if not cls._fsproxy_mirror_slow_overrides: + return + if name.startswith("_"): + return + try: + delattr(cls._fsproxy_slow_type, name) + except (AttributeError, TypeError): + pass + def __dir__(self): # Try to return the cached dir of the slow object, but if it # doesn't exist, fall back to the default implementation. diff --git a/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py b/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py index 5f3155adb71b..4f749565dfc8 100644 --- a/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py +++ b/python/cudf/cudf/pandas/scripts/pandas-testing-plugin.py @@ -2539,19 +2539,6 @@ def pytest_unconfigure(config): "tests/io/excel/test_odswriter.py::test_cell_value_type[test string-string-string-value-test string]": "TODO: Add a reason for failure", "tests/io/excel/test_odswriter.py::test_cell_value_type[value4-date-date-value-2010-10-10T10:10:10]": "TODO: Add a reason for failure", "tests/io/excel/test_odswriter.py::test_cell_value_type[value5-date-date-value-2010-10-10]": "TODO: Add a reason for failure", - "tests/io/excel/test_readers.py::TestReaders::test_engine_used[('calamine', '.ods')]": "TODO: Add a reason for failure", - "tests/io/excel/test_readers.py::TestReaders::test_engine_used[('calamine', '.xls')]": "TODO: Add a reason for failure", - "tests/io/excel/test_readers.py::TestReaders::test_engine_used[('calamine', '.xlsb')]": "TODO: Add a reason for failure", - "tests/io/excel/test_readers.py::TestReaders::test_engine_used[('calamine', '.xlsm')]": "TODO: Add a reason for failure", - "tests/io/excel/test_readers.py::TestReaders::test_engine_used[('calamine', '.xlsx')]": "TODO: Add a reason for failure", - "tests/io/excel/test_readers.py::TestReaders::test_engine_used[('odf', '.ods')]": "TODO: Add a reason for failure", - "tests/io/excel/test_readers.py::TestReaders::test_engine_used[('openpyxl', '.xlsm')]": "TODO: Add a reason for failure", - "tests/io/excel/test_readers.py::TestReaders::test_engine_used[('openpyxl', '.xlsx')]": "TODO: Add a reason for failure", - "tests/io/excel/test_readers.py::TestReaders::test_engine_used[('pyxlsb', '.xlsb')]": "TODO: Add a reason for failure", - "tests/io/excel/test_readers.py::TestReaders::test_engine_used[('xlrd', '.xls')]": "TODO: Add a reason for failure", - "tests/io/excel/test_readers.py::TestReaders::test_engine_used[(None, '.xls')]": "TODO: Add a reason for failure", - "tests/io/excel/test_readers.py::TestReaders::test_engine_used[(None, '.xlsm')]": "TODO: Add a reason for failure", - "tests/io/excel/test_readers.py::TestReaders::test_engine_used[(None, '.xlsx')]": "TODO: Add a reason for failure", "tests/io/excel/test_style.py::test_format_hierarchical_rows_periodindex[False]": "AttributeError: _compute. Did you mean: 'compare'?", "tests/io/excel/test_style.py::test_format_hierarchical_rows_periodindex[True]": "AttributeError: _compute. Did you mean: 'compare'?", "tests/io/excel/test_style.py::test_format_hierarchical_rows_periodindex[columns]": "AttributeError: _compute. Did you mean: 'compare'?", @@ -3685,7 +3672,6 @@ def pytest_unconfigure(config): "tests/test_algos.py::TestValueCounts::test_value_counts_dropna": "pandas keeps bool-with-None data as object dtype; cudf stores it as a masked bool column", "tests/test_algos.py::TestValueCounts::test_value_counts_stability": "asserts that kind='quicksort' produces an unstable order; cudf sorts are always stable", "tests/test_col.py::test_cached_property": "AssertionError: assert False", - "tests/test_col.py::test_custom_accessor": "AttributeError: 'Series' object has no attribute 'xyz'", "tests/test_common.py::test_serializable[obj0]": "TODO: Add a reason for failure", "tests/test_common.py::test_temp_setattr[False]": "TODO: Add a reason for failure", "tests/test_common.py::test_temp_setattr[True]": "TODO: Add a reason for failure", diff --git a/python/cudf/cudf_pandas_tests/test_cudf_pandas.py b/python/cudf/cudf_pandas_tests/test_cudf_pandas.py index cbf3e17bfdc4..792e4f2cf314 100644 --- a/python/cudf/cudf_pandas_tests/test_cudf_pandas.py +++ b/python/cudf/cudf_pandas_tests/test_cudf_pandas.py @@ -2186,6 +2186,38 @@ def test_module_proxy_write_through_config(monkeypatch): cf.register_option("foo", 1) +def test_class_monkeypatch_roundtrip_restores_real_pandas(monkeypatch): + # Class-level patches on proxy types are mirrored onto the real pandas + # type (so they stay visible to fallback code running under + # ``disable_module_accelerator``); undoing them must restore the real + # type's own attributes — including for attributes that cudf.pandas + # replaces on the proxy, like ``columns``/``eval``/``str``. + real_df = xpd.DataFrame._fsproxy_slow + real_series = xpd.Series._fsproxy_slow + orig_columns = real_df.__dict__["columns"] + orig_eval = real_df.__dict__["eval"] + orig_str = real_series.__dict__["str"] + + def fake_eval(self, *args, **kwargs): + return "patched" + + monkeypatch.setattr(xpd.DataFrame, "eval", fake_eval) + assert real_df.__dict__["eval"] is fake_eval + monkeypatch.setattr( + xpd.DataFrame, "columns", property(lambda self: "patched") + ) + monkeypatch.setattr(xpd.Series, "str", property(lambda self: "patched")) + monkeypatch.undo() + + assert real_df.__dict__["columns"] is orig_columns + assert real_df.__dict__["eval"] is orig_eval + assert real_series.__dict__["str"] is orig_str + # The real type must remain fully functional on the fallback path. + df = real_df({"a": [1, 2]}) + assert list(df.columns) == ["a"] + assert list(df.eval("b = a + 1").columns) == ["a", "b"] + + @pytest.mark.parametrize("box", ["Series", "array"]) @pytest.mark.parametrize("na_value", [pd.NA, np.nan], ids=["NA", "NaN"]) @pytest.mark.parametrize("storage", ["python", "pyarrow"]) diff --git a/python/cudf/cudf_pandas_tests/test_fast_slow_proxy.py b/python/cudf/cudf_pandas_tests/test_fast_slow_proxy.py index 30d2124edcbf..5e22badc4c0e 100644 --- a/python/cudf/cudf_pandas_tests/test_fast_slow_proxy.py +++ b/python/cudf/cudf_pandas_tests/test_fast_slow_proxy.py @@ -12,7 +12,9 @@ import cudf.pandas.fast_slow_proxy from cudf.pandas.fast_slow_proxy import ( _fast_arg, + _FastSlowAttribute, _FunctionProxy, + _setattr_fsproxy_no_mirror, _slow_arg, _transform_arg, _Unusable, @@ -747,3 +749,340 @@ def test_tuple_with_attrs_transform(): assert b is bprime assert c == cprime and c is not cprime assert d == dprime and d is not dprime + + +def _make_mirror_proxy(): + class Fast: + pass + + class SlowBase: + def inherited(self): + return "base" + + class Slow(SlowBase): + const = 42 + + def existing(self): + return "slow original" + + @property + def prop(self): + return "slow prop" + + @staticmethod + def smethod(x): + return x + 1 + + @classmethod + def cmethod(cls): + return cls.__name__ + + Pxy = make_final_proxy_type( + "Pxy", + Fast, + Slow, + fast_to_slow=lambda fast: Slow(), + slow_to_fast=lambda slow: Fast(), + ) + return Fast, Slow, Pxy + + +def test_class_attr_mirroring_enabled_after_construction(): + # ``make_*_proxy_type`` enables per-type mirroring once the proxy type is + # fully built (it starts disabled so the methods installed during + # construction are not forwarded to the real type). + _, _, Pxy = _make_mirror_proxy() + assert Pxy.__dict__["_fsproxy_mirror_slow_overrides"] is True + + +def test_class_attr_setattr_mirrored_to_slow(): + # A class-level attribute write on the proxy is mirrored onto the + # underlying "slow" (real) type so it is visible to fallback code. + _, Slow, Pxy = _make_mirror_proxy() + + def patched(self): + return "patched" + + Pxy.new_method = patched + assert Slow.__dict__.get("new_method") is patched + + +def test_class_attr_delattr_mirrored_to_slow(): + # Deleting a class-level attribute on the proxy mirrors the deletion + # onto the slow type (the ``__delattr__`` path). + _, Slow, Pxy = _make_mirror_proxy() + + def patched(self): + return "patched" + + Pxy.new_method = patched + assert "new_method" in Slow.__dict__ + + del Pxy.new_method + assert "new_method" not in Pxy.__dict__ + assert "new_method" not in Slow.__dict__ + + +def test_class_attr_delattr_existing_removes_from_slow(): + # Deleting is deleting, not restoring: patching an existing attribute + # and then deleting it removes it from both the proxy and the slow type, + # exactly as the same sequence would on a plain Python class. + _, Slow, Pxy = _make_mirror_proxy() + + def patched(self): + return "patched" + + Pxy.existing = patched + assert Slow.__dict__["existing"] is patched + + del Pxy.existing + assert "existing" not in Pxy.__dict__ + assert "existing" not in Slow.__dict__ + + +def test_class_attr_restore_existing_slow_attr(): + # Re-assigning the proxy's pristine class-dict entry (as ``monkeypatch`` + # and ``mock.patch`` teardown do) restores the slow type's pristine + # attribute. Note: no prior class-level getattr — the saved descriptor + # is unresolved, as in the ``mock.patch.object`` flow. + _, Slow, Pxy = _make_mirror_proxy() + saved = Pxy.__dict__["existing"] + assert isinstance(saved, _FastSlowAttribute) + original_slow = Slow.__dict__["existing"] + + def patched(self): + return "patched" + + Pxy.existing = patched + assert Slow.__dict__["existing"] is patched + + Pxy.existing = saved + assert Slow.__dict__["existing"] is original_slow + + +def test_class_attr_restore_via_saved_method_proxy(): + # Re-assigning a saved ``Pxy.method`` (a ``_MethodProxy`` over the + # resolved slow attribute, not the class-dict descriptor) also restores + # the slow type's pristine attribute. + _, Slow, Pxy = _make_mirror_proxy() + saved = getattr(Pxy, "existing") + original_slow = Slow.__dict__["existing"] + + def patched(self): + return "patched" + + Pxy.existing = patched + assert Slow.__dict__["existing"] is patched + + Pxy.existing = saved + assert Slow.__dict__["existing"] is original_slow + + +def test_class_attr_monkeypatch_roundtrip(monkeypatch): + # End-to-end: ``monkeypatch`` of a brand-new attribute mirrors onto the + # slow type, and teardown (which deletes it) mirrors the deletion. + _, Slow, Pxy = _make_mirror_proxy() + + def fake(self): + return "fake" + + monkeypatch.setattr(Pxy, "brand_new", fake, raising=False) + assert Slow.__dict__.get("brand_new") is fake + + monkeypatch.undo() + assert "brand_new" not in Pxy.__dict__ + assert "brand_new" not in Slow.__dict__ + + +def test_class_attr_monkeypatch_existing_roundtrip(monkeypatch): + # End-to-end: ``monkeypatch`` of a pre-existing method mirrors the patch + # onto the slow type, and teardown (which re-assigns the saved proxy + # descriptor) restores the slow type's original implementation. + _, Slow, Pxy = _make_mirror_proxy() + original_slow = Slow.__dict__["existing"] + + def fake(self): + return "fake" + + monkeypatch.setattr(Pxy, "existing", fake) + assert Slow.__dict__["existing"] is fake + + monkeypatch.undo() + assert isinstance(Pxy.__dict__["existing"], _FastSlowAttribute) + assert Slow.__dict__["existing"] is original_slow + + +def test_class_attr_nested_monkeypatch_existing_roundtrip(): + # Nested patches of the same pre-existing method unwind in order, + # each level restoring the slow type to the previous state. + from _pytest.monkeypatch import MonkeyPatch + + _, Slow, Pxy = _make_mirror_proxy() + original_slow = Slow.__dict__["existing"] + + def fake1(self): + return "fake1" + + def fake2(self): + return "fake2" + + mp1, mp2 = MonkeyPatch(), MonkeyPatch() + mp1.setattr(Pxy, "existing", fake1) + assert Slow.__dict__["existing"] is fake1 + mp2.setattr(Pxy, "existing", fake2) + assert Slow.__dict__["existing"] is fake2 + + mp2.undo() + assert Slow.__dict__["existing"] is fake1 + mp1.undo() + assert Slow.__dict__["existing"] is original_slow + + +def test_class_attr_mock_patch_object_roundtrip(): + # ``unittest.mock.patch.object`` saves the raw class-dict entry without + # a prior getattr (so the saved descriptor is never resolved); undo must + # still restore the slow type's pristine attribute. + from unittest import mock + + _, Slow, Pxy = _make_mirror_proxy() + original_slow = Slow.__dict__["existing"] + + def fake(self): + return "fake" + + with mock.patch.object(Pxy, "existing", fake): + assert Slow.__dict__["existing"] is fake + assert Slow.__dict__["existing"] is original_slow + + +def test_class_attr_property_monkeypatch_roundtrip(monkeypatch): + # Patching a property mirrors it onto the slow type; undo restores the + # slow type's pristine property object. + _, Slow, Pxy = _make_mirror_proxy() + original_slow = Slow.__dict__["prop"] + + fake = property(lambda self: "fake") + monkeypatch.setattr(Pxy, "prop", fake) + assert Slow.__dict__["prop"] is fake + assert Slow().prop == "fake" + + monkeypatch.undo() + assert Slow.__dict__["prop"] is original_slow + assert Slow().prop == "slow prop" + + +def test_class_attr_data_attr_monkeypatch_roundtrip(monkeypatch): + # Patching a plain class data attribute round-trips on the slow type. + _, Slow, Pxy = _make_mirror_proxy() + + monkeypatch.setattr(Pxy, "const", 99) + assert Slow.const == 99 + + monkeypatch.undo() + assert Slow.const == 42 + + +def test_class_attr_staticmethod_monkeypatch_roundtrip(monkeypatch): + # Undo restores the slow type's pristine ``staticmethod`` descriptor, + # not the plain function it resolves to (which would break instance + # calls by receiving ``self``). + _, Slow, Pxy = _make_mirror_proxy() + original_slow = Slow.__dict__["smethod"] + assert isinstance(original_slow, staticmethod) + + monkeypatch.setattr(Pxy, "smethod", staticmethod(lambda x: x - 1)) + assert Slow.smethod(1) == 0 + + monkeypatch.undo() + assert Slow.__dict__["smethod"] is original_slow + assert Slow().smethod(1) == 2 + + +def test_class_attr_classmethod_monkeypatch_roundtrip(monkeypatch): + # Undo restores the slow type's pristine ``classmethod`` descriptor, + # not the class-bound method it resolves to (which would pin ``cls`` + # for subclasses). + _, Slow, Pxy = _make_mirror_proxy() + original_slow = Slow.__dict__["cmethod"] + assert isinstance(original_slow, classmethod) + + monkeypatch.setattr(Pxy, "cmethod", classmethod(lambda cls: "fake")) + assert Slow.cmethod() == "fake" + + monkeypatch.undo() + assert Slow.__dict__["cmethod"] is original_slow + assert Slow.cmethod() == "Slow" + + +def test_class_attr_inherited_method_monkeypatch_roundtrip(monkeypatch): + # Patching a method the slow type only inherits mirrors it into the slow + # type's own dict; undo removes that entry again (rather than copying + # the base-class implementation into the subclass), leaving the + # inherited implementation visible. + _, Slow, Pxy = _make_mirror_proxy() + assert "inherited" not in Slow.__dict__ + + def fake(self): + return "fake" + + monkeypatch.setattr(Pxy, "inherited", fake) + assert Slow.__dict__["inherited"] is fake + + monkeypatch.undo() + assert "inherited" not in Slow.__dict__ + assert Slow().inherited() == "base" + + +def test_class_attr_monkeypatch_delattr_roundtrip(monkeypatch): + # ``monkeypatch.delattr`` mirrors the deletion onto the slow type, and + # undo (which re-assigns the saved pristine descriptor) restores the + # slow type's pristine attribute. + _, Slow, Pxy = _make_mirror_proxy() + original_slow = Slow.__dict__["existing"] + + monkeypatch.delattr(Pxy, "existing") + assert "existing" not in Pxy.__dict__ + assert "existing" not in Slow.__dict__ + + monkeypatch.undo() + assert Slow.__dict__["existing"] is original_slow + + +def test_setattr_fsproxy_no_mirror_skips_slow(): + # ``_setattr_fsproxy_no_mirror`` sets a class attribute on the proxy + # without forwarding it to the slow type (used for cudf.pandas' own custom + # methods such as ``DataFrame.query``/``eval``). + _, Slow, Pxy = _make_mirror_proxy() + + def custom(self): + return "custom" + + _setattr_fsproxy_no_mirror(Pxy, "custom_method", custom) + assert Pxy.__dict__["custom_method"] is custom + assert "custom_method" not in Slow.__dict__ + + +def test_setattr_fsproxy_no_mirror_monkeypatch_roundtrip(monkeypatch): + # A cudf-installed custom attribute (e.g. ``DataFrame.eval``/``query``) + # participates in the pristine state: monkeypatching it and undoing + # restores the proxy's custom object AND the slow type's own genuine + # implementation — the cudf-internal object is never forwarded to the + # slow type. + _, Slow, Pxy = _make_mirror_proxy() + original_slow = Slow.__dict__["existing"] + + def custom(self): + return "cudf custom" + + _setattr_fsproxy_no_mirror(Pxy, "existing", custom) + assert Slow.__dict__["existing"] is original_slow + + def fake(self): + return "fake" + + monkeypatch.setattr(Pxy, "existing", fake) + assert Slow.__dict__["existing"] is fake + + monkeypatch.undo() + assert Pxy.__dict__["existing"] is custom + assert Slow.__dict__["existing"] is original_slow From 0d69ccfa05a65851130152c8ea6f2a204f5f970b Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 21 Jul 2026 14:26:09 -0500 Subject: [PATCH 25/25] Empty commit to trigger a build (#23376) This is an empty commit to trigger a build. This is needed after the RMM ABI break in https://github.com/rapidsai/rmm/pull/2462.