From c4b80f193449e77a3c93f2e7ba349fa87c45070c Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 3 Apr 2025 01:29:22 +0000 Subject: [PATCH 01/14] First version of device capsule support --- .../pylibcudf/pylibcudf/_interop_helpers.pxd | 2 ++ .../pylibcudf/pylibcudf/_interop_helpers.pyx | 18 +++++++++- .../pylibcudf/pylibcudf/libcudf/interop.pxd | 33 ++++++++++++++--- python/pylibcudf/pylibcudf/table.pyx | 35 ++++++++++++++++++- 4 files changed, 82 insertions(+), 6 deletions(-) diff --git a/python/pylibcudf/pylibcudf/_interop_helpers.pxd b/python/pylibcudf/pylibcudf/_interop_helpers.pxd index 72036be4e775..457baa4a136a 100644 --- a/python/pylibcudf/pylibcudf/_interop_helpers.pxd +++ b/python/pylibcudf/pylibcudf/_interop_helpers.pxd @@ -6,4 +6,6 @@ cdef void _release_schema(object schema_capsule) noexcept cdef void _release_array(object array_capsule) noexcept +cdef void _release_device_array(object array_capsule) noexcept + cdef column_metadata _metadata_to_libcudf(metadata) diff --git a/python/pylibcudf/pylibcudf/_interop_helpers.pyx b/python/pylibcudf/pylibcudf/_interop_helpers.pyx index f2fa6ccf549d..f6a636c36fe7 100644 --- a/python/pylibcudf/pylibcudf/_interop_helpers.pyx +++ b/python/pylibcudf/pylibcudf/_interop_helpers.pyx @@ -1,12 +1,15 @@ # Copyright (c) 2025, NVIDIA CORPORATION. -from cpython.pycapsule cimport PyCapsule_GetPointer +from cpython.pycapsule cimport PyCapsule_GetPointer, PyCapsule_GetContext +from cpython cimport Py_DECREF from pylibcudf.libcudf.interop cimport ( ArrowArray, + ArrowDeviceArray, ArrowSchema, column_metadata, release_arrow_array_raw, + release_arrow_device_array_raw, release_arrow_schema_raw, ) @@ -39,6 +42,19 @@ cdef void _release_array(object array_capsule) noexcept: release_arrow_array_raw(array) +cdef void _release_device_array(object array_capsule) noexcept: + """Release the ArrowDeviceArray object stored in a PyCapsule.""" + cdef ArrowDeviceArray* array = PyCapsule_GetPointer( + array_capsule, 'arrow_device_array' + ) + release_arrow_device_array_raw(array) + + # TODO: Ultimately this logic actually needs to live in the array's release + # function. + cdef object obj = PyCapsule_GetContext(array_capsule) + Py_DECREF(obj) + + cdef column_metadata _metadata_to_libcudf(metadata): """Convert a ColumnMetadata object to C++ column_metadata. diff --git a/python/pylibcudf/pylibcudf/libcudf/interop.pxd b/python/pylibcudf/pylibcudf/libcudf/interop.pxd index ab1d5bf5d5b0..1215c2c67aee 100644 --- a/python/pylibcudf/pylibcudf/libcudf/interop.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/interop.pxd @@ -67,9 +67,10 @@ cdef extern from *: # Rather than exporting the underlying functions directly to Cython, we expose # these wrappers that handle the release to avoid needing to teach Cython how # to handle unique_ptrs with custom deleters that aren't default constructible. - # This will go away once we introduce cudf::arrow_column (need a - # cudf::arrow_schema as well), see - # https://github.com/rapidsai/cudf/issues/16104. + # We cannot use cudf's owning arrow types for this because pylibcudf's + # objects always manage data ownership independently of libcudf in order to + # support other data sources (e.g. cupy), so we must use the view-based + # C++ APIs and handle ownership in Python. """ #include #include @@ -102,7 +103,6 @@ cdef extern from *: cudf::table_view const& tbl, rmm::cuda_stream_view stream = cudf::get_default_stream(), rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()) { - // Assumes the sync event is null and the data is already on the host. ArrowArray *arr = new ArrowArray(); auto device_arr = cudf::to_arrow_host(tbl, stream, mr); ArrowArrayMove(&device_arr->array, arr); @@ -126,6 +126,25 @@ cdef extern from *: } delete array; } + + void release_arrow_device_array_raw(ArrowDeviceArray *array) { + // TODO: Probably needs a sync + if (array->array.release != nullptr) { + array->array.release(&array->array); + } + delete array; + } + + ArrowDeviceArray* to_arrow_device_raw( + cudf::table_view const& tbl, + rmm::cuda_stream_view stream = cudf::get_default_stream(), + rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()) { + // TODO: Technically need to call the sync event. + ArrowDeviceArray *arr = new ArrowDeviceArray(); + auto tmp = cudf::to_arrow_device(tbl, stream, mr); + ArrowDeviceArrayMove(tmp.get(), arr); + return arr; + } """ # The `to_*_raw` functions are all defined in the above extern block as wrappers # around libcudf functions that return unique_ptrs with non-default deleters, which @@ -158,3 +177,9 @@ cdef extern from *: cdef void release_arrow_array_raw( ArrowArray * ) except +libcudf_exception_handler nogil + cdef void release_arrow_device_array_raw( + ArrowDeviceArray * + ) except +libcudf_exception_handler nogil + cdef ArrowDeviceArray* to_arrow_device_raw( + const table_view& tbl, + ) except +libcudf_exception_handler nogil diff --git a/python/pylibcudf/pylibcudf/table.pyx b/python/pylibcudf/pylibcudf/table.pyx index 4992a02ab1c7..1dbaa0b1c4a5 100644 --- a/python/pylibcudf/pylibcudf/table.pyx +++ b/python/pylibcudf/pylibcudf/table.pyx @@ -2,9 +2,11 @@ from cython.operator cimport dereference +from cpython cimport Py_INCREF, Py_DECREF from cpython.pycapsule cimport ( PyCapsule_GetPointer, PyCapsule_New, + PyCapsule_SetContext, ) from libcpp.memory cimport unique_ptr, make_unique @@ -19,9 +21,11 @@ from pylibcudf.libcudf.table.table cimport table from pylibcudf.libcudf.interop cimport ( ArrowArray, ArrowArrayStream, + ArrowDeviceArray, ArrowSchema, arrow_table, column_metadata, + to_arrow_device_raw, to_arrow_host_raw, to_arrow_schema_raw, ) @@ -31,6 +35,7 @@ from .utils cimport _get_stream from pylibcudf._interop_helpers cimport ( _release_schema, _release_array, + _release_device_array, _metadata_to_libcudf, ) from ._interop_helpers import ColumnMetadata @@ -212,7 +217,7 @@ cdef class Table: with nogil: raw_schema_ptr = to_arrow_schema_raw(self.view(), c_metadata) - return PyCapsule_New(raw_schema_ptr, 'arrow_schema', _release_schema) + return PyCapsule_New(raw_schema_ptr, "arrow_schema", _release_schema) def _to_host_array(self): cdef ArrowArray* raw_host_array_ptr @@ -221,8 +226,36 @@ cdef class Table: return PyCapsule_New(raw_host_array_ptr, "arrow_array", _release_array) + def _to_device_array(self): + cdef ArrowDeviceArray* raw_device_array_ptr + with nogil: + raw_device_array_ptr = to_arrow_device_raw(self.view()) + + cdef object capsule = PyCapsule_New( + raw_device_array_ptr, + "arrow_device_array", + _release_device_array + ) + PyCapsule_SetContext(capsule, self) + Py_INCREF(self) + return capsule + def __arrow_c_array__(self, requested_schema=None): if requested_schema is not None: raise ValueError("pylibcudf.Table does not support alternative schema") return self._to_schema(), self._to_host_array() + + def __arrow_c_device_array__(self, requested_schema=None, **kwargs): + if requested_schema is not None: + raise ValueError("pylibcudf.Table does not support alternative schema") + + non_default_kwargs = [ + name for name, value in kwargs.items() if value is not None + ] + if non_default_kwargs: + raise NotImplementedError( + f"Received unsupported keyword argument(s): {non_default_kwargs}" + ) + + return self._to_schema(), self._to_device_array() From 911d3b703dfdf7c34ed6d95c51bcc740d2cb6561 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 10 Apr 2025 22:33:26 +0000 Subject: [PATCH 02/14] Properly handle lifetime management in the array rather than the capsule --- .../pylibcudf/pylibcudf/_interop_helpers.pyx | 8 +--- .../pylibcudf/pylibcudf/libcudf/interop.pxd | 40 +++++++++++++++++-- python/pylibcudf/pylibcudf/table.pyx | 9 +---- 3 files changed, 40 insertions(+), 17 deletions(-) diff --git a/python/pylibcudf/pylibcudf/_interop_helpers.pyx b/python/pylibcudf/pylibcudf/_interop_helpers.pyx index f6a636c36fe7..483c38d8f173 100644 --- a/python/pylibcudf/pylibcudf/_interop_helpers.pyx +++ b/python/pylibcudf/pylibcudf/_interop_helpers.pyx @@ -1,7 +1,6 @@ # Copyright (c) 2025, NVIDIA CORPORATION. -from cpython.pycapsule cimport PyCapsule_GetPointer, PyCapsule_GetContext -from cpython cimport Py_DECREF +from cpython.pycapsule cimport PyCapsule_GetPointer from pylibcudf.libcudf.interop cimport ( ArrowArray, @@ -49,11 +48,6 @@ cdef void _release_device_array(object array_capsule) noexcept: ) release_arrow_device_array_raw(array) - # TODO: Ultimately this logic actually needs to live in the array's release - # function. - cdef object obj = PyCapsule_GetContext(array_capsule) - Py_DECREF(obj) - cdef column_metadata _metadata_to_libcudf(metadata): """Convert a ColumnMetadata object to C++ column_metadata. diff --git a/python/pylibcudf/pylibcudf/libcudf/interop.pxd b/python/pylibcudf/pylibcudf/libcudf/interop.pxd index 1215c2c67aee..5da86d196367 100644 --- a/python/pylibcudf/pylibcudf/libcudf/interop.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/interop.pxd @@ -135,14 +135,47 @@ cdef extern from *: delete array; } + struct PylibcudfArrowDeviceArrayPrivateData { + ArrowArray parent; + PyObject* owner; + }; + + void PylibcudfArrowDeviceArrayRelease(ArrowArray* array) + { + // TODO: Figure out if synchronization needs to be handled here in addition to the + // parent. It probably does because we'll allocate an extra level of it. + auto private_data = reinterpret_cast( + array->private_data); + Py_DECREF(private_data->owner); + private_data->parent.release(&private_data->parent); + array->release = nullptr; + } + ArrowDeviceArray* to_arrow_device_raw( cudf::table_view const& tbl, + PyObject* owner, rmm::cuda_stream_view stream = cudf::get_default_stream(), rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()) { - // TODO: Technically need to call the sync event. - ArrowDeviceArray *arr = new ArrowDeviceArray(); auto tmp = cudf::to_arrow_device(tbl, stream, mr); - ArrowDeviceArrayMove(tmp.get(), arr); + + // TODO: Technically need to call the sync event before we do anything. + + // Instead of moving the whole device array, we move the underlying ArrowArray + // into the custom private data struct for managing its data then create a new + // device array from scratch. + auto private_data = new PylibcudfArrowDeviceArrayPrivateData(); + ArrowArrayMove(&tmp->array, &private_data->parent); + private_data->owner = owner; + Py_INCREF(owner); + + ArrowDeviceArray *arr = new ArrowDeviceArray(); + arr->device_id = tmp->device_id; + arr->device_type = tmp->device_type; + arr->sync_event = tmp->sync_event; + arr->array = private_data->parent; // shallow copy + arr->array.private_data = private_data; + arr->array.release = &PylibcudfArrowDeviceArrayRelease; + return arr; } """ @@ -182,4 +215,5 @@ cdef extern from *: ) except +libcudf_exception_handler nogil cdef ArrowDeviceArray* to_arrow_device_raw( const table_view& tbl, + object owner, ) except +libcudf_exception_handler nogil diff --git a/python/pylibcudf/pylibcudf/table.pyx b/python/pylibcudf/pylibcudf/table.pyx index 1dbaa0b1c4a5..1447b700201b 100644 --- a/python/pylibcudf/pylibcudf/table.pyx +++ b/python/pylibcudf/pylibcudf/table.pyx @@ -2,11 +2,9 @@ from cython.operator cimport dereference -from cpython cimport Py_INCREF, Py_DECREF from cpython.pycapsule cimport ( PyCapsule_GetPointer, PyCapsule_New, - PyCapsule_SetContext, ) from libcpp.memory cimport unique_ptr, make_unique @@ -229,16 +227,13 @@ cdef class Table: def _to_device_array(self): cdef ArrowDeviceArray* raw_device_array_ptr with nogil: - raw_device_array_ptr = to_arrow_device_raw(self.view()) + raw_device_array_ptr = to_arrow_device_raw(self.view(), self) - cdef object capsule = PyCapsule_New( + return PyCapsule_New( raw_device_array_ptr, "arrow_device_array", _release_device_array ) - PyCapsule_SetContext(capsule, self) - Py_INCREF(self) - return capsule def __arrow_c_array__(self, requested_schema=None): if requested_schema is not None: From 7fd98637adab3d005eca3e8894fef3288ec3651b Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 10 Apr 2025 23:03:21 +0000 Subject: [PATCH 03/14] Add initial test of just schema using nanoarrow --- .../all_cuda-118_arch-aarch64.yaml | 1 + .../all_cuda-118_arch-x86_64.yaml | 1 + .../all_cuda-128_arch-aarch64.yaml | 1 + .../all_cuda-128_arch-x86_64.yaml | 1 + dependencies.yaml | 1 + .../pylibcudf/pylibcudf/tests/test_interop.py | 33 ++++++++++++++++++- python/pylibcudf/pyproject.toml | 1 + 7 files changed, 38 insertions(+), 1 deletion(-) diff --git a/conda/environments/all_cuda-118_arch-aarch64.yaml b/conda/environments/all_cuda-118_arch-aarch64.yaml index e20be1953850..b436adaef819 100644 --- a/conda/environments/all_cuda-118_arch-aarch64.yaml +++ b/conda/environments/all_cuda-118_arch-aarch64.yaml @@ -47,6 +47,7 @@ dependencies: - moto>=4.0.8 - msgpack-python - myst-nb +- nanoarrow - nbconvert - nbformat - nbsphinx diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 040c3d0ba64e..4561194602f9 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -49,6 +49,7 @@ dependencies: - moto>=4.0.8 - msgpack-python - myst-nb +- nanoarrow - nbconvert - nbformat - nbsphinx diff --git a/conda/environments/all_cuda-128_arch-aarch64.yaml b/conda/environments/all_cuda-128_arch-aarch64.yaml index 264d60c2bd93..59d512f12c84 100644 --- a/conda/environments/all_cuda-128_arch-aarch64.yaml +++ b/conda/environments/all_cuda-128_arch-aarch64.yaml @@ -47,6 +47,7 @@ dependencies: - moto>=4.0.8 - msgpack-python - myst-nb +- nanoarrow - nbconvert - nbformat - nbsphinx diff --git a/conda/environments/all_cuda-128_arch-x86_64.yaml b/conda/environments/all_cuda-128_arch-x86_64.yaml index ea2e83116fb4..3d5d0c85be3d 100644 --- a/conda/environments/all_cuda-128_arch-x86_64.yaml +++ b/conda/environments/all_cuda-128_arch-x86_64.yaml @@ -48,6 +48,7 @@ dependencies: - moto>=4.0.8 - msgpack-python - myst-nb +- nanoarrow - nbconvert - nbformat - nbsphinx diff --git a/dependencies.yaml b/dependencies.yaml index d17ec2e75ca1..800591cbc78c 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -924,6 +924,7 @@ dependencies: - hypothesis - *numpy - pandas + - nanoarrow test_python_cudf: common: - output_types: [conda, requirements, pyproject] diff --git a/python/pylibcudf/pylibcudf/tests/test_interop.py b/python/pylibcudf/pylibcudf/tests/test_interop.py index ca42eacdfdb2..d92362a7249c 100644 --- a/python/pylibcudf/pylibcudf/tests/test_interop.py +++ b/python/pylibcudf/pylibcudf/tests/test_interop.py @@ -1,6 +1,8 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. +# Copyright (c) 2024-2025, NVIDIA CORPORATION. import cupy as cp +import nanoarrow +import nanoarrow.device import numpy as np import pyarrow as pa import pytest @@ -120,3 +122,32 @@ def test_to_dlpack_error(): def test_from_dlpack_error(): with pytest.raises(ValueError, match="Invalid PyCapsule object"): plc.interop.from_dlpack(1) + + +def test_device_interop_table(): + # Have to manually construct the schema to ensure that names match. pyarrow will + # assign names to nested types automatically otherwise. + schema = pa.schema( + [ + pa.field("", pa.int64()), + pa.field("", pa.float64()), + pa.field("", pa.string()), + pa.field("", pa.list_(pa.field("", pa.int64()))), + pa.field("", pa.struct([pa.field("", pa.float64())])), + ] + ) + pa_tbl = pa.table( + [ + [1, None, 3], + [1.0, 2.0, None], + ["a", "b", None], + [[1, None], None, [2]], + [{"a": 1.0}, None, {"b": 2.0}], + ], + schema=schema, + ) + plc_table = plc.interop.from_arrow(pa_tbl) + + na_arr = nanoarrow.device.c_device_array(plc_table) + actual_schema = pa.schema(na_arr.schema) + assert actual_schema.equals(pa_tbl.schema) diff --git a/python/pylibcudf/pyproject.toml b/python/pylibcudf/pyproject.toml index cb1b87c0b1c6..9b0d68afbd53 100644 --- a/python/pylibcudf/pyproject.toml +++ b/python/pylibcudf/pyproject.toml @@ -42,6 +42,7 @@ classifiers = [ test = [ "fastavro>=0.22.9", "hypothesis", + "nanoarrow", "numpy>=1.23,<3.0a0", "pandas", "pytest-cov", From a8bf16c55103fcdf104a14c18f3beca4f0e79235 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Thu, 10 Apr 2025 23:20:29 +0000 Subject: [PATCH 04/14] Enable ingestion of device array data in tables --- .../pylibcudf/pylibcudf/libcudf/interop.pxd | 6 ++++- python/pylibcudf/pylibcudf/table.pyx | 24 +++++++++++++++++-- .../pylibcudf/pylibcudf/tests/test_interop.py | 3 +++ 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/python/pylibcudf/pylibcudf/libcudf/interop.pxd b/python/pylibcudf/pylibcudf/libcudf/interop.pxd index 5da86d196367..d7cd495f064b 100644 --- a/python/pylibcudf/pylibcudf/libcudf/interop.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/interop.pxd @@ -59,7 +59,11 @@ cdef extern from "cudf/interop.hpp" namespace "cudf::interop" \ cdef cppclass arrow_table: arrow_table( ArrowArrayStream&& stream, - ) except +libcudf_exception_handler + ) except +libcudf_exception_handler + arrow_table( + ArrowSchema&& schema, + ArrowDeviceArray&& array, + ) except +libcudf_exception_handler table_view view() except +libcudf_exception_handler diff --git a/python/pylibcudf/pylibcudf/table.pyx b/python/pylibcudf/pylibcudf/table.pyx index 1447b700201b..638680565e5b 100644 --- a/python/pylibcudf/pylibcudf/table.pyx +++ b/python/pylibcudf/pylibcudf/table.pyx @@ -50,6 +50,7 @@ class _ArrowLikeMeta(type): return ( hasattr(other, "__arrow_c_stream__") or hasattr(other, "__arrow_c_array__") + or hasattr(other, "__arrow_c_device_array__") ) @@ -87,10 +88,29 @@ cdef class Table: @_init.register(_ArrowLike) def _(self, arrow_like): - cdef ArrowArrayStream* c_stream + cdef ArrowSchema* c_schema + cdef ArrowDeviceArray* c_array cdef _ArrowTableHolder result cdef unique_ptr[arrow_table] c_result - if hasattr(arrow_like, "__arrow_c_stream__"): + if hasattr(arrow_like, "__arrow_c_device_stream__"): + raise NotImplementedError("Device streams not yet supported") + elif hasattr(arrow_like, "__arrow_c_device_array__"): + schema, array = arrow_like.__arrow_c_device_array__() + c_schema = PyCapsule_GetPointer(schema, "arrow_schema") + c_array = ( + PyCapsule_GetPointer(array, "arrow_device_array") + ) + + result = _ArrowTableHolder() + with nogil: + c_result = make_unique[arrow_table]( + move(dereference(c_schema)), move(dereference(c_array)) + ) + result.tbl.swap(c_result) + + tmp = Table.from_table_view_of_arbitrary(result.tbl.get().view(), result) + self._columns = tmp.columns() + elif hasattr(arrow_like, "__arrow_c_stream__"): stream = arrow_like.__arrow_c_stream__() c_stream = ( PyCapsule_GetPointer(stream, "arrow_array_stream") diff --git a/python/pylibcudf/pylibcudf/tests/test_interop.py b/python/pylibcudf/pylibcudf/tests/test_interop.py index d92362a7249c..a0adc6a60a94 100644 --- a/python/pylibcudf/pylibcudf/tests/test_interop.py +++ b/python/pylibcudf/pylibcudf/tests/test_interop.py @@ -151,3 +151,6 @@ def test_device_interop_table(): na_arr = nanoarrow.device.c_device_array(plc_table) actual_schema = pa.schema(na_arr.schema) assert actual_schema.equals(pa_tbl.schema) + + new_tbl = plc.Table(na_arr) + assert_table_eq(pa_tbl, new_tbl) From ff21354d5206e689934b8b871d1847b1278352d6 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 16 Apr 2025 00:28:47 +0000 Subject: [PATCH 05/14] Add capsule support to columns --- python/pylibcudf/pylibcudf/column.pyx | 40 ++++++++++++++++--- .../pylibcudf/pylibcudf/libcudf/interop.pxd | 25 +++++------- python/pylibcudf/pylibcudf/table.pyx | 3 +- 3 files changed, 45 insertions(+), 23 deletions(-) diff --git a/python/pylibcudf/pylibcudf/column.pyx b/python/pylibcudf/pylibcudf/column.pyx index 434c8dbeb50b..7727dd7c7885 100644 --- a/python/pylibcudf/pylibcudf/column.pyx +++ b/python/pylibcudf/pylibcudf/column.pyx @@ -15,19 +15,21 @@ from libcpp.utility cimport move from pylibcudf.libcudf.column.column cimport column, column_contents from pylibcudf.libcudf.column.column_factories cimport make_column_from_scalar -from pylibcudf.libcudf.interop cimport ArrowArray, ArrowSchema, arrow_column -from pylibcudf.libcudf.scalar.scalar cimport scalar, numeric_scalar -from pylibcudf.libcudf.types cimport size_type, size_of as cpp_size_of, bitmask_type -from pylibcudf.libcudf.utilities.traits cimport is_fixed_width -from pylibcudf.libcudf.copying cimport get_element - from pylibcudf.libcudf.interop cimport ( ArrowArray, ArrowSchema, + ArrowDeviceArray, + arrow_column, column_metadata, to_arrow_host_raw, + to_arrow_device_raw, to_arrow_schema_raw, ) +from pylibcudf.libcudf.scalar.scalar cimport scalar, numeric_scalar +from pylibcudf.libcudf.types cimport size_type, size_of as cpp_size_of, bitmask_type +from pylibcudf.libcudf.utilities.traits cimport is_fixed_width +from pylibcudf.libcudf.copying cimport get_element + from rmm.librmm.device_buffer cimport device_buffer from rmm.pylibrmm.device_buffer cimport DeviceBuffer @@ -40,6 +42,7 @@ from .types cimport DataType, size_of, type_id from ._interop_helpers cimport ( _release_schema, _release_array, + _release_device_array, _metadata_to_libcudf, ) from .null_mask cimport bitmask_allocation_size_bytes @@ -752,12 +755,37 @@ cdef class Column: return PyCapsule_New(raw_host_array_ptr, "arrow_array", _release_array) + def _to_device_array(self): + cdef ArrowDeviceArray* raw_device_array_ptr + with nogil: + raw_device_array_ptr = to_arrow_device_raw(self.view(), self) + + return PyCapsule_New( + raw_device_array_ptr, + "arrow_device_array", + _release_device_array + ) + def __arrow_c_array__(self, requested_schema=None): if requested_schema is not None: raise ValueError("pylibcudf.Column does not support alternative schema") return self._to_schema(), self._to_host_array() + def __arrow_c_device_array__(self, requested_schema=None, **kwargs): + if requested_schema is not None: + raise ValueError("pylibcudf.Column does not support alternative schema") + + non_default_kwargs = [ + name for name, value in kwargs.items() if value is not None + ] + if non_default_kwargs: + raise NotImplementedError( + f"Received unsupported keyword argument(s): {non_default_kwargs}" + ) + + return self._to_schema(), self._to_device_array() + cdef class ListColumnView: """Accessor for methods of a Column that are specific to lists.""" diff --git a/python/pylibcudf/pylibcudf/libcudf/interop.pxd b/python/pylibcudf/pylibcudf/libcudf/interop.pxd index d7cd495f064b..65d8d5ea3e26 100644 --- a/python/pylibcudf/pylibcudf/libcudf/interop.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/interop.pxd @@ -103,23 +103,13 @@ cdef extern from *: delete schema; } + template ArrowArray* to_arrow_host_raw( - cudf::table_view const& tbl, + ViewType const& obj, rmm::cuda_stream_view stream = cudf::get_default_stream(), rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()) { ArrowArray *arr = new ArrowArray(); - auto device_arr = cudf::to_arrow_host(tbl, stream, mr); - ArrowArrayMove(&device_arr->array, arr); - return arr; - } - - ArrowArray* to_arrow_host_raw( - cudf::column_view const& col, - rmm::cuda_stream_view stream = cudf::get_default_stream(), - rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()) { - // Assumes the sync event is null and the data is already on the host. - ArrowArray *arr = new ArrowArray(); - auto device_arr = cudf::to_arrow_host(col, stream, mr); + auto device_arr = cudf::to_arrow_host(obj, stream, mr); ArrowArrayMove(&device_arr->array, arr); return arr; } @@ -155,12 +145,13 @@ cdef extern from *: array->release = nullptr; } + template ArrowDeviceArray* to_arrow_device_raw( - cudf::table_view const& tbl, + ViewType const& obj, PyObject* owner, rmm::cuda_stream_view stream = cudf::get_default_stream(), rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()) { - auto tmp = cudf::to_arrow_device(tbl, stream, mr); + auto tmp = cudf::to_arrow_device(obj, stream, mr); // TODO: Technically need to call the sync event before we do anything. @@ -221,3 +212,7 @@ cdef extern from *: const table_view& tbl, object owner, ) except +libcudf_exception_handler nogil + cdef ArrowDeviceArray* to_arrow_device_raw( + const column_view& tbl, + object owner, + ) except +libcudf_exception_handler nogil diff --git a/python/pylibcudf/pylibcudf/table.pyx b/python/pylibcudf/pylibcudf/table.pyx index 638680565e5b..bdb3544bb1ca 100644 --- a/python/pylibcudf/pylibcudf/table.pyx +++ b/python/pylibcudf/pylibcudf/table.pyx @@ -14,8 +14,6 @@ from libcpp.vector cimport vector from rmm.pylibrmm.stream cimport Stream from pylibcudf.libcudf.column.column cimport column from pylibcudf.libcudf.column.column_view cimport column_view -from pylibcudf.libcudf.table.table cimport table - from pylibcudf.libcudf.interop cimport ( ArrowArray, ArrowArrayStream, @@ -27,6 +25,7 @@ from pylibcudf.libcudf.interop cimport ( to_arrow_host_raw, to_arrow_schema_raw, ) +from pylibcudf.libcudf.table.table cimport table from .column cimport Column from .utils cimport _get_stream From 4a3d906e5ca918460e9006741f64f6b31320e9a1 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 16 Apr 2025 00:39:18 +0000 Subject: [PATCH 06/14] Enable ingestion of device array data in columns --- python/pylibcudf/pylibcudf/column.pyx | 78 +++++++++++++------ .../pylibcudf/pylibcudf/libcudf/interop.pxd | 4 + python/pylibcudf/pylibcudf/table.pyx | 13 ++-- 3 files changed, 66 insertions(+), 29 deletions(-) diff --git a/python/pylibcudf/pylibcudf/column.pyx b/python/pylibcudf/pylibcudf/column.pyx index 7727dd7c7885..0e507f3d08d7 100644 --- a/python/pylibcudf/pylibcudf/column.pyx +++ b/python/pylibcudf/pylibcudf/column.pyx @@ -58,7 +58,12 @@ __all__ = ["Column", "ListColumnView", "is_c_contiguous"] class _ArrowLikeMeta(type): def __subclasscheck__(cls, other): - return hasattr(other, "__arrow_c_array__") + # We cannot separate these types via singledispatch because the dispatch + # will often be ambiguous when objects expose multiple protocols. + return ( + hasattr(other, "__arrow_c_array__") + or hasattr(other, "__arrow_c_device_array__") + ) class _ArrowLike(metaclass=_ArrowLikeMeta): @@ -210,32 +215,59 @@ cdef class Column: @_init.register(_ArrowLike) def _(self, arrow_like): - schema, array = arrow_like.__arrow_c_array__() - cdef ArrowSchema* c_schema = ( - PyCapsule_GetPointer(schema, "arrow_schema") - ) - cdef ArrowArray* c_array = ( - PyCapsule_GetPointer(array, "arrow_array") - ) - - cdef _ArrowColumnHolder result = _ArrowColumnHolder() + cdef ArrowSchema* c_schema + cdef ArrowArray* c_array + cdef ArrowDeviceArray* c_device_array + cdef _ArrowColumnHolder result cdef unique_ptr[arrow_column] c_result - with nogil: - c_result = make_unique[arrow_column]( - move(dereference(c_schema)), move(dereference(c_array)) + if hasattr(arrow_like, "__arrow_c_array__"): + schema, array = arrow_like.__arrow_c_array__() + c_schema = PyCapsule_GetPointer(schema, "arrow_schema") + c_array = PyCapsule_GetPointer(array, "arrow_array") + + result = _ArrowColumnHolder() + with nogil: + c_result = make_unique[arrow_column]( + move(dereference(c_schema)), move(dereference(c_array)) + ) + result.col.swap(c_result) + + tmp = Column.from_column_view_of_arbitrary(result.col.get().view(), result) + self._init( + tmp.type(), + tmp.size(), + tmp.data(), + tmp.null_mask(), + tmp.null_count(), + tmp.offset(), + tmp.children(), + ) + elif hasattr(arrow_like, "__arrow_c_device_array__"): + schema, array = arrow_like.__arrow_c_device_array__() + c_schema = PyCapsule_GetPointer(schema, "arrow_schema") + c_device_array = ( + PyCapsule_GetPointer(array, "arrow_device_array") ) + + result = _ArrowColumnHolder() + with nogil: + c_result = make_unique[arrow_column]( + move(dereference(c_schema)), move(dereference(c_device_array)) + ) result.col.swap(c_result) - tmp = Column.from_column_view_of_arbitrary(result.col.get().view(), result) - self._init( - tmp.type(), - tmp.size(), - tmp.data(), - tmp.null_mask(), - tmp.null_count(), - tmp.offset(), - tmp.children(), - ) + tmp = Column.from_column_view_of_arbitrary(result.col.get().view(), result) + self._init( + tmp.type(), + tmp.size(), + tmp.data(), + tmp.null_mask(), + tmp.null_count(), + tmp.offset(), + tmp.children(), + ) + else: + raise ValueError("Invalid Arrow-like object") cdef column_view view(self) nogil: """Generate a libcudf column_view to pass to libcudf algorithms. diff --git a/python/pylibcudf/pylibcudf/libcudf/interop.pxd b/python/pylibcudf/pylibcudf/libcudf/interop.pxd index 65d8d5ea3e26..9106d3872884 100644 --- a/python/pylibcudf/pylibcudf/libcudf/interop.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/interop.pxd @@ -54,6 +54,10 @@ cdef extern from "cudf/interop.hpp" namespace "cudf::interop" \ ArrowSchema&& schema, ArrowArray&& array ) except +libcudf_exception_handler + arrow_column( + ArrowSchema&& schema, + ArrowDeviceArray&& array + ) except +libcudf_exception_handler column_view view() except +libcudf_exception_handler cdef cppclass arrow_table: diff --git a/python/pylibcudf/pylibcudf/table.pyx b/python/pylibcudf/pylibcudf/table.pyx index bdb3544bb1ca..d77c99d8429a 100644 --- a/python/pylibcudf/pylibcudf/table.pyx +++ b/python/pylibcudf/pylibcudf/table.pyx @@ -43,11 +43,12 @@ __all__ = ["Table"] class _ArrowLikeMeta(type): - # We cannot separate stream and array via singledispatch because the - # dispatch will often be ambiguous when objects expose both protocols. + # We cannot separate these types via singledispatch because the dispatch + # will often be ambiguous when objects expose multiple protocols. def __subclasscheck__(cls, other): return ( hasattr(other, "__arrow_c_stream__") + or hasattr(other, "__arrow_c_device_stream__") or hasattr(other, "__arrow_c_array__") or hasattr(other, "__arrow_c_device_array__") ) @@ -91,9 +92,7 @@ cdef class Table: cdef ArrowDeviceArray* c_array cdef _ArrowTableHolder result cdef unique_ptr[arrow_table] c_result - if hasattr(arrow_like, "__arrow_c_device_stream__"): - raise NotImplementedError("Device streams not yet supported") - elif hasattr(arrow_like, "__arrow_c_device_array__"): + if hasattr(arrow_like, "__arrow_c_device_array__"): schema, array = arrow_like.__arrow_c_device_array__() c_schema = PyCapsule_GetPointer(schema, "arrow_schema") c_array = ( @@ -122,8 +121,10 @@ cdef class Table: tmp = Table.from_table_view_of_arbitrary(result.tbl.get().view(), result) self._columns = tmp.columns() + elif hasattr(arrow_like, "__arrow_c_device_stream__"): + raise NotImplementedError("Device streams not yet supported") elif hasattr(arrow_like, "__arrow_c_array__"): - raise NotImplementedError("arrays not yet supported") + raise NotImplementedError("Arrow host arrays not yet supported") cdef table_view view(self) nogil: """Generate a libcudf table_view to pass to libcudf algorithms. From 6e0b4dc76ecc48ee17326aef8f3d59b745003b1e Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 16 Apr 2025 00:53:42 +0000 Subject: [PATCH 07/14] Add tests and fix a couple of bugs --- python/pylibcudf/pylibcudf/column.pyx | 20 +++++++++---------- .../pylibcudf/pylibcudf/tests/test_interop.py | 13 ++++++++++-- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/python/pylibcudf/pylibcudf/column.pyx b/python/pylibcudf/pylibcudf/column.pyx index 0e507f3d08d7..d1667e1e4341 100644 --- a/python/pylibcudf/pylibcudf/column.pyx +++ b/python/pylibcudf/pylibcudf/column.pyx @@ -220,15 +220,17 @@ cdef class Column: cdef ArrowDeviceArray* c_device_array cdef _ArrowColumnHolder result cdef unique_ptr[arrow_column] c_result - if hasattr(arrow_like, "__arrow_c_array__"): - schema, array = arrow_like.__arrow_c_array__() + if hasattr(arrow_like, "__arrow_c_device_array__"): + schema, array = arrow_like.__arrow_c_device_array__() c_schema = PyCapsule_GetPointer(schema, "arrow_schema") - c_array = PyCapsule_GetPointer(array, "arrow_array") + c_device_array = ( + PyCapsule_GetPointer(array, "arrow_device_array") + ) result = _ArrowColumnHolder() with nogil: c_result = make_unique[arrow_column]( - move(dereference(c_schema)), move(dereference(c_array)) + move(dereference(c_schema)), move(dereference(c_device_array)) ) result.col.swap(c_result) @@ -242,17 +244,15 @@ cdef class Column: tmp.offset(), tmp.children(), ) - elif hasattr(arrow_like, "__arrow_c_device_array__"): - schema, array = arrow_like.__arrow_c_device_array__() + elif hasattr(arrow_like, "__arrow_c_array__"): + schema, array = arrow_like.__arrow_c_array__() c_schema = PyCapsule_GetPointer(schema, "arrow_schema") - c_device_array = ( - PyCapsule_GetPointer(array, "arrow_device_array") - ) + c_array = PyCapsule_GetPointer(array, "arrow_array") result = _ArrowColumnHolder() with nogil: c_result = make_unique[arrow_column]( - move(dereference(c_schema)), move(dereference(c_device_array)) + move(dereference(c_schema)), move(dereference(c_array)) ) result.col.swap(c_result) diff --git a/python/pylibcudf/pylibcudf/tests/test_interop.py b/python/pylibcudf/pylibcudf/tests/test_interop.py index a0adc6a60a94..f946710320f4 100644 --- a/python/pylibcudf/pylibcudf/tests/test_interop.py +++ b/python/pylibcudf/pylibcudf/tests/test_interop.py @@ -6,7 +6,7 @@ import numpy as np import pyarrow as pa import pytest -from utils import assert_table_eq +from utils import assert_column_eq, assert_table_eq import pylibcudf as plc @@ -124,6 +124,15 @@ def test_from_dlpack_error(): plc.interop.from_dlpack(1) +def test_device_interop_column(): + pa_arr = pa.array([{"a": [1, None]}, None, {"b": [None, 4]}]) + plc_col = plc.Column(pa_arr) + + na_arr = nanoarrow.device.c_device_array(plc_col) + new_col = plc.Column(na_arr) + assert_column_eq(pa_arr, new_col) + + def test_device_interop_table(): # Have to manually construct the schema to ensure that names match. pyarrow will # assign names to nested types automatically otherwise. @@ -146,7 +155,7 @@ def test_device_interop_table(): ], schema=schema, ) - plc_table = plc.interop.from_arrow(pa_tbl) + plc_table = plc.Table(pa_tbl) na_arr = nanoarrow.device.c_device_array(plc_table) actual_schema = pa.schema(na_arr.schema) From f05f075efc9925ce222842538dd172f83e903150 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 16 Apr 2025 01:09:23 +0000 Subject: [PATCH 08/14] Remove invalid TODOs --- python/pylibcudf/pylibcudf/libcudf/interop.pxd | 5 ----- 1 file changed, 5 deletions(-) diff --git a/python/pylibcudf/pylibcudf/libcudf/interop.pxd b/python/pylibcudf/pylibcudf/libcudf/interop.pxd index 9106d3872884..13ace55f59d4 100644 --- a/python/pylibcudf/pylibcudf/libcudf/interop.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/interop.pxd @@ -126,7 +126,6 @@ cdef extern from *: } void release_arrow_device_array_raw(ArrowDeviceArray *array) { - // TODO: Probably needs a sync if (array->array.release != nullptr) { array->array.release(&array->array); } @@ -140,8 +139,6 @@ cdef extern from *: void PylibcudfArrowDeviceArrayRelease(ArrowArray* array) { - // TODO: Figure out if synchronization needs to be handled here in addition to the - // parent. It probably does because we'll allocate an extra level of it. auto private_data = reinterpret_cast( array->private_data); Py_DECREF(private_data->owner); @@ -157,8 +154,6 @@ cdef extern from *: rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()) { auto tmp = cudf::to_arrow_device(obj, stream, mr); - // TODO: Technically need to call the sync event before we do anything. - // Instead of moving the whole device array, we move the underlying ArrowArray // into the custom private data struct for managing its data then create a new // device array from scratch. From cf50a8776a20fdecb5922cd0477706606be1f610 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 16 Apr 2025 18:05:42 +0000 Subject: [PATCH 09/14] pylibcudf test dependencies must be installed for testing --- dependencies.yaml | 8 ++++---- python/pylibcudf/pyproject.toml | 1 - 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 800591cbc78c..9af8139489df 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -86,6 +86,7 @@ files: - test_python_common - test_python_cudf_common - test_python_dask_cudf + - test_python_pylibcudf - depends_on_cudf - depends_on_pylibcudf - depends_on_libcudf @@ -920,17 +921,16 @@ dependencies: common: - output_types: [conda, requirements, pyproject] packages: - - fastavro>=0.22.9 - - hypothesis + - &fastavro fastavro>=0.22.9 + - nanoarrow - *numpy - pandas - - nanoarrow test_python_cudf: common: - output_types: [conda, requirements, pyproject] packages: - cramjam - - fastavro>=0.22.9 + - *fastavro - hypothesis - mmh3 # Version 5.1 is incompatible with pytest<8.2. diff --git a/python/pylibcudf/pyproject.toml b/python/pylibcudf/pyproject.toml index 9b0d68afbd53..ecd536bcce8c 100644 --- a/python/pylibcudf/pyproject.toml +++ b/python/pylibcudf/pyproject.toml @@ -41,7 +41,6 @@ classifiers = [ [project.optional-dependencies] test = [ "fastavro>=0.22.9", - "hypothesis", "nanoarrow", "numpy>=1.23,<3.0a0", "pandas", From 167762bc0df5428e1ffc1d19b1d03970af033045 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 16 Apr 2025 19:15:48 +0000 Subject: [PATCH 10/14] No conda-forge arm packages for nanoarrow --- conda/environments/all_cuda-118_arch-aarch64.yaml | 1 - conda/environments/all_cuda-128_arch-aarch64.yaml | 1 - dependencies.yaml | 14 +++++++++++++- python/pylibcudf/pylibcudf/tests/test_interop.py | 8 ++++---- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-aarch64.yaml b/conda/environments/all_cuda-118_arch-aarch64.yaml index b436adaef819..e20be1953850 100644 --- a/conda/environments/all_cuda-118_arch-aarch64.yaml +++ b/conda/environments/all_cuda-118_arch-aarch64.yaml @@ -47,7 +47,6 @@ dependencies: - moto>=4.0.8 - msgpack-python - myst-nb -- nanoarrow - nbconvert - nbformat - nbsphinx diff --git a/conda/environments/all_cuda-128_arch-aarch64.yaml b/conda/environments/all_cuda-128_arch-aarch64.yaml index 59d512f12c84..264d60c2bd93 100644 --- a/conda/environments/all_cuda-128_arch-aarch64.yaml +++ b/conda/environments/all_cuda-128_arch-aarch64.yaml @@ -47,7 +47,6 @@ dependencies: - moto>=4.0.8 - msgpack-python - myst-nb -- nanoarrow - nbconvert - nbformat - nbsphinx diff --git a/dependencies.yaml b/dependencies.yaml index 9af8139489df..23e2f0d5b942 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -922,9 +922,21 @@ dependencies: - output_types: [conda, requirements, pyproject] packages: - &fastavro fastavro>=0.22.9 - - nanoarrow - *numpy - pandas + - output_types: [requirements, pyproject] + packages: + - nanoarrow + specific: + - output_types: conda + matrices: + - matrix: + arch: aarch64 + packages: + - matrix: + arch: x86_64 + packages: + - nanoarrow test_python_cudf: common: - output_types: [conda, requirements, pyproject] diff --git a/python/pylibcudf/pylibcudf/tests/test_interop.py b/python/pylibcudf/pylibcudf/tests/test_interop.py index f946710320f4..fd58ad9f3454 100644 --- a/python/pylibcudf/pylibcudf/tests/test_interop.py +++ b/python/pylibcudf/pylibcudf/tests/test_interop.py @@ -1,8 +1,6 @@ # Copyright (c) 2024-2025, NVIDIA CORPORATION. import cupy as cp -import nanoarrow -import nanoarrow.device import numpy as np import pyarrow as pa import pytest @@ -125,15 +123,17 @@ def test_from_dlpack_error(): def test_device_interop_column(): + nad = pytest.importorskip("nanoarrow.device") pa_arr = pa.array([{"a": [1, None]}, None, {"b": [None, 4]}]) plc_col = plc.Column(pa_arr) - na_arr = nanoarrow.device.c_device_array(plc_col) + na_arr = nad.c_device_array(plc_col) new_col = plc.Column(na_arr) assert_column_eq(pa_arr, new_col) def test_device_interop_table(): + nad = pytest.importorskip("nanoarrow.device") # Have to manually construct the schema to ensure that names match. pyarrow will # assign names to nested types automatically otherwise. schema = pa.schema( @@ -157,7 +157,7 @@ def test_device_interop_table(): ) plc_table = plc.Table(pa_tbl) - na_arr = nanoarrow.device.c_device_array(plc_table) + na_arr = nad.c_device_array(plc_table) actual_schema = pa.schema(na_arr.schema) assert actual_schema.equals(pa_tbl.schema) From 1cd438d21351b10277be89cb023cb31635f94e1d Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Wed, 16 Apr 2025 21:08:33 +0000 Subject: [PATCH 11/14] Remove pytest-benchmark bound now that the conda-forge fixes are merged --- conda/environments/all_cuda-118_arch-aarch64.yaml | 2 +- conda/environments/all_cuda-118_arch-x86_64.yaml | 2 +- conda/environments/all_cuda-128_arch-aarch64.yaml | 2 +- conda/environments/all_cuda-128_arch-x86_64.yaml | 2 +- dependencies.yaml | 9 +-------- python/cudf/pyproject.toml | 2 +- 6 files changed, 6 insertions(+), 13 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-aarch64.yaml b/conda/environments/all_cuda-118_arch-aarch64.yaml index e20be1953850..77edaeff186a 100644 --- a/conda/environments/all_cuda-118_arch-aarch64.yaml +++ b/conda/environments/all_cuda-118_arch-aarch64.yaml @@ -70,7 +70,7 @@ dependencies: - pyarrow>=14.0.0,<20.0.0a0 - pydata-sphinx-theme>=0.15.4 - pynvml>=12.0.0,<13.0.0a0 -- pytest-benchmark<5.1.0 +- pytest-benchmark - pytest-cases>=3.8.2 - pytest-cov - pytest-rerunfailures diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 4561194602f9..6fe5348816f2 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -73,7 +73,7 @@ dependencies: - pyarrow>=14.0.0,<20.0.0a0 - pydata-sphinx-theme>=0.15.4 - pynvml>=12.0.0,<13.0.0a0 -- pytest-benchmark<5.1.0 +- pytest-benchmark - pytest-cases>=3.8.2 - pytest-cov - pytest-rerunfailures diff --git a/conda/environments/all_cuda-128_arch-aarch64.yaml b/conda/environments/all_cuda-128_arch-aarch64.yaml index 264d60c2bd93..8921d72324dd 100644 --- a/conda/environments/all_cuda-128_arch-aarch64.yaml +++ b/conda/environments/all_cuda-128_arch-aarch64.yaml @@ -69,7 +69,7 @@ dependencies: - pydata-sphinx-theme>=0.15.4 - pynvjitlink>=0.0.0a0 - pynvml>=12.0.0,<13.0.0a0 -- pytest-benchmark<5.1.0 +- pytest-benchmark - pytest-cases>=3.8.2 - pytest-cov - pytest-rerunfailures diff --git a/conda/environments/all_cuda-128_arch-x86_64.yaml b/conda/environments/all_cuda-128_arch-x86_64.yaml index 3d5d0c85be3d..a43cf90497c8 100644 --- a/conda/environments/all_cuda-128_arch-x86_64.yaml +++ b/conda/environments/all_cuda-128_arch-x86_64.yaml @@ -71,7 +71,7 @@ dependencies: - pydata-sphinx-theme>=0.15.4 - pynvjitlink>=0.0.0a0 - pynvml>=12.0.0,<13.0.0a0 -- pytest-benchmark<5.1.0 +- pytest-benchmark - pytest-cases>=3.8.2 - pytest-cov - pytest-rerunfailures diff --git a/dependencies.yaml b/dependencies.yaml index 23e2f0d5b942..3923128edf87 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -945,14 +945,7 @@ dependencies: - *fastavro - hypothesis - mmh3 - # Version 5.1 is incompatible with pytest<8.2. - # https://github.com/ionelmc/pytest-benchmark/commit/8dfeeeca8a5c640a4dc4455904ace4f97f62e655 - # Remove upper bound when we unbound pytest - # https://github.com/rapidsai/build-planning/issues/105 - # or when a fixed version of pytest-benchmark 5.1 with the necessary bounds is released - # https://github.com/conda-forge/pytest-benchmark-feedstock/pull/27 - # https://github.com/conda-forge/conda-forge-repodata-patches-feedstock/pull/990 - - pytest-benchmark<5.1.0 + - pytest-benchmark - pytest-cases>=3.8.2 - scipy - zstandard diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index 5080a578b91d..e43cbd052799 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -56,7 +56,7 @@ test = [ "hypothesis", "mmh3", "msgpack", - "pytest-benchmark<5.1.0", + "pytest-benchmark", "pytest-cases>=3.8.2", "pytest-cov", "pytest-rerunfailures", From f015e2a81b03069c9eb52a32370281ecec300333 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 18 Apr 2025 21:31:52 +0000 Subject: [PATCH 12/14] PR review --- python/pylibcudf/pylibcudf/table.pyx | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/python/pylibcudf/pylibcudf/table.pyx b/python/pylibcudf/pylibcudf/table.pyx index d77c99d8429a..6d556e0196da 100644 --- a/python/pylibcudf/pylibcudf/table.pyx +++ b/python/pylibcudf/pylibcudf/table.pyx @@ -122,9 +122,14 @@ cdef class Table: tmp = Table.from_table_view_of_arbitrary(result.tbl.get().view(), result) self._columns = tmp.columns() elif hasattr(arrow_like, "__arrow_c_device_stream__"): + # TODO: When we add support for this case, it should be moved above + # the __arrow_c_stream__ case since we should prioritize device + # data if possible. raise NotImplementedError("Device streams not yet supported") elif hasattr(arrow_like, "__arrow_c_array__"): raise NotImplementedError("Arrow host arrays not yet supported") + else: + raise ValueError("Invalid Arrow-like object") cdef table_view view(self) nogil: """Generate a libcudf table_view to pass to libcudf algorithms. From 22c8a4c5c6bcb8e6bfd758561008096d6390240d Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Mon, 21 Apr 2025 18:27:34 +0000 Subject: [PATCH 13/14] Revert "No conda-forge arm packages for nanoarrow" This reverts commit 167762bc0df5428e1ffc1d19b1d03970af033045. --- conda/environments/all_cuda-118_arch-aarch64.yaml | 1 + conda/environments/all_cuda-128_arch-aarch64.yaml | 1 + dependencies.yaml | 14 +------------- python/pylibcudf/pylibcudf/tests/test_interop.py | 8 ++++---- 4 files changed, 7 insertions(+), 17 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-aarch64.yaml b/conda/environments/all_cuda-118_arch-aarch64.yaml index cbe3cb94f189..44b4df41b2d7 100644 --- a/conda/environments/all_cuda-118_arch-aarch64.yaml +++ b/conda/environments/all_cuda-118_arch-aarch64.yaml @@ -47,6 +47,7 @@ dependencies: - moto>=4.0.8 - msgpack-python - myst-nb +- nanoarrow - nbconvert - nbformat - nbsphinx diff --git a/conda/environments/all_cuda-128_arch-aarch64.yaml b/conda/environments/all_cuda-128_arch-aarch64.yaml index 3c431acbe9da..d5f53001c10a 100644 --- a/conda/environments/all_cuda-128_arch-aarch64.yaml +++ b/conda/environments/all_cuda-128_arch-aarch64.yaml @@ -47,6 +47,7 @@ dependencies: - moto>=4.0.8 - msgpack-python - myst-nb +- nanoarrow - nbconvert - nbformat - nbsphinx diff --git a/dependencies.yaml b/dependencies.yaml index 4cc3e6ec7fc0..35d086fd7033 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -922,21 +922,9 @@ dependencies: - output_types: [conda, requirements, pyproject] packages: - &fastavro fastavro>=0.22.9 + - nanoarrow - *numpy - pandas - - output_types: [requirements, pyproject] - packages: - - nanoarrow - specific: - - output_types: conda - matrices: - - matrix: - arch: aarch64 - packages: - - matrix: - arch: x86_64 - packages: - - nanoarrow test_python_cudf: common: - output_types: [conda, requirements, pyproject] diff --git a/python/pylibcudf/pylibcudf/tests/test_interop.py b/python/pylibcudf/pylibcudf/tests/test_interop.py index fd58ad9f3454..f946710320f4 100644 --- a/python/pylibcudf/pylibcudf/tests/test_interop.py +++ b/python/pylibcudf/pylibcudf/tests/test_interop.py @@ -1,6 +1,8 @@ # Copyright (c) 2024-2025, NVIDIA CORPORATION. import cupy as cp +import nanoarrow +import nanoarrow.device import numpy as np import pyarrow as pa import pytest @@ -123,17 +125,15 @@ def test_from_dlpack_error(): def test_device_interop_column(): - nad = pytest.importorskip("nanoarrow.device") pa_arr = pa.array([{"a": [1, None]}, None, {"b": [None, 4]}]) plc_col = plc.Column(pa_arr) - na_arr = nad.c_device_array(plc_col) + na_arr = nanoarrow.device.c_device_array(plc_col) new_col = plc.Column(na_arr) assert_column_eq(pa_arr, new_col) def test_device_interop_table(): - nad = pytest.importorskip("nanoarrow.device") # Have to manually construct the schema to ensure that names match. pyarrow will # assign names to nested types automatically otherwise. schema = pa.schema( @@ -157,7 +157,7 @@ def test_device_interop_table(): ) plc_table = plc.Table(pa_tbl) - na_arr = nad.c_device_array(plc_table) + na_arr = nanoarrow.device.c_device_array(plc_table) actual_schema = pa.schema(na_arr.schema) assert actual_schema.equals(pa_tbl.schema) From b57124f7b0412e37cdbe75d54517aeacf4ce91e2 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 22 Apr 2025 01:28:25 +0000 Subject: [PATCH 14/14] Add missed group --- dependencies.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/dependencies.yaml b/dependencies.yaml index 35d086fd7033..ced9b59ae5e1 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -75,6 +75,7 @@ files: - test_python_common - test_python_cudf_common - test_python_cudf + - test_python_pylibcudf - depends_on_cudf - depends_on_pylibcudf - depends_on_libcudf