diff --git a/conda/environments/all_cuda-118_arch-aarch64.yaml b/conda/environments/all_cuda-118_arch-aarch64.yaml index 435561ed8c17..44b4df41b2d7 100644 --- a/conda/environments/all_cuda-118_arch-aarch64.yaml +++ b/conda/environments/all_cuda-118_arch-aarch64.yaml @@ -47,6 +47,7 @@ dependencies: - moto>=4.0.8 - msgpack-python - myst-nb +- nanoarrow - nbconvert - nbformat - nbsphinx @@ -70,7 +71,7 @@ dependencies: - pyarrow>=14.0.0,<20.0.0a0 - pydata-sphinx-theme>=0.15.4 - pynvml>=12.0.0,<13.0.0a0 -- pytest-benchmark<5.1.0 +- pytest-benchmark - pytest-cases>=3.8.2 - pytest-cov - pytest-rerunfailures diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 24cc244efa64..2c6d43f8d547 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -49,6 +49,7 @@ dependencies: - moto>=4.0.8 - msgpack-python - myst-nb +- nanoarrow - nbconvert - nbformat - nbsphinx @@ -72,7 +73,7 @@ dependencies: - pyarrow>=14.0.0,<20.0.0a0 - pydata-sphinx-theme>=0.15.4 - pynvml>=12.0.0,<13.0.0a0 -- pytest-benchmark<5.1.0 +- pytest-benchmark - pytest-cases>=3.8.2 - pytest-cov - pytest-rerunfailures diff --git a/conda/environments/all_cuda-128_arch-aarch64.yaml b/conda/environments/all_cuda-128_arch-aarch64.yaml index ab237e095888..d5f53001c10a 100644 --- a/conda/environments/all_cuda-128_arch-aarch64.yaml +++ b/conda/environments/all_cuda-128_arch-aarch64.yaml @@ -47,6 +47,7 @@ dependencies: - moto>=4.0.8 - msgpack-python - myst-nb +- nanoarrow - nbconvert - nbformat - nbsphinx @@ -69,7 +70,7 @@ dependencies: - pydata-sphinx-theme>=0.15.4 - pynvjitlink>=0.0.0a0 - pynvml>=12.0.0,<13.0.0a0 -- pytest-benchmark<5.1.0 +- pytest-benchmark - pytest-cases>=3.8.2 - pytest-cov - pytest-rerunfailures diff --git a/conda/environments/all_cuda-128_arch-x86_64.yaml b/conda/environments/all_cuda-128_arch-x86_64.yaml index 7e05efaee8a8..092c61358613 100644 --- a/conda/environments/all_cuda-128_arch-x86_64.yaml +++ b/conda/environments/all_cuda-128_arch-x86_64.yaml @@ -48,6 +48,7 @@ dependencies: - moto>=4.0.8 - msgpack-python - myst-nb +- nanoarrow - nbconvert - nbformat - nbsphinx @@ -70,7 +71,7 @@ dependencies: - pydata-sphinx-theme>=0.15.4 - pynvjitlink>=0.0.0a0 - pynvml>=12.0.0,<13.0.0a0 -- pytest-benchmark<5.1.0 +- pytest-benchmark - pytest-cases>=3.8.2 - pytest-cov - pytest-rerunfailures diff --git a/dependencies.yaml b/dependencies.yaml index c31dec0e35fa..ced9b59ae5e1 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -75,6 +75,7 @@ files: - test_python_common - test_python_cudf_common - test_python_cudf + - test_python_pylibcudf - depends_on_cudf - depends_on_pylibcudf - depends_on_libcudf @@ -86,6 +87,7 @@ files: - test_python_common - test_python_cudf_common - test_python_dask_cudf + - test_python_pylibcudf - depends_on_cudf - depends_on_pylibcudf - depends_on_libcudf @@ -920,9 +922,8 @@ dependencies: common: - output_types: [conda, requirements, pyproject] packages: - - fastavro>=0.22.9 - # TODO: Remove hypothesis pinning once https://github.com/HypothesisWorks/hypothesis/issues/4365 is resolved - - hypothesis<6.131.1 + - &fastavro fastavro>=0.22.9 + - nanoarrow - *numpy - pandas test_python_cudf: @@ -930,18 +931,11 @@ dependencies: - output_types: [conda, requirements, pyproject] packages: - cramjam - - fastavro>=0.22.9 + - *fastavro # TODO: Remove hypothesis pinning once https://github.com/HypothesisWorks/hypothesis/issues/4365 is resolved - hypothesis<6.131.1 - mmh3 - # Version 5.1 is incompatible with pytest<8.2. - # https://github.com/ionelmc/pytest-benchmark/commit/8dfeeeca8a5c640a4dc4455904ace4f97f62e655 - # Remove upper bound when we unbound pytest - # https://github.com/rapidsai/build-planning/issues/105 - # or when a fixed version of pytest-benchmark 5.1 with the necessary bounds is released - # https://github.com/conda-forge/pytest-benchmark-feedstock/pull/27 - # https://github.com/conda-forge/conda-forge-repodata-patches-feedstock/pull/990 - - pytest-benchmark<5.1.0 + - pytest-benchmark - pytest-cases>=3.8.2 - scipy - zstandard diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index 1a151313d11c..e16ef6632377 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -56,7 +56,7 @@ test = [ "hypothesis<6.131.1", "mmh3", "msgpack", - "pytest-benchmark<5.1.0", + "pytest-benchmark", "pytest-cases>=3.8.2", "pytest-cov", "pytest-rerunfailures", diff --git a/python/pylibcudf/pylibcudf/_interop_helpers.pxd b/python/pylibcudf/pylibcudf/_interop_helpers.pxd index 72036be4e775..457baa4a136a 100644 --- a/python/pylibcudf/pylibcudf/_interop_helpers.pxd +++ b/python/pylibcudf/pylibcudf/_interop_helpers.pxd @@ -6,4 +6,6 @@ cdef void _release_schema(object schema_capsule) noexcept cdef void _release_array(object array_capsule) noexcept +cdef void _release_device_array(object array_capsule) noexcept + cdef column_metadata _metadata_to_libcudf(metadata) diff --git a/python/pylibcudf/pylibcudf/_interop_helpers.pyx b/python/pylibcudf/pylibcudf/_interop_helpers.pyx index f2fa6ccf549d..483c38d8f173 100644 --- a/python/pylibcudf/pylibcudf/_interop_helpers.pyx +++ b/python/pylibcudf/pylibcudf/_interop_helpers.pyx @@ -4,9 +4,11 @@ from cpython.pycapsule cimport PyCapsule_GetPointer from pylibcudf.libcudf.interop cimport ( ArrowArray, + ArrowDeviceArray, ArrowSchema, column_metadata, release_arrow_array_raw, + release_arrow_device_array_raw, release_arrow_schema_raw, ) @@ -39,6 +41,14 @@ cdef void _release_array(object array_capsule) noexcept: release_arrow_array_raw(array) +cdef void _release_device_array(object array_capsule) noexcept: + """Release the ArrowDeviceArray object stored in a PyCapsule.""" + cdef ArrowDeviceArray* array = PyCapsule_GetPointer( + array_capsule, 'arrow_device_array' + ) + release_arrow_device_array_raw(array) + + cdef column_metadata _metadata_to_libcudf(metadata): """Convert a ColumnMetadata object to C++ column_metadata. diff --git a/python/pylibcudf/pylibcudf/column.pyx b/python/pylibcudf/pylibcudf/column.pyx index 434c8dbeb50b..d1667e1e4341 100644 --- a/python/pylibcudf/pylibcudf/column.pyx +++ b/python/pylibcudf/pylibcudf/column.pyx @@ -15,19 +15,21 @@ from libcpp.utility cimport move from pylibcudf.libcudf.column.column cimport column, column_contents from pylibcudf.libcudf.column.column_factories cimport make_column_from_scalar -from pylibcudf.libcudf.interop cimport ArrowArray, ArrowSchema, arrow_column -from pylibcudf.libcudf.scalar.scalar cimport scalar, numeric_scalar -from pylibcudf.libcudf.types cimport size_type, size_of as cpp_size_of, bitmask_type -from pylibcudf.libcudf.utilities.traits cimport is_fixed_width -from pylibcudf.libcudf.copying cimport get_element - from pylibcudf.libcudf.interop cimport ( ArrowArray, ArrowSchema, + ArrowDeviceArray, + arrow_column, column_metadata, to_arrow_host_raw, + to_arrow_device_raw, to_arrow_schema_raw, ) +from pylibcudf.libcudf.scalar.scalar cimport scalar, numeric_scalar +from pylibcudf.libcudf.types cimport size_type, size_of as cpp_size_of, bitmask_type +from pylibcudf.libcudf.utilities.traits cimport is_fixed_width +from pylibcudf.libcudf.copying cimport get_element + from rmm.librmm.device_buffer cimport device_buffer from rmm.pylibrmm.device_buffer cimport DeviceBuffer @@ -40,6 +42,7 @@ from .types cimport DataType, size_of, type_id from ._interop_helpers cimport ( _release_schema, _release_array, + _release_device_array, _metadata_to_libcudf, ) from .null_mask cimport bitmask_allocation_size_bytes @@ -55,7 +58,12 @@ __all__ = ["Column", "ListColumnView", "is_c_contiguous"] class _ArrowLikeMeta(type): def __subclasscheck__(cls, other): - return hasattr(other, "__arrow_c_array__") + # We cannot separate these types via singledispatch because the dispatch + # will often be ambiguous when objects expose multiple protocols. + return ( + hasattr(other, "__arrow_c_array__") + or hasattr(other, "__arrow_c_device_array__") + ) class _ArrowLike(metaclass=_ArrowLikeMeta): @@ -207,32 +215,59 @@ cdef class Column: @_init.register(_ArrowLike) def _(self, arrow_like): - schema, array = arrow_like.__arrow_c_array__() - cdef ArrowSchema* c_schema = ( - PyCapsule_GetPointer(schema, "arrow_schema") - ) - cdef ArrowArray* c_array = ( - PyCapsule_GetPointer(array, "arrow_array") - ) - - cdef _ArrowColumnHolder result = _ArrowColumnHolder() + cdef ArrowSchema* c_schema + cdef ArrowArray* c_array + cdef ArrowDeviceArray* c_device_array + cdef _ArrowColumnHolder result cdef unique_ptr[arrow_column] c_result - with nogil: - c_result = make_unique[arrow_column]( - move(dereference(c_schema)), move(dereference(c_array)) + if hasattr(arrow_like, "__arrow_c_device_array__"): + schema, array = arrow_like.__arrow_c_device_array__() + c_schema = PyCapsule_GetPointer(schema, "arrow_schema") + c_device_array = ( + PyCapsule_GetPointer(array, "arrow_device_array") + ) + + result = _ArrowColumnHolder() + with nogil: + c_result = make_unique[arrow_column]( + move(dereference(c_schema)), move(dereference(c_device_array)) + ) + result.col.swap(c_result) + + tmp = Column.from_column_view_of_arbitrary(result.col.get().view(), result) + self._init( + tmp.type(), + tmp.size(), + tmp.data(), + tmp.null_mask(), + tmp.null_count(), + tmp.offset(), + tmp.children(), ) + elif hasattr(arrow_like, "__arrow_c_array__"): + schema, array = arrow_like.__arrow_c_array__() + c_schema = PyCapsule_GetPointer(schema, "arrow_schema") + c_array = PyCapsule_GetPointer(array, "arrow_array") + + result = _ArrowColumnHolder() + with nogil: + c_result = make_unique[arrow_column]( + move(dereference(c_schema)), move(dereference(c_array)) + ) result.col.swap(c_result) - tmp = Column.from_column_view_of_arbitrary(result.col.get().view(), result) - self._init( - tmp.type(), - tmp.size(), - tmp.data(), - tmp.null_mask(), - tmp.null_count(), - tmp.offset(), - tmp.children(), - ) + tmp = Column.from_column_view_of_arbitrary(result.col.get().view(), result) + self._init( + tmp.type(), + tmp.size(), + tmp.data(), + tmp.null_mask(), + tmp.null_count(), + tmp.offset(), + tmp.children(), + ) + else: + raise ValueError("Invalid Arrow-like object") cdef column_view view(self) nogil: """Generate a libcudf column_view to pass to libcudf algorithms. @@ -752,12 +787,37 @@ cdef class Column: return PyCapsule_New(raw_host_array_ptr, "arrow_array", _release_array) + def _to_device_array(self): + cdef ArrowDeviceArray* raw_device_array_ptr + with nogil: + raw_device_array_ptr = to_arrow_device_raw(self.view(), self) + + return PyCapsule_New( + raw_device_array_ptr, + "arrow_device_array", + _release_device_array + ) + def __arrow_c_array__(self, requested_schema=None): if requested_schema is not None: raise ValueError("pylibcudf.Column does not support alternative schema") return self._to_schema(), self._to_host_array() + def __arrow_c_device_array__(self, requested_schema=None, **kwargs): + if requested_schema is not None: + raise ValueError("pylibcudf.Column does not support alternative schema") + + non_default_kwargs = [ + name for name, value in kwargs.items() if value is not None + ] + if non_default_kwargs: + raise NotImplementedError( + f"Received unsupported keyword argument(s): {non_default_kwargs}" + ) + + return self._to_schema(), self._to_device_array() + cdef class ListColumnView: """Accessor for methods of a Column that are specific to lists.""" diff --git a/python/pylibcudf/pylibcudf/libcudf/interop.pxd b/python/pylibcudf/pylibcudf/libcudf/interop.pxd index ab1d5bf5d5b0..13ace55f59d4 100644 --- a/python/pylibcudf/pylibcudf/libcudf/interop.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/interop.pxd @@ -54,12 +54,20 @@ cdef extern from "cudf/interop.hpp" namespace "cudf::interop" \ ArrowSchema&& schema, ArrowArray&& array ) except +libcudf_exception_handler + arrow_column( + ArrowSchema&& schema, + ArrowDeviceArray&& array + ) except +libcudf_exception_handler column_view view() except +libcudf_exception_handler cdef cppclass arrow_table: arrow_table( ArrowArrayStream&& stream, - ) except +libcudf_exception_handler + ) except +libcudf_exception_handler + arrow_table( + ArrowSchema&& schema, + ArrowDeviceArray&& array, + ) except +libcudf_exception_handler table_view view() except +libcudf_exception_handler @@ -67,9 +75,10 @@ cdef extern from *: # Rather than exporting the underlying functions directly to Cython, we expose # these wrappers that handle the release to avoid needing to teach Cython how # to handle unique_ptrs with custom deleters that aren't default constructible. - # This will go away once we introduce cudf::arrow_column (need a - # cudf::arrow_schema as well), see - # https://github.com/rapidsai/cudf/issues/16104. + # We cannot use cudf's owning arrow types for this because pylibcudf's + # objects always manage data ownership independently of libcudf in order to + # support other data sources (e.g. cupy), so we must use the view-based + # C++ APIs and handle ownership in Python. """ #include #include @@ -98,24 +107,13 @@ cdef extern from *: delete schema; } + template ArrowArray* to_arrow_host_raw( - cudf::table_view const& tbl, - rmm::cuda_stream_view stream = cudf::get_default_stream(), - rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()) { - // Assumes the sync event is null and the data is already on the host. - ArrowArray *arr = new ArrowArray(); - auto device_arr = cudf::to_arrow_host(tbl, stream, mr); - ArrowArrayMove(&device_arr->array, arr); - return arr; - } - - ArrowArray* to_arrow_host_raw( - cudf::column_view const& col, + ViewType const& obj, rmm::cuda_stream_view stream = cudf::get_default_stream(), rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()) { - // Assumes the sync event is null and the data is already on the host. ArrowArray *arr = new ArrowArray(); - auto device_arr = cudf::to_arrow_host(col, stream, mr); + auto device_arr = cudf::to_arrow_host(obj, stream, mr); ArrowArrayMove(&device_arr->array, arr); return arr; } @@ -126,6 +124,54 @@ cdef extern from *: } delete array; } + + void release_arrow_device_array_raw(ArrowDeviceArray *array) { + if (array->array.release != nullptr) { + array->array.release(&array->array); + } + delete array; + } + + struct PylibcudfArrowDeviceArrayPrivateData { + ArrowArray parent; + PyObject* owner; + }; + + void PylibcudfArrowDeviceArrayRelease(ArrowArray* array) + { + auto private_data = reinterpret_cast( + array->private_data); + Py_DECREF(private_data->owner); + private_data->parent.release(&private_data->parent); + array->release = nullptr; + } + + template + ArrowDeviceArray* to_arrow_device_raw( + ViewType const& obj, + PyObject* owner, + rmm::cuda_stream_view stream = cudf::get_default_stream(), + rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()) { + auto tmp = cudf::to_arrow_device(obj, stream, mr); + + // Instead of moving the whole device array, we move the underlying ArrowArray + // into the custom private data struct for managing its data then create a new + // device array from scratch. + auto private_data = new PylibcudfArrowDeviceArrayPrivateData(); + ArrowArrayMove(&tmp->array, &private_data->parent); + private_data->owner = owner; + Py_INCREF(owner); + + ArrowDeviceArray *arr = new ArrowDeviceArray(); + arr->device_id = tmp->device_id; + arr->device_type = tmp->device_type; + arr->sync_event = tmp->sync_event; + arr->array = private_data->parent; // shallow copy + arr->array.private_data = private_data; + arr->array.release = &PylibcudfArrowDeviceArrayRelease; + + return arr; + } """ # The `to_*_raw` functions are all defined in the above extern block as wrappers # around libcudf functions that return unique_ptrs with non-default deleters, which @@ -158,3 +204,14 @@ cdef extern from *: cdef void release_arrow_array_raw( ArrowArray * ) except +libcudf_exception_handler nogil + cdef void release_arrow_device_array_raw( + ArrowDeviceArray * + ) except +libcudf_exception_handler nogil + cdef ArrowDeviceArray* to_arrow_device_raw( + const table_view& tbl, + object owner, + ) except +libcudf_exception_handler nogil + cdef ArrowDeviceArray* to_arrow_device_raw( + const column_view& tbl, + object owner, + ) except +libcudf_exception_handler nogil diff --git a/python/pylibcudf/pylibcudf/table.pyx b/python/pylibcudf/pylibcudf/table.pyx index 4992a02ab1c7..6d556e0196da 100644 --- a/python/pylibcudf/pylibcudf/table.pyx +++ b/python/pylibcudf/pylibcudf/table.pyx @@ -14,23 +14,25 @@ from libcpp.vector cimport vector from rmm.pylibrmm.stream cimport Stream from pylibcudf.libcudf.column.column cimport column from pylibcudf.libcudf.column.column_view cimport column_view -from pylibcudf.libcudf.table.table cimport table - from pylibcudf.libcudf.interop cimport ( ArrowArray, ArrowArrayStream, + ArrowDeviceArray, ArrowSchema, arrow_table, column_metadata, + to_arrow_device_raw, to_arrow_host_raw, to_arrow_schema_raw, ) +from pylibcudf.libcudf.table.table cimport table from .column cimport Column from .utils cimport _get_stream from pylibcudf._interop_helpers cimport ( _release_schema, _release_array, + _release_device_array, _metadata_to_libcudf, ) from ._interop_helpers import ColumnMetadata @@ -41,12 +43,14 @@ __all__ = ["Table"] class _ArrowLikeMeta(type): - # We cannot separate stream and array via singledispatch because the - # dispatch will often be ambiguous when objects expose both protocols. + # We cannot separate these types via singledispatch because the dispatch + # will often be ambiguous when objects expose multiple protocols. def __subclasscheck__(cls, other): return ( hasattr(other, "__arrow_c_stream__") + or hasattr(other, "__arrow_c_device_stream__") or hasattr(other, "__arrow_c_array__") + or hasattr(other, "__arrow_c_device_array__") ) @@ -84,10 +88,27 @@ cdef class Table: @_init.register(_ArrowLike) def _(self, arrow_like): - cdef ArrowArrayStream* c_stream + cdef ArrowSchema* c_schema + cdef ArrowDeviceArray* c_array cdef _ArrowTableHolder result cdef unique_ptr[arrow_table] c_result - if hasattr(arrow_like, "__arrow_c_stream__"): + if hasattr(arrow_like, "__arrow_c_device_array__"): + schema, array = arrow_like.__arrow_c_device_array__() + c_schema = PyCapsule_GetPointer(schema, "arrow_schema") + c_array = ( + PyCapsule_GetPointer(array, "arrow_device_array") + ) + + result = _ArrowTableHolder() + with nogil: + c_result = make_unique[arrow_table]( + move(dereference(c_schema)), move(dereference(c_array)) + ) + result.tbl.swap(c_result) + + tmp = Table.from_table_view_of_arbitrary(result.tbl.get().view(), result) + self._columns = tmp.columns() + elif hasattr(arrow_like, "__arrow_c_stream__"): stream = arrow_like.__arrow_c_stream__() c_stream = ( PyCapsule_GetPointer(stream, "arrow_array_stream") @@ -100,8 +121,15 @@ cdef class Table: tmp = Table.from_table_view_of_arbitrary(result.tbl.get().view(), result) self._columns = tmp.columns() + elif hasattr(arrow_like, "__arrow_c_device_stream__"): + # TODO: When we add support for this case, it should be moved above + # the __arrow_c_stream__ case since we should prioritize device + # data if possible. + raise NotImplementedError("Device streams not yet supported") elif hasattr(arrow_like, "__arrow_c_array__"): - raise NotImplementedError("arrays not yet supported") + raise NotImplementedError("Arrow host arrays not yet supported") + else: + raise ValueError("Invalid Arrow-like object") cdef table_view view(self) nogil: """Generate a libcudf table_view to pass to libcudf algorithms. @@ -212,7 +240,7 @@ cdef class Table: with nogil: raw_schema_ptr = to_arrow_schema_raw(self.view(), c_metadata) - return PyCapsule_New(raw_schema_ptr, 'arrow_schema', _release_schema) + return PyCapsule_New(raw_schema_ptr, "arrow_schema", _release_schema) def _to_host_array(self): cdef ArrowArray* raw_host_array_ptr @@ -221,8 +249,33 @@ cdef class Table: return PyCapsule_New(raw_host_array_ptr, "arrow_array", _release_array) + def _to_device_array(self): + cdef ArrowDeviceArray* raw_device_array_ptr + with nogil: + raw_device_array_ptr = to_arrow_device_raw(self.view(), self) + + return PyCapsule_New( + raw_device_array_ptr, + "arrow_device_array", + _release_device_array + ) + def __arrow_c_array__(self, requested_schema=None): if requested_schema is not None: raise ValueError("pylibcudf.Table does not support alternative schema") return self._to_schema(), self._to_host_array() + + def __arrow_c_device_array__(self, requested_schema=None, **kwargs): + if requested_schema is not None: + raise ValueError("pylibcudf.Table does not support alternative schema") + + non_default_kwargs = [ + name for name, value in kwargs.items() if value is not None + ] + if non_default_kwargs: + raise NotImplementedError( + f"Received unsupported keyword argument(s): {non_default_kwargs}" + ) + + return self._to_schema(), self._to_device_array() diff --git a/python/pylibcudf/pylibcudf/tests/test_interop.py b/python/pylibcudf/pylibcudf/tests/test_interop.py index ca42eacdfdb2..f946710320f4 100644 --- a/python/pylibcudf/pylibcudf/tests/test_interop.py +++ b/python/pylibcudf/pylibcudf/tests/test_interop.py @@ -1,10 +1,12 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. +# Copyright (c) 2024-2025, NVIDIA CORPORATION. import cupy as cp +import nanoarrow +import nanoarrow.device import numpy as np import pyarrow as pa import pytest -from utils import assert_table_eq +from utils import assert_column_eq, assert_table_eq import pylibcudf as plc @@ -120,3 +122,44 @@ def test_to_dlpack_error(): def test_from_dlpack_error(): with pytest.raises(ValueError, match="Invalid PyCapsule object"): plc.interop.from_dlpack(1) + + +def test_device_interop_column(): + pa_arr = pa.array([{"a": [1, None]}, None, {"b": [None, 4]}]) + plc_col = plc.Column(pa_arr) + + na_arr = nanoarrow.device.c_device_array(plc_col) + new_col = plc.Column(na_arr) + assert_column_eq(pa_arr, new_col) + + +def test_device_interop_table(): + # Have to manually construct the schema to ensure that names match. pyarrow will + # assign names to nested types automatically otherwise. + schema = pa.schema( + [ + pa.field("", pa.int64()), + pa.field("", pa.float64()), + pa.field("", pa.string()), + pa.field("", pa.list_(pa.field("", pa.int64()))), + pa.field("", pa.struct([pa.field("", pa.float64())])), + ] + ) + pa_tbl = pa.table( + [ + [1, None, 3], + [1.0, 2.0, None], + ["a", "b", None], + [[1, None], None, [2]], + [{"a": 1.0}, None, {"b": 2.0}], + ], + schema=schema, + ) + plc_table = plc.Table(pa_tbl) + + na_arr = nanoarrow.device.c_device_array(plc_table) + actual_schema = pa.schema(na_arr.schema) + assert actual_schema.equals(pa_tbl.schema) + + new_tbl = plc.Table(na_arr) + assert_table_eq(pa_tbl, new_tbl) diff --git a/python/pylibcudf/pyproject.toml b/python/pylibcudf/pyproject.toml index 2d63a671494f..ecd536bcce8c 100644 --- a/python/pylibcudf/pyproject.toml +++ b/python/pylibcudf/pyproject.toml @@ -41,7 +41,7 @@ classifiers = [ [project.optional-dependencies] test = [ "fastavro>=0.22.9", - "hypothesis<6.131.1", + "nanoarrow", "numpy>=1.23,<3.0a0", "pandas", "pytest-cov",