From 8707c0ac9178f794606b85155ddc0d02909d6e95 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Mon, 10 Aug 2026 13:03:54 -0500 Subject: [PATCH 1/9] Migrate Nanoarrow test helpers to memory_resources --- cpp/include/cudf_test/nanoarrow_utils.hpp | 78 ++++++++++++++++---- cpp/tests/interop/from_arrow_host_test.cpp | 8 +- cpp/tests/interop/from_arrow_stream_test.cpp | 39 +++++++++- cpp/tests/interop/from_arrow_test.cpp | 25 ++++--- cpp/tests/interop/to_arrow_device_test.cpp | 77 +++++++++++-------- 5 files changed, 164 insertions(+), 63 deletions(-) diff --git a/cpp/include/cudf_test/nanoarrow_utils.hpp b/cpp/include/cudf_test/nanoarrow_utils.hpp index d323d10ba39b..881dbf44d0ac 100644 --- a/cpp/include/cudf_test/nanoarrow_utils.hpp +++ b/cpp/include/cudf_test/nanoarrow_utils.hpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -100,8 +101,10 @@ std::enable_if_t() and !std::is_same_v, void> p // still represent boolean arrays differently, we have to use bools_to_mask // and give the ArrowArray object ownership of the device data. template -std::enable_if_t, void> populate_from_col(ArrowArray* arr, - cudf::column_view view) +std::enable_if_t, void> populate_from_col( + ArrowArray* arr, + cudf::column_view view, + cudf::memory_resources mr = cudf::get_current_device_resource_ref()) { arr->length = view.size(); arr->null_count = view.null_count(); @@ -112,7 +115,7 @@ std::enable_if_t, void> populate_from_col(ArrowArray* ar ArrowArrayValidityBitmap(arr)->buffer.data = const_cast(reinterpret_cast(view.null_mask())); - auto bitmask = cudf::bools_to_mask(view); + auto bitmask = cudf::bools_to_mask(view, cudf::get_default_stream(), mr.get_output_mr()); auto ptr = reinterpret_cast(bitmask.first->data()); NANOARROW_THROW_NOT_OK(ArrowBufferSetAllocator( ArrowArrayBuffer(arr, 1), @@ -131,7 +134,9 @@ std::enable_if_t, void> populate_from_col(ArrowArray* ar // of the device buffers. template std::enable_if_t, void> populate_from_col( - ArrowArray* arr, cudf::column_view view) + ArrowArray* arr, + cudf::column_view view, + cudf::memory_resources mr = cudf::get_current_device_resource_ref()) { arr->length = view.size(); arr->null_count = view.null_count(); @@ -151,14 +156,17 @@ std::enable_if_t, void> populate_from_col( ArrowArrayBuffer(arr, 2)->size_bytes = sview.chars_size(cudf::get_default_stream()); ArrowArrayBuffer(arr, 2)->data = const_cast(view.data()); } else { - auto zero = cudf::detail::device_scalar(0, cudf::get_default_stream()); + auto zero = + cudf::detail::device_scalar(0, cudf::get_default_stream(), mr.get_output_mr()); uint8_t const* ptr = reinterpret_cast(zero.data()); nanoarrow::BufferInitWrapped(ArrowArrayBuffer(arr, 1), std::move(zero), ptr, 4); } } template -void populate_dict_from_col(ArrowArray* arr, cudf::dictionary_column_view dview) +void populate_dict_from_col(ArrowArray* arr, + cudf::dictionary_column_view dview, + cudf::memory_resources mr = cudf::get_current_device_resource_ref()) { arr->length = dview.size(); arr->null_count = dview.null_count(); @@ -172,17 +180,37 @@ void populate_dict_from_col(ArrowArray* arr, cudf::dictionary_column_view dview) ArrowArrayBuffer(arr, 1)->size_bytes = sizeof(IND_TYPE) * dview.indices().size(); ArrowArrayBuffer(arr, 1)->data = const_cast(dview.indices().data()); - populate_from_col(arr->dictionary, dview.keys()); + if constexpr (std::is_same_v or std::is_same_v) { + populate_from_col(arr->dictionary, dview.keys(), mr); + } else { + static_cast(mr); + populate_from_col(arr->dictionary, dview.keys()); + } } using vector_of_columns = std::vector>; +/** + * @brief Create equivalent cuDF and device-backed nanoarrow tables. + * + * @param length Number of rows to generate + * @param mr Memory resources used for returned device allocations and helper temporaries + * @return cuDF table, Arrow schema, and Arrow array + */ std::tuple, nanoarrow::UniqueSchema, nanoarrow::UniqueArray> -get_nanoarrow_tables(cudf::size_type length = 10000); +get_nanoarrow_tables(cudf::size_type length = 10000, + cudf::memory_resources mr = cudf::get_current_device_resource_ref()); void populate_list_from_col(ArrowArray* arr, cudf::lists_column_view view); -std::unique_ptr get_cudf_table(); +/** + * @brief Create the standard cuDF table used by Arrow interop tests. + * + * @param mr Memory resources used for returned table allocations and helper temporaries + * @return Generated cuDF table + */ +std::unique_ptr get_cudf_table( + cudf::memory_resources mr = cudf::get_current_device_resource_ref()); template struct nanoarrow_storage_type {}; @@ -388,11 +416,27 @@ nanoarrow::UniqueArray get_nanoarrow_list_array(std::initializer_list data, return get_nanoarrow_list_array(data_vector, offset, data_mask, list_mask); } +/** + * @brief Create a cuDF table, matching Arrow schema, and source host data. + * + * @param length Number of rows to generate + * @param mr Memory resources used for returned table allocations and helper temporaries + * @return cuDF table, Arrow schema, and generated host data + */ std::tuple, nanoarrow::UniqueSchema, generated_test_data> -get_nanoarrow_cudf_table(cudf::size_type length); - +get_nanoarrow_cudf_table(cudf::size_type length, + cudf::memory_resources mr = cudf::get_current_device_resource_ref()); + +/** + * @brief Create equivalent cuDF and host-backed nanoarrow tables. + * + * @param length Number of rows to generate + * @param mr Memory resources used for returned table allocations and helper temporaries + * @return cuDF table, Arrow schema, and Arrow array + */ std::tuple, nanoarrow::UniqueSchema, nanoarrow::UniqueArray> -get_nanoarrow_host_tables(cudf::size_type length); +get_nanoarrow_host_tables(cudf::size_type length, + cudf::memory_resources mr = cudf::get_current_device_resource_ref()); void slice_host_nanoarrow(ArrowArray* arr, int64_t start, int64_t end); @@ -442,5 +486,13 @@ void makeStreamFromArrays(std::vector arrays, nanoarrow::UniqueSchema schema, ArrowArrayStream* out); +/** + * @brief Create a cuDF table and equivalent nanoarrow stream. + * + * @param num_copies Number of record batches in the stream + * @param mr Memory resources used for returned table allocations and helper temporaries + * @return Concatenated cuDF table, Arrow schema, and Arrow stream + */ std::tuple, nanoarrow::UniqueSchema, ArrowArrayStream> -get_nanoarrow_stream(int num_copies); +get_nanoarrow_stream(int num_copies, + cudf::memory_resources mr = cudf::get_current_device_resource_ref()); diff --git a/cpp/tests/interop/from_arrow_host_test.cpp b/cpp/tests/interop/from_arrow_host_test.cpp index 3cb451165f79..7715ee4bb096 100644 --- a/cpp/tests/interop/from_arrow_host_test.cpp +++ b/cpp/tests/interop/from_arrow_host_test.cpp @@ -26,16 +26,16 @@ // create a cudf::table and equivalent arrow table with host memory std::tuple, nanoarrow::UniqueSchema, nanoarrow::UniqueArray> -get_nanoarrow_host_tables(cudf::size_type length) +get_nanoarrow_host_tables(cudf::size_type length, cudf::memory_resources mr) { - auto [table, schema, test_data] = get_nanoarrow_cudf_table(length); + auto [table, schema, test_data] = get_nanoarrow_cudf_table(length, mr); auto int64_array = get_nanoarrow_array(test_data.int64_data, test_data.validity); auto string_array = get_nanoarrow_array(test_data.string_data, test_data.validity); cudf::dictionary_column_view view(table->get_column(2).view()); - auto keys = cudf::test::to_host(view.keys()).first; - auto indices = cudf::test::to_host(view.indices()).first; + auto keys = cudf::test::to_host(view.keys(), mr).first; + auto indices = cudf::test::to_host(view.indices(), mr).first; auto dict_array = get_nanoarrow_dict_array(std::vector(keys.begin(), keys.end()), std::vector(indices.begin(), indices.end()), test_data.validity); diff --git a/cpp/tests/interop/from_arrow_stream_test.cpp b/cpp/tests/interop/from_arrow_stream_test.cpp index 74f9fc1df31d..d3335d8ba25c 100644 --- a/cpp/tests/interop/from_arrow_stream_test.cpp +++ b/cpp/tests/interop/from_arrow_stream_test.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -14,6 +14,8 @@ #include #include +#include + struct FromArrowStreamTest : public cudf::test::BaseFixture {}; void makeStreamFromArrays(std::vector arrays, @@ -29,14 +31,16 @@ void makeStreamFromArrays(std::vector arrays, } std::tuple, nanoarrow::UniqueSchema, ArrowArrayStream> -get_nanoarrow_stream(int num_copies) +get_nanoarrow_stream(int num_copies, cudf::memory_resources mr) { + auto const temporary_mr = mr.get_temporary_mr(); + auto const temporary_resources = cudf::memory_resources{temporary_mr, temporary_mr}; std::vector> tables; // The schema is unique across all tables. nanoarrow::UniqueSchema schema; std::vector arrays; for (auto i = 0; i < num_copies; ++i) { - auto [tbl, sch, arr] = get_nanoarrow_host_tables(3); + auto [tbl, sch, arr] = get_nanoarrow_host_tables(3, temporary_resources); tables.push_back(std::move(tbl)); arrays.push_back(std::move(arr)); if (i == 0) { sch.move(schema.get()); } @@ -45,7 +49,7 @@ get_nanoarrow_stream(int num_copies) for (auto const& table : tables) { table_views.push_back(table->view()); } - auto expected = cudf::concatenate(table_views); + auto expected = cudf::concatenate(table_views, cudf::get_default_stream(), mr.get_output_mr()); ArrowArrayStream stream; makeStreamFromArrays(std::move(arrays), std::move(schema), &stream); @@ -87,6 +91,33 @@ TEST_F(FromArrowStreamTest, BasicTest) CUDF_TEST_EXPECT_TABLES_EQUAL(tbl->view(), result->view()); } +TEST_F(FromArrowStreamTest, TestUtilityMemoryResourceControl) +{ + auto upstream = this->mr(); + auto output_mr = rmm::mr::statistics_resource_adaptor(upstream); + auto temporary_mr = rmm::mr::statistics_resource_adaptor(upstream); + auto resources = cudf::memory_resources{output_mr, temporary_mr}; + + { + auto direct_table = get_cudf_table(resources); + auto [generated_table, generated_schema, test_data] = get_nanoarrow_cudf_table(3, resources); + auto [device_table, device_schema, device_array] = get_nanoarrow_tables(0, resources); + auto [host_table, host_schema, host_array] = get_nanoarrow_host_tables(3, resources); + auto [stream_table, stream_schema, stream] = get_nanoarrow_stream(2, resources); + + cudf::get_default_stream().synchronize(); + EXPECT_GT(output_mr.get_bytes_counter().value, 0); + EXPECT_EQ(temporary_mr.get_bytes_counter().value, 0); + EXPECT_GT(temporary_mr.get_bytes_counter().total, 0); + + if (stream.release != nullptr) { stream.release(&stream); } + } + + cudf::get_default_stream().synchronize(); + EXPECT_EQ(output_mr.get_bytes_counter().value, 0); + EXPECT_EQ(temporary_mr.get_bytes_counter().value, 0); +} + TEST_F(FromArrowStreamTest, EmptyTest) { auto [tbl, sch, arr] = get_nanoarrow_host_tables(0); diff --git a/cpp/tests/interop/from_arrow_test.cpp b/cpp/tests/interop/from_arrow_test.cpp index 3ee6c378f558..5a4d2e4bf395 100644 --- a/cpp/tests/interop/from_arrow_test.cpp +++ b/cpp/tests/interop/from_arrow_test.cpp @@ -27,23 +27,27 @@ #include -std::unique_ptr get_cudf_table() +std::unique_ptr get_cudf_table(cudf::memory_resources mr) { + auto const temporary_mr = mr.get_temporary_mr(); std::vector> columns; columns.emplace_back(cudf::test::fixed_width_column_wrapper( - {1, 2, 5, 2, 7}, {true, false, true, true, true}) + {1, 2, 5, 2, 7}, {true, false, true, true, true}, mr) .release()); - columns.emplace_back(cudf::test::fixed_width_column_wrapper({1, 2, 3, 4, 5}).release()); - columns.emplace_back(cudf::test::strings_column_wrapper({"fff", "aaa", "", "fff", "ccc"}, - {true, true, true, false, true}) + columns.emplace_back( + cudf::test::fixed_width_column_wrapper({1, 2, 3, 4, 5}, mr).release()); + columns.emplace_back(cudf::test::strings_column_wrapper( + {"fff", "aaa", "", "fff", "ccc"}, {true, true, true, false, true}, mr) .release()); - auto keys = cudf::test::fixed_width_column_wrapper({1, 2, 5, 7}); - auto indices = cudf::test::fixed_width_column_wrapper({0, 1, 2, 1, 3}, {1, 0, 1, 1, 1}); - columns.emplace_back(cudf::make_dictionary_column(keys, indices)); + auto keys = cudf::test::fixed_width_column_wrapper({1, 2, 5, 7}, temporary_mr); + auto indices = + cudf::test::fixed_width_column_wrapper({0, 1, 2, 1, 3}, {1, 0, 1, 1, 1}, temporary_mr); + columns.emplace_back( + cudf::make_dictionary_column(keys, indices, cudf::get_default_stream(), mr.get_output_mr())); columns.emplace_back(cudf::test::fixed_width_column_wrapper( - {true, false, true, false, true}, {true, false, true, true, false}) + {true, false, true, false, true}, {true, false, true, true, false}, mr) .release()); columns.emplace_back(cudf::test::strings_column_wrapper( { @@ -53,7 +57,8 @@ std::unique_ptr get_cudf_table() "1", "2", }, - {0, 1, 1, 1, 1}) + {0, 1, 1, 1, 1}, + mr) .release()); // columns.emplace_back(cudf::test::lists_column_wrapper({{1, 2}, {3, 4}, {}, {6}, {7, 8, // 9}}).release()); diff --git a/cpp/tests/interop/to_arrow_device_test.cpp b/cpp/tests/interop/to_arrow_device_test.cpp index 7c4e5ea043f0..a73efaab3ff9 100644 --- a/cpp/tests/interop/to_arrow_device_test.cpp +++ b/cpp/tests/interop/to_arrow_device_test.cpp @@ -19,35 +19,42 @@ #include std::tuple, nanoarrow::UniqueSchema, generated_test_data> -get_nanoarrow_cudf_table(cudf::size_type length) +get_nanoarrow_cudf_table(cudf::size_type length, cudf::memory_resources mr) { + auto const temporary_mr = mr.get_temporary_mr(); generated_test_data test_data(length); std::vector> columns; - columns.emplace_back(cudf::test::fixed_width_column_wrapper(test_data.int64_data.begin(), - test_data.int64_data.end(), - test_data.validity.begin()) - .release()); - columns.emplace_back(cudf::test::strings_column_wrapper(test_data.string_data.begin(), - test_data.string_data.end(), - test_data.validity.begin()) - .release()); - auto col4 = cudf::test::fixed_width_column_wrapper( - test_data.int64_data.begin(), test_data.int64_data.end(), test_data.validity.begin()); - columns.emplace_back(cudf::dictionary::encode(col4)); - columns.emplace_back(cudf::test::fixed_width_column_wrapper(test_data.bool_data.begin(), - test_data.bool_data.end(), - test_data.bool_validity.begin()) - .release()); + columns.emplace_back( + cudf::test::fixed_width_column_wrapper( + test_data.int64_data.begin(), test_data.int64_data.end(), test_data.validity.begin(), mr) + .release()); + columns.emplace_back( + cudf::test::strings_column_wrapper( + test_data.string_data.begin(), test_data.string_data.end(), test_data.validity.begin(), mr) + .release()); + auto col4 = cudf::test::fixed_width_column_wrapper(test_data.int64_data.begin(), + test_data.int64_data.end(), + test_data.validity.begin(), + temporary_mr); + columns.emplace_back(cudf::dictionary::encode( + col4, cudf::data_type{cudf::type_id::INT32}, cudf::get_default_stream(), mr.get_output_mr())); + columns.emplace_back( + cudf::test::fixed_width_column_wrapper( + test_data.bool_data.begin(), test_data.bool_data.end(), test_data.bool_validity.begin(), mr) + .release()); auto list_child_column = cudf::test::fixed_width_column_wrapper(test_data.list_int64_data.begin(), test_data.list_int64_data.end(), - test_data.list_int64_data_validity.begin()); + test_data.list_int64_data_validity.begin(), + mr); auto list_offsets_column = cudf::test::fixed_width_column_wrapper( - test_data.list_offsets.begin(), test_data.list_offsets.end()); - auto [list_mask, list_nulls] = cudf::bools_to_mask(cudf::test::fixed_width_column_wrapper( - test_data.list_validity.begin(), test_data.list_validity.end())); + test_data.list_offsets.begin(), test_data.list_offsets.end(), mr); + auto list_validity = cudf::test::fixed_width_column_wrapper( + test_data.list_validity.begin(), test_data.list_validity.end(), temporary_mr); + auto [list_mask, list_nulls] = + cudf::bools_to_mask(list_validity, cudf::get_default_stream(), mr.get_output_mr()); columns.emplace_back(cudf::make_lists_column(length, list_offsets_column.release(), list_child_column.release(), @@ -55,19 +62,25 @@ get_nanoarrow_cudf_table(cudf::size_type length) std::move(*list_mask))); auto int_column = cudf::test::fixed_width_column_wrapper( - test_data.int64_data.begin(), test_data.int64_data.end(), test_data.validity.begin()) + test_data.int64_data.begin(), test_data.int64_data.end(), test_data.validity.begin(), mr) .release(); auto str_column = cudf::test::strings_column_wrapper( - test_data.string_data.begin(), test_data.string_data.end(), test_data.validity.begin()) + test_data.string_data.begin(), test_data.string_data.end(), test_data.validity.begin(), mr) .release(); vector_of_columns cols; cols.push_back(std::move(int_column)); cols.push_back(std::move(str_column)); - auto [null_mask, null_count] = cudf::bools_to_mask(cudf::test::fixed_width_column_wrapper( - test_data.bool_data_validity.begin(), test_data.bool_data_validity.end())); - columns.emplace_back( - cudf::make_structs_column(length, std::move(cols), null_count, std::move(*null_mask))); + auto struct_validity = cudf::test::fixed_width_column_wrapper( + test_data.bool_data_validity.begin(), test_data.bool_data_validity.end(), temporary_mr); + auto [null_mask, null_count] = + cudf::bools_to_mask(struct_validity, cudf::get_default_stream(), mr.get_output_mr()); + columns.emplace_back(cudf::make_structs_column(length, + std::move(cols), + null_count, + std::move(*null_mask), + cudf::get_default_stream(), + mr.get_output_mr())); nanoarrow::UniqueSchema schema; ArrowSchemaInit(schema.get()); @@ -157,27 +170,27 @@ get_nanoarrow_cudf_table(cudf::size_type length) } std::tuple, nanoarrow::UniqueSchema, nanoarrow::UniqueArray> -get_nanoarrow_tables(cudf::size_type length) +get_nanoarrow_tables(cudf::size_type length, cudf::memory_resources mr) { - auto [table, schema, test_data] = get_nanoarrow_cudf_table(length); + auto [table, schema, test_data] = get_nanoarrow_cudf_table(length, mr); nanoarrow::UniqueArray arrow; NANOARROW_THROW_NOT_OK(ArrowArrayInitFromSchema(arrow.get(), schema.get(), nullptr)); arrow->length = length; populate_from_col(arrow->children[0], table->get_column(0).view()); - populate_from_col(arrow->children[1], table->get_column(1).view()); + populate_from_col(arrow->children[1], table->get_column(1).view(), mr); populate_dict_from_col( - arrow->children[2], cudf::dictionary_column_view(table->get_column(2).view())); + arrow->children[2], cudf::dictionary_column_view(table->get_column(2).view()), mr); - populate_from_col(arrow->children[3], table->get_column(3).view()); + populate_from_col(arrow->children[3], table->get_column(3).view(), mr); cudf::lists_column_view list_view{table->get_column(4).view()}; populate_list_from_col(arrow->children[4], list_view); populate_from_col(arrow->children[4]->children[0], list_view.child()); cudf::structs_column_view struct_view{table->get_column(5).view()}; populate_from_col(arrow->children[5]->children[0], struct_view.child(0)); - populate_from_col(arrow->children[5]->children[1], struct_view.child(1)); + populate_from_col(arrow->children[5]->children[1], struct_view.child(1), mr); arrow->children[5]->length = struct_view.size(); arrow->children[5]->null_count = struct_view.null_count(); NANOARROW_THROW_NOT_OK( From 171c2bb4c2a902a73fe45d801cdce22c05456a9e Mon Sep 17 00:00:00 2001 From: niranda perera Date: Tue, 18 Aug 2026 16:23:43 -0700 Subject: [PATCH 2/9] adding threading stream arg throgh, adding concepts --- cpp/include/cudf_test/nanoarrow_utils.hpp | 72 ++++++++++------ cpp/tests/interop/from_arrow_host_test.cpp | 10 ++- cpp/tests/interop/from_arrow_stream_test.cpp | 32 +++---- cpp/tests/interop/from_arrow_test.cpp | 30 ++++--- cpp/tests/interop/to_arrow_device_test.cpp | 89 +++++++++++--------- 5 files changed, 131 insertions(+), 102 deletions(-) diff --git a/cpp/include/cudf_test/nanoarrow_utils.hpp b/cpp/include/cudf_test/nanoarrow_utils.hpp index 881dbf44d0ac..fa523006b2ad 100644 --- a/cpp/include/cudf_test/nanoarrow_utils.hpp +++ b/cpp/include/cudf_test/nanoarrow_utils.hpp @@ -13,14 +13,19 @@ #include #include #include +#include #include #include #include #include +#include + #include #include +#include + struct generated_test_data { generated_test_data(cudf::size_type length) : int64_data(length), @@ -82,8 +87,8 @@ static ArrowBufferAllocator noop_alloc = (struct ArrowBufferAllocator){ // populate an ArrowArray with pointers to the raw device buffers of a cudf::column_view // and use the no-op alloc so that the ArrowArray doesn't presume ownership of the data template -std::enable_if_t() and !std::is_same_v, void> populate_from_col( - ArrowArray* arr, cudf::column_view view) +void populate_from_col(ArrowArray* arr, cudf::column_view view) + requires(cudf::is_fixed_width() && !cudf::is_boolean()) { arr->length = view.size(); arr->null_count = view.null_count(); @@ -101,10 +106,11 @@ std::enable_if_t() and !std::is_same_v, void> p // still represent boolean arrays differently, we have to use bools_to_mask // and give the ArrowArray object ownership of the device data. template -std::enable_if_t, void> populate_from_col( - ArrowArray* arr, - cudf::column_view view, - cudf::memory_resources mr = cudf::get_current_device_resource_ref()) +void populate_from_col(ArrowArray* arr, + cudf::column_view view, + cuda::stream_ref stream = cudf::get_default_stream(), + cudf::memory_resources mr = cudf::get_current_device_resource_ref()) + requires(cudf::is_boolean()) { arr->length = view.size(); arr->null_count = view.null_count(); @@ -115,7 +121,7 @@ std::enable_if_t, void> populate_from_col( ArrowArrayValidityBitmap(arr)->buffer.data = const_cast(reinterpret_cast(view.null_mask())); - auto bitmask = cudf::bools_to_mask(view, cudf::get_default_stream(), mr.get_output_mr()); + auto bitmask = cudf::bools_to_mask(view, stream, mr.get_output_mr()); auto ptr = reinterpret_cast(bitmask.first->data()); NANOARROW_THROW_NOT_OK(ArrowBufferSetAllocator( ArrowArrayBuffer(arr, 1), @@ -133,10 +139,11 @@ std::enable_if_t, void> populate_from_col( // using no-op allocator so the ArrowArray knows it doesn't have ownership // of the device buffers. template -std::enable_if_t, void> populate_from_col( - ArrowArray* arr, - cudf::column_view view, - cudf::memory_resources mr = cudf::get_current_device_resource_ref()) +void populate_from_col(ArrowArray* arr, + cudf::column_view view, + cuda::stream_ref stream = cudf::get_default_stream(), + cudf::memory_resources mr = cudf::get_current_device_resource_ref()) + requires(std::same_as) { arr->length = view.size(); arr->null_count = view.null_count(); @@ -153,11 +160,10 @@ std::enable_if_t, void> populate_from_col( ArrowArrayBuffer(arr, 1)->size_bytes = sizeof(int32_t) * sview.offsets().size(); ArrowArrayBuffer(arr, 1)->data = const_cast(sview.offsets().data()); NANOARROW_THROW_NOT_OK(ArrowBufferSetAllocator(ArrowArrayBuffer(arr, 2), noop_alloc)); - ArrowArrayBuffer(arr, 2)->size_bytes = sview.chars_size(cudf::get_default_stream()); + ArrowArrayBuffer(arr, 2)->size_bytes = sview.chars_size(stream); ArrowArrayBuffer(arr, 2)->data = const_cast(view.data()); } else { - auto zero = - cudf::detail::device_scalar(0, cudf::get_default_stream(), mr.get_output_mr()); + auto zero = cudf::detail::device_scalar(0, stream, mr.get_output_mr()); uint8_t const* ptr = reinterpret_cast(zero.data()); nanoarrow::BufferInitWrapped(ArrowArrayBuffer(arr, 1), std::move(zero), ptr, 4); } @@ -166,6 +172,7 @@ std::enable_if_t, void> populate_from_col( template void populate_dict_from_col(ArrowArray* arr, cudf::dictionary_column_view dview, + cuda::stream_ref stream = cudf::get_default_stream(), cudf::memory_resources mr = cudf::get_current_device_resource_ref()) { arr->length = dview.size(); @@ -180,10 +187,9 @@ void populate_dict_from_col(ArrowArray* arr, ArrowArrayBuffer(arr, 1)->size_bytes = sizeof(IND_TYPE) * dview.indices().size(); ArrowArrayBuffer(arr, 1)->data = const_cast(dview.indices().data()); - if constexpr (std::is_same_v or std::is_same_v) { - populate_from_col(arr->dictionary, dview.keys(), mr); + if constexpr (cudf::is_boolean() or std::same_as) { + populate_from_col(arr->dictionary, dview.keys(), stream, mr); } else { - static_cast(mr); populate_from_col(arr->dictionary, dview.keys()); } } @@ -194,11 +200,13 @@ using vector_of_columns = std::vector>; * @brief Create equivalent cuDF and device-backed nanoarrow tables. * * @param length Number of rows to generate + * @param stream CUDA stream used for device memory operations and kernel launches * @param mr Memory resources used for returned device allocations and helper temporaries * @return cuDF table, Arrow schema, and Arrow array */ std::tuple, nanoarrow::UniqueSchema, nanoarrow::UniqueArray> get_nanoarrow_tables(cudf::size_type length = 10000, + cuda::stream_ref stream = cudf::get_default_stream(), cudf::memory_resources mr = cudf::get_current_device_resource_ref()); void populate_list_from_col(ArrowArray* arr, cudf::lists_column_view view); @@ -206,10 +214,12 @@ void populate_list_from_col(ArrowArray* arr, cudf::lists_column_view view); /** * @brief Create the standard cuDF table used by Arrow interop tests. * + * @param stream CUDA stream used for device memory operations and kernel launches * @param mr Memory resources used for returned table allocations and helper temporaries * @return Generated cuDF table */ std::unique_ptr get_cudf_table( + cuda::stream_ref stream = cudf::get_default_stream(), cudf::memory_resources mr = cudf::get_current_device_resource_ref()); template @@ -257,8 +267,9 @@ struct nanoarrow_decimal_type<__int128_t> { }; template -std::enable_if_t() and !std::is_same_v, nanoarrow::UniqueArray> -get_nanoarrow_array(std::vector const& data, std::vector const& mask = {}) +nanoarrow::UniqueArray get_nanoarrow_array(std::vector const& data, + std::vector const& mask = {}) + requires(cudf::is_fixed_width() && !cudf::is_boolean()) { nanoarrow::UniqueArray tmp; NANOARROW_THROW_NOT_OK(ArrowArrayInitFromType(tmp.get(), nanoarrow_storage_type::type)); @@ -287,8 +298,9 @@ get_nanoarrow_array(std::vector const& data, std::vector const& mask } template -std::enable_if_t, nanoarrow::UniqueArray> get_nanoarrow_array( - std::vector const& data, std::vector const& mask = {}) +nanoarrow::UniqueArray get_nanoarrow_array(std::vector const& data, + std::vector const& mask = {}) + requires(cudf::is_boolean()) { nanoarrow::UniqueArray tmp; NANOARROW_THROW_NOT_OK(ArrowArrayInitFromType(tmp.get(), NANOARROW_TYPE_BOOL)); @@ -333,8 +345,9 @@ nanoarrow::UniqueArray get_nanoarrow_array(std::initializer_list elements, } template -std::enable_if_t, nanoarrow::UniqueArray> get_nanoarrow_array( - std::vector const& data, std::vector const& mask = {}) +nanoarrow::UniqueArray get_nanoarrow_array(std::vector const& data, + std::vector const& mask = {}) + requires(std::same_as) { nanoarrow::UniqueArray tmp; NANOARROW_THROW_NOT_OK(ArrowArrayInitFromType(tmp.get(), NANOARROW_TYPE_STRING)); @@ -420,32 +433,33 @@ nanoarrow::UniqueArray get_nanoarrow_list_array(std::initializer_list data, * @brief Create a cuDF table, matching Arrow schema, and source host data. * * @param length Number of rows to generate + * @param stream CUDA stream used for device memory operations and kernel launches * @param mr Memory resources used for returned table allocations and helper temporaries * @return cuDF table, Arrow schema, and generated host data */ std::tuple, nanoarrow::UniqueSchema, generated_test_data> get_nanoarrow_cudf_table(cudf::size_type length, + cuda::stream_ref stream = cudf::get_default_stream(), cudf::memory_resources mr = cudf::get_current_device_resource_ref()); /** * @brief Create equivalent cuDF and host-backed nanoarrow tables. * * @param length Number of rows to generate + * @param stream CUDA stream used for device memory operations and kernel launches * @param mr Memory resources used for returned table allocations and helper temporaries * @return cuDF table, Arrow schema, and Arrow array */ std::tuple, nanoarrow::UniqueSchema, nanoarrow::UniqueArray> get_nanoarrow_host_tables(cudf::size_type length, + cuda::stream_ref stream = cudf::get_default_stream(), cudf::memory_resources mr = cudf::get_current_device_resource_ref()); void slice_host_nanoarrow(ArrowArray* arr, int64_t start, int64_t end); template -std::enable_if_t, - std::is_same, - std::is_same>, - std::size_t> -get_decimal_precision() +std::size_t get_decimal_precision() + requires(std::same_as || std::same_as || std::same_as) { return std::numeric_limits::digits10; } @@ -490,9 +504,11 @@ void makeStreamFromArrays(std::vector arrays, * @brief Create a cuDF table and equivalent nanoarrow stream. * * @param num_copies Number of record batches in the stream + * @param stream CUDA stream used for device memory operations and kernel launches * @param mr Memory resources used for returned table allocations and helper temporaries * @return Concatenated cuDF table, Arrow schema, and Arrow stream */ std::tuple, nanoarrow::UniqueSchema, ArrowArrayStream> get_nanoarrow_stream(int num_copies, + cuda::stream_ref stream = cudf::get_default_stream(), cudf::memory_resources mr = cudf::get_current_device_resource_ref()); diff --git a/cpp/tests/interop/from_arrow_host_test.cpp b/cpp/tests/interop/from_arrow_host_test.cpp index 7715ee4bb096..d0a8458b3002 100644 --- a/cpp/tests/interop/from_arrow_host_test.cpp +++ b/cpp/tests/interop/from_arrow_host_test.cpp @@ -26,16 +26,18 @@ // create a cudf::table and equivalent arrow table with host memory std::tuple, nanoarrow::UniqueSchema, nanoarrow::UniqueArray> -get_nanoarrow_host_tables(cudf::size_type length, cudf::memory_resources mr) +get_nanoarrow_host_tables(cudf::size_type length, + cuda::stream_ref stream, + cudf::memory_resources mr) { - auto [table, schema, test_data] = get_nanoarrow_cudf_table(length, mr); + auto [table, schema, test_data] = get_nanoarrow_cudf_table(length, stream, mr); auto int64_array = get_nanoarrow_array(test_data.int64_data, test_data.validity); auto string_array = get_nanoarrow_array(test_data.string_data, test_data.validity); cudf::dictionary_column_view view(table->get_column(2).view()); - auto keys = cudf::test::to_host(view.keys(), mr).first; - auto indices = cudf::test::to_host(view.indices(), mr).first; + auto keys = cudf::test::to_host(view.keys(), stream, mr).first; + auto indices = cudf::test::to_host(view.indices(), stream, mr).first; auto dict_array = get_nanoarrow_dict_array(std::vector(keys.begin(), keys.end()), std::vector(indices.begin(), indices.end()), test_data.validity); diff --git a/cpp/tests/interop/from_arrow_stream_test.cpp b/cpp/tests/interop/from_arrow_stream_test.cpp index d3335d8ba25c..34cd8206212f 100644 --- a/cpp/tests/interop/from_arrow_stream_test.cpp +++ b/cpp/tests/interop/from_arrow_stream_test.cpp @@ -31,7 +31,7 @@ void makeStreamFromArrays(std::vector arrays, } std::tuple, nanoarrow::UniqueSchema, ArrowArrayStream> -get_nanoarrow_stream(int num_copies, cudf::memory_resources mr) +get_nanoarrow_stream(int num_copies, cuda::stream_ref stream, cudf::memory_resources mr) { auto const temporary_mr = mr.get_temporary_mr(); auto const temporary_resources = cudf::memory_resources{temporary_mr, temporary_mr}; @@ -40,7 +40,7 @@ get_nanoarrow_stream(int num_copies, cudf::memory_resources mr) nanoarrow::UniqueSchema schema; std::vector arrays; for (auto i = 0; i < num_copies; ++i) { - auto [tbl, sch, arr] = get_nanoarrow_host_tables(3, temporary_resources); + auto [tbl, sch, arr] = get_nanoarrow_host_tables(3, stream, temporary_resources); tables.push_back(std::move(tbl)); arrays.push_back(std::move(arr)); if (i == 0) { sch.move(schema.get()); } @@ -49,11 +49,11 @@ get_nanoarrow_stream(int num_copies, cudf::memory_resources mr) for (auto const& table : tables) { table_views.push_back(table->view()); } - auto expected = cudf::concatenate(table_views, cudf::get_default_stream(), mr.get_output_mr()); + auto expected = cudf::concatenate(table_views, stream, mr.get_output_mr()); - ArrowArrayStream stream; - makeStreamFromArrays(std::move(arrays), std::move(schema), &stream); - return std::make_tuple(std::move(expected), std::move(schema), stream); + ArrowArrayStream arrow_stream; + makeStreamFromArrays(std::move(arrays), std::move(schema), &arrow_stream); + return std::make_tuple(std::move(expected), std::move(schema), arrow_stream); } std::tuple, nanoarrow::UniqueSchema, ArrowArrayStream> @@ -97,23 +97,25 @@ TEST_F(FromArrowStreamTest, TestUtilityMemoryResourceControl) auto output_mr = rmm::mr::statistics_resource_adaptor(upstream); auto temporary_mr = rmm::mr::statistics_resource_adaptor(upstream); auto resources = cudf::memory_resources{output_mr, temporary_mr}; + auto stream = cudf::get_default_stream(); { - auto direct_table = get_cudf_table(resources); - auto [generated_table, generated_schema, test_data] = get_nanoarrow_cudf_table(3, resources); - auto [device_table, device_schema, device_array] = get_nanoarrow_tables(0, resources); - auto [host_table, host_schema, host_array] = get_nanoarrow_host_tables(3, resources); - auto [stream_table, stream_schema, stream] = get_nanoarrow_stream(2, resources); - - cudf::get_default_stream().synchronize(); + auto direct_table = get_cudf_table(stream, resources); + auto [generated_table, generated_schema, test_data] = + get_nanoarrow_cudf_table(3, stream, resources); + auto [device_table, device_schema, device_array] = get_nanoarrow_tables(0, stream, resources); + auto [host_table, host_schema, host_array] = get_nanoarrow_host_tables(3, stream, resources); + auto [stream_table, stream_schema, arrow_stream] = get_nanoarrow_stream(2, stream, resources); + + stream.synchronize(); EXPECT_GT(output_mr.get_bytes_counter().value, 0); EXPECT_EQ(temporary_mr.get_bytes_counter().value, 0); EXPECT_GT(temporary_mr.get_bytes_counter().total, 0); - if (stream.release != nullptr) { stream.release(&stream); } + if (arrow_stream.release != nullptr) { arrow_stream.release(&arrow_stream); } } - cudf::get_default_stream().synchronize(); + stream.synchronize(); EXPECT_EQ(output_mr.get_bytes_counter().value, 0); EXPECT_EQ(temporary_mr.get_bytes_counter().value, 0); } diff --git a/cpp/tests/interop/from_arrow_test.cpp b/cpp/tests/interop/from_arrow_test.cpp index 5a4d2e4bf395..1dfba55e524f 100644 --- a/cpp/tests/interop/from_arrow_test.cpp +++ b/cpp/tests/interop/from_arrow_test.cpp @@ -27,28 +27,29 @@ #include -std::unique_ptr get_cudf_table(cudf::memory_resources mr) +std::unique_ptr get_cudf_table(cuda::stream_ref stream, cudf::memory_resources mr) { auto const temporary_mr = mr.get_temporary_mr(); std::vector> columns; columns.emplace_back(cudf::test::fixed_width_column_wrapper( - {1, 2, 5, 2, 7}, {true, false, true, true, true}, mr) + {1, 2, 5, 2, 7}, {true, false, true, true, true}, stream, mr) .release()); columns.emplace_back( - cudf::test::fixed_width_column_wrapper({1, 2, 3, 4, 5}, mr).release()); - columns.emplace_back(cudf::test::strings_column_wrapper( - {"fff", "aaa", "", "fff", "ccc"}, {true, true, true, false, true}, mr) - .release()); - - auto keys = cudf::test::fixed_width_column_wrapper({1, 2, 5, 7}, temporary_mr); - auto indices = - cudf::test::fixed_width_column_wrapper({0, 1, 2, 1, 3}, {1, 0, 1, 1, 1}, temporary_mr); + cudf::test::fixed_width_column_wrapper({1, 2, 3, 4, 5}, stream, mr).release()); columns.emplace_back( - cudf::make_dictionary_column(keys, indices, cudf::get_default_stream(), mr.get_output_mr())); + cudf::test::strings_column_wrapper( + {"fff", "aaa", "", "fff", "ccc"}, {true, true, true, false, true}, stream, mr) + .release()); - columns.emplace_back(cudf::test::fixed_width_column_wrapper( - {true, false, true, false, true}, {true, false, true, true, false}, mr) - .release()); + auto keys = cudf::test::fixed_width_column_wrapper({1, 2, 5, 7}, stream, temporary_mr); + auto indices = cudf::test::fixed_width_column_wrapper( + {0, 1, 2, 1, 3}, {1, 0, 1, 1, 1}, stream, temporary_mr); + columns.emplace_back(cudf::make_dictionary_column(keys, indices, stream, mr.get_output_mr())); + + columns.emplace_back( + cudf::test::fixed_width_column_wrapper( + {true, false, true, false, true}, {true, false, true, true, false}, stream, mr) + .release()); columns.emplace_back(cudf::test::strings_column_wrapper( { "", @@ -58,6 +59,7 @@ std::unique_ptr get_cudf_table(cudf::memory_resources mr) "2", }, {0, 1, 1, 1, 1}, + stream, mr) .release()); // columns.emplace_back(cudf::test::lists_column_wrapper({{1, 2}, {3, 4}, {}, {6}, {7, 8, diff --git a/cpp/tests/interop/to_arrow_device_test.cpp b/cpp/tests/interop/to_arrow_device_test.cpp index a73efaab3ff9..72c4a75d2b58 100644 --- a/cpp/tests/interop/to_arrow_device_test.cpp +++ b/cpp/tests/interop/to_arrow_device_test.cpp @@ -19,68 +19,74 @@ #include std::tuple, nanoarrow::UniqueSchema, generated_test_data> -get_nanoarrow_cudf_table(cudf::size_type length, cudf::memory_resources mr) +get_nanoarrow_cudf_table(cudf::size_type length, cuda::stream_ref stream, cudf::memory_resources mr) { auto const temporary_mr = mr.get_temporary_mr(); generated_test_data test_data(length); std::vector> columns; - columns.emplace_back( - cudf::test::fixed_width_column_wrapper( - test_data.int64_data.begin(), test_data.int64_data.end(), test_data.validity.begin(), mr) - .release()); - columns.emplace_back( - cudf::test::strings_column_wrapper( - test_data.string_data.begin(), test_data.string_data.end(), test_data.validity.begin(), mr) - .release()); + columns.emplace_back(cudf::test::fixed_width_column_wrapper(test_data.int64_data.begin(), + test_data.int64_data.end(), + test_data.validity.begin(), + stream, + mr) + .release()); + columns.emplace_back(cudf::test::strings_column_wrapper(test_data.string_data.begin(), + test_data.string_data.end(), + test_data.validity.begin(), + stream, + mr) + .release()); auto col4 = cudf::test::fixed_width_column_wrapper(test_data.int64_data.begin(), test_data.int64_data.end(), test_data.validity.begin(), + stream, temporary_mr); columns.emplace_back(cudf::dictionary::encode( - col4, cudf::data_type{cudf::type_id::INT32}, cudf::get_default_stream(), mr.get_output_mr())); - columns.emplace_back( - cudf::test::fixed_width_column_wrapper( - test_data.bool_data.begin(), test_data.bool_data.end(), test_data.bool_validity.begin(), mr) - .release()); + col4, cudf::data_type{cudf::type_id::INT32}, stream, mr.get_output_mr())); + columns.emplace_back(cudf::test::fixed_width_column_wrapper(test_data.bool_data.begin(), + test_data.bool_data.end(), + test_data.bool_validity.begin(), + stream, + mr) + .release()); auto list_child_column = cudf::test::fixed_width_column_wrapper(test_data.list_int64_data.begin(), test_data.list_int64_data.end(), test_data.list_int64_data_validity.begin(), + stream, mr); auto list_offsets_column = cudf::test::fixed_width_column_wrapper( - test_data.list_offsets.begin(), test_data.list_offsets.end(), mr); + test_data.list_offsets.begin(), test_data.list_offsets.end(), stream, mr); auto list_validity = cudf::test::fixed_width_column_wrapper( - test_data.list_validity.begin(), test_data.list_validity.end(), temporary_mr); - auto [list_mask, list_nulls] = - cudf::bools_to_mask(list_validity, cudf::get_default_stream(), mr.get_output_mr()); + test_data.list_validity.begin(), test_data.list_validity.end(), stream, temporary_mr); + auto [list_mask, list_nulls] = cudf::bools_to_mask(list_validity, stream, mr.get_output_mr()); columns.emplace_back(cudf::make_lists_column(length, list_offsets_column.release(), list_child_column.release(), list_nulls, std::move(*list_mask))); - auto int_column = - cudf::test::fixed_width_column_wrapper( - test_data.int64_data.begin(), test_data.int64_data.end(), test_data.validity.begin(), mr) - .release(); - auto str_column = - cudf::test::strings_column_wrapper( - test_data.string_data.begin(), test_data.string_data.end(), test_data.validity.begin(), mr) - .release(); + auto int_column = cudf::test::fixed_width_column_wrapper(test_data.int64_data.begin(), + test_data.int64_data.end(), + test_data.validity.begin(), + stream, + mr) + .release(); + auto str_column = cudf::test::strings_column_wrapper(test_data.string_data.begin(), + test_data.string_data.end(), + test_data.validity.begin(), + stream, + mr) + .release(); vector_of_columns cols; cols.push_back(std::move(int_column)); cols.push_back(std::move(str_column)); auto struct_validity = cudf::test::fixed_width_column_wrapper( - test_data.bool_data_validity.begin(), test_data.bool_data_validity.end(), temporary_mr); - auto [null_mask, null_count] = - cudf::bools_to_mask(struct_validity, cudf::get_default_stream(), mr.get_output_mr()); - columns.emplace_back(cudf::make_structs_column(length, - std::move(cols), - null_count, - std::move(*null_mask), - cudf::get_default_stream(), - mr.get_output_mr())); + test_data.bool_data_validity.begin(), test_data.bool_data_validity.end(), stream, temporary_mr); + auto [null_mask, null_count] = cudf::bools_to_mask(struct_validity, stream, mr.get_output_mr()); + columns.emplace_back(cudf::make_structs_column( + length, std::move(cols), null_count, std::move(*null_mask), stream, mr.get_output_mr())); nanoarrow::UniqueSchema schema; ArrowSchemaInit(schema.get()); @@ -170,27 +176,28 @@ get_nanoarrow_cudf_table(cudf::size_type length, cudf::memory_resources mr) } std::tuple, nanoarrow::UniqueSchema, nanoarrow::UniqueArray> -get_nanoarrow_tables(cudf::size_type length, cudf::memory_resources mr) +get_nanoarrow_tables(cudf::size_type length, cuda::stream_ref stream, cudf::memory_resources mr) { - auto [table, schema, test_data] = get_nanoarrow_cudf_table(length, mr); + auto [table, schema, test_data] = get_nanoarrow_cudf_table(length, stream, mr); nanoarrow::UniqueArray arrow; NANOARROW_THROW_NOT_OK(ArrowArrayInitFromSchema(arrow.get(), schema.get(), nullptr)); arrow->length = length; populate_from_col(arrow->children[0], table->get_column(0).view()); - populate_from_col(arrow->children[1], table->get_column(1).view(), mr); + populate_from_col(arrow->children[1], table->get_column(1).view(), stream, mr); populate_dict_from_col( - arrow->children[2], cudf::dictionary_column_view(table->get_column(2).view()), mr); + arrow->children[2], cudf::dictionary_column_view(table->get_column(2).view()), stream, mr); - populate_from_col(arrow->children[3], table->get_column(3).view(), mr); + populate_from_col(arrow->children[3], table->get_column(3).view(), stream, mr); cudf::lists_column_view list_view{table->get_column(4).view()}; populate_list_from_col(arrow->children[4], list_view); populate_from_col(arrow->children[4]->children[0], list_view.child()); cudf::structs_column_view struct_view{table->get_column(5).view()}; populate_from_col(arrow->children[5]->children[0], struct_view.child(0)); - populate_from_col(arrow->children[5]->children[1], struct_view.child(1), mr); + populate_from_col( + arrow->children[5]->children[1], struct_view.child(1), stream, mr); arrow->children[5]->length = struct_view.size(); arrow->children[5]->null_count = struct_view.null_count(); NANOARROW_THROW_NOT_OK( From 039c87ca56bf36c6a095789d8ac88e7e861365ce Mon Sep 17 00:00:00 2001 From: niranda perera Date: Tue, 25 Aug 2026 17:03:46 -0700 Subject: [PATCH 3/9] precommit Signed-off-by: niranda perera --- cpp/tests/interop/from_arrow_stream_test.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cpp/tests/interop/from_arrow_stream_test.cpp b/cpp/tests/interop/from_arrow_stream_test.cpp index 79696ed34e96..3f8d056abc50 100644 --- a/cpp/tests/interop/from_arrow_stream_test.cpp +++ b/cpp/tests/interop/from_arrow_stream_test.cpp @@ -1,5 +1,4 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -18,11 +17,11 @@ #include #include +#include + #include #include -#include - struct FromArrowStreamTest : public cudf::test::BaseFixture {}; void makeStreamFromArrays(std::vector arrays, From 6221c7b298e7c1aac26c7df0b81f3c8f0925778c Mon Sep 17 00:00:00 2001 From: niranda perera Date: Tue, 25 Aug 2026 18:07:54 -0700 Subject: [PATCH 4/9] fix get_cudf_table Signed-off-by: niranda perera --- cpp/tests/interop/from_arrow_stream_test.cpp | 42 ++++++++++++++++++++ cpp/tests/interop/from_arrow_test.cpp | 37 +++++++---------- 2 files changed, 57 insertions(+), 22 deletions(-) diff --git a/cpp/tests/interop/from_arrow_stream_test.cpp b/cpp/tests/interop/from_arrow_stream_test.cpp index 3f8d056abc50..a127c57185c2 100644 --- a/cpp/tests/interop/from_arrow_stream_test.cpp +++ b/cpp/tests/interop/from_arrow_stream_test.cpp @@ -11,6 +11,7 @@ #include #include +#include #include #include #include @@ -24,6 +25,47 @@ struct FromArrowStreamTest : public cudf::test::BaseFixture {}; +// Defined here rather than in from_arrow_test.cpp, which historically held this helper. That file +// still depends on Arrow C++ and was dropped from the INTEROP_TEST target when the Arrow C++ +// dependency was removed from the C++ tests, so a definition placed there never reaches the link. +std::unique_ptr get_cudf_table(cuda::stream_ref stream, cudf::memory_resources mr) +{ + auto const temporary_mr = mr.get_temporary_mr(); + std::vector> columns; + columns.emplace_back(cudf::test::fixed_width_column_wrapper( + {1, 2, 5, 2, 7}, {true, false, true, true, true}, stream, mr) + .release()); + columns.emplace_back( + cudf::test::fixed_width_column_wrapper({1, 2, 3, 4, 5}, stream, mr).release()); + columns.emplace_back( + cudf::test::strings_column_wrapper( + {"fff", "aaa", "", "fff", "ccc"}, {true, true, true, false, true}, stream, mr) + .release()); + + auto keys = cudf::test::fixed_width_column_wrapper({1, 2, 5, 7}, stream, temporary_mr); + auto indices = cudf::test::fixed_width_column_wrapper( + {0, 1, 2, 1, 3}, {1, 0, 1, 1, 1}, stream, temporary_mr); + columns.emplace_back(cudf::make_dictionary_column(keys, indices, stream, mr.get_output_mr())); + + columns.emplace_back( + cudf::test::fixed_width_column_wrapper( + {true, false, true, false, true}, {true, false, true, true, false}, stream, mr) + .release()); + columns.emplace_back(cudf::test::strings_column_wrapper( + { + "", + "abc", + "def", + "1", + "2", + }, + {0, 1, 1, 1, 1}, + stream, + mr) + .release()); + return std::make_unique(std::move(columns)); +} + void makeStreamFromArrays(std::vector arrays, nanoarrow::UniqueSchema schema, ArrowArrayStream* out) diff --git a/cpp/tests/interop/from_arrow_test.cpp b/cpp/tests/interop/from_arrow_test.cpp index 1dfba55e524f..3ee6c378f558 100644 --- a/cpp/tests/interop/from_arrow_test.cpp +++ b/cpp/tests/interop/from_arrow_test.cpp @@ -27,29 +27,24 @@ #include -std::unique_ptr get_cudf_table(cuda::stream_ref stream, cudf::memory_resources mr) +std::unique_ptr get_cudf_table() { - auto const temporary_mr = mr.get_temporary_mr(); std::vector> columns; columns.emplace_back(cudf::test::fixed_width_column_wrapper( - {1, 2, 5, 2, 7}, {true, false, true, true, true}, stream, mr) + {1, 2, 5, 2, 7}, {true, false, true, true, true}) + .release()); + columns.emplace_back(cudf::test::fixed_width_column_wrapper({1, 2, 3, 4, 5}).release()); + columns.emplace_back(cudf::test::strings_column_wrapper({"fff", "aaa", "", "fff", "ccc"}, + {true, true, true, false, true}) + .release()); + + auto keys = cudf::test::fixed_width_column_wrapper({1, 2, 5, 7}); + auto indices = cudf::test::fixed_width_column_wrapper({0, 1, 2, 1, 3}, {1, 0, 1, 1, 1}); + columns.emplace_back(cudf::make_dictionary_column(keys, indices)); + + columns.emplace_back(cudf::test::fixed_width_column_wrapper( + {true, false, true, false, true}, {true, false, true, true, false}) .release()); - columns.emplace_back( - cudf::test::fixed_width_column_wrapper({1, 2, 3, 4, 5}, stream, mr).release()); - columns.emplace_back( - cudf::test::strings_column_wrapper( - {"fff", "aaa", "", "fff", "ccc"}, {true, true, true, false, true}, stream, mr) - .release()); - - auto keys = cudf::test::fixed_width_column_wrapper({1, 2, 5, 7}, stream, temporary_mr); - auto indices = cudf::test::fixed_width_column_wrapper( - {0, 1, 2, 1, 3}, {1, 0, 1, 1, 1}, stream, temporary_mr); - columns.emplace_back(cudf::make_dictionary_column(keys, indices, stream, mr.get_output_mr())); - - columns.emplace_back( - cudf::test::fixed_width_column_wrapper( - {true, false, true, false, true}, {true, false, true, true, false}, stream, mr) - .release()); columns.emplace_back(cudf::test::strings_column_wrapper( { "", @@ -58,9 +53,7 @@ std::unique_ptr get_cudf_table(cuda::stream_ref stream, cudf::memor "1", "2", }, - {0, 1, 1, 1, 1}, - stream, - mr) + {0, 1, 1, 1, 1}) .release()); // columns.emplace_back(cudf::test::lists_column_wrapper({{1, 2}, {3, 4}, {}, {6}, {7, 8, // 9}}).release()); From e7ec74681da85b3a9c49be91c07c71bc98960157 Mon Sep 17 00:00:00 2001 From: niranda perera Date: Wed, 26 Aug 2026 09:01:51 -0700 Subject: [PATCH 5/9] move impls to a cpp Signed-off-by: niranda perera --- cpp/tests/CMakeLists.txt | 1 + cpp/tests/interop/from_arrow_host_test.cpp | 84 ---- cpp/tests/interop/from_arrow_stream_test.cpp | 81 ---- cpp/tests/interop/from_arrow_test.cpp | 33 -- cpp/tests/interop/nanoarrow_utils.cpp | 392 +++++++++++++++++++ cpp/tests/interop/to_arrow_device_test.cpp | 217 ---------- 6 files changed, 393 insertions(+), 415 deletions(-) create mode 100644 cpp/tests/interop/nanoarrow_utils.cpp diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index d268ed76d3b2..38827d502f9b 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -302,6 +302,7 @@ ConfigureTest(FILTER_TEST filter/filter_test.cpp) ConfigureTest( INTEROP_TEST interop/arrow_data_structures_test.cpp + interop/nanoarrow_utils.cpp interop/to_arrow_device_test.cpp interop/to_arrow_host_test.cpp interop/from_arrow_device_test.cpp diff --git a/cpp/tests/interop/from_arrow_host_test.cpp b/cpp/tests/interop/from_arrow_host_test.cpp index 70663a932d6f..a1be56b8d81a 100644 --- a/cpp/tests/interop/from_arrow_host_test.cpp +++ b/cpp/tests/interop/from_arrow_host_test.cpp @@ -115,64 +115,6 @@ struct direct_arrow_c_producer { } // namespace -// create a cudf::table and equivalent arrow table with host memory -std::tuple, nanoarrow::UniqueSchema, nanoarrow::UniqueArray> -get_nanoarrow_host_tables(cudf::size_type length, - cuda::stream_ref stream, - cudf::memory_resources mr) -{ - auto [table, schema, test_data] = get_nanoarrow_cudf_table(length, stream, mr); - - auto int64_array = get_nanoarrow_array(test_data.int64_data, test_data.validity); - auto string_array = - get_nanoarrow_array(test_data.string_data, test_data.validity); - cudf::dictionary_column_view view(table->get_column(2).view()); - auto keys = cudf::test::to_host(view.keys(), stream, mr).first; - auto indices = cudf::test::to_host(view.indices(), stream, mr).first; - auto dict_array = get_nanoarrow_dict_array(std::vector(keys.begin(), keys.end()), - std::vector(indices.begin(), indices.end()), - test_data.validity); - auto boolarray = get_nanoarrow_array(test_data.bool_data, test_data.bool_validity); - auto list_array = get_nanoarrow_list_array(test_data.list_int64_data, - test_data.list_offsets, - test_data.list_int64_data_validity, - test_data.list_validity); - - nanoarrow::UniqueArray arrow; - NANOARROW_THROW_NOT_OK(ArrowArrayInitFromSchema(arrow.get(), schema.get(), nullptr)); - arrow->length = length; - - int64_array.move(arrow->children[0]); - string_array.move(arrow->children[1]); - dict_array.move(arrow->children[2]); - boolarray.move(arrow->children[3]); - list_array.move(arrow->children[4]); - - int64_array = get_nanoarrow_array(test_data.int64_data, test_data.validity); - string_array = get_nanoarrow_array(test_data.string_data, test_data.validity); - int64_array.move(arrow->children[5]->children[0]); - string_array.move(arrow->children[5]->children[1]); - - ArrowBitmap struct_validity; - ArrowBitmapInit(&struct_validity); - NANOARROW_THROW_NOT_OK(ArrowBitmapReserve(&struct_validity, length)); - ArrowBitmapAppendInt8Unsafe( - &struct_validity, reinterpret_cast(test_data.bool_data_validity.data()), length); - arrow->children[5]->length = length; - ArrowArraySetValidityBitmap(arrow->children[5], &struct_validity); - arrow->children[5]->null_count = - length - ArrowBitCountSet(ArrowArrayValidityBitmap(arrow->children[5])->buffer.data, 0, length); - - ArrowError error; - if (ArrowArrayFinishBuilding(arrow.get(), NANOARROW_VALIDATION_LEVEL_MINIMAL, &error) != - NANOARROW_OK) { - std::cerr << ArrowErrorMessage(&error) << std::endl; - CUDF_FAIL("failed to build example arrays"); - } - - return std::make_tuple(std::move(table), std::move(schema), std::move(arrow)); -} - struct FromArrowHostDeviceTest : public cudf::test::BaseFixture {}; template @@ -1084,32 +1026,6 @@ TEST_F(FromArrowHostDeviceTest, DictionaryIndicesType) CUDF_TEST_EXPECT_TABLES_EQUAL(got_cudf_table->view(), from_struct); } -void slice_host_nanoarrow(ArrowArray* arr, int64_t start, int64_t end) -{ - auto op = [&](ArrowArray* array) { - // slicing only needs to happen at the top level of an array - array->offset = start; - array->length = end - start; - if (array->null_count != 0) { - array->null_count = - array->length - - ArrowBitCountSet(ArrowArrayValidityBitmap(array)->buffer.data, start, end - start); - } - }; - - if (arr->n_children == 0) { - op(arr); - return; - } - - // since we want to simulate a sliced table where the children are sliced, - // we slice each individual child of the record batch - arr->length = end - start; - for (int64_t i = 0; i < arr->n_children; ++i) { - op(arr->children[i]); - } -} - TEST_F(FromArrowHostDeviceTest, StringViewType) { auto data = std::vector({"hello", diff --git a/cpp/tests/interop/from_arrow_stream_test.cpp b/cpp/tests/interop/from_arrow_stream_test.cpp index a127c57185c2..08cd25cfd05c 100644 --- a/cpp/tests/interop/from_arrow_stream_test.cpp +++ b/cpp/tests/interop/from_arrow_stream_test.cpp @@ -4,14 +4,12 @@ */ #include -#include #include #include #include #include #include -#include #include #include #include @@ -25,85 +23,6 @@ struct FromArrowStreamTest : public cudf::test::BaseFixture {}; -// Defined here rather than in from_arrow_test.cpp, which historically held this helper. That file -// still depends on Arrow C++ and was dropped from the INTEROP_TEST target when the Arrow C++ -// dependency was removed from the C++ tests, so a definition placed there never reaches the link. -std::unique_ptr get_cudf_table(cuda::stream_ref stream, cudf::memory_resources mr) -{ - auto const temporary_mr = mr.get_temporary_mr(); - std::vector> columns; - columns.emplace_back(cudf::test::fixed_width_column_wrapper( - {1, 2, 5, 2, 7}, {true, false, true, true, true}, stream, mr) - .release()); - columns.emplace_back( - cudf::test::fixed_width_column_wrapper({1, 2, 3, 4, 5}, stream, mr).release()); - columns.emplace_back( - cudf::test::strings_column_wrapper( - {"fff", "aaa", "", "fff", "ccc"}, {true, true, true, false, true}, stream, mr) - .release()); - - auto keys = cudf::test::fixed_width_column_wrapper({1, 2, 5, 7}, stream, temporary_mr); - auto indices = cudf::test::fixed_width_column_wrapper( - {0, 1, 2, 1, 3}, {1, 0, 1, 1, 1}, stream, temporary_mr); - columns.emplace_back(cudf::make_dictionary_column(keys, indices, stream, mr.get_output_mr())); - - columns.emplace_back( - cudf::test::fixed_width_column_wrapper( - {true, false, true, false, true}, {true, false, true, true, false}, stream, mr) - .release()); - columns.emplace_back(cudf::test::strings_column_wrapper( - { - "", - "abc", - "def", - "1", - "2", - }, - {0, 1, 1, 1, 1}, - stream, - mr) - .release()); - return std::make_unique(std::move(columns)); -} - -void makeStreamFromArrays(std::vector arrays, - nanoarrow::UniqueSchema schema, - ArrowArrayStream* out) -{ - auto* private_data = new VectorOfArrays{std::move(arrays), std::move(schema)}; - out->get_schema = VectorOfArrays::get_schema; - out->get_next = VectorOfArrays::get_next; - out->get_last_error = VectorOfArrays::get_last_error; - out->release = VectorOfArrays::release; - out->private_data = private_data; -} - -std::tuple, nanoarrow::UniqueSchema, ArrowArrayStream> -get_nanoarrow_stream(int num_copies, cuda::stream_ref stream, cudf::memory_resources mr) -{ - auto const temporary_mr = mr.get_temporary_mr(); - auto const temporary_resources = cudf::memory_resources{temporary_mr, temporary_mr}; - std::vector> tables; - // The schema is unique across all tables. - nanoarrow::UniqueSchema schema; - std::vector arrays; - for (auto i = 0; i < num_copies; ++i) { - auto [tbl, sch, arr] = get_nanoarrow_host_tables(3, stream, temporary_resources); - tables.push_back(std::move(tbl)); - arrays.push_back(std::move(arr)); - if (i == 0) { sch.move(schema.get()); } - } - std::vector table_views; - for (auto const& table : tables) { - table_views.push_back(table->view()); - } - auto expected = cudf::concatenate(table_views, stream, mr.get_output_mr()); - - ArrowArrayStream arrow_stream; - makeStreamFromArrays(std::move(arrays), std::move(schema), &arrow_stream); - return std::make_tuple(std::move(expected), std::move(schema), arrow_stream); -} - std::tuple, nanoarrow::UniqueSchema, ArrowArrayStream> get_nanoarrow_chunked_stream(int num_copies, cudf::size_type length) { diff --git a/cpp/tests/interop/from_arrow_test.cpp b/cpp/tests/interop/from_arrow_test.cpp index 3ee6c378f558..e342167169f4 100644 --- a/cpp/tests/interop/from_arrow_test.cpp +++ b/cpp/tests/interop/from_arrow_test.cpp @@ -27,39 +27,6 @@ #include -std::unique_ptr get_cudf_table() -{ - std::vector> columns; - columns.emplace_back(cudf::test::fixed_width_column_wrapper( - {1, 2, 5, 2, 7}, {true, false, true, true, true}) - .release()); - columns.emplace_back(cudf::test::fixed_width_column_wrapper({1, 2, 3, 4, 5}).release()); - columns.emplace_back(cudf::test::strings_column_wrapper({"fff", "aaa", "", "fff", "ccc"}, - {true, true, true, false, true}) - .release()); - - auto keys = cudf::test::fixed_width_column_wrapper({1, 2, 5, 7}); - auto indices = cudf::test::fixed_width_column_wrapper({0, 1, 2, 1, 3}, {1, 0, 1, 1, 1}); - columns.emplace_back(cudf::make_dictionary_column(keys, indices)); - - columns.emplace_back(cudf::test::fixed_width_column_wrapper( - {true, false, true, false, true}, {true, false, true, true, false}) - .release()); - columns.emplace_back(cudf::test::strings_column_wrapper( - { - "", - "abc", - "def", - "1", - "2", - }, - {0, 1, 1, 1, 1}) - .release()); - // columns.emplace_back(cudf::test::lists_column_wrapper({{1, 2}, {3, 4}, {}, {6}, {7, 8, - // 9}}).release()); - return std::make_unique(std::move(columns)); -} - std::shared_ptr get_arrow_large_string_array( std::vector const& data, std::vector const& mask = {}) { diff --git a/cpp/tests/interop/nanoarrow_utils.cpp b/cpp/tests/interop/nanoarrow_utils.cpp new file mode 100644 index 000000000000..71b4de2f3ff5 --- /dev/null +++ b/cpp/tests/interop/nanoarrow_utils.cpp @@ -0,0 +1,392 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +std::unique_ptr get_cudf_table(cuda::stream_ref stream, cudf::memory_resources mr) +{ + auto const temporary_mr = mr.get_temporary_mr(); + std::vector> columns; + columns.emplace_back(cudf::test::fixed_width_column_wrapper( + {1, 2, 5, 2, 7}, {true, false, true, true, true}, stream, mr) + .release()); + columns.emplace_back( + cudf::test::fixed_width_column_wrapper({1, 2, 3, 4, 5}, stream, mr).release()); + columns.emplace_back( + cudf::test::strings_column_wrapper( + {"fff", "aaa", "", "fff", "ccc"}, {true, true, true, false, true}, stream, mr) + .release()); + + auto keys = cudf::test::fixed_width_column_wrapper({1, 2, 5, 7}, stream, temporary_mr); + auto indices = cudf::test::fixed_width_column_wrapper( + {0, 1, 2, 1, 3}, {1, 0, 1, 1, 1}, stream, temporary_mr); + columns.emplace_back(cudf::make_dictionary_column(keys, indices, stream, mr.get_output_mr())); + + columns.emplace_back( + cudf::test::fixed_width_column_wrapper( + {true, false, true, false, true}, {true, false, true, true, false}, stream, mr) + .release()); + columns.emplace_back(cudf::test::strings_column_wrapper( + { + "", + "abc", + "def", + "1", + "2", + }, + {0, 1, 1, 1, 1}, + stream, + mr) + .release()); + return std::make_unique(std::move(columns)); +} + +void populate_list_from_col(ArrowArray* arr, cudf::lists_column_view view) +{ + arr->length = view.size(); + arr->null_count = view.null_count(); + + NANOARROW_THROW_NOT_OK(ArrowBufferSetAllocator(ArrowArrayBuffer(arr, 0), noop_alloc)); + ArrowArrayValidityBitmap(arr)->buffer.size_bytes = + cudf::bitmask_allocation_size_bytes(view.size()); + ArrowArrayValidityBitmap(arr)->buffer.data = + const_cast(reinterpret_cast(view.null_mask())); + + NANOARROW_THROW_NOT_OK(ArrowBufferSetAllocator(ArrowArrayBuffer(arr, 1), noop_alloc)); + ArrowArrayBuffer(arr, 1)->size_bytes = sizeof(int32_t) * view.offsets().size(); + ArrowArrayBuffer(arr, 1)->data = const_cast(view.offsets().data()); +} + +std::tuple, nanoarrow::UniqueSchema, generated_test_data> +get_nanoarrow_cudf_table(cudf::size_type length, cuda::stream_ref stream, cudf::memory_resources mr) +{ + auto const temporary_mr = mr.get_temporary_mr(); + generated_test_data test_data(length); + + std::vector> columns; + + columns.emplace_back(cudf::test::fixed_width_column_wrapper(test_data.int64_data.begin(), + test_data.int64_data.end(), + test_data.validity.begin(), + stream, + mr) + .release()); + columns.emplace_back(cudf::test::strings_column_wrapper(test_data.string_data.begin(), + test_data.string_data.end(), + test_data.validity.begin(), + stream, + mr) + .release()); + auto col4 = cudf::test::fixed_width_column_wrapper(test_data.int64_data.begin(), + test_data.int64_data.end(), + test_data.validity.begin(), + stream, + temporary_mr); + columns.emplace_back(cudf::dictionary::encode( + col4, cudf::data_type{cudf::type_id::INT32}, stream, mr.get_output_mr())); + columns.emplace_back(cudf::test::fixed_width_column_wrapper(test_data.bool_data.begin(), + test_data.bool_data.end(), + test_data.bool_validity.begin(), + stream, + mr) + .release()); + auto list_child_column = + cudf::test::fixed_width_column_wrapper(test_data.list_int64_data.begin(), + test_data.list_int64_data.end(), + test_data.list_int64_data_validity.begin(), + stream, + mr); + auto list_offsets_column = cudf::test::fixed_width_column_wrapper( + test_data.list_offsets.begin(), test_data.list_offsets.end(), stream, mr); + auto list_validity = cudf::test::fixed_width_column_wrapper( + test_data.list_validity.begin(), test_data.list_validity.end(), stream, temporary_mr); + auto [list_mask, list_nulls] = cudf::bools_to_mask(list_validity, stream, mr.get_output_mr()); + columns.emplace_back(cudf::make_lists_column(length, + list_offsets_column.release(), + list_child_column.release(), + list_nulls, + std::move(*list_mask))); + auto int_column = cudf::test::fixed_width_column_wrapper(test_data.int64_data.begin(), + test_data.int64_data.end(), + test_data.validity.begin(), + stream, + mr) + .release(); + auto str_column = cudf::test::strings_column_wrapper(test_data.string_data.begin(), + test_data.string_data.end(), + test_data.validity.begin(), + stream, + mr) + .release(); + vector_of_columns cols; + cols.push_back(std::move(int_column)); + cols.push_back(std::move(str_column)); + auto struct_validity = cudf::test::fixed_width_column_wrapper( + test_data.bool_data_validity.begin(), test_data.bool_data_validity.end(), stream, temporary_mr); + auto [null_mask, null_count] = cudf::bools_to_mask(struct_validity, stream, mr.get_output_mr()); + columns.emplace_back(cudf::make_structs_column( + length, std::move(cols), null_count, std::move(*null_mask), stream, mr.get_output_mr())); + + nanoarrow::UniqueSchema schema; + ArrowSchemaInit(schema.get()); + NANOARROW_THROW_NOT_OK(ArrowSchemaSetTypeStruct(schema.get(), 6)); + + NANOARROW_THROW_NOT_OK(ArrowSchemaInitFromType(schema->children[0], NANOARROW_TYPE_INT64)); + NANOARROW_THROW_NOT_OK(ArrowSchemaSetName(schema->children[0], "a")); + if (columns[0]->null_count() > 0) { + schema->children[0]->flags |= ARROW_FLAG_NULLABLE; + } else { + schema->children[0]->flags = 0; + } + + NANOARROW_THROW_NOT_OK(ArrowSchemaInitFromType(schema->children[1], NANOARROW_TYPE_STRING)); + NANOARROW_THROW_NOT_OK(ArrowSchemaSetName(schema->children[1], "b")); + if (columns[1]->null_count() > 0) { + schema->children[1]->flags |= ARROW_FLAG_NULLABLE; + } else { + schema->children[1]->flags = 0; + } + + NANOARROW_THROW_NOT_OK(ArrowSchemaInitFromType(schema->children[2], NANOARROW_TYPE_INT32)); + NANOARROW_THROW_NOT_OK(ArrowSchemaAllocateDictionary(schema->children[2])); + NANOARROW_THROW_NOT_OK( + ArrowSchemaInitFromType(schema->children[2]->dictionary, NANOARROW_TYPE_INT64)); + NANOARROW_THROW_NOT_OK(ArrowSchemaSetName(schema->children[2], "c")); + if (columns[2]->null_count() > 0) { + schema->children[2]->flags |= ARROW_FLAG_NULLABLE; + } else { + schema->children[2]->flags = 0; + } + + NANOARROW_THROW_NOT_OK(ArrowSchemaInitFromType(schema->children[3], NANOARROW_TYPE_BOOL)); + NANOARROW_THROW_NOT_OK(ArrowSchemaSetName(schema->children[3], "d")); + if (columns[3]->null_count() > 0) { + schema->children[3]->flags |= ARROW_FLAG_NULLABLE; + } else { + schema->children[3]->flags = 0; + } + + NANOARROW_THROW_NOT_OK(ArrowSchemaInitFromType(schema->children[4], NANOARROW_TYPE_LIST)); + NANOARROW_THROW_NOT_OK( + ArrowSchemaInitFromType(schema->children[4]->children[0], NANOARROW_TYPE_INT64)); + NANOARROW_THROW_NOT_OK(ArrowSchemaSetName(schema->children[4]->children[0], "element")); + if (columns[4]->child(1).null_count() > 0) { + schema->children[4]->children[0]->flags |= ARROW_FLAG_NULLABLE; + } else { + schema->children[4]->children[0]->flags = 0; + } + + NANOARROW_THROW_NOT_OK(ArrowSchemaSetName(schema->children[4], "e")); + if (columns[4]->has_nulls()) { + schema->children[4]->flags |= ARROW_FLAG_NULLABLE; + } else { + schema->children[4]->flags = 0; + } + + ArrowSchemaInit(schema->children[5]); + NANOARROW_THROW_NOT_OK(ArrowSchemaSetTypeStruct(schema->children[5], 2)); + NANOARROW_THROW_NOT_OK( + ArrowSchemaInitFromType(schema->children[5]->children[0], NANOARROW_TYPE_INT64)); + NANOARROW_THROW_NOT_OK(ArrowSchemaSetName(schema->children[5]->children[0], "integral")); + if (columns[5]->child(0).has_nulls()) { + schema->children[5]->children[0]->flags |= ARROW_FLAG_NULLABLE; + } else { + schema->children[5]->children[0]->flags = 0; + } + + NANOARROW_THROW_NOT_OK( + ArrowSchemaInitFromType(schema->children[5]->children[1], NANOARROW_TYPE_STRING)); + NANOARROW_THROW_NOT_OK(ArrowSchemaSetName(schema->children[5]->children[1], "string")); + if (columns[5]->child(1).has_nulls()) { + schema->children[5]->children[1]->flags |= ARROW_FLAG_NULLABLE; + } else { + schema->children[5]->children[1]->flags = 0; + } + + NANOARROW_THROW_NOT_OK(ArrowSchemaSetName(schema->children[5], "f")); + if (columns[5]->has_nulls()) { + schema->children[5]->flags |= ARROW_FLAG_NULLABLE; + } else { + schema->children[5]->flags = 0; + } + + return std::make_tuple( + std::make_unique(std::move(columns)), std::move(schema), std::move(test_data)); +} + +std::tuple, nanoarrow::UniqueSchema, nanoarrow::UniqueArray> +get_nanoarrow_tables(cudf::size_type length, cuda::stream_ref stream, cudf::memory_resources mr) +{ + auto [table, schema, test_data] = get_nanoarrow_cudf_table(length, stream, mr); + + nanoarrow::UniqueArray arrow; + NANOARROW_THROW_NOT_OK(ArrowArrayInitFromSchema(arrow.get(), schema.get(), nullptr)); + arrow->length = length; + + populate_from_col(arrow->children[0], table->get_column(0).view()); + populate_from_col(arrow->children[1], table->get_column(1).view(), stream, mr); + populate_dict_from_col( + arrow->children[2], cudf::dictionary_column_view(table->get_column(2).view()), stream, mr); + + populate_from_col(arrow->children[3], table->get_column(3).view(), stream, mr); + cudf::lists_column_view list_view{table->get_column(4).view()}; + populate_list_from_col(arrow->children[4], list_view); + populate_from_col(arrow->children[4]->children[0], list_view.child()); + + cudf::structs_column_view struct_view{table->get_column(5).view()}; + populate_from_col(arrow->children[5]->children[0], struct_view.child(0)); + populate_from_col( + arrow->children[5]->children[1], struct_view.child(1), stream, mr); + arrow->children[5]->length = struct_view.size(); + arrow->children[5]->null_count = struct_view.null_count(); + NANOARROW_THROW_NOT_OK( + ArrowBufferSetAllocator(ArrowArrayBuffer(arrow->children[5], 0), noop_alloc)); + ArrowArrayValidityBitmap(arrow->children[5])->buffer.size_bytes = + cudf::bitmask_allocation_size_bytes(struct_view.size()); + ArrowArrayValidityBitmap(arrow->children[5])->buffer.data = + const_cast(reinterpret_cast(struct_view.null_mask())); + + ArrowError error; + if (ArrowArrayFinishBuilding(arrow.get(), NANOARROW_VALIDATION_LEVEL_MINIMAL, &error) != + NANOARROW_OK) { + std::cerr << ArrowErrorMessage(&error) << std::endl; + CUDF_FAIL("failed to build example arrays"); + } + + return std::make_tuple(std::move(table), std::move(schema), std::move(arrow)); +} + +std::tuple, nanoarrow::UniqueSchema, nanoarrow::UniqueArray> +get_nanoarrow_host_tables(cudf::size_type length, + cuda::stream_ref stream, + cudf::memory_resources mr) +{ + auto [table, schema, test_data] = get_nanoarrow_cudf_table(length, stream, mr); + + auto int64_array = get_nanoarrow_array(test_data.int64_data, test_data.validity); + auto string_array = + get_nanoarrow_array(test_data.string_data, test_data.validity); + cudf::dictionary_column_view view(table->get_column(2).view()); + auto keys = cudf::test::to_host(view.keys(), stream, mr).first; + auto indices = cudf::test::to_host(view.indices(), stream, mr).first; + auto dict_array = get_nanoarrow_dict_array(std::vector(keys.begin(), keys.end()), + std::vector(indices.begin(), indices.end()), + test_data.validity); + auto boolarray = get_nanoarrow_array(test_data.bool_data, test_data.bool_validity); + auto list_array = get_nanoarrow_list_array(test_data.list_int64_data, + test_data.list_offsets, + test_data.list_int64_data_validity, + test_data.list_validity); + + nanoarrow::UniqueArray arrow; + NANOARROW_THROW_NOT_OK(ArrowArrayInitFromSchema(arrow.get(), schema.get(), nullptr)); + arrow->length = length; + + int64_array.move(arrow->children[0]); + string_array.move(arrow->children[1]); + dict_array.move(arrow->children[2]); + boolarray.move(arrow->children[3]); + list_array.move(arrow->children[4]); + + int64_array = get_nanoarrow_array(test_data.int64_data, test_data.validity); + string_array = get_nanoarrow_array(test_data.string_data, test_data.validity); + int64_array.move(arrow->children[5]->children[0]); + string_array.move(arrow->children[5]->children[1]); + + ArrowBitmap struct_validity; + ArrowBitmapInit(&struct_validity); + NANOARROW_THROW_NOT_OK(ArrowBitmapReserve(&struct_validity, length)); + ArrowBitmapAppendInt8Unsafe( + &struct_validity, reinterpret_cast(test_data.bool_data_validity.data()), length); + arrow->children[5]->length = length; + ArrowArraySetValidityBitmap(arrow->children[5], &struct_validity); + arrow->children[5]->null_count = + length - ArrowBitCountSet(ArrowArrayValidityBitmap(arrow->children[5])->buffer.data, 0, length); + + ArrowError error; + if (ArrowArrayFinishBuilding(arrow.get(), NANOARROW_VALIDATION_LEVEL_MINIMAL, &error) != + NANOARROW_OK) { + std::cerr << ArrowErrorMessage(&error) << std::endl; + CUDF_FAIL("failed to build example arrays"); + } + + return std::make_tuple(std::move(table), std::move(schema), std::move(arrow)); +} + +void slice_host_nanoarrow(ArrowArray* arr, int64_t start, int64_t end) +{ + auto op = [&](ArrowArray* array) { + // slicing only needs to happen at the top level of an array + array->offset = start; + array->length = end - start; + if (array->null_count != 0) { + array->null_count = + array->length - + ArrowBitCountSet(ArrowArrayValidityBitmap(array)->buffer.data, start, end - start); + } + }; + + if (arr->n_children == 0) { + op(arr); + return; + } + + // since we want to simulate a sliced table where the children are sliced, + // we slice each individual child of the record batch + arr->length = end - start; + for (int64_t i = 0; i < arr->n_children; ++i) { + op(arr->children[i]); + } +} + +void makeStreamFromArrays(std::vector arrays, + nanoarrow::UniqueSchema schema, + ArrowArrayStream* out) +{ + auto* private_data = new VectorOfArrays{std::move(arrays), std::move(schema)}; + out->get_schema = VectorOfArrays::get_schema; + out->get_next = VectorOfArrays::get_next; + out->get_last_error = VectorOfArrays::get_last_error; + out->release = VectorOfArrays::release; + out->private_data = private_data; +} + +std::tuple, nanoarrow::UniqueSchema, ArrowArrayStream> +get_nanoarrow_stream(int num_copies, cuda::stream_ref stream, cudf::memory_resources mr) +{ + auto const temporary_mr = mr.get_temporary_mr(); + auto const temporary_resources = cudf::memory_resources{temporary_mr, temporary_mr}; + std::vector> tables; + // The schema is unique across all tables. + nanoarrow::UniqueSchema schema; + std::vector arrays; + for (auto i = 0; i < num_copies; ++i) { + auto [tbl, sch, arr] = get_nanoarrow_host_tables(3, stream, temporary_resources); + tables.push_back(std::move(tbl)); + arrays.push_back(std::move(arr)); + if (i == 0) { sch.move(schema.get()); } + } + std::vector table_views; + for (auto const& table : tables) { + table_views.push_back(table->view()); + } + auto expected = cudf::concatenate(table_views, stream, mr.get_output_mr()); + + ArrowArrayStream arrow_stream; + makeStreamFromArrays(std::move(arrays), std::move(schema), &arrow_stream); + return std::make_tuple(std::move(expected), std::move(schema), arrow_stream); +} diff --git a/cpp/tests/interop/to_arrow_device_test.cpp b/cpp/tests/interop/to_arrow_device_test.cpp index 72c4a75d2b58..f19dc606768b 100644 --- a/cpp/tests/interop/to_arrow_device_test.cpp +++ b/cpp/tests/interop/to_arrow_device_test.cpp @@ -18,223 +18,6 @@ #include #include -std::tuple, nanoarrow::UniqueSchema, generated_test_data> -get_nanoarrow_cudf_table(cudf::size_type length, cuda::stream_ref stream, cudf::memory_resources mr) -{ - auto const temporary_mr = mr.get_temporary_mr(); - generated_test_data test_data(length); - - std::vector> columns; - - columns.emplace_back(cudf::test::fixed_width_column_wrapper(test_data.int64_data.begin(), - test_data.int64_data.end(), - test_data.validity.begin(), - stream, - mr) - .release()); - columns.emplace_back(cudf::test::strings_column_wrapper(test_data.string_data.begin(), - test_data.string_data.end(), - test_data.validity.begin(), - stream, - mr) - .release()); - auto col4 = cudf::test::fixed_width_column_wrapper(test_data.int64_data.begin(), - test_data.int64_data.end(), - test_data.validity.begin(), - stream, - temporary_mr); - columns.emplace_back(cudf::dictionary::encode( - col4, cudf::data_type{cudf::type_id::INT32}, stream, mr.get_output_mr())); - columns.emplace_back(cudf::test::fixed_width_column_wrapper(test_data.bool_data.begin(), - test_data.bool_data.end(), - test_data.bool_validity.begin(), - stream, - mr) - .release()); - auto list_child_column = - cudf::test::fixed_width_column_wrapper(test_data.list_int64_data.begin(), - test_data.list_int64_data.end(), - test_data.list_int64_data_validity.begin(), - stream, - mr); - auto list_offsets_column = cudf::test::fixed_width_column_wrapper( - test_data.list_offsets.begin(), test_data.list_offsets.end(), stream, mr); - auto list_validity = cudf::test::fixed_width_column_wrapper( - test_data.list_validity.begin(), test_data.list_validity.end(), stream, temporary_mr); - auto [list_mask, list_nulls] = cudf::bools_to_mask(list_validity, stream, mr.get_output_mr()); - columns.emplace_back(cudf::make_lists_column(length, - list_offsets_column.release(), - list_child_column.release(), - list_nulls, - std::move(*list_mask))); - auto int_column = cudf::test::fixed_width_column_wrapper(test_data.int64_data.begin(), - test_data.int64_data.end(), - test_data.validity.begin(), - stream, - mr) - .release(); - auto str_column = cudf::test::strings_column_wrapper(test_data.string_data.begin(), - test_data.string_data.end(), - test_data.validity.begin(), - stream, - mr) - .release(); - vector_of_columns cols; - cols.push_back(std::move(int_column)); - cols.push_back(std::move(str_column)); - auto struct_validity = cudf::test::fixed_width_column_wrapper( - test_data.bool_data_validity.begin(), test_data.bool_data_validity.end(), stream, temporary_mr); - auto [null_mask, null_count] = cudf::bools_to_mask(struct_validity, stream, mr.get_output_mr()); - columns.emplace_back(cudf::make_structs_column( - length, std::move(cols), null_count, std::move(*null_mask), stream, mr.get_output_mr())); - - nanoarrow::UniqueSchema schema; - ArrowSchemaInit(schema.get()); - NANOARROW_THROW_NOT_OK(ArrowSchemaSetTypeStruct(schema.get(), 6)); - - NANOARROW_THROW_NOT_OK(ArrowSchemaInitFromType(schema->children[0], NANOARROW_TYPE_INT64)); - NANOARROW_THROW_NOT_OK(ArrowSchemaSetName(schema->children[0], "a")); - if (columns[0]->null_count() > 0) { - schema->children[0]->flags |= ARROW_FLAG_NULLABLE; - } else { - schema->children[0]->flags = 0; - } - - NANOARROW_THROW_NOT_OK(ArrowSchemaInitFromType(schema->children[1], NANOARROW_TYPE_STRING)); - NANOARROW_THROW_NOT_OK(ArrowSchemaSetName(schema->children[1], "b")); - if (columns[1]->null_count() > 0) { - schema->children[1]->flags |= ARROW_FLAG_NULLABLE; - } else { - schema->children[1]->flags = 0; - } - - NANOARROW_THROW_NOT_OK(ArrowSchemaInitFromType(schema->children[2], NANOARROW_TYPE_INT32)); - NANOARROW_THROW_NOT_OK(ArrowSchemaAllocateDictionary(schema->children[2])); - NANOARROW_THROW_NOT_OK( - ArrowSchemaInitFromType(schema->children[2]->dictionary, NANOARROW_TYPE_INT64)); - NANOARROW_THROW_NOT_OK(ArrowSchemaSetName(schema->children[2], "c")); - if (columns[2]->null_count() > 0) { - schema->children[2]->flags |= ARROW_FLAG_NULLABLE; - } else { - schema->children[2]->flags = 0; - } - - NANOARROW_THROW_NOT_OK(ArrowSchemaInitFromType(schema->children[3], NANOARROW_TYPE_BOOL)); - NANOARROW_THROW_NOT_OK(ArrowSchemaSetName(schema->children[3], "d")); - if (columns[3]->null_count() > 0) { - schema->children[3]->flags |= ARROW_FLAG_NULLABLE; - } else { - schema->children[3]->flags = 0; - } - - NANOARROW_THROW_NOT_OK(ArrowSchemaInitFromType(schema->children[4], NANOARROW_TYPE_LIST)); - NANOARROW_THROW_NOT_OK( - ArrowSchemaInitFromType(schema->children[4]->children[0], NANOARROW_TYPE_INT64)); - NANOARROW_THROW_NOT_OK(ArrowSchemaSetName(schema->children[4]->children[0], "element")); - if (columns[4]->child(1).null_count() > 0) { - schema->children[4]->children[0]->flags |= ARROW_FLAG_NULLABLE; - } else { - schema->children[4]->children[0]->flags = 0; - } - - NANOARROW_THROW_NOT_OK(ArrowSchemaSetName(schema->children[4], "e")); - if (columns[4]->has_nulls()) { - schema->children[4]->flags |= ARROW_FLAG_NULLABLE; - } else { - schema->children[4]->flags = 0; - } - - ArrowSchemaInit(schema->children[5]); - NANOARROW_THROW_NOT_OK(ArrowSchemaSetTypeStruct(schema->children[5], 2)); - NANOARROW_THROW_NOT_OK( - ArrowSchemaInitFromType(schema->children[5]->children[0], NANOARROW_TYPE_INT64)); - NANOARROW_THROW_NOT_OK(ArrowSchemaSetName(schema->children[5]->children[0], "integral")); - if (columns[5]->child(0).has_nulls()) { - schema->children[5]->children[0]->flags |= ARROW_FLAG_NULLABLE; - } else { - schema->children[5]->children[0]->flags = 0; - } - - NANOARROW_THROW_NOT_OK( - ArrowSchemaInitFromType(schema->children[5]->children[1], NANOARROW_TYPE_STRING)); - NANOARROW_THROW_NOT_OK(ArrowSchemaSetName(schema->children[5]->children[1], "string")); - if (columns[5]->child(1).has_nulls()) { - schema->children[5]->children[1]->flags |= ARROW_FLAG_NULLABLE; - } else { - schema->children[5]->children[1]->flags = 0; - } - - NANOARROW_THROW_NOT_OK(ArrowSchemaSetName(schema->children[5], "f")); - if (columns[5]->has_nulls()) { - schema->children[5]->flags |= ARROW_FLAG_NULLABLE; - } else { - schema->children[5]->flags = 0; - } - - return std::make_tuple( - std::make_unique(std::move(columns)), std::move(schema), std::move(test_data)); -} - -std::tuple, nanoarrow::UniqueSchema, nanoarrow::UniqueArray> -get_nanoarrow_tables(cudf::size_type length, cuda::stream_ref stream, cudf::memory_resources mr) -{ - auto [table, schema, test_data] = get_nanoarrow_cudf_table(length, stream, mr); - - nanoarrow::UniqueArray arrow; - NANOARROW_THROW_NOT_OK(ArrowArrayInitFromSchema(arrow.get(), schema.get(), nullptr)); - arrow->length = length; - - populate_from_col(arrow->children[0], table->get_column(0).view()); - populate_from_col(arrow->children[1], table->get_column(1).view(), stream, mr); - populate_dict_from_col( - arrow->children[2], cudf::dictionary_column_view(table->get_column(2).view()), stream, mr); - - populate_from_col(arrow->children[3], table->get_column(3).view(), stream, mr); - cudf::lists_column_view list_view{table->get_column(4).view()}; - populate_list_from_col(arrow->children[4], list_view); - populate_from_col(arrow->children[4]->children[0], list_view.child()); - - cudf::structs_column_view struct_view{table->get_column(5).view()}; - populate_from_col(arrow->children[5]->children[0], struct_view.child(0)); - populate_from_col( - arrow->children[5]->children[1], struct_view.child(1), stream, mr); - arrow->children[5]->length = struct_view.size(); - arrow->children[5]->null_count = struct_view.null_count(); - NANOARROW_THROW_NOT_OK( - ArrowBufferSetAllocator(ArrowArrayBuffer(arrow->children[5], 0), noop_alloc)); - ArrowArrayValidityBitmap(arrow->children[5])->buffer.size_bytes = - cudf::bitmask_allocation_size_bytes(struct_view.size()); - ArrowArrayValidityBitmap(arrow->children[5])->buffer.data = - const_cast(reinterpret_cast(struct_view.null_mask())); - - ArrowError error; - if (ArrowArrayFinishBuilding(arrow.get(), NANOARROW_VALIDATION_LEVEL_MINIMAL, &error) != - NANOARROW_OK) { - std::cerr << ArrowErrorMessage(&error) << std::endl; - CUDF_FAIL("failed to build example arrays"); - } - - return std::make_tuple(std::move(table), std::move(schema), std::move(arrow)); -} - -// populate an ArrowArray list array from device buffers using a no-op -// allocator so that the ArrowArray doesn't have ownership of the buffers -void populate_list_from_col(ArrowArray* arr, cudf::lists_column_view view) -{ - arr->length = view.size(); - arr->null_count = view.null_count(); - - NANOARROW_THROW_NOT_OK(ArrowBufferSetAllocator(ArrowArrayBuffer(arr, 0), noop_alloc)); - ArrowArrayValidityBitmap(arr)->buffer.size_bytes = - cudf::bitmask_allocation_size_bytes(view.size()); - ArrowArrayValidityBitmap(arr)->buffer.data = - const_cast(reinterpret_cast(view.null_mask())); - - NANOARROW_THROW_NOT_OK(ArrowBufferSetAllocator(ArrowArrayBuffer(arr, 1), noop_alloc)); - ArrowArrayBuffer(arr, 1)->size_bytes = sizeof(int32_t) * view.offsets().size(); - ArrowArrayBuffer(arr, 1)->data = const_cast(view.offsets().data()); -} - struct BaseArrowFixture : public cudf::test::BaseFixture { void compare_schemas(ArrowSchema const* expected, ArrowSchema const* actual) { From 62e1af64d4e934bab888007cdc63c9ce05ad71be Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 26 Aug 2026 18:46:16 +0000 Subject: [PATCH 6/9] Remove dead code --- cpp/tests/interop/from_arrow_test.cpp | 649 ---------------------- cpp/tests/interop/to_arrow_test.cpp | 742 -------------------------- 2 files changed, 1391 deletions(-) delete mode 100644 cpp/tests/interop/from_arrow_test.cpp delete mode 100644 cpp/tests/interop/to_arrow_test.cpp diff --git a/cpp/tests/interop/from_arrow_test.cpp b/cpp/tests/interop/from_arrow_test.cpp deleted file mode 100644 index e342167169f4..000000000000 --- a/cpp/tests/interop/from_arrow_test.cpp +++ /dev/null @@ -1,649 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "arrow_utils.hpp" - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include - -std::shared_ptr get_arrow_large_string_array( - std::vector const& data, std::vector const& mask = {}) -{ - std::shared_ptr large_string_array; - arrow::LargeStringBuilder large_string_builder; - - auto const append_status = large_string_builder.AppendValues(data, mask.data()); - CUDF_EXPECTS(append_status.ok(), "Failed to append values to string builder"); - auto const finish_status = large_string_builder.Finish(&large_string_array); - CUDF_EXPECTS(finish_status.ok(), "Failed to create arrow string array"); - - return large_string_array; -} - -struct FromArrowTest : public cudf::test::BaseFixture {}; - -template -struct FromArrowTestDurationsTest : public cudf::test::BaseFixture {}; - -template -struct FromArrowTestDecimalsTest : public cudf::test::BaseFixture {}; - -std::optional> export_table(std::shared_ptr arrow_table) -{ - ArrowSchema schema; - if (!arrow::ExportSchema(*arrow_table->schema(), &schema).ok()) { return std::nullopt; } - auto batch = arrow_table->CombineChunksToBatch().ValueOrDie(); - ArrowArray arr; - if (!arrow::ExportRecordBatch(*batch, &arr).ok()) { return std::nullopt; } - auto ret = cudf::from_arrow(&schema, &arr); - arr.release(&arr); - schema.release(&schema); - return {std::move(ret)}; -} - -std::optional> export_scalar(arrow::Scalar const& arrow_scalar) -{ - auto maybe_array = arrow::MakeArrayFromScalar(arrow_scalar, 1); - if (!maybe_array.ok()) { return std::nullopt; } - auto array = *maybe_array; - - ArrowSchema schema; - if (!arrow::ExportType(*array->type(), &schema).ok()) { return std::nullopt; } - - ArrowArray arr; - if (!arrow::ExportArray(*array, &arr).ok()) { return std::nullopt; } - - auto col = cudf::from_arrow_column(&schema, &arr); - auto ret = cudf::get_element(col->view(), 0); - - arr.release(&arr); - schema.release(&schema); - return {std::move(ret)}; -} - -std::optional> export_scalar( - std::shared_ptr const arrow_scalar) -{ - return export_scalar(*arrow_scalar); -} - -TYPED_TEST_SUITE(FromArrowTestDurationsTest, cudf::test::DurationTypes); -using FixedPointTypes = cudf::test::Types; -TYPED_TEST_SUITE(FromArrowTestDecimalsTest, FixedPointTypes); - -TEST_F(FromArrowTest, EmptyTable) -{ - auto tables = get_tables(0); - - auto expected_cudf_table = tables.first->view(); - auto arrow_table = tables.second; - - auto got_cudf_table = export_table(arrow_table); - ASSERT_TRUE(got_cudf_table.has_value()); - - CUDF_TEST_EXPECT_TABLES_EQUAL(expected_cudf_table, got_cudf_table.value()->view()); -} - -TEST_F(FromArrowTest, DateTimeTable) -{ - auto data = std::vector{1, 2, 3, 4, 5, 6}; - - auto col = cudf::test::fixed_width_column_wrapper( - data.begin(), data.end(), cuda::constant_iterator(true)); - - cudf::table_view expected_table_view({col}); - - std::shared_ptr arr; - arrow::TimestampBuilder timestamp_builder(arrow::timestamp(arrow::TimeUnit::type::MILLI), - arrow::default_memory_pool()); - ASSERT_TRUE(timestamp_builder.AppendValues(data).ok()); - ASSERT_TRUE(timestamp_builder.Finish(&arr).ok()); - - std::vector> schema_vector({arrow::field("a", arr->type())}); - auto schema = std::make_shared(schema_vector); - - auto arrow_table = arrow::Table::Make(schema, {arr}); - - auto got_cudf_table = export_table(arrow_table); - ASSERT_TRUE(got_cudf_table.has_value()); - - CUDF_TEST_EXPECT_TABLES_EQUAL(expected_table_view, got_cudf_table.value()->view()); -} - -TYPED_TEST(FromArrowTestDurationsTest, DurationTable) -{ - using T = TypeParam; - - auto data = {T{1}, T{2}, T{3}, T{4}, T{5}, T{6}}; - auto col = cudf::test::fixed_width_column_wrapper(data, cuda::constant_iterator(true)); - - std::shared_ptr arr; - cudf::table_view expected_table_view({col}); - arrow::TimeUnit::type arrow_unit; - - switch (cudf::type_to_id()) { - case cudf::type_id::DURATION_SECONDS: arrow_unit = arrow::TimeUnit::type::SECOND; break; - case cudf::type_id::DURATION_MILLISECONDS: arrow_unit = arrow::TimeUnit::type::MILLI; break; - case cudf::type_id::DURATION_MICROSECONDS: arrow_unit = arrow::TimeUnit::type::MICRO; break; - case cudf::type_id::DURATION_NANOSECONDS: arrow_unit = arrow::TimeUnit::type::NANO; break; - case cudf::type_id::DURATION_DAYS: return; - default: CUDF_FAIL("Unsupported duration unit in arrow"); - } - arrow::DurationBuilder duration_builder(duration(arrow_unit), arrow::default_memory_pool()); - ASSERT_TRUE(duration_builder.AppendValues(std::vector{1, 2, 3, 4, 5, 6}).ok()); - ASSERT_TRUE(duration_builder.Finish(&arr).ok()); - - std::vector> schema_vector({arrow::field("a", arr->type())}); - auto schema = std::make_shared(schema_vector); - - auto arrow_table = arrow::Table::Make(schema, {arr}); - - auto got_cudf_table = export_table(arrow_table); - ASSERT_TRUE(got_cudf_table.has_value()); - - CUDF_TEST_EXPECT_TABLES_EQUAL(expected_table_view, got_cudf_table.value()->view()); -} - -TEST_F(FromArrowTest, NestedList) -{ - auto valids = cudf::test::iterators::nulls_at_multiples_of(3); - auto col = cudf::test::lists_column_wrapper( - {{{{{1, 2}, valids}, {{3, 4}, valids}, {5}}, {{6}, {{7, 8, 9}, valids}}}, valids}); - cudf::table_view expected_table_view({col}); - - auto list_arr = get_arrow_list_array({6, 7, 8, 9}, {0, 1, 4}, {1, 0, 1, 1}); - std::vector offset{0, 0, 2}; - auto mask_buffer = arrow::internal::BytesToBits(std::vector({0, 1})).ValueOrDie(); - auto nested_list_arr = std::make_shared(arrow::list(list(arrow::int64())), - offset.size() - 1, - arrow::Buffer::Wrap(offset), - list_arr, - mask_buffer); - - std::vector> schema_vector( - {arrow::field("a", nested_list_arr->type())}); - auto schema = std::make_shared(schema_vector); - - auto arrow_table = arrow::Table::Make(schema, {nested_list_arr}); - - auto got_cudf_table = export_table(arrow_table); - ASSERT_TRUE(got_cudf_table.has_value()); - CUDF_TEST_EXPECT_TABLES_EQUIVALENT(expected_table_view, got_cudf_table.value()->view()); -} - -TEST_F(FromArrowTest, StructColumn) -{ - using vector_of_columns = std::vector>; - - // Create cudf table - auto nested_type_field_names = - std::vector>{{"string", "integral", "bool", "nested_list", "struct"}}; - auto str_col = - cudf::test::strings_column_wrapper{ - "Samuel Vimes", "Carrot Ironfoundersson", "Angua von Überwald"} - .release(); - auto str_col2 = - cudf::test::strings_column_wrapper{{"", "ROCKS", ""}, {false, true, false}}.release(); - int num_rows{str_col->size()}; - auto int_col = cudf::test::fixed_width_column_wrapper{{48, 27, 25}}.release(); - auto int_col2 = - cudf::test::fixed_width_column_wrapper{{12, 24, 47}, {true, false, true}} - .release(); - auto bool_col = cudf::test::fixed_width_column_wrapper{{true, true, false}}.release(); - auto list_col = cudf::test::lists_column_wrapper( - {{{1, 2}, {3, 4}, {5}}, {{{6}}}, {{7}, {8, 9}}}) // NOLINT - .release(); - vector_of_columns cols2; - cols2.push_back(std::move(str_col2)); - cols2.push_back(std::move(int_col2)); - auto [null_mask, null_count] = - cudf::bools_to_mask(cudf::test::fixed_width_column_wrapper{{true, true, false}}); - auto sub_struct_col = - cudf::make_structs_column(num_rows, std::move(cols2), null_count, std::move(*null_mask)); - vector_of_columns cols; - cols.push_back(std::move(str_col)); - cols.push_back(std::move(int_col)); - cols.push_back(std::move(bool_col)); - cols.push_back(std::move(list_col)); - cols.push_back(std::move(sub_struct_col)); - - auto struct_col = cudf::make_structs_column(num_rows, std::move(cols), 0, {}); - cudf::table_view expected_cudf_table({struct_col->view()}); - - // Create Arrow table - std::vector str{"Samuel Vimes", "Carrot Ironfoundersson", "Angua von Überwald"}; - std::vector str2{"CUDF", "ROCKS", "EVERYWHERE"}; - auto str_array = get_arrow_array(str); - auto int_array = get_arrow_array({48, 27, 25}); - auto str2_array = get_arrow_array(str2, {0, 1, 0}); - auto int2_array = get_arrow_array({12, 24, 47}, {1, 0, 1}); - auto bool_array = get_arrow_array({true, true, false}); - auto list_arr = get_arrow_list_array({1, 2, 3, 4, 5, 6, 7, 8, 9}, {0, 2, 4, 5, 6, 7, 9}); - std::vector offset{0, 3, 4, 6}; - auto nested_list_arr = std::make_shared( - arrow::list(list(arrow::field("element", arrow::int64(), false))), - offset.size() - 1, - arrow::Buffer::Wrap(offset), - list_arr); - - std::vector> child_arrays2({str2_array, int2_array}); - auto fields2 = std::vector>{ - std::make_shared("string2", str2_array->type(), str2_array->null_count() > 0), - std::make_shared("integral2", int2_array->type(), int2_array->null_count() > 0)}; - std::shared_ptr mask_buffer = - arrow::internal::BytesToBits(std::vector({1, 1, 0})).ValueOrDie(); - auto dtype2 = std::make_shared(fields2); - auto struct_array2 = std::make_shared( - dtype2, static_cast(expected_cudf_table.num_rows()), child_arrays2, mask_buffer); - - std::vector> child_arrays( - {str_array, int_array, bool_array, nested_list_arr, struct_array2}); - std::vector> fields; - std::transform(child_arrays.cbegin(), - child_arrays.cend(), - nested_type_field_names[0].cbegin(), - std::back_inserter(fields), - [](auto const array, auto const name) { - return std::make_shared( - name, array->type(), array->null_count() > 0); - }); - auto dtype = std::make_shared(fields); - - auto struct_array = std::make_shared( - dtype, static_cast(expected_cudf_table.num_rows()), child_arrays); - std::vector> schema_vector( - {arrow::field("a", struct_array->type())}); - auto schema = std::make_shared(schema_vector); - auto input = arrow::Table::Make(schema, {struct_array}); - - auto got_cudf_table = export_table(input); - ASSERT_TRUE(got_cudf_table.has_value()); - - CUDF_TEST_EXPECT_TABLES_EQUIVALENT(expected_cudf_table, got_cudf_table.value()->view()); -} - -TEST_F(FromArrowTest, DictionaryIndicesType) -{ - auto keys = std::initializer_list{1, 2, 5, 7}; - auto indices1 = std::initializer_list{0, 1, 2, 1, 3}; - auto indices2 = std::initializer_list{0, 1, 2, 1, 3}; - auto indices3 = std::initializer_list{0, 1, 2, 1, 3}; - auto valids = std::initializer_list{1, 0, 1, 1, 1}; - auto bvalids = std::initializer_list{true, false, true, true, true}; - - auto array1 = get_arrow_dict_array(keys, indices1, valids); - auto array2 = get_arrow_dict_array(keys, indices2, valids); - auto array3 = get_arrow_dict_array(keys, indices3, valids); - - std::vector> schema_vector({arrow::field("a", array1->type()), - arrow::field("b", array2->type()), - arrow::field("c", array3->type())}); - auto schema = std::make_shared(schema_vector); - - auto arrow_table = arrow::Table::Make(schema, {array1, array2, array3}); - - std::vector> columns; - auto keys_cw = cudf::test::fixed_width_column_wrapper(keys); - auto indices1_cw = cudf::test::fixed_width_column_wrapper(indices1, bvalids); - auto indices2_cw = cudf::test::fixed_width_column_wrapper(indices2, bvalids); - auto indices3_cw = cudf::test::fixed_width_column_wrapper(indices3, bvalids); - columns.emplace_back(cudf::make_dictionary_column(keys_cw, indices1_cw)); - columns.emplace_back(cudf::make_dictionary_column(keys_cw, indices2_cw)); - columns.emplace_back(cudf::make_dictionary_column(keys_cw, indices3_cw)); - - cudf::table expected_table(std::move(columns)); - - auto got_cudf_table = export_table(arrow_table); - ASSERT_TRUE(got_cudf_table.has_value()); - - CUDF_TEST_EXPECT_TABLES_EQUAL(expected_table.view(), got_cudf_table.value()->view()); -} - -TEST_F(FromArrowTest, ChunkedArray) -{ - auto int64array = get_arrow_array({1, 2, 3, 4, 5}); - auto int32array_1 = get_arrow_array({1, 2}, {1, 0}); - auto int32array_2 = get_arrow_array({5, 2, 7}, {1, 1, 1}); - auto string_array_1 = get_arrow_array({ - "fff", - "aaa", - "", - }); - auto string_array_2 = get_arrow_array( - { - "fff", - "ccc", - }, - {0, 1}); - auto large_string_array_1 = get_arrow_large_string_array( - { - "", - "abc", - "def", - "1", - "2", - }, - {0, 1, 1, 1, 1}); - auto dict_array1 = get_arrow_dict_array({1, 2, 5, 7}, {0, 1, 2}, {1, 0, 1}); - auto dict_array2 = get_arrow_dict_array({1, 2, 5, 7}, {1, 3}); - - auto int64_chunked_array = std::make_shared(int64array); - auto int32_chunked_array = std::make_shared( - std::vector>{int32array_1, int32array_2}); - auto string_chunked_array = std::make_shared( - std::vector>{string_array_1, string_array_2}); - auto dict_chunked_array = std::make_shared( - std::vector>{dict_array1, dict_array2}); - auto boolean_array = - get_arrow_array({true, false, true, false, true}, {true, false, true, true, false}); - auto boolean_chunked_array = std::make_shared(boolean_array); - auto large_string_chunked_array = std::make_shared( - std::vector>{large_string_array_1}); - - std::vector> schema_vector( - {arrow::field("a", int32_chunked_array->type()), - arrow::field("b", int64array->type()), - arrow::field("c", string_array_1->type()), - arrow::field("d", dict_chunked_array->type()), - arrow::field("e", boolean_chunked_array->type()), - arrow::field("f", large_string_array_1->type())}); - auto schema = std::make_shared(schema_vector); - - auto arrow_table = arrow::Table::Make(schema, - {int32_chunked_array, - int64_chunked_array, - string_chunked_array, - dict_chunked_array, - boolean_chunked_array, - large_string_chunked_array}); - - auto expected_cudf_table = get_cudf_table(); - - auto got_cudf_table = export_table(arrow_table); - ASSERT_TRUE(got_cudf_table.has_value()); - - CUDF_TEST_EXPECT_TABLES_EQUIVALENT(expected_cudf_table->view(), got_cudf_table.value()->view()); -} - -struct FromArrowTestSlice - : public FromArrowTest, - public ::testing::WithParamInterface> {}; - -TEST_P(FromArrowTestSlice, SliceTest) -{ - auto tables = get_tables(10000); - auto cudf_table_view = tables.first->view(); - auto arrow_table = tables.second; - auto const [start, end] = GetParam(); - - auto sliced_cudf_table = cudf::slice(cudf_table_view, {start, end})[0]; - auto expected_cudf_table = cudf::table{sliced_cudf_table}; - auto sliced_arrow_table = arrow_table->Slice(start, end - start); - auto got_cudf_table = export_table(sliced_arrow_table); - ASSERT_TRUE(got_cudf_table.has_value()); - - // This has been added to take-care of empty string column issue with no children - if (got_cudf_table.value()->num_rows() == 0 and expected_cudf_table.num_rows() == 0) { - CUDF_TEST_EXPECT_TABLES_EQUIVALENT(expected_cudf_table.view(), got_cudf_table.value()->view()); - } else { - CUDF_TEST_EXPECT_TABLES_EQUIVALENT(expected_cudf_table.view(), got_cudf_table.value()->view()); - } -} - -template -using fp_wrapper = cudf::test::fixed_point_column_wrapper; - -TYPED_TEST(FromArrowTestDecimalsTest, FixedPointTable) -{ - using T = TypeParam; - using namespace numeric; - - auto const precision = get_decimal_precision(); - for (auto const scale : {3, 2, 1, 0, -1, -2, -3}) { - auto const data = std::vector{1, 2, 3, 4, 5, 6}; - auto const col = fp_wrapper( - data.cbegin(), data.cend(), cuda::constant_iterator(true), scale_type{scale}); - auto const expected = cudf::table_view({col}); - - auto const arr = get_decimal_arrow_array(data, std::nullopt, precision, scale); - - auto const field = arrow::field("a", arr->type()); - auto const schema_vector = std::vector>({field}); - auto const schema = std::make_shared(schema_vector); - auto const arrow_table = arrow::Table::Make(schema, {arr}); - - auto got_cudf_table = export_table(arrow_table); - ASSERT_TRUE(got_cudf_table.has_value()); - - CUDF_TEST_EXPECT_TABLES_EQUAL(expected, got_cudf_table.value()->view()); - } -} - -TYPED_TEST(FromArrowTestDecimalsTest, FixedPointTableLarge) -{ - using T = TypeParam; - using namespace numeric; - - auto const precision = get_decimal_precision(); - auto constexpr NUM_ELEMENTS = 1000; - - for (auto const scale : {3, 2, 1, 0, -1, -2, -3}) { - auto data = std::vector(NUM_ELEMENTS); - std::iota(data.begin(), data.end(), T{1}); - auto const col = fp_wrapper( - data.begin(), data.end(), cuda::constant_iterator(true), scale_type{scale}); - auto const expected = cudf::table_view({col}); - - auto const arr = get_decimal_arrow_array(data, std::nullopt, precision, scale); - - auto const field = arrow::field("a", arr->type()); - auto const schema_vector = std::vector>({field}); - auto const schema = std::make_shared(schema_vector); - auto const arrow_table = arrow::Table::Make(schema, {arr}); - - auto got_cudf_table = export_table(arrow_table); - ASSERT_TRUE(got_cudf_table.has_value()); - - CUDF_TEST_EXPECT_TABLES_EQUAL(expected, got_cudf_table.value()->view()); - } -} - -TYPED_TEST(FromArrowTestDecimalsTest, FixedPointTableNulls) -{ - using T = TypeParam; - using namespace numeric; - - auto const precision = get_decimal_precision(); - for (auto const scale : {3, 2, 1, 0, -1, -2, -3}) { - auto const data = std::vector{1, 2, 3, 4, 5, 6, 0, 0}; - auto const validity = std::vector{1, 1, 1, 1, 1, 1, 0, 0}; - auto const col = fp_wrapper({1, 2, 3, 4, 5, 6, 0, 0}, - {true, true, true, true, true, true, false, false}, - scale_type{scale}); - auto const expected = cudf::table_view({col}); - - auto const arr = get_decimal_arrow_array(data, validity, precision, scale); - - auto const field = arrow::field("a", arr->type()); - auto const schema_vector = std::vector>({field}); - auto const schema = std::make_shared(schema_vector); - auto const arrow_table = arrow::Table::Make(schema, {arr}); - - auto got_cudf_table = export_table(arrow_table); - ASSERT_TRUE(got_cudf_table.has_value()); - - CUDF_TEST_EXPECT_TABLES_EQUAL(expected, got_cudf_table.value()->view()); - } -} - -TYPED_TEST(FromArrowTestDecimalsTest, FixedPointTableNullsLarge) -{ - using T = TypeParam; - using namespace numeric; - - auto const precision = get_decimal_precision(); - auto constexpr NUM_ELEMENTS = 1000; - - for (auto const scale : {3, 2, 1, 0, -1, -2, -3}) { - auto every_other = [](auto i) { return i % 2 ? 0 : 1; }; - auto validity = cudf::detail::make_counting_transform_iterator(0, every_other); - auto iota = cudf::detail::make_counting_transform_iterator(1, [](int i) { return T{i}; }); - auto const data = std::vector(iota, iota + NUM_ELEMENTS); - auto const col = fp_wrapper(iota, iota + NUM_ELEMENTS, validity, scale_type{scale}); - auto const expected = cudf::table_view({col}); - - auto const arr = get_decimal_arrow_array( - data, std::vector(validity, validity + NUM_ELEMENTS), precision, scale); - - auto const field = arrow::field("a", arr->type()); - auto const schema_vector = std::vector>({field}); - auto const schema = std::make_shared(schema_vector); - auto const arrow_table = arrow::Table::Make(schema, {arr}); - - auto got_cudf_table = export_table(arrow_table); - ASSERT_TRUE(got_cudf_table.has_value()); - - CUDF_TEST_EXPECT_TABLES_EQUAL(expected, got_cudf_table.value()->view()); - } -} - -INSTANTIATE_TEST_CASE_P(FromArrowTest, - FromArrowTestSlice, - ::testing::Values(std::make_tuple(0, 10000), - std::make_tuple(2912, 2915), - std::make_tuple(100, 3000), - std::make_tuple(0, 0), - std::make_tuple(0, 3000), - std::make_tuple(10000, 10000))); - -template -struct FromArrowNumericScalarTest : public cudf::test::BaseFixture {}; - -using NumericTypesNotBool = - cudf::test::Concat; -TYPED_TEST_SUITE(FromArrowNumericScalarTest, NumericTypesNotBool); - -TYPED_TEST(FromArrowNumericScalarTest, Basic) -{ - TypeParam const value{42}; - auto const arrow_scalar = arrow::MakeScalar(value); - - auto const cudf_scalar = export_scalar(arrow_scalar); - ASSERT_TRUE(cudf_scalar.has_value()); - - auto const cudf_numeric_scalar = - dynamic_cast*>(cudf_scalar.value().get()); - CUDF_EXPECTS(cudf_numeric_scalar != nullptr, "Attempted to test with a non-numeric type."); - EXPECT_EQ(cudf_numeric_scalar->type(), cudf::data_type(cudf::type_to_id())); - EXPECT_EQ(cudf_numeric_scalar->value(), value); -} - -struct FromArrowDecimalScalarTest : public cudf::test::BaseFixture {}; - -template -void check_decimal_scalar(int const value, ScalarType const& arrow_scalar) -{ - auto const scale{4}; - auto const cudf_scalar = export_scalar(arrow_scalar); - ASSERT_TRUE(cudf_scalar.has_value()); - - auto const cudf_decimal_scalar = - dynamic_cast*>(cudf_scalar.value().get()); - EXPECT_EQ(cudf_decimal_scalar->type(), cudf::data_type(cudf::type_to_id(), scale)); - EXPECT_EQ(cudf_decimal_scalar->value(), value); -} - -TEST_F(FromArrowDecimalScalarTest, Basic) -{ - auto const value{42}; - auto const precision{8}; - auto const scale{4}; - auto arrow_scalar32 = arrow::Decimal32Scalar(value, arrow::decimal32(precision, -scale)); - auto arrow_scalar64 = arrow::Decimal64Scalar(value, arrow::decimal64(precision, -scale)); - auto arrow_scalar128 = arrow::Decimal128Scalar(value, arrow::decimal128(precision, -scale)); - - check_decimal_scalar(value, arrow_scalar32); - check_decimal_scalar(value, arrow_scalar64); - check_decimal_scalar(value, arrow_scalar128); -} - -struct FromArrowStringScalarTest : public cudf::test::BaseFixture {}; - -TEST_F(FromArrowStringScalarTest, Basic) -{ - auto const value = std::string("hello world"); - auto const arrow_scalar = arrow::StringScalar(value); - auto const cudf_scalar = export_scalar(arrow_scalar); - ASSERT_TRUE(cudf_scalar.has_value()); - - auto const cudf_string_scalar = dynamic_cast(cudf_scalar.value().get()); - EXPECT_EQ(cudf_string_scalar->type(), cudf::data_type(cudf::type_id::STRING)); - EXPECT_EQ(cudf_string_scalar->to_string(), value); -} - -struct FromArrowListScalarTest : public cudf::test::BaseFixture {}; - -TEST_F(FromArrowListScalarTest, Basic) -{ - std::vector host_values = {1, 2, 3, 5, 6, 7, 8}; - std::vector host_validity = {true, true, true, false, true, true, true}; - - arrow::Int64Builder builder; - auto const status = builder.AppendValues(host_values, host_validity); - auto const maybe_array = builder.Finish(); - auto const array = *maybe_array; - - auto const arrow_scalar = arrow::ListScalar(array); - auto const cudf_scalar = export_scalar(arrow_scalar); - ASSERT_TRUE(cudf_scalar.has_value()); - - auto const cudf_list_scalar = dynamic_cast(cudf_scalar.value().get()); - EXPECT_EQ(cudf_list_scalar->type(), cudf::data_type(cudf::type_id::LIST)); - - cudf::test::fixed_width_column_wrapper const lhs( - host_values.begin(), host_values.end(), host_validity.begin()); - CUDF_TEST_EXPECT_COLUMNS_EQUAL(lhs, cudf_list_scalar->view()); -} - -struct FromArrowStructScalarTest : public cudf::test::BaseFixture {}; - -TEST_F(FromArrowStructScalarTest, Basic) -{ - int64_t const value{42}; - auto const underlying_arrow_scalar = arrow::MakeScalar(value); - - auto const field = arrow::field("", underlying_arrow_scalar->type); - auto const arrow_type = arrow::struct_({field}); - auto const arrow_scalar = arrow::StructScalar({underlying_arrow_scalar}, arrow_type); - auto const cudf_scalar = export_scalar(arrow_scalar); - ASSERT_TRUE(cudf_scalar.has_value()); - - auto const cudf_struct_scalar = dynamic_cast(cudf_scalar.value().get()); - EXPECT_EQ(cudf_struct_scalar->type(), cudf::data_type(cudf::type_id::STRUCT)); - - cudf::test::fixed_width_column_wrapper const col({value}, {true}); - cudf::table_view const lhs({col}); - - CUDF_TEST_EXPECT_TABLES_EQUAL(lhs, cudf_struct_scalar->view()); -} diff --git a/cpp/tests/interop/to_arrow_test.cpp b/cpp/tests/interop/to_arrow_test.cpp deleted file mode 100644 index 5659def59a1e..000000000000 --- a/cpp/tests/interop/to_arrow_test.cpp +++ /dev/null @@ -1,742 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include - -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include - -using vector_of_columns = std::vector>; - -std::pair, std::shared_ptr> get_tables( - cudf::size_type length) -{ - std::vector int64_data(length); - std::vector bool_data(length); - std::vector string_data(length); - std::vector validity(length); - std::vector bool_validity(length); - std::vector bool_data_validity; - - std::vector> columns; - - auto validity_generator = []() { return rand() % 7 != 0; }; - - std::generate(int64_data.begin(), int64_data.end(), []() { return rand() % 500000; }); - std::generate(bool_data.begin(), bool_data.end(), validity_generator); - std::generate( - string_data.begin(), string_data.end(), []() { return rand() % 7 != 0 ? "CUDF" : "Rocks"; }); - std::generate(validity.begin(), validity.end(), validity_generator); - std::generate(bool_validity.begin(), bool_validity.end(), validity_generator); - - std::transform(bool_validity.cbegin(), - bool_validity.cend(), - std::back_inserter(bool_data_validity), - [](auto val) { return static_cast(val); }); - - std::vector list_validity = bool_data_validity; - std::vector list_offsets(length + 1); - list_offsets[0] = 0; - - constexpr cudf::size_type length_of_individual_list = 3; - for (cudf::size_type i = 0; i < length; ++i) { - list_offsets[i + 1] = list_offsets[i] + (list_validity[i] ? length_of_individual_list : 0); - } - std::vector list_int64_data_validity(list_offsets.back()); - std::vector list_int64_data(list_offsets.back()); - std::generate(list_int64_data.begin(), list_int64_data.end(), []() { return rand() % 500000; }); - std::generate( - list_int64_data_validity.begin(), list_int64_data_validity.end(), validity_generator); - - columns.emplace_back(cudf::test::fixed_width_column_wrapper( - int64_data.begin(), int64_data.end(), validity.begin()) - .release()); - columns.emplace_back( - cudf::test::strings_column_wrapper(string_data.begin(), string_data.end(), validity.begin()) - .release()); - auto dict_column = cudf::test::dictionary_column_wrapper( - int64_data.begin(), int64_data.end(), validity.begin()); - // make a copy of dict_column since it is used again below - columns.emplace_back(std::make_unique(dict_column)); - columns.emplace_back(cudf::test::fixed_width_column_wrapper( - bool_data.begin(), bool_data.end(), bool_validity.begin()) - .release()); - auto list_child_column = cudf::test::fixed_width_column_wrapper( - list_int64_data.begin(), list_int64_data.end(), list_int64_data_validity.begin()); - auto list_offsets_column = - cudf::test::fixed_width_column_wrapper(list_offsets.begin(), list_offsets.end()); - auto [list_mask, list_nulls] = cudf::bools_to_mask( - cudf::test::fixed_width_column_wrapper(list_validity.begin(), list_validity.end())); - columns.emplace_back(cudf::make_lists_column(length, - list_offsets_column.release(), - list_child_column.release(), - list_nulls, - std::move(*list_mask))); - auto int_column = cudf::test::fixed_width_column_wrapper( - int64_data.begin(), int64_data.end(), validity.begin()) - .release(); - auto str_column = - cudf::test::strings_column_wrapper(string_data.begin(), string_data.end(), validity.begin()) - .release(); - vector_of_columns cols; - cols.push_back(std::move(int_column)); - cols.push_back(std::move(str_column)); - auto [null_mask, null_count] = cudf::bools_to_mask(cudf::test::fixed_width_column_wrapper( - bool_data_validity.begin(), bool_data_validity.end())); - columns.emplace_back( - cudf::make_structs_column(length, std::move(cols), null_count, std::move(*null_mask))); - - auto int64array = get_arrow_array(int64_data, validity); - auto boolarray = get_arrow_array(bool_data, bool_validity); - auto list_array = get_arrow_list_array( - list_int64_data, list_offsets, list_int64_data_validity, list_validity); - - cudf::dictionary_column_view view(dict_column); - auto keys = cudf::test::to_host(view.keys()).first; - auto indices = cudf::test::to_host(view.indices()).first; - auto dict_array = get_arrow_dict_array(std::vector(keys.begin(), keys.end()), - std::vector(indices.begin(), indices.end()), - validity); - - auto string_array = get_arrow_array(string_data, validity); - arrow::ArrayVector child_arrays({int64array, string_array}); - std::vector> fields = { - arrow::field("integral", int64array->type(), int64array->null_count() > 0), - arrow::field("string", string_array->type(), string_array->null_count() > 0)}; - auto dtype = std::make_shared(fields); - std::shared_ptr mask_buffer = - arrow::internal::BytesToBits(static_cast>(bool_data_validity)) - .ValueOrDie(); - auto struct_array = - std::make_shared(dtype, length, child_arrays, mask_buffer); - - std::vector> schema_vector = { - arrow::field("a", int64array->type()), - arrow::field("b", string_array->type()), - arrow::field("c", dict_array->type()), - arrow::field("d", boolarray->type()), - arrow::field("e", list_array->type()), - arrow::field("f", struct_array->type())}; - - auto schema = std::make_shared(schema_vector); - - return std::pair( - std::make_unique(std::move(columns)), - arrow::Table::Make( - schema, {int64array, string_array, dict_array, boolarray, list_array, struct_array})); -} - -struct ToArrowTest : public cudf::test::BaseFixture {}; - -template -struct ToArrowTestDurationsTest : public cudf::test::BaseFixture {}; - -auto is_equal(cudf::table_view const& table, - cudf::host_span metadata, - std::shared_ptr expected_arrow_table) -{ - auto got_arrow_schema = cudf::to_arrow_schema(table, metadata); - auto got_arrow_table = cudf::to_arrow_host(table); - - for (auto i = 0; i < got_arrow_schema->n_children; ++i) { - auto arr = arrow::ImportArray(got_arrow_table->array.children[i], got_arrow_schema->children[i]) - .ValueOrDie(); - if (!expected_arrow_table->column(i)->Equals(arrow::ChunkedArray(arr))) { return false; } - } - return true; -} - -TYPED_TEST_SUITE(ToArrowTestDurationsTest, cudf::test::DurationTypes); - -TEST_F(ToArrowTest, EmptyTable) -{ - auto tables = get_tables(0); - - auto cudf_table_view = tables.first->view(); - auto expected_arrow_table = tables.second; - auto struct_meta = cudf::column_metadata{"f"}; - struct_meta.children_meta = {{"integral"}, {"string"}}; - - std::vector const metadata = { - {"a"}, {"b"}, {"c"}, {"d"}, {"e"}, struct_meta}; - ASSERT_TRUE(is_equal(cudf_table_view, metadata, expected_arrow_table)); -} - -TEST_F(ToArrowTest, DateTimeTable) -{ - auto data = {1, 2, 3, 4, 5, 6}; - - auto col = - cudf::test::fixed_width_column_wrapper(data); - - cudf::table_view input_view({col}); - - std::shared_ptr arr; - arrow::TimestampBuilder timestamp_builder(timestamp(arrow::TimeUnit::type::MILLI), - arrow::default_memory_pool()); - ASSERT_TRUE(timestamp_builder.AppendValues(std::vector{1, 2, 3, 4, 5, 6}).ok()); - ASSERT_TRUE(timestamp_builder.Finish(&arr).ok()); - - std::vector> schema_vector({arrow::field("a", arr->type())}); - auto schema = std::make_shared(schema_vector); - - auto expected_arrow_table = arrow::Table::Make(schema, {arr}); - - std::vector const metadata = {{"a"}}; - ASSERT_TRUE(is_equal(input_view, metadata, expected_arrow_table)); -} - -TYPED_TEST(ToArrowTestDurationsTest, DurationTable) -{ - using T = TypeParam; - - auto data = {T{1}, T{2}, T{3}, T{4}, T{5}, T{6}}; - auto col = cudf::test::fixed_width_column_wrapper(data); - - cudf::table_view input_view({col}); - - std::shared_ptr arr; - arrow::TimeUnit::type arrow_unit; - switch (cudf::type_to_id()) { - case cudf::type_id::DURATION_SECONDS: arrow_unit = arrow::TimeUnit::type::SECOND; break; - case cudf::type_id::DURATION_MILLISECONDS: arrow_unit = arrow::TimeUnit::type::MILLI; break; - case cudf::type_id::DURATION_MICROSECONDS: arrow_unit = arrow::TimeUnit::type::MICRO; break; - case cudf::type_id::DURATION_NANOSECONDS: arrow_unit = arrow::TimeUnit::type::NANO; break; - case cudf::type_id::DURATION_DAYS: return; - default: CUDF_FAIL("Unsupported duration unit in arrow"); - } - arrow::DurationBuilder duration_builder(duration(arrow_unit), arrow::default_memory_pool()); - ASSERT_TRUE(duration_builder.AppendValues(std::vector{1, 2, 3, 4, 5, 6}).ok()); - ASSERT_TRUE(duration_builder.Finish(&arr).ok()); - - std::vector> schema_vector({arrow::field("a", arr->type())}); - auto schema = std::make_shared(schema_vector); - - auto expected_arrow_table = arrow::Table::Make(schema, {arr}); - - std::vector const metadata = {{"a"}}; - ASSERT_TRUE(is_equal(input_view, metadata, expected_arrow_table)); -} - -TEST_F(ToArrowTest, NestedList) -{ - auto valids = cudf::test::iterators::nulls_at_multiples_of(3); - auto col = cudf::test::lists_column_wrapper( - {{{{{1, 2}, valids}, {{3, 4}, valids}, {5}}, {{6}, {{7, 8, 9}, valids}}}, valids}); - cudf::table_view input_view({col}); - - auto list_arr = get_arrow_list_array({6, 7, 8, 9}, {0, 1, 4}, {1, 0, 1, 1}); - std::vector offset{0, 0, 2}; - auto mask_buffer = arrow::internal::BytesToBits(std::vector({0, 1})).ValueOrDie(); - auto nested_list_arr = std::make_shared( - arrow::list(arrow::field("element", arrow::list(arrow::int64()), false)), - offset.size() - 1, - arrow::Buffer::Wrap(offset), - list_arr, - mask_buffer); - - std::vector> schema_vector( - {arrow::field("a", nested_list_arr->type(), false)}); - auto schema = std::make_shared(schema_vector); - - auto expected_arrow_table = arrow::Table::Make(schema, {nested_list_arr}); - std::vector const metadata = {{"a"}}; - ASSERT_TRUE(is_equal(input_view, metadata, expected_arrow_table)); -} - -TEST_F(ToArrowTest, StructColumn) -{ - // Create cudf table - auto nested_type_field_names = - std::vector>{{"string", "integral", "bool", "nested_list", "struct"}}; - auto str_col = - cudf::test::strings_column_wrapper{ - "Samuel Vimes", "Carrot Ironfoundersson", "Angua von Überwald"} - .release(); - auto str_col2 = - cudf::test::strings_column_wrapper{{"CUDF", "ROCKS", "EVERYWHERE"}, {0, 1, 0}}.release(); - int num_rows{str_col->size()}; - auto int_col = cudf::test::fixed_width_column_wrapper{{48, 27, 25}}.release(); - auto int_col2 = - cudf::test::fixed_width_column_wrapper{{12, 24, 47}, {1, 0, 1}}.release(); - auto bool_col = cudf::test::fixed_width_column_wrapper{{true, true, false}}.release(); - auto list_col = cudf::test::lists_column_wrapper( - {{{1, 2}, {3, 4}, {5}}, {{{6}}}, {{7}, {8, 9}}}) // NOLINT - .release(); - vector_of_columns cols2; - cols2.push_back(std::move(str_col2)); - cols2.push_back(std::move(int_col2)); - auto [null_mask, null_count] = - cudf::bools_to_mask(cudf::test::fixed_width_column_wrapper{{true, true, false}}); - auto sub_struct_col = - cudf::make_structs_column(num_rows, std::move(cols2), null_count, std::move(*null_mask)); - vector_of_columns cols; - cols.push_back(std::move(str_col)); - cols.push_back(std::move(int_col)); - cols.push_back(std::move(bool_col)); - cols.push_back(std::move(list_col)); - cols.push_back(std::move(sub_struct_col)); - - auto struct_col = cudf::make_structs_column(num_rows, std::move(cols), 0, {}); - cudf::table_view input_view({struct_col->view()}); - - // Create name metadata - auto sub_metadata = cudf::column_metadata{"struct"}; - sub_metadata.children_meta = {{"string2"}, {"integral2"}}; - auto metadata = cudf::column_metadata{"a"}; - metadata.children_meta = {{"string"}, {"integral"}, {"bool"}, {"nested_list"}, sub_metadata}; - - // Create Arrow table - std::vector str{"Samuel Vimes", "Carrot Ironfoundersson", "Angua von Überwald"}; - std::vector str2{"CUDF", "ROCKS", "EVERYWHERE"}; - auto str_array = get_arrow_array(str); - auto int_array = get_arrow_array({48, 27, 25}); - auto str2_array = get_arrow_array(str2, {0, 1, 0}); - auto int2_array = get_arrow_array({12, 24, 47}, {1, 0, 1}); - auto bool_array = get_arrow_array({true, true, false}); - auto list_arr = get_arrow_list_array({1, 2, 3, 4, 5, 6, 7, 8, 9}, {0, 2, 4, 5, 6, 7, 9}); - std::vector offset{0, 3, 4, 6}; - auto nested_list_arr = std::make_shared( - arrow::list(arrow::field("a", arrow::list(arrow::field("a", arrow::int64(), false)), false)), - offset.size() - 1, - arrow::Buffer::Wrap(offset), - list_arr); - - std::vector> child_arrays2({str2_array, int2_array}); - auto fields2 = std::vector>{ - std::make_shared("string2", str2_array->type(), str2_array->null_count() > 0), - std::make_shared("integral2", int2_array->type(), int2_array->null_count() > 0)}; - auto dtype2 = std::make_shared(fields2); - std::shared_ptr mask_buffer = - arrow::internal::BytesToBits(std::vector({1, 1, 0})).ValueOrDie(); - auto struct_array2 = std::make_shared( - dtype2, static_cast(input_view.num_rows()), child_arrays2, mask_buffer); - - std::vector> child_arrays( - {str_array, int_array, bool_array, nested_list_arr, struct_array2}); - std::vector> fields; - std::transform(child_arrays.cbegin(), - child_arrays.cend(), - nested_type_field_names[0].cbegin(), - std::back_inserter(fields), - [](auto const array, auto const name) { - return std::make_shared( - name, array->type(), array->null_count() > 0); - }); - auto dtype = std::make_shared(fields); - - auto struct_array = std::make_shared( - dtype, static_cast(input_view.num_rows()), child_arrays); - std::vector> schema_vector( - {arrow::field("a", struct_array->type())}); - auto schema = std::make_shared(schema_vector); - - auto expected_arrow_table = arrow::Table::Make(schema, {struct_array}); - - std::vector const meta = {metadata}; - ASSERT_TRUE(is_equal(input_view, meta, expected_arrow_table)); -} - -template -using fp_wrapper = cudf::test::fixed_point_column_wrapper; - -TEST_F(ToArrowTest, FixedPoint64Table) -{ - using namespace numeric; - - for (auto const scale : {3, 2, 1, 0, -1, -2, -3}) { - auto const col = fp_wrapper({-1, 2, 3, 4, 5, 6}, scale_type{scale}); - auto const input = cudf::table_view({col}); - auto const expect_data = std::vector{-1, 2, 3, 4, 5, 6}; - - auto const arr = get_decimal_arrow_array(expect_data, std::nullopt, 18, scale); - - auto const field = arrow::field("a", arr->type()); - auto const schema_vector = std::vector>({field}); - auto const schema = std::make_shared(schema_vector); - auto const expected_arrow_table = arrow::Table::Make(schema, {arr}); - - std::vector const metadata = {{"a"}}; - ASSERT_TRUE(is_equal(input, metadata, expected_arrow_table)); - } -} - -TEST_F(ToArrowTest, FixedPoint128Table) -{ - using namespace numeric; - - for (auto const scale : {3, 2, 1, 0, -1, -2, -3}) { - auto const col = fp_wrapper<__int128_t>({-1, 2, 3, 4, 5, 6}, scale_type{scale}); - auto const input = cudf::table_view({col}); - auto const expect_data = std::vector<__int128_t>{-1, 2, 3, 4, 5, 6}; - - auto const arr = get_decimal_arrow_array(expect_data, std::nullopt, 38, scale); - - auto const field = arrow::field("a", arr->type()); - auto const schema_vector = std::vector>({field}); - auto const schema = std::make_shared(schema_vector); - auto const expected_arrow_table = arrow::Table::Make(schema, {arr}); - - std::vector const metadata = {{"a"}}; - ASSERT_TRUE(is_equal(input, metadata, expected_arrow_table)); - } -} - -TEST_F(ToArrowTest, FixedPoint64TableLarge) -{ - using namespace numeric; - auto constexpr NUM_ELEMENTS = 1000; - - for (auto const scale : {3, 2, 1, 0, -1, -2, -3}) { - auto const iota = cuda::counting_iterator{1}; - auto const col = fp_wrapper(iota, iota + NUM_ELEMENTS, scale_type{scale}); - auto const input = cudf::table_view({col}); - auto const expect_data = std::vector{iota, iota + NUM_ELEMENTS}; - - auto const arr = get_decimal_arrow_array(expect_data, std::nullopt, 18, scale); - - auto const field = arrow::field("a", arr->type()); - auto const schema_vector = std::vector>({field}); - auto const schema = std::make_shared(schema_vector); - auto const expected_arrow_table = arrow::Table::Make(schema, {arr}); - - std::vector const metadata = {{"a"}}; // NOLINT - ASSERT_TRUE(is_equal(input, metadata, expected_arrow_table)); - } -} - -TEST_F(ToArrowTest, FixedPoint128TableLarge) -{ - using namespace numeric; - auto constexpr NUM_ELEMENTS = 1000; - - for (auto const scale : {3, 2, 1, 0, -1, -2, -3}) { - auto const iota = cuda::counting_iterator<__int128_t>{1}; - auto const col = fp_wrapper<__int128_t>(iota, iota + NUM_ELEMENTS, scale_type{scale}); - auto const input = cudf::table_view({col}); - auto const expect_data = std::vector<__int128_t>{iota, iota + NUM_ELEMENTS}; - - auto const arr = get_decimal_arrow_array(expect_data, std::nullopt, 38, scale); - - auto const field = arrow::field("a", arr->type()); - auto const schema_vector = std::vector>({field}); - auto const schema = std::make_shared(schema_vector); - auto const expected_arrow_table = arrow::Table::Make(schema, {arr}); - - std::vector const metadata = {{"a"}}; - ASSERT_TRUE(is_equal(input, metadata, expected_arrow_table)); - } -} - -TEST_F(ToArrowTest, FixedPoint64TableNullsSimple) -{ - using namespace numeric; - - for (auto const scale : {3, 2, 1, 0, -1, -2, -3}) { - auto const data = std::vector{1, 2, 3, 4, 5, 6, 0, 0}; - auto const validity = std::vector{1, 1, 1, 1, 1, 1, 0, 0}; - auto const col = - fp_wrapper({1, 2, 3, 4, 5, 6, 0, 0}, {1, 1, 1, 1, 1, 1, 0, 0}, scale_type{scale}); - auto const input = cudf::table_view({col}); - - auto const arr = get_decimal_arrow_array(data, validity, 18, scale); - - auto const field = arrow::field("a", arr->type()); - auto const schema_vector = std::vector>({field}); - auto const schema = std::make_shared(schema_vector); - auto const arrow_table = arrow::Table::Make(schema, {arr}); - - std::vector const metadata = {{"a"}}; - ASSERT_TRUE(is_equal(input, metadata, arrow_table)); - } -} - -TEST_F(ToArrowTest, FixedPoint128TableNullsSimple) -{ - using namespace numeric; - - for (auto const scale : {3, 2, 1, 0, -1, -2, -3}) { - auto const data = std::vector<__int128_t>{1, 2, 3, 4, 5, 6, 0, 0}; - auto const validity = std::vector{1, 1, 1, 1, 1, 1, 0, 0}; - auto const col = - fp_wrapper<__int128_t>({1, 2, 3, 4, 5, 6, 0, 0}, {1, 1, 1, 1, 1, 1, 0, 0}, scale_type{scale}); - auto const input = cudf::table_view({col}); - - auto const arr = get_decimal_arrow_array(data, validity, 38, scale); - - auto const field = arrow::field("a", arr->type()); - auto const schema_vector = std::vector>({field}); - auto const schema = std::make_shared(schema_vector); - auto const arrow_table = arrow::Table::Make(schema, {arr}); - - std::vector const metadata = {{"a"}}; - ASSERT_TRUE(is_equal(input, metadata, arrow_table)); - } -} - -TEST_F(ToArrowTest, FixedPoint64TableNulls) -{ - using namespace numeric; - - for (auto const scale : {3, 2, 1, 0, -1, -2, -3}) { - auto const col = fp_wrapper( - {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, {1, 0, 1, 0, 1, 0, 1, 0, 1, 0}, scale_type{scale}); - auto const input = cudf::table_view({col}); - - auto const expect_data = std::vector{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; - auto const validity = std::vector{1, 0, 1, 0, 1, 0, 1, 0, 1, 0}; - - auto const arr = get_decimal_arrow_array(expect_data, validity, 18, scale); - - auto const field = arrow::field("a", arr->type()); - auto const schema_vector = std::vector>({field}); - auto const schema = std::make_shared(schema_vector); - auto const expected_arrow_table = arrow::Table::Make(schema, {arr}); - - std::vector const metadata = {{"a"}}; - ASSERT_TRUE(is_equal(input, metadata, expected_arrow_table)); - } -} - -TEST_F(ToArrowTest, FixedPoint128TableNulls) -{ - using namespace numeric; - - for (auto const scale : {3, 2, 1, 0, -1, -2, -3}) { - auto const col = fp_wrapper<__int128_t>( - {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, {1, 0, 1, 0, 1, 0, 1, 0, 1, 0}, scale_type{scale}); - auto const input = cudf::table_view({col}); - - auto const expect_data = std::vector<__int128_t>{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; - auto const validity = std::vector{1, 0, 1, 0, 1, 0, 1, 0, 1, 0}; - - auto const arr = get_decimal_arrow_array(expect_data, validity, 38, scale); - - auto const field = arrow::field("a", arr->type()); - auto const schema_vector = std::vector>({field}); - auto const schema = std::make_shared(schema_vector); - auto const expected_arrow_table = arrow::Table::Make(schema, {arr}); - - std::vector const metadata = {{"a"}}; - ASSERT_TRUE(is_equal(input, metadata, expected_arrow_table)); - } -} - -struct ToArrowTestSlice - : public ToArrowTest, - public ::testing::WithParamInterface> {}; - -TEST_P(ToArrowTestSlice, SliceTest) -{ - auto tables = get_tables(10000); - auto cudf_table_view = tables.first->view(); - auto arrow_table = tables.second; - auto const [start, end] = GetParam(); - - auto sliced_cudf_table = cudf::slice(cudf_table_view, {start, end})[0]; - auto expected_arrow_table = arrow_table->Slice(start, end - start); - auto struct_meta = cudf::column_metadata{"f"}; - struct_meta.children_meta = {{"integral"}, {"string"}}; - - std::vector const metadata = { - {"a"}, {"b"}, {"c"}, {"d"}, {"e"}, struct_meta}; - ASSERT_TRUE(is_equal(sliced_cudf_table, metadata, expected_arrow_table)); -} - -INSTANTIATE_TEST_CASE_P(ToArrowTest, - ToArrowTestSlice, - ::testing::Values(std::make_tuple(0, 10000), - std::make_tuple(100, 3000), - std::make_tuple(0, 0), - std::make_tuple(0, 3000))); - -template -struct ToArrowNumericScalarTest : public cudf::test::BaseFixture {}; - -using NumericTypesNotBool = - cudf::test::Concat; -TYPED_TEST_SUITE(ToArrowNumericScalarTest, NumericTypesNotBool); - -auto col_to_arrow_type(cudf::column_view const& col) -{ - switch (col.type().id()) { - case cudf::type_id::BOOL8: return arrow::boolean(); - case cudf::type_id::INT8: return arrow::int8(); - case cudf::type_id::INT16: return arrow::int16(); - case cudf::type_id::INT32: return arrow::int32(); - case cudf::type_id::INT64: return arrow::int64(); - case cudf::type_id::UINT8: return arrow::uint8(); - case cudf::type_id::UINT16: return arrow::uint16(); - case cudf::type_id::UINT32: return arrow::uint32(); - case cudf::type_id::UINT64: return arrow::uint64(); - case cudf::type_id::FLOAT32: return arrow::float32(); - case cudf::type_id::FLOAT64: return arrow::float64(); - case cudf::type_id::TIMESTAMP_DAYS: return arrow::date32(); - case cudf::type_id::STRING: return arrow::utf8(); - case cudf::type_id::LIST: - return arrow::list(col_to_arrow_type(col.child(cudf::lists_column_view::child_column_index))); - case cudf::type_id::DECIMAL32: return arrow::decimal32(9, -col.type().scale()); - case cudf::type_id::DECIMAL64: return arrow::decimal64(18, -col.type().scale()); - case cudf::type_id::DECIMAL128: return arrow::decimal128(38, -col.type().scale()); - default: CUDF_FAIL("Unsupported type_id conversion to arrow type", cudf::data_type_error); - } -} - -std::optional> cudf_scalar_to_arrow( - cudf::scalar const& scalar, std::optional metadata = std::nullopt) -{ - auto const cudf_column = cudf::make_column_from_scalar(scalar, 1); - auto const c_arrow_array = cudf::to_arrow_host(*cudf_column); - auto const arrow_array = [&]() { - if (metadata.has_value()) { - auto const table = cudf::table_view({cudf_column->view()}); - std::vector const table_metadata = {metadata.value()}; - auto const arrow_schema = cudf::to_arrow_schema(table, table_metadata); - return arrow::ImportArray(&c_arrow_array->array, arrow_schema->children[0]).ValueOrDie(); - } else { - auto const arrow_type = col_to_arrow_type(cudf_column->view()); - return arrow::ImportArray(&c_arrow_array->array, arrow_type).ValueOrDie(); - } - }(); - auto const maybe_scalar = arrow_array->GetScalar(0); - if (!maybe_scalar.ok()) { return std::nullopt; } - return maybe_scalar.ValueOrDie(); -} - -TYPED_TEST(ToArrowNumericScalarTest, Basic) -{ - TypeParam const value{42}; - auto const cudf_scalar = cudf::make_fixed_width_scalar(value); - - auto const maybe_scalar = cudf_scalar_to_arrow(*cudf_scalar); - ASSERT_TRUE(maybe_scalar.has_value()); - auto const arrow_scalar = *maybe_scalar; - - auto const ref_arrow_scalar = arrow::MakeScalar(value); - EXPECT_TRUE(arrow_scalar->Equals(*ref_arrow_scalar)); -} - -struct ToArrowDecimalScalarTest : public cudf::test::BaseFixture {}; - -template -void check_decimal_scalar(int const value, arrow::Scalar const& ref_scalar, int32_t const scale) -{ - auto const cudf_scalar = - cudf::make_fixed_point_scalar(value, numeric::scale_type{scale}); - - auto const maybe_scalar = cudf_scalar_to_arrow(*cudf_scalar); - ASSERT_TRUE(maybe_scalar.has_value()); - auto const arrow_scalar = *maybe_scalar; - EXPECT_TRUE(arrow_scalar->Equals(ref_scalar)); -} - -TEST_F(ToArrowDecimalScalarTest, Basic) -{ - auto const value{42}; - int32_t const scale{4}; - - auto const get_ref_scalar = [&](std::shared_ptr type) { - auto const maybe_ref_scalar = arrow::MakeScalar(type, value); - CUDF_EXPECTS(maybe_ref_scalar.ok(), "Failed to construct reference scalar"); - return *maybe_ref_scalar; - }; - - auto const decimal32_scalar = get_ref_scalar(arrow::decimal32(9, -scale)); - auto const decimal64_scalar = get_ref_scalar(arrow::decimal64(18, -scale)); - auto const decimal128_scalar = get_ref_scalar(arrow::decimal128(38, -scale)); - - check_decimal_scalar(value, *decimal32_scalar, scale); - check_decimal_scalar(value, *decimal64_scalar, scale); - check_decimal_scalar(value, *decimal128_scalar, scale); -} - -struct ToArrowStringScalarTest : public cudf::test::BaseFixture {}; - -TEST_F(ToArrowStringScalarTest, Basic) -{ - std::string const value{"hello world"}; - auto const cudf_scalar = cudf::make_string_scalar(value); - auto const maybe_scalar = cudf_scalar_to_arrow(*cudf_scalar); - ASSERT_TRUE(maybe_scalar.has_value()); - auto const arrow_scalar = *maybe_scalar; - - auto const ref_arrow_scalar = arrow::MakeScalar(value); - EXPECT_TRUE(arrow_scalar->Equals(*ref_arrow_scalar)); -} - -struct ToArrowListScalarTest : public cudf::test::BaseFixture {}; - -TEST_F(ToArrowListScalarTest, Basic) -{ - std::vector const host_values = {1, 2, 3, 5, 6, 7, 8}; - std::vector const host_validity = {true, true, true, false, true, true, true}; - - cudf::test::fixed_width_column_wrapper const col( - host_values.begin(), host_values.end(), host_validity.begin()); - - auto const cudf_scalar = cudf::make_list_scalar(col); - - auto const maybe_scalar = cudf_scalar_to_arrow(*cudf_scalar); - ASSERT_TRUE(maybe_scalar.has_value()); - auto const arrow_scalar = *maybe_scalar; - - arrow::Int64Builder builder; - auto const status = builder.AppendValues(host_values, host_validity); - auto const maybe_array = builder.Finish(); - auto const array = *maybe_array; - - auto const ref_arrow_scalar = arrow::ListScalar(array); - - EXPECT_TRUE(arrow_scalar->Equals(ref_arrow_scalar)); -} - -struct ToArrowStructScalarTest : public cudf::test::BaseFixture {}; - -TEST_F(ToArrowStructScalarTest, Basic) -{ - int64_t const value{42}; - auto const field_name{"a"}; - - cudf::test::fixed_width_column_wrapper const col{value}; - cudf::table_view const tbl({col}); - auto const cudf_scalar = cudf::make_struct_scalar(tbl); - - cudf::column_metadata metadata{""}; - metadata.children_meta.emplace_back(field_name); - - auto const maybe_scalar = cudf_scalar_to_arrow(*cudf_scalar, metadata); - ASSERT_TRUE(maybe_scalar.has_value()); - auto const arrow_scalar = *maybe_scalar; - - auto const underlying_arrow_scalar = arrow::MakeScalar(value); - auto const field = arrow::field(field_name, underlying_arrow_scalar->type, false); - auto const arrow_type = arrow::struct_({field}); - auto const ref_arrow_scalar = arrow::StructScalar({underlying_arrow_scalar}, arrow_type); - - EXPECT_TRUE(arrow_scalar->Equals(ref_arrow_scalar)); -} - -CUDF_TEST_PROGRAM_MAIN() From fa0a801a41b523062b4ac098d389d1e1cf6cce65 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 26 Aug 2026 18:57:58 +0000 Subject: [PATCH 7/9] Refactor to have a single overload and use MR test harness --- cpp/include/cudf_test/nanoarrow_utils.hpp | 12 +++++----- cpp/tests/interop/from_arrow_stream_test.cpp | 25 ++++++++------------ 2 files changed, 16 insertions(+), 21 deletions(-) diff --git a/cpp/include/cudf_test/nanoarrow_utils.hpp b/cpp/include/cudf_test/nanoarrow_utils.hpp index b7180b20b609..05058e11ec68 100644 --- a/cpp/include/cudf_test/nanoarrow_utils.hpp +++ b/cpp/include/cudf_test/nanoarrow_utils.hpp @@ -150,7 +150,11 @@ static ArrowBufferAllocator noop_alloc = (struct ArrowBufferAllocator){ // populate an ArrowArray with pointers to the raw device buffers of a cudf::column_view // and use the no-op alloc so that the ArrowArray doesn't presume ownership of the data template -void populate_from_col(ArrowArray* arr, cudf::column_view view) +void populate_from_col( + ArrowArray* arr, + cudf::column_view view, + [[maybe_unused]] cuda::stream_ref stream = cudf::get_default_stream(), + [[maybe_unused]] cudf::memory_resources mr = cudf::get_current_device_resource_ref()) requires(cudf::is_fixed_width() && !cudf::is_boolean()) { arr->length = view.size(); @@ -250,11 +254,7 @@ void populate_dict_from_col(ArrowArray* arr, ArrowArrayBuffer(arr, 1)->size_bytes = sizeof(IND_TYPE) * dview.indices().size(); ArrowArrayBuffer(arr, 1)->data = const_cast(dview.indices().data()); - if constexpr (cudf::is_boolean() or std::same_as) { - populate_from_col(arr->dictionary, dview.keys(), stream, mr); - } else { - populate_from_col(arr->dictionary, dview.keys()); - } + populate_from_col(arr->dictionary, dview.keys(), stream, mr); } using vector_of_columns = std::vector>; diff --git a/cpp/tests/interop/from_arrow_stream_test.cpp b/cpp/tests/interop/from_arrow_stream_test.cpp index 08cd25cfd05c..c98253a0ac8c 100644 --- a/cpp/tests/interop/from_arrow_stream_test.cpp +++ b/cpp/tests/interop/from_arrow_stream_test.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -16,8 +17,6 @@ #include #include -#include - #include #include @@ -60,31 +59,27 @@ TEST_F(FromArrowStreamTest, BasicTest) TEST_F(FromArrowStreamTest, TestUtilityMemoryResourceControl) { - auto upstream = this->mr(); - auto output_mr = rmm::mr::statistics_resource_adaptor(upstream); - auto temporary_mr = rmm::mr::statistics_resource_adaptor(upstream); - auto resources = cudf::memory_resources{output_mr, temporary_mr}; - auto stream = cudf::get_default_stream(); + auto harness = cudf::test::memory_resource_test_harness{this->mr()}; + auto resources = harness.resources(); + auto stream = cudf::get_default_stream(); { - auto direct_table = get_cudf_table(stream, resources); + auto current_scope = harness.fail_on_current_device_resource_use(); + auto direct_table = get_cudf_table(stream, resources); auto [generated_table, generated_schema, test_data] = get_nanoarrow_cudf_table(3, stream, resources); auto [device_table, device_schema, device_array] = get_nanoarrow_tables(0, stream, resources); auto [host_table, host_schema, host_array] = get_nanoarrow_host_tables(3, stream, resources); auto [stream_table, stream_schema, arrow_stream] = get_nanoarrow_stream(2, stream, resources); - stream.synchronize(); - EXPECT_GT(output_mr.get_bytes_counter().value, 0); - EXPECT_EQ(temporary_mr.get_bytes_counter().value, 0); - EXPECT_GT(temporary_mr.get_bytes_counter().total, 0); + harness.expect_output_allocations_live(stream); + harness.expect_temporary_allocation_activity(stream); + harness.expect_temporary_allocations_released(stream); if (arrow_stream.release != nullptr) { arrow_stream.release(&arrow_stream); } } - stream.synchronize(); - EXPECT_EQ(output_mr.get_bytes_counter().value, 0); - EXPECT_EQ(temporary_mr.get_bytes_counter().value, 0); + harness.expect_no_live_allocations(stream); } TEST_F(FromArrowStreamTest, EmptyTest) From 02bc18efeaa935f33f93fe917e514f83585170ee Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 26 Aug 2026 19:01:29 +0000 Subject: [PATCH 8/9] Remove dead code. --- cpp/include/cudf_test/nanoarrow_utils.hpp | 11 ------ cpp/tests/interop/from_arrow_stream_test.cpp | 26 ------------- cpp/tests/interop/nanoarrow_utils.cpp | 39 -------------------- 3 files changed, 76 deletions(-) diff --git a/cpp/include/cudf_test/nanoarrow_utils.hpp b/cpp/include/cudf_test/nanoarrow_utils.hpp index 05058e11ec68..3251fd2eee47 100644 --- a/cpp/include/cudf_test/nanoarrow_utils.hpp +++ b/cpp/include/cudf_test/nanoarrow_utils.hpp @@ -274,17 +274,6 @@ get_nanoarrow_tables(cudf::size_type length = 10000, void populate_list_from_col(ArrowArray* arr, cudf::lists_column_view view); -/** - * @brief Create the standard cuDF table used by Arrow interop tests. - * - * @param stream CUDA stream used for device memory operations and kernel launches - * @param mr Memory resources used for returned table allocations and helper temporaries - * @return Generated cuDF table - */ -std::unique_ptr get_cudf_table( - cuda::stream_ref stream = cudf::get_default_stream(), - cudf::memory_resources mr = cudf::get_current_device_resource_ref()); - template struct nanoarrow_storage_type {}; diff --git a/cpp/tests/interop/from_arrow_stream_test.cpp b/cpp/tests/interop/from_arrow_stream_test.cpp index c98253a0ac8c..a416a8d8b811 100644 --- a/cpp/tests/interop/from_arrow_stream_test.cpp +++ b/cpp/tests/interop/from_arrow_stream_test.cpp @@ -5,7 +5,6 @@ #include #include -#include #include #include @@ -57,31 +56,6 @@ TEST_F(FromArrowStreamTest, BasicTest) CUDF_TEST_EXPECT_TABLES_EQUAL(tbl->view(), result->view()); } -TEST_F(FromArrowStreamTest, TestUtilityMemoryResourceControl) -{ - auto harness = cudf::test::memory_resource_test_harness{this->mr()}; - auto resources = harness.resources(); - auto stream = cudf::get_default_stream(); - - { - auto current_scope = harness.fail_on_current_device_resource_use(); - auto direct_table = get_cudf_table(stream, resources); - auto [generated_table, generated_schema, test_data] = - get_nanoarrow_cudf_table(3, stream, resources); - auto [device_table, device_schema, device_array] = get_nanoarrow_tables(0, stream, resources); - auto [host_table, host_schema, host_array] = get_nanoarrow_host_tables(3, stream, resources); - auto [stream_table, stream_schema, arrow_stream] = get_nanoarrow_stream(2, stream, resources); - - harness.expect_output_allocations_live(stream); - harness.expect_temporary_allocation_activity(stream); - harness.expect_temporary_allocations_released(stream); - - if (arrow_stream.release != nullptr) { arrow_stream.release(&arrow_stream); } - } - - harness.expect_no_live_allocations(stream); -} - TEST_F(FromArrowStreamTest, EmptyTest) { auto [tbl, sch, arr] = get_nanoarrow_host_tables(0); diff --git a/cpp/tests/interop/nanoarrow_utils.cpp b/cpp/tests/interop/nanoarrow_utils.cpp index 71b4de2f3ff5..31f55d8b9991 100644 --- a/cpp/tests/interop/nanoarrow_utils.cpp +++ b/cpp/tests/interop/nanoarrow_utils.cpp @@ -9,7 +9,6 @@ #include #include -#include #include #include #include @@ -17,44 +16,6 @@ #include -std::unique_ptr get_cudf_table(cuda::stream_ref stream, cudf::memory_resources mr) -{ - auto const temporary_mr = mr.get_temporary_mr(); - std::vector> columns; - columns.emplace_back(cudf::test::fixed_width_column_wrapper( - {1, 2, 5, 2, 7}, {true, false, true, true, true}, stream, mr) - .release()); - columns.emplace_back( - cudf::test::fixed_width_column_wrapper({1, 2, 3, 4, 5}, stream, mr).release()); - columns.emplace_back( - cudf::test::strings_column_wrapper( - {"fff", "aaa", "", "fff", "ccc"}, {true, true, true, false, true}, stream, mr) - .release()); - - auto keys = cudf::test::fixed_width_column_wrapper({1, 2, 5, 7}, stream, temporary_mr); - auto indices = cudf::test::fixed_width_column_wrapper( - {0, 1, 2, 1, 3}, {1, 0, 1, 1, 1}, stream, temporary_mr); - columns.emplace_back(cudf::make_dictionary_column(keys, indices, stream, mr.get_output_mr())); - - columns.emplace_back( - cudf::test::fixed_width_column_wrapper( - {true, false, true, false, true}, {true, false, true, true, false}, stream, mr) - .release()); - columns.emplace_back(cudf::test::strings_column_wrapper( - { - "", - "abc", - "def", - "1", - "2", - }, - {0, 1, 1, 1, 1}, - stream, - mr) - .release()); - return std::make_unique(std::move(columns)); -} - void populate_list_from_col(ArrowArray* arr, cudf::lists_column_view view) { arr->length = view.size(); From a709de073c6413a5d4eae1de08fd2e280472ff45 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 26 Aug 2026 19:13:45 +0000 Subject: [PATCH 9/9] Remove unused schemas from Nanoarrow stream helpers --- cpp/include/cudf_test/nanoarrow_utils.hpp | 10 +++++----- .../interop/arrow_data_structures_test.cpp | 4 ++-- cpp/tests/interop/from_arrow_stream_test.cpp | 20 +++++++++---------- cpp/tests/interop/nanoarrow_utils.cpp | 6 +++--- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/cpp/include/cudf_test/nanoarrow_utils.hpp b/cpp/include/cudf_test/nanoarrow_utils.hpp index 3251fd2eee47..8194cabf2810 100644 --- a/cpp/include/cudf_test/nanoarrow_utils.hpp +++ b/cpp/include/cudf_test/nanoarrow_utils.hpp @@ -558,9 +558,9 @@ void makeStreamFromArrays(std::vector arrays, * @param num_copies Number of record batches in the stream * @param stream CUDA stream used for device memory operations and kernel launches * @param mr Memory resources used for returned table allocations and helper temporaries - * @return Concatenated cuDF table, Arrow schema, and Arrow stream + * @return Concatenated cuDF table and Arrow stream */ -std::tuple, nanoarrow::UniqueSchema, ArrowArrayStream> -get_nanoarrow_stream(int num_copies, - cuda::stream_ref stream = cudf::get_default_stream(), - cudf::memory_resources mr = cudf::get_current_device_resource_ref()); +std::pair, ArrowArrayStream> get_nanoarrow_stream( + int num_copies, + cuda::stream_ref stream = cudf::get_default_stream(), + cudf::memory_resources mr = cudf::get_current_device_resource_ref()); diff --git a/cpp/tests/interop/arrow_data_structures_test.cpp b/cpp/tests/interop/arrow_data_structures_test.cpp index cc7a9abd8fa2..75caaafba75b 100644 --- a/cpp/tests/interop/arrow_data_structures_test.cpp +++ b/cpp/tests/interop/arrow_data_structures_test.cpp @@ -340,8 +340,8 @@ TEST_F(ArrowTableTest, ToFromHost) TEST_F(ArrowTableTest, FromArrowArrayStream) { - auto num_copies = 3; - auto [tbl, sch, stream] = get_nanoarrow_stream(num_copies); + auto num_copies = 3; + auto [tbl, stream] = get_nanoarrow_stream(num_copies); auto result = cudf::interop::arrow_table(std::move(stream)); CUDF_TEST_EXPECT_TABLES_EQUAL(tbl->view(), result.view()); diff --git a/cpp/tests/interop/from_arrow_stream_test.cpp b/cpp/tests/interop/from_arrow_stream_test.cpp index a416a8d8b811..3311b3d68417 100644 --- a/cpp/tests/interop/from_arrow_stream_test.cpp +++ b/cpp/tests/interop/from_arrow_stream_test.cpp @@ -21,8 +21,8 @@ struct FromArrowStreamTest : public cudf::test::BaseFixture {}; -std::tuple, nanoarrow::UniqueSchema, ArrowArrayStream> -get_nanoarrow_chunked_stream(int num_copies, cudf::size_type length) +std::pair, ArrowArrayStream> get_nanoarrow_chunked_stream( + int num_copies, cudf::size_type length) { std::vector> columns; std::vector arrays; @@ -44,13 +44,13 @@ get_nanoarrow_chunked_stream(int num_copies, cudf::size_type length) ArrowArrayStream stream; makeStreamFromArrays(std::move(arrays), std::move(schema), &stream); - return std::make_tuple(std::move(expected), std::move(schema), stream); + return std::make_pair(std::move(expected), stream); } TEST_F(FromArrowStreamTest, BasicTest) { constexpr auto num_copies = 3; - auto [tbl, sch, stream] = get_nanoarrow_stream(num_copies); + auto [tbl, stream] = get_nanoarrow_stream(num_copies); auto result = cudf::from_arrow_stream(&stream); CUDF_TEST_EXPECT_TABLES_EQUAL(tbl->view(), result->view()); @@ -70,9 +70,9 @@ TEST_F(FromArrowStreamTest, EmptyTest) TEST_F(FromArrowStreamTest, ChunkedTest) { - constexpr auto num_copies = 3; - constexpr auto length = 3; - auto [expected, schema, stream] = get_nanoarrow_chunked_stream(num_copies, length); + constexpr auto num_copies = 3; + constexpr auto length = 3; + auto [expected, stream] = get_nanoarrow_chunked_stream(num_copies, length); auto result = cudf::from_arrow_stream_column(&stream); CUDF_TEST_EXPECT_COLUMNS_EQUAL(expected->view(), result->view()); @@ -80,9 +80,9 @@ TEST_F(FromArrowStreamTest, ChunkedTest) TEST_F(FromArrowStreamTest, EmptyChunkedTest) { - constexpr auto num_copies = 3; - constexpr auto length = 0; - auto [expected, schema, stream] = get_nanoarrow_chunked_stream(num_copies, length); + constexpr auto num_copies = 3; + constexpr auto length = 0; + auto [expected, stream] = get_nanoarrow_chunked_stream(num_copies, length); auto result = cudf::from_arrow_stream_column(&stream); CUDF_TEST_EXPECT_COLUMNS_EQUAL(result->view(), expected->view()); diff --git a/cpp/tests/interop/nanoarrow_utils.cpp b/cpp/tests/interop/nanoarrow_utils.cpp index 31f55d8b9991..3b5256637cdf 100644 --- a/cpp/tests/interop/nanoarrow_utils.cpp +++ b/cpp/tests/interop/nanoarrow_utils.cpp @@ -326,8 +326,8 @@ void makeStreamFromArrays(std::vector arrays, out->private_data = private_data; } -std::tuple, nanoarrow::UniqueSchema, ArrowArrayStream> -get_nanoarrow_stream(int num_copies, cuda::stream_ref stream, cudf::memory_resources mr) +std::pair, ArrowArrayStream> get_nanoarrow_stream( + int num_copies, cuda::stream_ref stream, cudf::memory_resources mr) { auto const temporary_mr = mr.get_temporary_mr(); auto const temporary_resources = cudf::memory_resources{temporary_mr, temporary_mr}; @@ -349,5 +349,5 @@ get_nanoarrow_stream(int num_copies, cuda::stream_ref stream, cudf::memory_resou ArrowArrayStream arrow_stream; makeStreamFromArrays(std::move(arrays), std::move(schema), &arrow_stream); - return std::make_tuple(std::move(expected), std::move(schema), arrow_stream); + return std::make_pair(std::move(expected), arrow_stream); }