diff --git a/cpp/include/cudf/io/experimental/variant_spec.hpp b/cpp/include/cudf/io/experimental/variant_spec.hpp new file mode 100644 index 000000000000..6b71dc57385c --- /dev/null +++ b/cpp/include/cudf/io/experimental/variant_spec.hpp @@ -0,0 +1,49 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +namespace cudf::io::parquet::experimental { + +/** + * @brief Low 2 bits of a VARIANT value's metadata byte: the basic type. + */ +enum class variant_basic_type : uint8_t { + PRIMITIVE = 0, + SHORT_STRING = 1, + OBJECT = 2, + ARRAY = 3, +}; + +/** + * @brief Physical type ID carried in the value_header of a primitive VARIANT value. + */ +enum class variant_primitive_type : uint8_t { + NULLVAL = 0, + BOOLEAN_TRUE = 1, + BOOLEAN_FALSE = 2, + INT8 = 3, + INT16 = 4, + INT32 = 5, + INT64 = 6, + FLOAT64 = 7, + DECIMAL4 = 8, + DECIMAL8 = 9, + DECIMAL16 = 10, + DATE = 11, + TIMESTAMP_MICROS = 12, + TIMESTAMP_NTZ_MICROS = 13, + FLOAT32 = 14, + BINARY = 15, + LONG_STRING = 16, + TIME_NTZ_MICROS = 17, + TIMESTAMP_NANOS = 18, + TIMESTAMP_NTZ_NANOS = 19, + UUID = 20, +}; + +} // namespace cudf::io::parquet::experimental diff --git a/cpp/src/io/parquet/experimental/variant_extract.cu b/cpp/src/io/parquet/experimental/variant_extract.cu index 5c7e7b470867..5eefa1b47e79 100644 --- a/cpp/src/io/parquet/experimental/variant_extract.cu +++ b/cpp/src/io/parquet/experimental/variant_extract.cu @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -55,32 +56,10 @@ constexpr int variant_version_v1 = 1; constexpr size_type variant_header_bytes = 1; // Low 2 bits of a value's metadata byte: the basic type. -enum class basic_type : uint8_t { primitive = 0, short_string = 1, object = 2, array = 3 }; +using basic_type = variant_basic_type; // For a primitive value, the value_header is the physical type id of the payload. -enum class primitive_type : uint8_t { - null = 0, - boolean_true = 1, - boolean_false = 2, - int8 = 3, - int16 = 4, - int32 = 5, - int64 = 6, - float64 = 7, - decimal4 = 8, - decimal8 = 9, - decimal16 = 10, - date = 11, - timestamp_micros = 12, - timestamp_ntz_micros = 13, - float32 = 14, - binary = 15, - long_string = 16, - time_ntz_micros = 17, - timestamp_nanos = 18, - timestamp_ntz_nanos = 19, - uuid = 20, -}; +using primitive_type = variant_primitive_type; __device__ cuda::std::optional read_uint64(device_span data, size_type pos, @@ -102,7 +81,7 @@ __device__ cuda::std::optional narrow_cast(cuda::std::optional(value.value()); } -__device__ basic_type variant_basic_type(uint8_t value_metadata) +__device__ basic_type decode_basic_type(uint8_t value_metadata) { return static_cast(value_metadata & 0x03); } @@ -166,34 +145,34 @@ __device__ cuda::std::optional variant_value_length(device_span(value_header)) { - case primitive_type::null: - case primitive_type::boolean_true: - case primitive_type::boolean_false: break; // no payload - case primitive_type::int8: payload = 1; break; - case primitive_type::int16: payload = 2; break; - case primitive_type::int32: - case primitive_type::date: - case primitive_type::float32: payload = 4; break; - case primitive_type::int64: - case primitive_type::float64: - case primitive_type::timestamp_micros: - case primitive_type::timestamp_ntz_micros: - case primitive_type::time_ntz_micros: - case primitive_type::timestamp_nanos: - case primitive_type::timestamp_ntz_nanos: payload = 8; break; - case primitive_type::decimal4: payload = 1 + 4; break; // scale + int32 - case primitive_type::decimal8: payload = 1 + 8; break; // scale + int64 - case primitive_type::decimal16: payload = 1 + 16; break; // scale + int128 - case primitive_type::uuid: payload = 16; break; - case primitive_type::binary: - case primitive_type::long_string: { + case primitive_type::NULLVAL: + case primitive_type::BOOLEAN_TRUE: + case primitive_type::BOOLEAN_FALSE: break; // no payload + case primitive_type::INT8: payload = 1; break; + case primitive_type::INT16: payload = 2; break; + case primitive_type::INT32: + case primitive_type::DATE: + case primitive_type::FLOAT32: payload = 4; break; + case primitive_type::INT64: + case primitive_type::FLOAT64: + case primitive_type::TIMESTAMP_MICROS: + case primitive_type::TIMESTAMP_NTZ_MICROS: + case primitive_type::TIME_NTZ_MICROS: + case primitive_type::TIMESTAMP_NANOS: + case primitive_type::TIMESTAMP_NTZ_NANOS: payload = 8; break; + case primitive_type::DECIMAL4: payload = 1 + 4; break; // scale + int32 + case primitive_type::DECIMAL8: payload = 1 + 8; break; // scale + int64 + case primitive_type::DECIMAL16: payload = 1 + 16; break; // scale + int128 + case primitive_type::UUID: payload = 16; break; + case primitive_type::BINARY: + case primitive_type::LONG_STRING: { constexpr int length_prefix_bytes = 4; auto const len = read_uint64(enc, variant_header_bytes, length_prefix_bytes); if (!len.has_value()) { return cuda::std::nullopt; } @@ -205,14 +184,14 @@ __device__ cuda::std::optional variant_value_length(device_span(value_header); } // Object / array: the encoded size is the header bytes (metadata byte, element count, optional // field-id list, and offset list) - bool const is_object = btype == basic_type::object; + bool const is_object = btype == basic_type::OBJECT; auto const [offset_size, id_size, num_elements_size] = decode_object_array_header(value_header, is_object); @@ -322,7 +301,7 @@ __device__ device_span locate_object_field(device_span(val.size()); if (val_len < 1) { return {}; } auto const value_metadata = val[0]; - if (variant_basic_type(value_metadata) != basic_type::object) { return {}; } + if (decode_basic_type(value_metadata) != basic_type::OBJECT) { return {}; } auto const [offset_size, id_size, num_elements_size] = decode_object_array_header(variant_value_header(value_metadata), true); @@ -392,7 +371,7 @@ __device__ device_span locate_array_element(device_span(value.size()); if (value_size < 1) { return {}; } uint8_t const value_metadata = value[0]; - if (variant_basic_type(value_metadata) != basic_type::array) { return {}; } + if (decode_basic_type(value_metadata) != basic_type::ARRAY) { return {}; } int const value_header = variant_value_header(value_metadata); [[maybe_unused]] auto const [offset_size, _, num_elements_size] = @@ -450,20 +429,20 @@ template __device__ constexpr primitive_type primitive_type_for() { if constexpr (cuda::std::is_same_v) { - return primitive_type::int8; + return primitive_type::INT8; } else if constexpr (cuda::std::is_same_v) { - return primitive_type::int16; + return primitive_type::INT16; } else if constexpr (cuda::std::is_same_v) { - return primitive_type::int32; + return primitive_type::INT32; } else if constexpr (cuda::std::is_same_v) { - return primitive_type::int64; + return primitive_type::INT64; } else if constexpr (cuda::std::is_same_v) { - return primitive_type::float32; + return primitive_type::FLOAT32; } else if constexpr (cuda::std::is_same_v) { - return primitive_type::float64; + return primitive_type::FLOAT64; } else { CUDF_UNREACHABLE("primitive_type_for: T is not a supported variant primitive type"); - return primitive_type::null; + return primitive_type::NULLVAL; } } @@ -478,7 +457,7 @@ __device__ inline cuda::std::optional decode_primitive(device_span(primitive_type_for())) { return cuda::std::nullopt; } @@ -546,17 +525,17 @@ __device__ cuda::std::optional> decode_string( auto const len = enc.size(); if (len < 1) { return cuda::std::nullopt; } uint8_t const value_metadata = enc[0]; - auto const btype = variant_basic_type(value_metadata); + auto const btype = decode_basic_type(value_metadata); auto const value_header = variant_value_header(value_metadata); - if (btype == basic_type::short_string) { + if (btype == basic_type::SHORT_STRING) { // Short string: value_header = length std::size_t const str_len = value_header; if (1 + str_len > len) { return cuda::std::nullopt; } return enc.subspan(1, str_len); } - if (btype == basic_type::primitive && - value_header == static_cast(primitive_type::long_string)) { + if (btype == basic_type::PRIMITIVE && + value_header == static_cast(primitive_type::LONG_STRING)) { // Long string: 1-byte header + 4-byte LE length + char bytes constexpr std::size_t long_string_prefix_bytes = 1 + sizeof(uint32_t); if (len < long_string_prefix_bytes) { return cuda::std::nullopt; } diff --git a/cpp/tests/io/experimental/variant_extract_test.cpp b/cpp/tests/io/experimental/variant_extract_test.cpp index 4356fb40d1c9..55f2291cbb76 100644 --- a/cpp/tests/io/experimental/variant_extract_test.cpp +++ b/cpp/tests/io/experimental/variant_extract_test.cpp @@ -12,12 +12,14 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include #include @@ -26,6 +28,56 @@ namespace avf = cudf::test::apache_variant_fixtures; namespace { +// --------------------------------------------------------------------------- +// VARIANT value-header factory helpers. +// +// Every VARIANT value begins with a one-byte "value metadata" header. Its bits +// are (per the Apache Parquet variant spec [1]): +// +// bit index: 7 6 5 4 3 2 | 1 0 +// field: <- value_header ->|basic +// +// - basic_type (low 2 bits): 0=primitive, 1=short_string, 2=object, 3=array +// - value_header (high 6 bits): meaning depends on basic_type +// * primitive -> physical type id (variant_primitive_type below) +// * short_string -> string length in bytes (0..63) +// * object/array -> field-id / field-offset size flags +// +// The enums below let tests spell header bytes out by name (and avoid +// endianness ambiguity in the bit layout) instead of using magic numbers. +// +// [1] https://github.com/apache/parquet-format/blob/master/VariantEncoding.md +// --------------------------------------------------------------------------- +using cudf::io::parquet::experimental::variant_basic_type; +using cudf::io::parquet::experimental::variant_primitive_type; + +// Compose a value-metadata header byte from a basic type and its 6-bit value_header. +constexpr uint8_t make_variant_header(variant_basic_type basic, uint8_t value_header) +{ + CUDF_EXPECTS(value_header <= 0x3F, "VARIANT value_header must fit in 6 bits"); + return static_cast(static_cast(basic) | (value_header << 2)); +} + +// Header byte for a primitive value of the given physical type. +constexpr uint8_t make_variant_primitive(variant_primitive_type type) +{ + return make_variant_header(variant_basic_type::PRIMITIVE, static_cast(type)); +} + +// Header byte for a short string of the given length (must fit in 6 bits: 0..63). +constexpr uint8_t make_variant_short_string_header(std::size_t length) +{ + CUDF_EXPECTS(length <= 0x3F, "VARIANT short string length must fit in 6 bits"); + return make_variant_header(variant_basic_type::SHORT_STRING, static_cast(length)); +} + +// Header byte for an object value with 1-byte field ids and 1-byte offsets +// (is_large=false), i.e. value_header == 0. +constexpr uint8_t make_variant_object_header() +{ + return make_variant_header(variant_basic_type::OBJECT, 0); +} + // Build a struct `column_view` over (metadata, value) without copying. inline cudf::column_view wrap_variant_view(cudf::column_view const& metadata, cudf::column_view const& value) @@ -332,11 +384,11 @@ TEST_F(ExtractVariantFieldTest, BareNameEqualsDollarPath) namespace { -// INT32 primitive blob: header 0x14, little-endian 4-byte payload. +// INT32 primitive blob: primitive int32 header + little-endian 4-byte payload. inline std::vector enc_int32(int32_t v) { auto const u = static_cast(v); - return {0x14, + return {make_variant_primitive(variant_primitive_type::INT32), static_cast(u & 0xff), static_cast((u >> 8) & 0xff), static_cast((u >> 16) & 0xff), @@ -347,11 +399,68 @@ inline std::vector enc_int32(int32_t v) inline std::vector enc_short_string(std::string_view s) { CUDF_EXPECTS(s.size() < 64, "short-string length must fit in 6 bits of the single-byte header"); - std::vector out{static_cast(0x01 | (s.size() << 2))}; + std::vector out{make_variant_short_string_header(s.size())}; out.insert(out.end(), s.begin(), s.end()); return out; } +// Append `width` little-endian bytes of `bits` to `out`. +inline void append_le(std::vector& out, uint64_t bits, int width) +{ + for (int i = 0; i < width; ++i) { + out.push_back(static_cast((bits >> (8 * i)) & 0xff)); + } +} + +// Primitive value blobs (header + fixed payload) for every physical type the cast matrix exercises. +inline std::vector enc_null() +{ + return {make_variant_primitive(variant_primitive_type::NULLVAL)}; +} + +inline std::vector enc_bool(bool b) +{ + return {make_variant_primitive(b ? variant_primitive_type::BOOLEAN_TRUE + : variant_primitive_type::BOOLEAN_FALSE)}; +} + +inline std::vector enc_int8(int8_t v) +{ + return {make_variant_primitive(variant_primitive_type::INT8), static_cast(v)}; +} + +inline std::vector enc_int16(int16_t v) +{ + std::vector out{make_variant_primitive(variant_primitive_type::INT16)}; + append_le(out, static_cast(v), 2); + return out; +} + +inline std::vector enc_int64(int64_t v) +{ + std::vector out{make_variant_primitive(variant_primitive_type::INT64)}; + append_le(out, static_cast(v), 8); + return out; +} + +inline std::vector enc_float64(double v) +{ + std::vector out{make_variant_primitive(variant_primitive_type::FLOAT64)}; + append_le(out, std::bit_cast(v), 8); + return out; +} + +// Long-string primitive blob: header + 4-byte LE length + payload. +inline std::vector enc_long_string(std::string_view s) +{ + std::vector out{make_variant_primitive(variant_primitive_type::LONG_STRING)}; + append_le(out, s.size(), 4); + out.insert(out.end(), + reinterpret_cast(s.begin()), + reinterpret_cast(s.end())); + return out; +} + // Build a single-field object value wrapping `inner` under field id `fid`. // field_off_size=1, field_id_size=1, is_large=false. inline std::vector build_single_field_object(uint8_t fid, @@ -359,7 +468,8 @@ inline std::vector build_single_field_object(uint8_t fid, { CUDF_EXPECTS(inner.size() < 256, "inner blob too large for 1-byte offset header"); // Header, num_elements, field_id, offset 0, sentinel = inner.size(). - std::vector out{0x02, 0x01, fid, 0x00, static_cast(inner.size())}; + std::vector out{ + make_variant_object_header(), 0x01, fid, 0x00, static_cast(inner.size())}; out.insert(out.end(), inner.begin(), inner.end()); return out; } @@ -370,7 +480,7 @@ inline std::vector build_single_field_object(uint8_t fid, // <= 51 so the total value bytes (5 * n_fields) still fit in 1-byte offsets. inline std::vector build_sequential_int32_object(int n_fields) { - std::vector out{0x02, static_cast(n_fields)}; + std::vector out{make_variant_object_header(), static_cast(n_fields)}; for (int fid = 0; fid < n_fields; ++fid) { out.push_back(static_cast(fid)); } @@ -637,22 +747,60 @@ TEST_F(ExtractVariantFieldTest, MixedObjectArrayTraversal) TEST_F(ExtractVariantFieldTest, LargeDictionaryAndObjectScan) { - auto const keys = make_numeric_keys(50); - auto const meta = build_metadata(keys); - auto const val = build_sequential_int32_object(50); - auto col = wrap_single_variant(meta, val); - auto stream = cudf::test::get_default_stream(); - auto const i32 = cudf::data_type{cudf::type_id::INT32}; + auto const keys = make_numeric_keys(50); + auto const meta = build_metadata(keys); + auto const val = build_sequential_int32_object(50); + auto col = wrap_single_variant(meta, val); + auto stream = cudf::test::get_default_stream(); + auto const int32_dtype = cudf::data_type{cudf::type_id::INT32}; // First, middle, and last keys each decode to their own field id. - auto first = cudf::io::parquet::experimental::extract_variant_field(col, "k00", i32, stream); - auto mid = cudf::io::parquet::experimental::extract_variant_field(col, "k24", i32, stream); - auto last = cudf::io::parquet::experimental::extract_variant_field(col, "k49", i32, stream); + auto first = + cudf::io::parquet::experimental::extract_variant_field(col, "k00", int32_dtype, stream); + auto mid = + cudf::io::parquet::experimental::extract_variant_field(col, "k24", int32_dtype, stream); + auto last = + cudf::io::parquet::experimental::extract_variant_field(col, "k49", int32_dtype, stream); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*first, cudf::test::fixed_width_column_wrapper{0}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*mid, cudf::test::fixed_width_column_wrapper{24}); CUDF_TEST_EXPECT_COLUMNS_EQUAL(*last, cudf::test::fixed_width_column_wrapper{49}); } +TEST_F(ExtractVariantFieldTest, MalformedVariantDataYieldsNull) +{ + // The column shape is a valid STRUCT, list>, but the VARIANT bytes are + // internally inconsistent. Each such row must resolve to a null result rather than throwing or + // reading out of bounds. + auto stream = cudf::test::get_default_stream(); + auto const int32_dtype = cudf::data_type{cudf::type_id::INT32}; + + struct data_case { + std::string label; + std::vector meta; + std::vector val; + }; + auto const valid_object = build_single_field_object(/*fid=*/0, enc_int32(1)); + std::vector const cases{ + // Metadata claims 5 dictionary entries but carries no offset/string bytes for them. + {"metadata dictionary size overruns the buffer", {0x01, 0x05}, valid_object}, + // Single-key dict whose trailing offset (0xFF) points far past the string payload. + {"metadata offset points past the string payload", {0x01, 0x01, 0x00, 0xFF, 'x'}, valid_object}, + // Object header declares 255 fields but carries no field-id/offset bytes. + {"object declares more fields than the value buffer holds", + build_metadata({"x"}), + {make_variant_object_header(), 0xFF}}, + }; + + for (auto const& c : cases) { + SCOPED_TRACE(c.label); + auto col = wrap_single_variant(c.meta, c.val); + auto got = + cudf::io::parquet::experimental::extract_variant_field(col, "x", int32_dtype, stream); + ASSERT_EQ(got->size(), 1); + EXPECT_EQ(got->null_count(), 1); + } +} + TEST_F(ExtractVariantFieldTest, NullsAtDifferentDepths) { std::vector const dict = {"a", "b", "c", "d"}; // fids: a=0,b=1,c=2,d=3 @@ -883,3 +1031,273 @@ TEST_F(CastVariantTest, EmptyInput) EXPECT_EQ(got->null_count(), 0); } } + +TEST_F(CastVariantTest, CastToUnsupportedTargetThrows) +{ + // cast_variant only supports INT8/16/32/64 and STRING targets. Every other target is rejected at + // compile-time dispatch on the requested output type, independent of the input bytes, so a single + // well-formed placeholder row triggers the same throw for all of them. + auto stream = cudf::test::get_default_stream(); + std::vector const val{make_variant_primitive(variant_primitive_type::NULLVAL)}; + cudf::test::lists_column_wrapper values(val.begin(), val.end()); + + std::vector const ids{cudf::type_id::UINT8, + cudf::type_id::UINT16, + cudf::type_id::UINT32, + cudf::type_id::UINT64, + cudf::type_id::TIMESTAMP_DAYS, + cudf::type_id::TIMESTAMP_SECONDS, + cudf::type_id::TIMESTAMP_MICROSECONDS, + cudf::type_id::DURATION_SECONDS, + cudf::type_id::DECIMAL32, + cudf::type_id::DECIMAL64, + cudf::type_id::DECIMAL128}; + + for (auto const id : ids) { + SCOPED_TRACE(std::string{"target type_id: "} + std::to_string(static_cast(id))); + EXPECT_THROW(static_cast(cudf::io::parquet::experimental::cast_variant( + values, cudf::data_type{id}, stream)), + std::invalid_argument); + } +} + +TEST_F(CastVariantTest, CastSourceTargetMatrix) +{ + // Exhaustively covers (source physical type) x (supported target) casts. The supported targets + // are INT8/16/32/64 and STRING. Expected behaviour: + // - integer targets: only a source whose physical type has the *exact* same width decodes; + // every + // other source (including narrower/wider ints) yields null — cast_variant does not widen. + // - STRING target: short_string and long_string sources decode; every other source yields null. + auto const stream = cudf::test::get_default_stream(); + + struct source_blob { + std::string label; + std::vector bytes; + }; + std::vector const sources{ + {"null", enc_null()}, + {"bool_true", enc_bool(true)}, + {"bool_false", enc_bool(false)}, + {"int8", enc_int8(42)}, + {"int16", enc_int16(1234)}, + {"int32", enc_int32(123456)}, + {"int64", enc_int64(1234567890123456789LL)}, + {"float64", enc_float64(2.5)}, + {"short_string", enc_short_string("hi")}, + {"long_string", enc_long_string(std::string(70, 'a'))}, + }; + + auto values_of = [](std::vector const& b) { + return cudf::test::lists_column_wrapper(b.begin(), b.end()); + }; + + // Integer targets: exactly one source label decodes to `match_value`; the rest are null. + auto check_int_target = [&](char const* match_label, T match_value) { + auto const target = cudf::data_type{cudf::type_to_id()}; + for (auto const& src : sources) { + SCOPED_TRACE(std::string{"int target "} + match_label + ", source " + src.label); + auto values = values_of(src.bytes); + auto got = cudf::io::parquet::experimental::cast_variant(values, target, stream); + if (std::string_view{src.label} == match_label) { + cudf::test::fixed_width_column_wrapper const expected{match_value}; + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); + } else { + ASSERT_EQ(got->size(), 1); + EXPECT_EQ(got->null_count(), 1); + } + } + }; + check_int_target.template operator()("int8", int8_t{42}); + check_int_target.template operator()("int16", int16_t{1234}); + check_int_target.template operator()("int32", int32_t{123456}); + check_int_target.template operator()("int64", int64_t{1234567890123456789LL}); + + // STRING target: short_string and long_string decode; every other source is null. + auto const string_type = cudf::data_type{cudf::type_id::STRING}; + for (auto const& src : sources) { + SCOPED_TRACE(std::string{"string target, source "} + src.label); + auto values = values_of(src.bytes); + auto got = cudf::io::parquet::experimental::cast_variant(values, string_type, stream); + std::string_view const label{src.label}; + if (label == "short_string" || label == "long_string") { + std::string const expected_str = (label == "short_string") ? "hi" : std::string(70, 'a'); + cudf::test::strings_column_wrapper const expected({expected_str}); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); + } else { + ASSERT_EQ(got->size(), 1); + EXPECT_EQ(got->null_count(), 1); + } + } +} + +TEST_F(CastVariantTest, ShortStringLengthZero) +{ + // Short string with length 0 (lower boundary of the 6-bit length field): header only, no payload. + auto stream = cudf::test::get_default_stream(); + std::vector const val{make_variant_short_string_header(0)}; + cudf::test::lists_column_wrapper values(val.begin(), val.end()); + auto got = cudf::io::parquet::experimental::cast_variant( + values, cudf::data_type{cudf::type_id::STRING}, stream); + cudf::test::strings_column_wrapper expected({""}); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); +} + +TEST_F(CastVariantTest, ShortStringMaxLength) +{ + // Short string with length 63, the max value a 6-bit length field can hold, then 63 bytes. + auto stream = cudf::test::get_default_stream(); + std::vector val; + val.push_back(make_variant_short_string_header(63)); + std::string const payload(63, 'z'); + val.insert(val.end(), payload.begin(), payload.end()); + cudf::test::lists_column_wrapper values(val.begin(), val.end()); + auto got = cudf::io::parquet::experimental::cast_variant( + values, cudf::data_type{cudf::type_id::STRING}, stream); + cudf::test::strings_column_wrapper expected({payload}); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); +} + +TEST_F(CastVariantTest, LongStringLengthZero) +{ + // Long string: primitive long_string header, 4-byte LE length = 0, no payload. + auto stream = cudf::test::get_default_stream(); + std::vector const val{ + make_variant_primitive(variant_primitive_type::LONG_STRING), 0x00, 0x00, 0x00, 0x00}; + cudf::test::lists_column_wrapper values(val.begin(), val.end()); + auto got = cudf::io::parquet::experimental::cast_variant( + values, cudf::data_type{cudf::type_id::STRING}, stream); + cudf::test::strings_column_wrapper expected({""}); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); +} + +TEST_F(CastVariantTest, LongStringDeclaredLengthExceedsPayloadYieldsNull) +{ + // decode_string rejects any long string whose declared LE length exceeds the payload bytes + // actually present, whether the payload is partially present or entirely absent. Both shapes + // below declare length=10 (0x0000000A). + auto stream = cudf::test::get_default_stream(); + auto const hdr = make_variant_primitive(variant_primitive_type::LONG_STRING); + std::vector> const cases{ + {hdr, 0x0A, 0x00, 0x00, 0x00, 'a', 'b', 'c'}, // 3 of 10 payload bytes present + {hdr, 0x0A, 0x00, 0x00, 0x00}, // 0 of 10 payload bytes present + }; + for (auto const& val : cases) { + SCOPED_TRACE(std::string{"payload bytes present: "} + std::to_string(val.size() - 5)); + cudf::test::lists_column_wrapper values(val.begin(), val.end()); + auto got = cudf::io::parquet::experimental::cast_variant( + values, cudf::data_type{cudf::type_id::STRING}, stream); + ASSERT_EQ(got->size(), 1); + EXPECT_EQ(got->null_count(), 1); + } +} + +TEST_F(CastVariantTest, LongStringPayloadExceedsDeclaredLength) +{ + // When more bytes are present than the declared length, decode_string should read exactly the + // declared number of bytes and ignore the trailing ones. + auto stream = cudf::test::get_default_stream(); + auto const hdr = make_variant_primitive(variant_primitive_type::LONG_STRING); + // Declared length = 3 ("abc"), followed by 5 extra bytes that must be ignored. + std::vector const val{ + hdr, 0x03, 0x00, 0x00, 0x00, 'a', 'b', 'c', 'x', 'x', 'x', 'x', 'x'}; + cudf::test::lists_column_wrapper values(val.begin(), val.end()); + auto got = cudf::io::parquet::experimental::cast_variant( + values, cudf::data_type{cudf::type_id::STRING}, stream); + cudf::test::strings_column_wrapper expected({"abc"}); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*got, expected); +} + +struct InvalidInputShapeTest : public cudf::test::BaseFixture {}; + +namespace { + +// A well-formed VARIANT child: a single-row list holding `bytes`. +inline std::unique_ptr list_u8(std::vector const& bytes) +{ + return cudf::test::lists_column_wrapper(bytes.begin(), bytes.end()).release(); +} + +// A single-row list (wrong element type for a VARIANT child). +inline std::unique_ptr list_i32(std::vector const& values) +{ + return cudf::test::lists_column_wrapper(values.begin(), values.end()).release(); +} + +// A single-row fixed-width int32 column (a non-list child). +inline std::unique_ptr scalar_i32() +{ + return cudf::test::fixed_width_column_wrapper{42}.release(); +} + +// A single-row STRUCT column adopting `children`. +inline std::unique_ptr struct_of(std::vector> children) +{ + return cudf::make_structs_column(1, std::move(children), 0, rmm::device_buffer{}); +} + +inline std::vector> two_children(std::unique_ptr a, + std::unique_ptr b) +{ + std::vector> v; + v.push_back(std::move(a)); + v.push_back(std::move(b)); + return v; +} + +// A malformed-shape case: a human-readable label plus the offending column. +struct broken_shape { + std::string label; + std::unique_ptr column; +}; + +} // namespace + +// A VARIANT column must be a STRUCT whose first two children are each a list. Enumerate the +// distinct ways that column-shape contract can be broken; get_variant_field must reject every one +// with std::invalid_argument. +TEST_F(InvalidInputShapeTest, GetVariantFieldRejectsMalformedInput) +{ + auto stream = cudf::test::get_default_stream(); + + std::vector cases; + cases.push_back({"input column is not a struct", scalar_i32()}); + { + std::vector> one; + one.push_back(list_u8({0x01, 0x00, 0x00})); + cases.push_back({"struct has fewer than two children", struct_of(std::move(one))}); + } + cases.push_back({"metadata child has wrong column type (not a list)", + struct_of(two_children(scalar_i32(), list_u8({0x00})))}); + cases.push_back({"metadata child has wrong list element type (not uint8)", + struct_of(two_children(list_i32({1, 2, 3}), list_u8({0x00})))}); + cases.push_back({"value child has wrong column type (not a list)", + struct_of(two_children(list_u8({0x01, 0x00, 0x00}), scalar_i32()))}); + cases.push_back({"value child has wrong list element type (not uint8)", + struct_of(two_children(list_u8({0x01, 0x00, 0x00}), list_i32({1, 2, 3})))}); + + for (auto const& c : cases) { + SCOPED_TRACE(c.label); + EXPECT_THROW(static_cast(cudf::io::parquet::experimental::get_variant_field( + c.column->view(), "x", stream)), + std::invalid_argument); + } +} + +// cast_variant requires a list input; every other shape must be rejected with +// std::invalid_argument. +TEST_F(InvalidInputShapeTest, CastVariantRejectsMalformedInput) +{ + auto stream = cudf::test::get_default_stream(); + + std::vector cases; + cases.push_back({"input is not a list", scalar_i32()}); + cases.push_back({"input list has wrong element type (not uint8)", list_i32({1, 2, 3})}); + + for (auto const& c : cases) { + SCOPED_TRACE(c.label); + EXPECT_THROW(static_cast(cudf::io::parquet::experimental::cast_variant( + c.column->view(), cudf::data_type{cudf::type_id::INT32}, stream)), + std::invalid_argument); + } +}