Support fixed point types in Parquet predicate pushdown

cpp/src/io/parquet/experimental/page_index_filter.cu

@@ -38,6 +38,7 @@
 
 #include <algorithm>
 #include <limits>
+#include <numeric>
 
 namespace cudf::io::parquet::experimental::detail {
 
@@ -911,7 +912,7 @@ std::unique_ptr<cudf::column> aggregate_reader_metadata::build_row_mask_with_pag
     [&](auto col_idx) {
       auto const schema_idx = output_column_schemas[col_idx];
       auto const& dtype     = output_dtypes[col_idx];
-      // Only participating columns and comparable types except fixed point are supported
+      // Only participating columns and comparable types are supported
       if (not stats_columns_mask[col_idx] or
           (cudf::is_compound(dtype) && dtype.id() != cudf::type_id::STRING)) {
         // Placeholder for unsupported types and non-participating columns

cpp/src/io/parquet/predicate_pushdown.cpp

@@ -146,7 +146,7 @@ std::optional<std::vector<std::vector<size_type>>> aggregate_reader_metadata::ap
   for (size_t col_idx = 0; col_idx < output_dtypes.size(); col_idx++) {
     auto const schema_idx = output_column_schemas[col_idx];
     auto const& dtype     = output_dtypes[col_idx];
-    // Only participating columns and comparable types except fixed point are supported
+    // Only participating columns and comparable types are supported
     if (not stats_columns_mask[col_idx] or
         (cudf::is_compound(dtype) && dtype.id() != cudf::type_id::STRING)) {
       // Placeholder for unsupported types and non-participating columns

cpp/src/io/parquet/stats_filter_helpers.hpp

@@ -18,8 +18,9 @@
 #include <cudf/utilities/traits.hpp>
 
 #include <algorithm>
-#include <numeric>
-#include <string>
+#include <array>
+#include <bit>
+#include <string_view>
 
 namespace cudf::io::parquet::detail {
 
@@ -38,8 +39,34 @@ constexpr size_t initial_chars_capacity = 1024;
  */
 class stats_caster_base {
  protected:
+  static inline numeric::decimal128::rep decode_flba_decimal128(uint8_t const* stats_val)
+  {
+    auto constexpr endianness = std::endian::native;
+    static_assert(endianness == std::endian::little or endianness == std::endian::big,
+                  "Encountered unsupported endianness while decoding decimal128 from FLBA");
+    using RepType = numeric::decimal128::rep;
+    auto value    = RepType{};
+    std::memcpy(&value, stats_val, sizeof(RepType));
+    auto value_rep = std::bit_cast<std::array<std::byte, sizeof(RepType)>>(value);
+    // byte-swap to native representation on little-endian platforms
+    if constexpr (endianness == std::endian::little) { std::ranges::reverse(value_rep); }
+    return std::bit_cast<RepType>(value_rep);
+  }
+
+  template <typename T>
+  static inline T decode_fixed_width_value(uint8_t const* stats_val, size_t stats_size)
+    requires((cudf::is_integral<T>() and !cudf::is_boolean<T>()) or cudf::is_fixed_point<T>() or
+             cudf::is_chrono<T>())
+  {
+    CUDF_EXPECTS(stats_size == sizeof(T),
+                 "Parquet reader encountered a statistics vector larger than the type's size");
+    auto value = T{};
+    std::memcpy(&value, stats_val, std::min(stats_size, sizeof(T)));
+    return value;
+  }
+
   template <typename ToType, typename FromType>
-  static inline ToType targetType(FromType const value)
+  static inline ToType target_type(FromType const value)
   {
     if constexpr (cudf::is_timestamp<ToType>()) {
       return static_cast<ToType>(
@@ -52,64 +79,76 @@ class stats_caster_base {
   }
 
   // uses storage type as T
-  template <typename T, CUDF_ENABLE_IF(cudf::is_dictionary<T>() or cudf::is_nested<T>())>
+  template <typename T>
   static inline T convert(uint8_t const* stats_val, size_t stats_size, Type const type)
+    requires(cudf::is_dictionary<T>() or cudf::is_nested<T>())
   {
     CUDF_FAIL("unsupported type for stats casting");
   }
 
-  template <typename T, CUDF_ENABLE_IF(cudf::is_boolean<T>())>
+  template <typename T>
   static inline T convert(uint8_t const* stats_val, size_t stats_size, Type const type)
+    requires(cudf::is_boolean<T>())
   {
     CUDF_EXPECTS(type == Type::BOOLEAN, "Invalid type and stats combination");
-    return stats_caster_base::targetType<T>(*reinterpret_cast<bool const*>(stats_val));
+    return stats_caster_base::target_type<T>(*reinterpret_cast<bool const*>(stats_val));
   }
 
   // integral but not boolean, and fixed_point, and chrono.
-  template <typename T,
-            CUDF_ENABLE_IF((cudf::is_integral<T>() and !cudf::is_boolean<T>()) or
-                           cudf::is_fixed_point<T>() or cudf::is_chrono<T>())>
+  template <typename T>
   static inline T convert(uint8_t const* stats_val, size_t stats_size, Type const type)
+    requires((cudf::is_integral<T>() and !cudf::is_boolean<T>()) or cudf::is_fixed_point<T>() or
+             cudf::is_chrono<T>())
   {
     switch (type) {
       case Type::INT32:
-        return stats_caster_base::targetType<T>(*reinterpret_cast<int32_t const*>(stats_val));
+        return stats_caster_base::target_type<T>(
+          decode_fixed_width_value<int32_t>(stats_val, stats_size));
       case Type::INT64:
-        return stats_caster_base::targetType<T>(*reinterpret_cast<int64_t const*>(stats_val));
+        return stats_caster_base::target_type<T>(
+          decode_fixed_width_value<int64_t>(stats_val, stats_size));
       case Type::INT96:  // Deprecated in parquet specification
-        return stats_caster_base::targetType<T>(
-          static_cast<__int128_t>(reinterpret_cast<int64_t const*>(stats_val)[0]) << 32 |
-          reinterpret_cast<int32_t const*>(stats_val)[2]);
+        return stats_caster_base::target_type<T>(
+          static_cast<__int128_t>(decode_fixed_width_value<int64_t>(stats_val, stats_size)) << 32 |
+          decode_fixed_width_value<int32_t>(stats_val + sizeof(int64_t), stats_size));
       case Type::BYTE_ARRAY: [[fallthrough]];
       case Type::FIXED_LEN_BYTE_ARRAY:
         if (stats_size == sizeof(T)) {
-          // if type size == length of stats_val. then typecast and return.
           if constexpr (cudf::is_chrono<T>()) {
-            return stats_caster_base::targetType<T>(
-              *reinterpret_cast<typename T::rep const*>(stats_val));
+            return stats_caster_base::target_type<T>(
+              decode_fixed_width_value<typename T::rep>(stats_val, stats_size));
+          } else if constexpr (std::is_same_v<T, numeric::decimal128::rep>) {
+            // Decimals with physical type FLBA/BYTE_ARRAY are stored as two's complement using
+            // big-endian.
+            return stats_caster_base::target_type<T>(decode_flba_decimal128(stats_val));
           } else {
-            return stats_caster_base::targetType<T>(*reinterpret_cast<T const*>(stats_val));
+            // TODO(mh): We may need to add support for `decimal256` (two's complement using
+            // big-endian) and `UUID` types (big-endian)
+            return stats_caster_base::target_type<T>(
+              decode_fixed_width_value<T>(stats_val, stats_size));
           }
         }
         // unsupported type
       default: CUDF_FAIL("Invalid type and stats combination");
     }
   }
 
-  template <typename T, CUDF_ENABLE_IF(cudf::is_floating_point<T>())>
+  template <typename T>
   static inline T convert(uint8_t const* stats_val, size_t stats_size, Type const type)
+    requires(cudf::is_floating_point<T>())
   {
     switch (type) {
       case Type::FLOAT:
-        return stats_caster_base::targetType<T>(*reinterpret_cast<float const*>(stats_val));
+        return stats_caster_base::target_type<T>(*reinterpret_cast<float const*>(stats_val));
       case Type::DOUBLE:
-        return stats_caster_base::targetType<T>(*reinterpret_cast<double const*>(stats_val));
+        return stats_caster_base::target_type<T>(*reinterpret_cast<double const*>(stats_val));
       default: CUDF_FAIL("Invalid type and stats combination");
     }
   }
 
-  template <typename T, CUDF_ENABLE_IF(std::is_same_v<T, string_view>)>
+  template <typename T>
   static inline T convert(uint8_t const* stats_val, size_t stats_size, Type const type)
+    requires(std::is_same_v<T, string_view>)
   {
     switch (type) {
       case Type::BYTE_ARRAY: [[fallthrough]];

cpp/tests/io/experimental/hybrid_scan_filters_test.cpp

@@ -1193,9 +1193,9 @@ TEST_F(HybridScanFiltersTest, FilterRowGroupsWithDictionary)
 template <typename T>
 struct RowGroupFilteringWithDictTest : public HybridScanFiltersTest {};
 
-// Booleans are not supported for dictionary based filtering
+// Booleans and fixed-point types are not supported for dictionary based filtering
 using DictionaryTestTypes =
-  cudf::test::RemoveIf<cudf::test::ContainedIn<cudf::test::Types<bool>>, SupportedTestTypes>;
+  cudf::test::RemoveIf<cudf::test::ContainedIn<cudf::test::Types<bool>>, SupportedTestTypesJIT>;
 
 TYPED_TEST_SUITE(RowGroupFilteringWithDictTest, DictionaryTestTypes);
 

cpp/tests/io/parquet_common.hpp

@@ -45,9 +45,14 @@ using ByteLikeTypes = cudf::test::Types<int8_t, char, uint8_t, unsigned char, st
 // them.
 using UnsupportedChronoTypes =
   cudf::test::Types<cudf::timestamp_s, cudf::duration_D, cudf::duration_s>;
-// Also fixed point types unsupported, because AST does not support them yet.
-using SupportedTestTypes = cudf::test::RemoveIf<cudf::test::ContainedIn<UnsupportedChronoTypes>,
-                                                cudf::test::ComparableTypes>;
+
+// Support types for AST expression evaluator
+using SupportedTestTypesAST =
+  cudf::test::RemoveIf<cudf::test::ContainedIn<UnsupportedChronoTypes>, ComparableAndFixedTypes>;
+
+// JIT does not yet support fixed point types
+using SupportedTestTypesJIT =
+  cudf::test::RemoveIf<cudf::test::ContainedIn<cudf::test::FixedPointTypes>, SupportedTestTypesAST>;
 
 // removing duration_D, duration_s, and timestamp_s as they don't appear to be supported properly.
 // see definition of UnsupportedChronoTypes above.