[blocked] Switch to Utf8View for TPC-H

Cargo.toml

@@ -104,6 +104,7 @@ serde = "1.0.197"
 serde_json = "1.0.116"
 serde_test = "1.0.176"
 simplelog = { version = "0.12.2", features = ["paris"] }
+static_assertions = "1.1.0"
 thiserror = "1.0.58"
 tokio = "1.37.0"
 uninit = "0.6.2"

vortex-array/Cargo.toml

@@ -50,6 +50,7 @@ vortex-scalar = { path = "../vortex-scalar", features = [
     "serde",
 ] }
 serde = { workspace = true, features = ["derive"] }
+static_assertions = { workspace = true }
 
 [target.'cfg(target_arch = "wasm32")'.dependencies]
 # Enable the JS feature of getrandom (via rand) to supprt wasm32 target

vortex-array/src/array/chunked/canonical.rs

@@ -196,33 +196,32 @@ fn pack_primitives(
 ///
 /// It is expected this function is only called from [try_canonicalize_chunks], and thus all chunks have
 /// been checked to have the same DType already.
-#[allow(unused)]
 fn pack_views(
     chunks: &[Array],
     dtype: &DType,
     nullability: Nullability,
 ) -> VortexResult<VarBinViewArray> {
-    let length: usize = chunks.iter().map(|c| c.len()).sum();
     let validity = validity_from_chunks(chunks, nullability);
     let mut views = Vec::new();
     let mut buffers = Vec::new();
     for chunk in chunks {
-        // Each chunk's views have block IDs that are zero-referenced.
+        // Each chunk's views have buffer IDs that are zero-referenced.
         // As part of the packing operation, we need to rewrite them to be referenced to the global
-        // merged blocks list.
+        // merged buffers list.
         let buffers_offset = buffers.len();
         let canonical_chunk = chunk.clone().into_varbinview().unwrap();
 
-        for block in canonical_chunk.buffers() {
-            let canonical_buffer = block.into_canonical().unwrap().into_array();
+        for buffer in canonical_chunk.buffers() {
+            let canonical_buffer = buffer.into_canonical().unwrap().into_array();
             buffers.push(canonical_buffer);
         }
 
-        // Write a new primitive array with all of the view outputs.
         for view in canonical_chunk.view_slice() {
             if view.is_inlined() {
+                // Inlined views can be copied directly into the output
                 views.push(*view);
             } else {
+                // Referencing views must have their buffer_index adjusted with new offsets
                 let view_ref = view.as_view();
                 views.push(BinaryView::new_view(
                     view.len(),
@@ -235,7 +234,8 @@ fn pack_views(
     }
 
     let (view_ptr, view_len, view_cap) = views.into_raw_parts();
-    // Transmute the pointer to be to a set of u128, which is of identical size.
+    // Transmute the pointer to target type u128, which is of identical size and is
+    // an Arrow native type.
     let views_u128 = unsafe {
         Vec::from_raw_parts(
             std::mem::transmute::<*mut BinaryView, *mut u128>(view_ptr),

vortex-array/src/array/varbinview/compute.rs

@@ -3,7 +3,7 @@ use vortex_error::VortexResult;
 use vortex_scalar::Scalar;
 
 use crate::array::varbin::varbin_scalar;
-use crate::array::varbinview::{VarBinViewArray, VIEW_SIZE};
+use crate::array::varbinview::{VarBinViewArray, VIEW_SIZE_BYTES};
 use crate::arrow::FromArrowArray;
 use crate::compute::unary::ScalarAtFn;
 use crate::compute::{slice, ArrayCompute, SliceFn, TakeFn};
@@ -38,7 +38,11 @@ impl ScalarAtFn for VarBinViewArray {
 impl SliceFn for VarBinViewArray {
     fn slice(&self, start: usize, stop: usize) -> VortexResult<Array> {
         Ok(Self::try_new(
-            slice(&self.views(), start * VIEW_SIZE, stop * VIEW_SIZE)?,
+            slice(
+                &self.views(),
+                start * VIEW_SIZE_BYTES,
+                stop * VIEW_SIZE_BYTES,
+            )?,
             (0..self.metadata().buffer_lens.len())
                 .map(|i| self.buffer(i))
                 .collect::<Vec<_>>(),

vortex-array/src/array/varbinview/mod.rs

@@ -7,6 +7,7 @@ use arrow_array::builder::{BinaryViewBuilder, StringViewBuilder};
 use arrow_array::{ArrayRef, BinaryViewArray, StringViewArray};
 use arrow_buffer::ScalarBuffer;
 use itertools::Itertools;
+use static_assertions::{assert_eq_align, assert_eq_size};
 use vortex_dtype::{DType, PType};
 use vortex_error::{vortex_bail, VortexResult};
 
@@ -96,6 +97,12 @@ pub union BinaryView {
     _ref: Ref,
 }
 
+/// BinaryView must be 16 bytes and have 8 byte alignment
+assert_eq_size!(BinaryView, [u8; 16]);
+assert_eq_size!(Inlined, [u8; 16]);
+assert_eq_size!(Ref, [u8; 16]);
+assert_eq_align!(BinaryView, u64);
+
 impl BinaryView {
     pub const MAX_INLINED_SIZE: usize = 12;
 
@@ -159,7 +166,7 @@ impl Debug for BinaryView {
 }
 
 // reminder: views are 16 bytes with 8-byte alignment
-pub(crate) const VIEW_SIZE: usize = size_of::<BinaryView>();
+pub(crate) const VIEW_SIZE_BYTES: usize = size_of::<BinaryView>();
 
 impl_encoding!("vortex.varbinview", 5u16, VarBinView);
 
@@ -218,7 +225,7 @@ impl VarBinViewArray {
             vortex_bail!("incorrect validity {:?}", validity);
         }
 
-        let num_views = views.len() / VIEW_SIZE;
+        let num_views = views.len() / VIEW_SIZE_BYTES;
         let metadata = VarBinViewMetadata {
             validity: validity.to_metadata(num_views)?,
             buffer_lens: buffers.iter().map(|a| a.len()).collect_vec(),
@@ -250,7 +257,7 @@ impl VarBinViewArray {
                     .expect("Views must be a primitive array")
                     .maybe_null_slice::<u8>()
                     .as_ptr() as _,
-                self.views().len() / VIEW_SIZE,
+                self.views().len() / VIEW_SIZE_BYTES,
             )
         }
     }
@@ -267,7 +274,7 @@ impl VarBinViewArray {
     #[inline]
     pub fn views(&self) -> Array {
         self.array()
-            .child(0, &DType::BYTES, self.len() * VIEW_SIZE)
+            .child(0, &DType::BYTES, self.len() * VIEW_SIZE_BYTES)
             .unwrap()
     }
 
@@ -481,7 +488,7 @@ impl<'a> FromIterator<Option<&'a str>> for VarBinViewArray {
 mod test {
     use vortex_scalar::Scalar;
 
-    use crate::array::varbinview::{BinaryView, VarBinViewArray, VIEW_SIZE};
+    use crate::array::varbinview::{BinaryView, VarBinViewArray, VIEW_SIZE_BYTES};
     use crate::compute::slice;
     use crate::compute::unary::scalar_at;
     use crate::{Canonical, IntoArray, IntoCanonical};
@@ -530,7 +537,7 @@ mod test {
 
     #[test]
     pub fn binary_view_size_and_alignment() {
-        assert_eq!(size_of::<BinaryView>(), VIEW_SIZE);
+        assert_eq!(size_of::<BinaryView>(), VIEW_SIZE_BYTES);
         assert_eq!(size_of::<BinaryView>(), 16);
         assert_eq!(align_of::<BinaryView>(), 8);
     }